From d142ccb159c70da4b1c1e4159f955f06b5deb7c2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 06:46:08 -0500 Subject: [PATCH 0001/1054] Use twisted.web.template in web/storage.py Related to ticket:3247. Nevow usage has been removed, and generated page looks the same as its former self, but tests are failing because test_storage.py assumes that we're using nevow. --- src/allmydata/web/storage.py | 258 ++++++++++++++----------- src/allmydata/web/storage_status.xhtml | 60 +++--- 2 files changed, 182 insertions(+), 136 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 79c0a8c38..3000fb70d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,10 +1,10 @@ import time, json -from nevow import rend, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile from allmydata.web.common import ( - getxmlfile, abbreviate_time, - MultiFormatPage, + MultiFormatResource ) from allmydata.util.abbreviate import abbreviate_space from allmydata.util import time_format, idlib @@ -16,91 +16,100 @@ def remove_prefix(s, prefix): return s[len(prefix):] -class StorageStatus(MultiFormatPage): - docFactory = getxmlfile("storage_status.xhtml") - # the default 'data' argument is the StorageServer instance +class StorageStatusElement(Element): + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname=""): - rend.Page.__init__(self, storage) + def __init__(self, storage, nickname): + super(StorageStatusElement, self).__init__() self.storage = storage - self.nickname = nickname + self.nick = nickname - def render_JSON(self, req): - req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), - } - return json.dumps(d, indent=1) + "\n" + @renderer + def nickname(self, req, tag): + return self.nick - def data_nickname(self, ctx, storage): - return self.nickname - def data_nodeid(self, ctx, storage): + @renderer + def nodeid(self, req, tag): return idlib.nodeid_b2a(self.storage.my_nodeid) - def render_storage_running(self, ctx, storage): - if storage: - return ctx.tag - else: - return T.h1["No Storage Server Running"] + def get_stat(self, key): + return self.storage.get_stats().get(key) - def render_bool(self, ctx, data): - return {True: "Yes", False: "No"}[bool(data)] + def str(self, tag, val): + if val is None: + return tag("?") + return tag(str(val)) - def render_abbrev_space(self, ctx, size): - if size is None: - return "?" - return abbreviate_space(size) + def abbr(self, tag, val): + if val is None: + return tag("?") + return tag(abbreviate_space(val)) - def render_space(self, ctx, size): - if size is None: - return "?" - return "%d" % size + @renderer + def disk_total(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_total")) - def data_stats(self, ctx, data): - # FYI: 'data' appears to be self, rather than the StorageServer - # object in self.original that gets passed to render_* methods. I - # still don't understand Nevow. + @renderer + def disk_total_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_total")) - # Nevow has nevow.accessors.DictionaryContainer: Any data= directive - # that appears in a context in which the current data is a dictionary - # will be looked up as keys in that dictionary. So if data_stats() - # returns a dictionary, then we can use something like this: - # - # + @renderer + def disk_used(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_used")) - # to use get_stats()["storage_server.disk_total"] . However, - # DictionaryContainer does a raw d[] instead of d.get(), so any - # missing keys will cause an error, even if the renderer can tolerate - # None values. To overcome this, we either need a dict-like object - # that always returns None for unknown keys, or we must pre-populate - # our dict with those missing keys, or we should get rid of data_ - # methods that return dicts (or find some way to override Nevow's - # handling of dictionaries). + @renderer + def disk_used_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_used")) - d = dict([ (remove_prefix(k, "storage_server."), v) - for k,v in self.storage.get_stats().items() ]) - d.setdefault("disk_total", None) - d.setdefault("disk_used", None) - d.setdefault("disk_free_for_root", None) - d.setdefault("disk_free_for_nonroot", None) - d.setdefault("reserved_space", None) - d.setdefault("disk_avail", None) - return d + @renderer + def disk_free_for_root(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) - def data_last_complete_bucket_count(self, ctx, data): + @renderer + def disk_free_for_root_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + + @renderer + def disk_free_for_nonroot(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def disk_free_for_nonroot_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def reserved_space(self, req, tag): + return self.str(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def reserved_space_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def disk_avail(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def disk_avail_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def accepting_immutable_shares(self, req, tag): + accepting = self.get_stat("storage_server.accepting_immutable_shares") + return {True: "Yes", False: "No"}[bool(accepting)] + + @renderer + def last_complete_bucket_count(self, req, tag): s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" - return count + return str(count) - def render_count_crawler_status(self, ctx, storage): + @renderer + def count_crawler_status(self, req, tag): p = self.storage.bucket_counter.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return self.format_crawler_progress(p) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] @@ -127,55 +136,51 @@ class StorageStatus(MultiFormatPage): return ["Next crawl in %s" % abbreviate_time(soon), cycletime_s] - def render_lease_expiration_enabled(self, ctx, data): + @renderer + def storage_running(self, req, tag): + if self.storage: + return tag + return tag("No Storage Server Running") + + @renderer + def lease_expiration_enabled(self, req, tag): lc = self.storage.lease_checker if lc.expiration_enabled: - return ctx.tag["Enabled: expired leases will be removed"] + return tag("Enabled: expired leases will be removed") else: - return ctx.tag["Disabled: scan-only mode, no leases will be removed"] + return tag("Disabled: scan-only mode, no leases will be removed") - def render_lease_expiration_mode(self, ctx, data): + @renderer + def lease_expiration_mode(self, req, tag): lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: - ctx.tag["Leases will expire naturally, probably 31 days after " - "creation or renewal."] + tag("Leases will expire naturally, probably 31 days after " + "creation or renewal.") else: - ctx.tag["Leases created or last renewed more than %s ago " - "will be considered expired." - % abbreviate_time(lc.override_lease_duration)] + tag("Leases created or last renewed more than %s ago " + "will be considered expired." + % abbreviate_time(lc.override_lease_duration)) else: assert lc.mode == "cutoff-date" localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date)) isoutcdate = time_format.iso_utc_date(lc.cutoff_date) - ctx.tag["Leases created or last renewed before %s (%s) UTC " - "will be considered expired." % (isoutcdate, localizedutcdate, )] + tag("Leases created or last renewed before %s (%s) UTC " + "will be considered expired." + % (isoutcdate, localizedutcdate, )) if len(lc.mode) > 2: - ctx.tag[" The following sharetypes will be expired: ", - " ".join(sorted(lc.sharetypes_to_expire)), "."] - return ctx.tag + tag(" The following sharetypes will be expired: ", + " ".join(sorted(lc.sharetypes_to_expire)), ".") + return tag - def format_recovered(self, sr, a): - def maybe(d): - if d is None: - return "?" - return "%d" % d - return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ - (maybe(sr["%s-shares" % a]), - maybe(sr["%s-buckets" % a]), - maybe(sr["%s-buckets-mutable" % a]), - maybe(sr["%s-buckets-immutable" % a]), - abbreviate_space(sr["%s-diskbytes" % a]), - abbreviate_space(sr["%s-diskbytes-mutable" % a]), - abbreviate_space(sr["%s-diskbytes-immutable" % a]), - ) - - def render_lease_current_cycle_progress(self, ctx, data): + @renderer + def lease_current_cycle_progress(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return tag(self.format_crawler_progress(p)) - def render_lease_current_cycle_results(self, ctx, data): + @renderer + def lease_current_cycle_results(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -229,10 +234,10 @@ class StorageStatus(MultiFormatPage): T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] ]]]) + return tag("Current cycle:", p) - return ctx.tag["Current cycle:", p] - - def render_lease_last_cycle_results(self, ctx, data): + @renderer + def lease_last_cycle_results(self, req, tag): lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: @@ -240,15 +245,15 @@ class StorageStatus(MultiFormatPage): last = h[max(h.keys())] start, end = last["cycle-start-finish-times"] - ctx.tag["Last complete cycle (which took %s and finished %s ago)" - " recovered: " % (abbreviate_time(end-start), - abbreviate_time(time.time() - end)), - self.format_recovered(last["space-recovered"], "actual") - ] + tag("Last complete cycle (which took %s and finished %s ago)" + " recovered: " % (abbreviate_time(end-start), + abbreviate_time(time.time() - end)), + self.format_recovered(last["space-recovered"], "actual")) p = T.ul() + def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) saw = self.format_recovered(last["space-recovered"], "examined") add("and saw a total of ", saw) @@ -264,4 +269,37 @@ class StorageStatus(MultiFormatPage): for corrupt_share in last["corrupt-shares"] ] ]]]) - return ctx.tag[p] + return tag(p) + + def format_recovered(self, sr, a): + def maybe(d): + if d is None: + return "?" + return "%d" % d + return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ + (maybe(sr["%s-shares" % a]), + maybe(sr["%s-buckets" % a]), + maybe(sr["%s-buckets-mutable" % a]), + maybe(sr["%s-buckets-immutable" % a]), + abbreviate_space(sr["%s-diskbytes" % a]), + abbreviate_space(sr["%s-diskbytes-mutable" % a]), + abbreviate_space(sr["%s-diskbytes-immutable" % a]), + ) + +class StorageStatus(MultiFormatResource): + def __init__(self, storage, nickname=""): + super(StorageStatus, self).__init__() + self.storage = storage + self.nickname = nickname + + def render_HTML(self, req): + return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") + d = {"stats": self.storage.get_stats(), + "bucket-counter": self.storage.bucket_counter.get_state(), + "lease-checker": self.storage.lease_checker.get_state(), + "lease-checker-progress": self.storage.lease_checker.get_progress(), + } + return json.dumps(d, indent=1) + "\n" diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d97daf9af..cfd7a860c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Storage Server Status @@ -7,19 +7,19 @@ -
+

Storage Server Status

- +
- - + + - - + + - - + + - - + + - - + + - - + +
Total disk space:()()
Disk space used:- ()- ()
@@ -28,18 +28,18 @@
Disk space free (root):()() [see 1]
Disk space free (non-root):()() [see 2]
Reserved space:- ()- ()
@@ -48,23 +48,31 @@
Space Available to Tahoe:()()
    -
  • Server Nickname:
    • -
  • Server Nodeid:
    • -
  • Accepting new shares: -
    • +
  • Server Nickname: + + + +
    • +
  • Server Nodeid: + + + +
    • +
  • Accepting new shares: +
  • Total buckets: - + (the number of files and directories for which this server is holding a share)
      -
    • +
@@ -72,11 +80,11 @@

Lease Expiration Crawler

    -
  • Expiration
    • -
  • -
  • -
  • -
  • +
  • Expiration
    • +
  • +
  • +
  • +
From b29652e0f0d27aafd52771b3f7b57f511fcec076 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 22:59:45 -0500 Subject: [PATCH 0002/1054] Add `StorageStatus::renderSynchronously` Related to ticket:3247 test_storage.py wants a `StorageStatus::renderSynchronously()` method and a `StorageStatus::renderHTTP()` method. Let us begin with the goofy first-cut. Both these methods are not only wrong, but they will also not please the test suite. However error messages produced in CI can be shared, and that way I can hopefully get unstuck. --- src/allmydata/web/storage.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 3000fb70d..8f36cf44d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,7 +1,8 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile +from twisted.web.template import tags as T, \ + renderer, Element, renderElement, XMLFile from allmydata.web.common import ( abbreviate_time, MultiFormatResource @@ -303,3 +304,14 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self.storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" + + def renderSynchronously(self): + # to appease the test suite. + elem = StorageStatusElement(self.storage, self.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result + + def renderHTTP(self, ctx=None): + # to appease the test suite. + self.renderSynchronously() From d3790a4d4227f31f06a35042448a1a119fa4658e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 23:16:48 -0500 Subject: [PATCH 0003/1054] Add missing `flattenString` import --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 8f36cf44d..a912d79f6 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -2,7 +2,7 @@ import time, json from twisted.python.filepath import FilePath from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile + renderer, Element, renderElement, XMLFile, flattenString from allmydata.web.common import ( abbreviate_time, MultiFormatResource From c019c7e9556e31c599997ce6463fe679668c3a73 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:05:33 -0500 Subject: [PATCH 0004/1054] Second version of renderSynchronously --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a912d79f6..4ad5a2baf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -310,7 +310,7 @@ class StorageStatus(MultiFormatResource): elem = StorageStatusElement(self.storage, self.nickname) result = [] flattenString(None, elem).addCallback(result.append) - return result + return result[0] def renderHTTP(self, ctx=None): # to appease the test suite. From 4e81a3a0a2906a2e2c3774a126afa750b7f9bf3c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:09:16 -0500 Subject: [PATCH 0005/1054] Check storage server status before using it --- src/allmydata/web/storage.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 4ad5a2baf..9a99286bf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -31,9 +31,13 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): + if not self.storage: + return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def get_stat(self, key): + if not self.storage: + return None return self.storage.get_stats().get(key) def str(self, tag, val): @@ -101,6 +105,8 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): + if not self.storage: + return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -109,6 +115,8 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): + if not self.storage: + return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -145,6 +153,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -153,6 +163,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -176,12 +188,16 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -239,6 +255,8 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From c88c97aad540b928129944bdcd349df6c7df72de Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 23:09:39 -0500 Subject: [PATCH 0006/1054] Use right syntax for twisted.web.template tags --- src/allmydata/web/storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9a99286bf..1e206d8cf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -212,7 +212,7 @@ class StorageStatusElement(Element): p = T.ul() def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) def maybe(d): if d is None: @@ -248,9 +248,9 @@ class StorageStatusElement(Element): if so_far["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] - ]]]) + )))) return tag("Current cycle:", p) @renderer @@ -284,9 +284,9 @@ class StorageStatusElement(Element): if last["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in last["corrupt-shares"] ] - ]]]) + )))) return tag(p) From c7a63f957d4a476092f3bcff70e57dacc9d47c60 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:17:35 -0500 Subject: [PATCH 0007/1054] Refactor so that test_util pass --- src/allmydata/test/test_storage.py | 5 +++-- src/allmydata/web/storage.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9f3aee9b8..85f4dd2bd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,7 +30,8 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, remove_prefix +from allmydata.web.storage import StorageStatus, StorageStatusElement, \ + remove_prefix from allmydata.storage_client import ( _StorageServer, ) @@ -4208,7 +4209,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) def test_util(self): - w = StorageStatus(None) + w = StorageStatusElement(None, None) self.failUnlessEqual(w.render_space(None, None), "?") self.failUnlessEqual(w.render_space(None, 10e6), "10000000") self.failUnlessEqual(w.render_abbrev_space(None, None), "?") diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1e206d8cf..6b033bc5a 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,6 +50,16 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) + def render_abbrev_space(self, ctx, size): + if size is None: + return "?" + return abbreviate_space(size) + + def render_space(self, ctx, size): + if size is None: + return "?" + return "%d" % size + @renderer def disk_total(self, req, tag): return self.str(tag, self.get_stat("storage_server.disk_total")) From 0993e610468afe97876c9e4af4367ac9a6a7bde4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:20:45 -0500 Subject: [PATCH 0008/1054] Drop unused `ctx` argument from render_space methods --- src/allmydata/test/test_storage.py | 8 ++++---- src/allmydata/web/storage.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 85f4dd2bd..6f4c455f2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -4210,9 +4210,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_util(self): w = StorageStatusElement(None, None) - self.failUnlessEqual(w.render_space(None, None), "?") - self.failUnlessEqual(w.render_space(None, 10e6), "10000000") - self.failUnlessEqual(w.render_abbrev_space(None, None), "?") - self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB") + self.failUnlessEqual(w.render_space(None), "?") + self.failUnlessEqual(w.render_space(10e6), "10000000") + self.failUnlessEqual(w.render_abbrev_space(None), "?") + self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB") self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 6b033bc5a..88f099b24 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,12 +50,12 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) - def render_abbrev_space(self, ctx, size): + def render_abbrev_space(self, size): if size is None: return "?" return abbreviate_space(size) - def render_space(self, ctx, size): + def render_space(self, size): if size is None: return "?" return "%d" % size From d46df30bd059633d68dee3eb5d42ba5e375d0d5c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:49:11 -0500 Subject: [PATCH 0009/1054] Use render_space methods to render space --- src/allmydata/web/storage.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 88f099b24..496766b92 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -40,16 +40,6 @@ class StorageStatusElement(Element): return None return self.storage.get_stats().get(key) - def str(self, tag, val): - if val is None: - return tag("?") - return tag(str(val)) - - def abbr(self, tag, val): - if val is None: - return tag("?") - return tag(abbreviate_space(val)) - def render_abbrev_space(self, size): if size is None: return "?" @@ -62,51 +52,63 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - return self.str(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): From b14f36082c148119cc9814c56a02a4155447032a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 06:22:25 -0500 Subject: [PATCH 0010/1054] Use within table cells --- src/allmydata/web/storage_status.xhtml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index cfd7a860c..6e0e26008 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -13,13 +13,13 @@ - - + + - - + + - - + + - - + + - - + + - - + +
Total disk space:()()
Disk space used:- ()- ()
@@ -28,18 +28,18 @@
Disk space free (root):()() [see 1]
Disk space free (non-root):()() [see 2]
Reserved space:- ()- ()
@@ -48,8 +48,8 @@
Space Available to Tahoe:()()
From 2df2ae92d56722ee9d810a678d9b74fe197cb1ab Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 07:34:55 -0500 Subject: [PATCH 0011/1054] Fix test_storage.WebStatus.test_no_server failure --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 496766b92..c2e517dce 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -161,7 +161,7 @@ class StorageStatusElement(Element): def storage_running(self, req, tag): if self.storage: return tag - return tag("No Storage Server Running") + return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): From b1c78244abeab83e0061e7586b26d4b44911a85d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 08:19:29 -0500 Subject: [PATCH 0012/1054] Use parentheses in import statement --- src/allmydata/web/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index c2e517dce..d45541457 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,8 +1,14 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile, flattenString +from twisted.web.template import ( + Element, + XMLFile, + tags as T, + renderer, + renderElement, + flattenString +) from allmydata.web.common import ( abbreviate_time, MultiFormatResource From d3ff578640f85181fffd787b5209e7847033d8e9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 09:55:33 -0500 Subject: [PATCH 0013/1054] Use parentheses in test suite's import statement --- src/allmydata/test/test_storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 6f4c455f2..6956fc757 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,8 +30,11 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, StorageStatusElement, \ +from allmydata.web.storage import ( + StorageStatus, + StorageStatusElement, remove_prefix +) from allmydata.storage_client import ( _StorageServer, ) From 227d06fe64533ba45b1319b7f498e0df3492f670 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 17:02:14 -0500 Subject: [PATCH 0014/1054] Add docstrings to StorageStatusElement --- src/allmydata/web/storage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d45541457..df8354a38 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -24,9 +24,15 @@ def remove_prefix(s, prefix): class StorageStatusElement(Element): + """Class to render a storage status page.""" + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname): + def __init__(self, storage, nickname=""): + """ + :param _StorageServer storage: data about storage. + :param string nickname: friendly name for storage. + """ super(StorageStatusElement, self).__init__() self.storage = storage self.nick = nickname From f22417e51b4fadbba09cf2ad882605c5aecaef20 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:14:24 -0500 Subject: [PATCH 0015/1054] Rename function for clarity --- src/allmydata/web/storage.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index df8354a38..a5d65fa85 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -47,7 +47,7 @@ class StorageStatusElement(Element): return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) - def get_stat(self, key): + def _get_storage_stat(self, key): if not self.storage: return None return self.storage.get_stats().get(key) @@ -64,67 +64,67 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): - accepting = self.get_stat("storage_server.accepting_immutable_shares") + accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") return {True: "Yes", False: "No"}[bool(accepting)] @renderer From 6e9a4e30d733c2a08d801df03f1107d9177ec76e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:15:26 -0500 Subject: [PATCH 0016/1054] Add a docstring --- src/allmydata/web/storage.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a5d65fa85..263fa9344 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -48,6 +48,30 @@ class StorageStatusElement(Element): return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): + """Get storage server statistics. + + Storage Server keeps a dict that contains various usage and + latency statistics. The dict looks like this: + + { + 'storage_server.accepting_immutable_shares': 1, + 'storage_server.allocated': 0, + 'storage_server.disk_avail': 106539192320, + 'storage_server.disk_free_for_nonroot': 106539192320, + 'storage_server.disk_free_for_root': 154415284224, + 'storage_server.disk_total': 941088460800, + 'storage_server.disk_used': 786673176576, + 'storage_server.latencies.add-lease.01_0_percentile': None, + 'storage_server.latencies.add-lease.10_0_percentile': None, + ... + } + + ``StorageServer.get_stats()`` returns the above dict. Storage + status page uses a subset of the items in the dict, concerning + disk usage. + + :param str key: storage server statistic we want to know. + """ if not self.storage: return None return self.storage.get_stats().get(key) From 9b51bdf7faca3314e5dff2de22b8e035b9114fec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 12:25:20 -0500 Subject: [PATCH 0017/1054] Return result from renderHTTP Bogus renderHTTP, but test failures are now down to three. We just need to handle requests for JSON now. --- src/allmydata/web/storage.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 263fa9344..1656df70b 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -378,6 +378,18 @@ class StorageStatus(MultiFormatResource): flattenString(None, elem).addCallback(result.append) return result[0] + # to appease the test suite def renderHTTP(self, ctx=None): - # to appease the test suite. - self.renderSynchronously() + """Send HTML or JSON formatted data, based on request. + + This function contains a bit of nevow-ism, but since this is + only called from the test suite, the nevow-ism should go away + as we update things. + + :param _nevow.context.WovenContext ctx: context is passed on + from the test suite. We get a request out of this + context, and use the request to render a result. + + """ + from nevow.inevow import IRequest + return self.render(IRequest(ctx)) From 7a053ddeff85d32376c12582ca35a825e708704a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 15:29:42 -0500 Subject: [PATCH 0018/1054] Use explicit `None`-check on self.storage --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1656df70b..76a09fda3 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -43,7 +43,7 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if not self.storage: + if self.storage is None: return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) From e2fc1fc07ea56cd6b4805864d1d275170e0c261e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 16:49:15 -0500 Subject: [PATCH 0019/1054] Move `renderSynchronously` to test suite --- src/allmydata/test/test_storage.py | 36 +++++++++++++++++++----------- src/allmydata/web/storage.py | 10 +-------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 6956fc757..f37401cc2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -4,6 +4,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service +from twisted.web.template import flattenString from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -2963,6 +2964,15 @@ def remove_tags(s): s = re.sub(r'\s+', ' ', s) return s +def renderSynchronously(ss): + """ + :param _StorageStatus ss: a StorageStatus instance. + """ + elem = StorageStatusElement(ss.storage, ss.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result[0] + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -2999,7 +3009,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): w = StorageStatus(ss) # this sample is before the crawler has started doing anything - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -3022,7 +3032,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(state["last-complete-prefix"], ss.bucket_counter.prefixes[0]) ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn(" Current crawl ", s) self.failUnlessIn(" (next work in ", s) @@ -3034,7 +3044,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): d.addCallback(lambda ignored: self.poll(_watch)) def _check2(ignored): ss.bucket_counter.cpu_slice = orig_cpu_slice - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("Total buckets: 0 (the number of", s) self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) @@ -3096,20 +3106,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): def _check_1(ignored): # no ETA is available yet - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (next work", s) def _check_2(ignored): # one prefix has finished, so an ETA based upon that elapsed time # should be available. - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) def _check_3(ignored): # two prefixes have finished - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) d.callback("done") @@ -4064,7 +4074,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_no_server(self): w = StorageStatus(None) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

No Storage Server Running

", html) def test_status(self): @@ -4110,7 +4120,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -4130,7 +4140,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4166,7 +4176,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) @@ -4184,7 +4194,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4195,7 +4205,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) @@ -4206,7 +4216,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 76a09fda3..e9c2f6bb5 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -6,8 +6,7 @@ from twisted.web.template import ( XMLFile, tags as T, renderer, - renderElement, - flattenString + renderElement ) from allmydata.web.common import ( abbreviate_time, @@ -371,13 +370,6 @@ class StorageStatus(MultiFormatResource): } return json.dumps(d, indent=1) + "\n" - def renderSynchronously(self): - # to appease the test suite. - elem = StorageStatusElement(self.storage, self.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] - # to appease the test suite def renderHTTP(self, ctx=None): """Send HTML or JSON formatted data, based on request. From c061f6830e66ff7770bc0af43eb0442fbdf0f441 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:00:54 -0500 Subject: [PATCH 0020/1054] Use `successResultOf` in `renderSynchronously` Get rid of [].append trick when dealing with the deferred. --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f37401cc2..0a176bd60 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2969,9 +2969,8 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss.storage, ss.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] + deferred = flattenString(None, elem) + return unittest.TestCase().successResultOf(deferred) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): From aab940f65fb95815c9998b79876adeda5a0fb5f8 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:18:37 -0500 Subject: [PATCH 0021/1054] Remove redundant `None`-checks on `self.storage` When no storage is up, `storage_running()` renderer will return a big honking `no storage server running` message, and no further renderers will be invoked. Therefore the extra defense is probably not required. (I tested this hypothesis. The extra defense is not required, unless there's something I have not seen.) --- src/allmydata/web/storage.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index e9c2f6bb5..cbee2780f 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -42,8 +42,6 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if self.storage is None: - return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): @@ -71,8 +69,6 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - if not self.storage: - return None return self.storage.get_stats().get(key) def render_abbrev_space(self, size): @@ -152,8 +148,6 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - if not self.storage: - return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -162,8 +156,6 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - if not self.storage: - return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -200,8 +192,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -210,8 +200,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -235,16 +223,12 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -302,8 +286,6 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From 0382b1ec63cfdcebf5f76f03ee939edbd0901d35 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:23:35 -0500 Subject: [PATCH 0022/1054] Use Unicode strings to render space --- src/allmydata/web/storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index cbee2780f..607eeff79 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -73,13 +73,13 @@ class StorageStatusElement(Element): def render_abbrev_space(self, size): if size is None: - return "?" + return u"?" return abbreviate_space(size) def render_space(self, size): if size is None: - return "?" - return "%d" % size + return u"?" + return u"%d" % size @renderer def disk_total(self, req, tag): From 36a486426e970d86fb2dea42ab9ffba0dfc6238c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 22:02:30 -0500 Subject: [PATCH 0023/1054] Mark `format_recovered` as static method --- src/allmydata/web/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 607eeff79..d670c4231 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -319,7 +319,8 @@ class StorageStatusElement(Element): return tag(p) - def format_recovered(self, sr, a): + @staticmethod + def format_recovered(sr, a): def maybe(d): if d is None: return "?" From 9bb7812148b7aa17cfa641f080ab8e3eb2bce642 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 07:25:49 -0500 Subject: [PATCH 0024/1054] Add news fragment --- newsfragments/3247.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3247.minor diff --git a/newsfragments/3247.minor b/newsfragments/3247.minor new file mode 100644 index 000000000..e69de29bb From 7625d959bc2735816710af01e80073b102312e17 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 08:26:44 -0500 Subject: [PATCH 0025/1054] Use to render node nickname and id CI did not like the old way, but it passed in my system. Odd. --- src/allmydata/web/storage_status.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 6e0e26008..8f74c478c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,12 +57,12 @@
  • Server Nickname: - +
  • Server Nodeid: - +
  • Accepting new shares: From ff019e5b12d3210046e269519a1e474f1d99ceb3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 10:10:48 -0500 Subject: [PATCH 0026/1054] Use BeautifulSoup to check favicon in storage page --- src/allmydata/test/web/test_web.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f84923521..19753d3b7 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -963,8 +963,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_storage(self): d = self.GET("/storage") def _check(res): - self.failUnlessIn('Storage Server Status', res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_text(self, soup, 'Storage Server Status') + assert_soup_has_favicon(self, soup) res_u = res.decode('utf-8') self.failUnlessIn(u'
  • Server Nickname: fake_nickname \u263A
    • ', res_u) d.addCallback(_check) From 0cbe2871fd80ffd186d54848995c6b6c91ee5ca1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 11:42:42 -0500 Subject: [PATCH 0027/1054] Give the suite the precise string it wants --- src/allmydata/web/storage_status.xhtml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 8f74c478c..d052fbabd 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -55,16 +55,8 @@
      -
    • Server Nickname: - - - -
      • -
    • Server Nodeid: - - - -
      • +
    • Server Nickname:
      • +
    • Server Nodeid:
    • Accepting new shares:
    • Total buckets: From 6c3256517a0fa8cd1b8c00267d0f80325f868a70 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Feb 2020 12:52:48 -0500 Subject: [PATCH 0028/1054] Use slots to render storage stats table --- src/allmydata/web/storage.py | 80 ++++++++------------------ src/allmydata/web/storage_status.xhtml | 26 ++++----- 2 files changed, 36 insertions(+), 70 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d670c4231..d26f4a5dd 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -82,64 +82,30 @@ class StorageStatusElement(Element): return u"%d" % size @renderer - def disk_total(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_space(val)) + def storage_stats(self, req, tag): + # Render storage status table that appears near the top of the page. + total = self._get_storage_stat("storage_server.disk_total") + used = self._get_storage_stat("storage_server.disk_used") + free_root = self._get_storage_stat("storage_server.disk_free_for_root") + free_nonroot = self._get_storage_stat("storage_server.disk_free_for_nonroot") + reserved = self._get_storage_stat("storage_server.reserved_space") + available = self._get_storage_stat("storage_server.disk_avail") - @renderer - def disk_total_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_used(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_space(val)) - - @renderer - def disk_used_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_root(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_root_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_nonroot(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_nonroot_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_abbrev_space(val)) - - @renderer - def reserved_space(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_space(val)) - - @renderer - def reserved_space_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_avail(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_space(val)) - - @renderer - def disk_avail_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_abbrev_space(val)) + tag.fillSlots( + disk_total = self.render_space(total), + disk_total_abbrev = self.render_abbrev_space(total), + disk_used = self.render_space(used), + disk_used_abbrev = self.render_abbrev_space(used), + disk_free_for_root = self.render_space(free_root), + disk_free_for_root_abbrev = self.render_abbrev_space(free_root), + disk_free_for_nonroot = self.render_space(free_nonroot), + disk_free_for_nonroot_abbrev = self.render_abbrev_space(free_nonroot), + reserved_space = self.render_space(reserved), + reserved_space_abbrev = self.render_abbrev_space(reserved), + disk_avail = self.render_space(available), + disk_avail_abbrev = self.render_abbrev_space(available) + ) + return tag @renderer def accepting_immutable_shares(self, req, tag): diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d052fbabd..bbf3d2c8a 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -11,15 +11,15 @@

      Storage Server Status

      - +
      - - + + - - + + - - + + - - + + - - + + - - + +
      Total disk space:()()
      Disk space used:- ()- ()
      @@ -28,18 +28,18 @@
      Disk space free (root):()() [see 1]
      Disk space free (non-root):()() [see 2]
      Reserved space:- ()- ()
      @@ -48,8 +48,8 @@
      Space Available to Tahoe:()()
      From 554c477cea5a3c777794923d406137aa1be46f18 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 25 Feb 2020 08:52:45 -0500 Subject: [PATCH 0029/1054] Prefix member variable with "_" --- src/allmydata/test/test_storage.py | 2 +- src/allmydata/web/storage.py | 40 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0a176bd60..56d281688 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2968,7 +2968,7 @@ def renderSynchronously(ss): """ :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss.storage, ss.nickname) + elem = StorageStatusElement(ss._storage, ss._nickname) deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d26f4a5dd..9b3daa0cb 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -33,16 +33,16 @@ class StorageStatusElement(Element): :param string nickname: friendly name for storage. """ super(StorageStatusElement, self).__init__() - self.storage = storage - self.nick = nickname + self._storage = storage + self._nickname = nickname @renderer def nickname(self, req, tag): - return self.nick + return self._nickname @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self.storage.my_nodeid) + return idlib.nodeid_b2a(self._storage.my_nodeid) def _get_storage_stat(self, key): """Get storage server statistics. @@ -69,7 +69,7 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - return self.storage.get_stats().get(key) + return self._storage.get_stats().get(key) def render_abbrev_space(self, size): if size is None: @@ -114,7 +114,7 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - s = self.storage.bucket_counter.get_state() + s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" @@ -122,7 +122,7 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - p = self.storage.bucket_counter.get_progress() + p = self._storage.bucket_counter.get_progress() return self.format_crawler_progress(p) def format_crawler_progress(self, p): @@ -152,13 +152,13 @@ class StorageStatusElement(Element): @renderer def storage_running(self, req, tag): - if self.storage: + if self._storage: return tag return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") else: @@ -166,7 +166,7 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: tag("Leases will expire naturally, probably 31 days after " @@ -189,13 +189,13 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: return "" @@ -252,7 +252,7 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker h = lc.get_state()["history"] if not h: return "" @@ -304,18 +304,18 @@ class StorageStatusElement(Element): class StorageStatus(MultiFormatResource): def __init__(self, storage, nickname=""): super(StorageStatus, self).__init__() - self.storage = storage - self.nickname = nickname + self._storage = storage + self._nickname = nickname def render_HTML(self, req): - return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + return renderElement(req, StorageStatusElement(self._storage, self._nickname)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), + d = {"stats": self._storage.get_stats(), + "bucket-counter": self._storage.bucket_counter.get_state(), + "lease-checker": self._storage.lease_checker.get_state(), + "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" From 110734daf065e7b2706e7d92b2361c21954c11eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:45:18 -0500 Subject: [PATCH 0030/1054] Use a helper to exercise render() in storage test cases --- src/allmydata/test/test_storage.py | 50 ++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 56d281688..86b23cd48 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,6 +30,7 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin +from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2972,6 +2973,29 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) +def renderDeferred(resource, **kwargs): + """ + Use this to exercise an overridden MultiFormatResource.render(), + usually for output=json or render_GET. It returns a Deferred. + + :param _MultiFormatResource resource: an HTTP resource to be rendered. + + """ + # We should be using twisted.web's DummyRequest here instead of + # nevow's FakeRequest, but right now it is a bit of a problem: see + # web/common.py. MultiFormatResource.render() makes a get_arg() + # call, which does a IRequest(ctx_or_req). IRequest can handle + # FakeRequest, but it can't handle DummyRequest. + req = FakeRequest(**kwargs) + req.fields = None + d = defer.maybeDeferred(resource.render, req) + def _done(res): + if isinstance(res, str): + return res + req.v + return req.v + d.addCallback(_done) + return d + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -3291,7 +3315,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failIfEqual(sr2["configured-diskbytes"], None) self.failIfEqual(sr2["original-sharebytes"], None) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) self.failUnlessIn("So far, this cycle has examined " @@ -3366,7 +3390,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_leases(mutable_si_2), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("recovered: 0 shares, 0 buckets " @@ -3466,7 +3490,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3525,7 +3549,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) @@ -3610,7 +3634,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3671,7 +3695,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled:" @@ -3733,7 +3757,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_3), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: immutable.", s) @@ -3790,7 +3814,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_2), 0) self.failUnlessEqual(count_shares(mutable_si_3), 0) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: mutable.", s) @@ -4021,7 +4045,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): # it also turns all tuples into lists self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4046,7 +4070,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): corrupt_shares = last["corrupt-shares"] self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json_history) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html_history(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4060,7 +4084,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): @@ -4083,7 +4107,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, nodeid) ss.setServiceParent(self.s) w = StorageStatus(ss, "nickname") - d = self.render1(w) + d = renderDeferred(w) def _check_html(html): self.failUnlessIn("

      Storage Server Status

      ", html) s = remove_tags(html) @@ -4104,7 +4128,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d def test_status_no_disk_stats(self): From f1fe3a75884c2073310727a5f4c4127bfb928645 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:46:09 -0500 Subject: [PATCH 0031/1054] Get rid of WebRenderingMixin in storage test --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 86b23cd48..cff8c3e22 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -29,7 +29,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from allmydata.test.common_web import WebRenderingMixin from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( @@ -3185,7 +3184,7 @@ class InstrumentedStorageServer(StorageServer): class No_ST_BLOCKS_StorageServer(StorageServer): LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler -class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() @@ -4087,7 +4086,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d = renderDeferred(page, args={"t": ["json"]}) return d -class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class WebStatus(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() From bae32179bf467e8802d235e5bcd4c1c18411a5c9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:57:12 -0500 Subject: [PATCH 0032/1054] Remove StorageStatus.renderHTTP This was added to please the test suite. Pleased to remove it! --- src/allmydata/web/storage.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9b3daa0cb..63ac47282 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -318,19 +318,3 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" - - # to appease the test suite - def renderHTTP(self, ctx=None): - """Send HTML or JSON formatted data, based on request. - - This function contains a bit of nevow-ism, but since this is - only called from the test suite, the nevow-ism should go away - as we update things. - - :param _nevow.context.WovenContext ctx: context is passed on - from the test suite. We get a request out of this - context, and use the request to render a result. - - """ - from nevow.inevow import IRequest - return self.render(IRequest(ctx)) From 9c7357bc61dd74e4b15ae0da2d28a5275acd6b5e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 30 Mar 2020 16:15:44 -0400 Subject: [PATCH 0033/1054] Remove an extraneous directive `t:data` is not really a Twisted template directive. Added my mistake, removing now. --- src/allmydata/web/storage_status.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index bbf3d2c8a..354f9f177 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,7 +57,7 @@
      • Server Nickname:
      • Server Nodeid:
        • -
      • Accepting new shares: +
      • Accepting new shares:
      • Total buckets: From 8c92187d9263fa8d092c62c1305200191e850ec5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:00:55 -0400 Subject: [PATCH 0034/1054] Avoid using nevow FakeRequest in storage test. Use twisted.web.server.Request instead, with a DummyChannel. There's still one line of inevitable nevow now, because of code in web/common.py; but that should be easily replaceable once we switch that over. --- src/allmydata/test/test_storage.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index cff8c3e22..40b9b52e9 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -5,6 +5,16 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString + +# We need to use `nevow.inevow.IRequest` for now for compatibility +# with the code in web/common.py. Once nevow bits are gone from +# web/common.py, we can use `twisted.web.iweb.IRequest` here. +from nevow.inevow import IRequest + +from twisted.web.server import Request +from twisted.web.test.test_web import DummyChannel +from zope.interface import implements + from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -29,7 +39,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2972,28 +2981,20 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) -def renderDeferred(resource, **kwargs): - """ - Use this to exercise an overridden MultiFormatResource.render(), - usually for output=json or render_GET. It returns a Deferred. +def renderDeferred(ss): + elem = StorageStatusElement(ss._storage, ss._nickname) + return flattenString(None, elem) - :param _MultiFormatResource resource: an HTTP resource to be rendered. +class JSONRequest(Request): + implements(IRequest) - """ - # We should be using twisted.web's DummyRequest here instead of - # nevow's FakeRequest, but right now it is a bit of a problem: see - # web/common.py. MultiFormatResource.render() makes a get_arg() - # call, which does a IRequest(ctx_or_req). IRequest can handle - # FakeRequest, but it can't handle DummyRequest. - req = FakeRequest(**kwargs) - req.fields = None - d = defer.maybeDeferred(resource.render, req) - def _done(res): - if isinstance(res, str): - return res + req.v - return req.v - d.addCallback(_done) - return d + def __init__(self, **kwargs): + Request.__init__(self, DummyChannel(), **kwargs) + self.args = {"t": ["json"]} + self.fields = {} + +def renderJSON(resource): + return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): @@ -4083,7 +4084,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4127,7 +4128,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d def test_status_no_disk_stats(self): From 3e7dea7dda3bee3dfb2713297c10f3fac1b2698f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:01:22 -0400 Subject: [PATCH 0035/1054] Wrap renderer results in tags --- src/allmydata/web/storage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 63ac47282..ba6609456 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -38,11 +38,11 @@ class StorageStatusElement(Element): @renderer def nickname(self, req, tag): - return self._nickname + return tag(self._nickname) @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self._storage.my_nodeid) + return tag(idlib.nodeid_b2a(self._storage.my_nodeid)) def _get_storage_stat(self, key): """Get storage server statistics. @@ -110,20 +110,20 @@ class StorageStatusElement(Element): @renderer def accepting_immutable_shares(self, req, tag): accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") - return {True: "Yes", False: "No"}[bool(accepting)] + return tag({True: "Yes", False: "No"}[bool(accepting)]) @renderer def last_complete_bucket_count(self, req, tag): s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: - return "Not computed yet" - return str(count) + return tag("Not computed yet") + return tag(str(count)) @renderer def count_crawler_status(self, req, tag): p = self._storage.bucket_counter.get_progress() - return self.format_crawler_progress(p) + return tag(self.format_crawler_progress(p)) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] From 8b7ef33b3d3b7caf6a91f6f803111288d2bb5ed4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:05:17 -0400 Subject: [PATCH 0036/1054] Remove redundant render_json() method --- src/allmydata/test/test_storage.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 40b9b52e9..e20a89da6 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3399,7 +3399,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): "(2 mutable / 2 immutable),", s) self.failUnlessIn("but expiration was not enabled", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(webstatus)) + d.addCallback(lambda ign: renderJSON(webstatus)) def _check_json(raw): data = json.loads(raw) self.failUnlessIn("lease-checker", data) @@ -4036,7 +4036,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) # grr. json turns all dict keys into strings. @@ -4063,7 +4063,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(rec["examined-shares"], 3) self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json_history(raw): data = json.loads(raw) last = data["lease-checker"]["history"]["0"] @@ -4083,9 +4083,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d - def render_json(self, page): - d = renderJSON(page) - return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4116,7 +4113,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): self.failUnlessIn("Accepting new shares: Yes", s) self.failUnlessIn("Reserved space: - 0 B (0)", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) s = data["stats"] @@ -4127,9 +4124,6 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): d.addCallback(_check_json) return d - def render_json(self, page): - d = renderJSON(page) - return d def test_status_no_disk_stats(self): def call_get_disk_stats(whichdir, reserved_space=0): From 72b8f720802f91d5aef25643b326d1ba98b54cc2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:13:08 -0400 Subject: [PATCH 0037/1054] Add docstrings to storage test helpers --- src/allmydata/test/test_storage.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e20a89da6..b69713df5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2975,6 +2975,8 @@ def remove_tags(s): def renderSynchronously(ss): """ + Return fully rendered HTML document. + :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss._storage, ss._nickname) @@ -2982,10 +2984,20 @@ def renderSynchronously(ss): return unittest.TestCase().successResultOf(deferred) def renderDeferred(ss): + """ + Return a `Deferred` HTML renderer. + + :param _StorageStatus ss: a StorageStatus instance. + """ elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) class JSONRequest(Request): + """ + A Request with t=json argument added to it. + + This is useful to invoke a Resouce.render_JSON() method. + """ implements(IRequest) def __init__(self, **kwargs): @@ -2994,6 +3006,11 @@ class JSONRequest(Request): self.fields = {} def renderJSON(resource): + """Exercise resouce.render_JSON() + + :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` + that contains a render_JSON() method. + """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From b2b706198042fa39ddc4959d3804647340c8bb24 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:18:15 -0400 Subject: [PATCH 0038/1054] Refactor storage test helpers Rewrite `renderSynchronously()` to use `renderDeferred()` --- src/allmydata/test/test_storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index b69713df5..f50c3b352 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2979,9 +2979,7 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss._storage, ss._nickname) - deferred = flattenString(None, elem) - return unittest.TestCase().successResultOf(deferred) + return unittest.TestCase().successResultOf(renderDeferred(ss)) def renderDeferred(ss): """ From 82cd5a87fe9582dd1de8e021b8b8a8459d3ba499 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 21:48:49 -0400 Subject: [PATCH 0039/1054] Use DummyRequest in storage tests Using twisted.web.server.Request causes test_new_style_classes to fail like so: Traceback (most recent call last): Failure: testtools.testresult.real._StringException: Traceback (most recent call last): File ".tox/coverage/lib/python2.7/site-packages/allmydata/test/test_python2_regressions.py", line 69, in test_new_style_classes "Expected to find no classic classes.", File ".tox/coverage/lib/python2.7/site-packages/testtools/testcase.py", line 502, in assertThat raise mismatch_error testtools.matchers._impl.MismatchError: !=: reference = set([]) actual = set([]) : Expected to find no classic classes. Seems that `DummyRequest` is an acceptable new style class. --- src/allmydata/test/test_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f50c3b352..a0816a6ce 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,8 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.server import Request -from twisted.web.test.test_web import DummyChannel +# from twisted.web.server import Request +from twisted.web.test.test_web import DummyRequest from zope.interface import implements from foolscap.api import fireEventually @@ -2990,7 +2990,7 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -class JSONRequest(Request): +class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. @@ -2999,7 +2999,7 @@ class JSONRequest(Request): implements(IRequest) def __init__(self, **kwargs): - Request.__init__(self, DummyChannel(), **kwargs) + DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} self.fields = {} From 201c08dbe5d0e7c585329261e24082670d999f88 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 20 Apr 2020 14:35:32 -0400 Subject: [PATCH 0040/1054] Declare JSON request interface using @implementer "zope.interface.implements(IRequest)" is deprectated in favor of "@zope.interface.implementer(IRequest)" decorator. --- src/allmydata/test/test_storage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index a0816a6ce..23920a27a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,9 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -# from twisted.web.server import Request -from twisted.web.test.test_web import DummyRequest -from zope.interface import implements +from twisted.web.test.requesthelper import DummyRequest +from zope.interface import implementer from foolscap.api import fireEventually import itertools @@ -2990,14 +2989,13 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) +@implementer(IRequest) class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. This is useful to invoke a Resouce.render_JSON() method. """ - implements(IRequest) - def __init__(self, **kwargs): DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} From b3feaae644de727bf564a50df83e1ff4b733f76a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Apr 2020 10:29:18 -0400 Subject: [PATCH 0041/1054] Use an inner JSONRequest class with renderJSON Once nevow is removed from web/common.py, we can simplify renderJSON(), like so: def renderJSON(): req = Request() req.args = {"t": ["json"]} req.fields = {} return resource.render(req) But for now we have to live with an inner class that implements the nevow.inevow.IRequest interface. --- src/allmydata/test/test_storage.py | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 23920a27a..6c5f6e937 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,7 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.test.requesthelper import DummyRequest +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel from zope.interface import implementer from foolscap.api import fireEventually @@ -2989,24 +2990,20 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -@implementer(IRequest) -class JSONRequest(DummyRequest): - """ - A Request with t=json argument added to it. - - This is useful to invoke a Resouce.render_JSON() method. - """ - def __init__(self, **kwargs): - DummyRequest.__init__(self, b"/", **kwargs) - self.args = {"t": ["json"]} - self.fields = {} - def renderJSON(resource): - """Exercise resouce.render_JSON() + """Render a JSON from the given resource.""" + + @implementer(IRequest) + class JSONRequest(Request): + """ + A Request with t=json argument added to it. This is useful to + invoke a Resouce.render_JSON() method. + """ + def __init__(self): + Request.__init__(self, DummyChannel()) + self.args = {"t": ["json"]} + self.fields = {} - :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` - that contains a render_JSON() method. - """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From 8d8281cd3eda32ae6fa158170ca6039a56e608fd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 24 Apr 2020 14:23:15 -0400 Subject: [PATCH 0042/1054] Add autobahn derivation for nixos Copied from nixos-19.09-small/pkgs/development/python-modules/autobahn/default.nix --- nix/autobahn.nix | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 nix/autobahn.nix diff --git a/nix/autobahn.nix b/nix/autobahn.nix new file mode 100644 index 000000000..7d1e365b2 --- /dev/null +++ b/nix/autobahn.nix @@ -0,0 +1,34 @@ +{ lib, buildPythonPackage, fetchPypi, isPy3k, + six, txaio, twisted, zope_interface, cffi, trollius, futures, + mock, pytest, cryptography, pynacl +}: +buildPythonPackage rec { + pname = "autobahn"; + version = "19.8.1"; + + src = fetchPypi { + inherit pname version; + sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304"; + }; + + propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++ + (lib.optionals (!isPy3k) [ trollius futures ]); + + checkInputs = [ mock pytest ]; + checkPhase = '' + runHook preCheck + USE_TWISTED=true py.test $out + runHook postCheck + ''; + + # Tests do no seem to be compatible yet with pytest 5.1 + # https://github.com/crossbario/autobahn-python/issues/1235 + doCheck = false; + + meta = with lib; { + description = "WebSocket and WAMP in Python for Twisted and asyncio."; + homepage = "https://crossbar.io/autobahn"; + license = licenses.mit; + maintainers = with maintainers; [ nand0p ]; + }; +} From e7c1c78581b79309c726ebfe082f4aade62feb69 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 24 Apr 2020 14:24:01 -0400 Subject: [PATCH 0043/1054] Remove trollius from nixos autobahn derivation --- nix/autobahn.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/autobahn.nix b/nix/autobahn.nix index 7d1e365b2..83148c4f8 100644 --- a/nix/autobahn.nix +++ b/nix/autobahn.nix @@ -1,5 +1,5 @@ { lib, buildPythonPackage, fetchPypi, isPy3k, - six, txaio, twisted, zope_interface, cffi, trollius, futures, + six, txaio, twisted, zope_interface, cffi, futures, mock, pytest, cryptography, pynacl }: buildPythonPackage rec { @@ -12,7 +12,7 @@ buildPythonPackage rec { }; propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++ - (lib.optionals (!isPy3k) [ trollius futures ]); + (lib.optionals (!isPy3k) [ futures ]); checkInputs = [ mock pytest ]; checkPhase = '' From 756dd2524702c20cc97cf783dd5027e38349e1ec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 24 Apr 2020 14:29:59 -0400 Subject: [PATCH 0044/1054] Use our own autobahn nixos derivation --- nix/overlays.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/overlays.nix b/nix/overlays.nix index 08d11306e..2472c1a37 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -7,6 +7,10 @@ self: super: { # conflicts with the packaged version of Twisted. Supply our own # slightly newer version. nevow = python-super.callPackage ./nevow.nix { }; + # NixOS autobahn package has trollius as a dependency, although + # it is optional. Trollius is no longer maintained and fails on + # CI. + autobahn = python-super.callPackage ./autobahn.nix { }; }; }; } From d27e5e0afe1e8224f1572dd5df50b9a3738bfddb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 24 Apr 2020 19:01:49 -0400 Subject: [PATCH 0045/1054] Add newsfragment --- newsfragments/3304.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3304.minor diff --git a/newsfragments/3304.minor b/newsfragments/3304.minor new file mode 100644 index 000000000..e69de29bb From 5fd855a600123ce8ee6eb3aa6757465399d146b3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 24 Apr 2020 21:16:57 -0400 Subject: [PATCH 0046/1054] Update comment about trollius --- nix/overlays.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nix/overlays.nix b/nix/overlays.nix index 2472c1a37..2a9fe8e5e 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -8,8 +8,7 @@ self: super: { # slightly newer version. nevow = python-super.callPackage ./nevow.nix { }; # NixOS autobahn package has trollius as a dependency, although - # it is optional. Trollius is no longer maintained and fails on - # CI. + # it is optional. Trollius is unmaintained and fails on CI. autobahn = python-super.callPackage ./autobahn.nix { }; }; }; From 4b0c730adf8c7f9240c50cef156627a1a6269c80 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Dec 2019 03:18:46 -0700 Subject: [PATCH 0047/1054] use 'with open' correctly --- src/allmydata/scripts/cli.py | 3 +- src/allmydata/storage/immutable.py | 68 ++++++++++++++---------------- src/allmydata/test/no_network.py | 31 ++++++++------ 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 31ef26510..379e1d212 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -38,7 +38,8 @@ class FileStoreOptions(BaseOptions): raise usage.UsageError(msg) else: node_url_file = os.path.join(self['node-directory'], "node.url") - self['node-url'] = open(node_url_file, "r").read().strip() + with open(node_url_file, "r") as f: + self['node-url'] = f.read().strip() if self['node-url'][-1] != "/": self['node-url'] += "/" diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index f66eec594..7f69fb3b8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -50,7 +50,6 @@ class ShareFile(object): # it. Also construct the metadata. assert not os.path.exists(self.home) fileutil.make_dirs(os.path.dirname(self.home)) - f = open(self.home, 'wb') # The second field -- the four-byte share data length -- is no # longer used as of Tahoe v1.3.0, but we continue to write it in # there in case someone downgrades a storage server from >= @@ -60,15 +59,14 @@ class ShareFile(object): # the largest length that can fit into the field. That way, even # if this does happen, the old < v1.3.0 server will still allow # clients to read the first part of the share. - f.write(struct.pack(">LLL", 1, min(2**32-1, max_size), 0)) - f.close() + with open(self.home, 'wb') as f: + f.write(struct.pack(">LLL", 1, min(2**32-1, max_size), 0)) self._lease_offset = max_size + 0x0c self._num_leases = 0 else: - f = open(self.home, 'rb') - filesize = os.path.getsize(self.home) - (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - f.close() + with open(self.home, 'rb') as f: + filesize = os.path.getsize(self.home) + (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) if version != 1: msg = "sharefile %s had version %d but we wanted 1" % \ (filename, version) @@ -88,21 +86,20 @@ class ShareFile(object): actuallength = max(0, min(length, self._lease_offset-seekpos)) if actuallength == 0: return "" - f = open(self.home, 'rb') - f.seek(seekpos) - return f.read(actuallength) + with open(self.home, 'rb') as f: + f.seek(seekpos) + return f.read(actuallength) def write_share_data(self, offset, data): length = len(data) precondition(offset >= 0, offset) if self._max_size is not None and offset+length > self._max_size: raise DataTooLargeError(self._max_size, offset, length) - f = open(self.home, 'rb+') - real_offset = self._data_offset+offset - f.seek(real_offset) - assert f.tell() == real_offset - f.write(data) - f.close() + with open(self.home, 'rb+') as f: + real_offset = self._data_offset+offset + f.seek(real_offset) + assert f.tell() == real_offset + f.write(data) def _write_lease_record(self, f, lease_number, lease_info): offset = self._lease_offset + lease_number * self.LEASE_SIZE @@ -124,20 +121,19 @@ class ShareFile(object): def get_leases(self): """Yields a LeaseInfo instance for all leases.""" - f = open(self.home, 'rb') - (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - f.seek(self._lease_offset) - for i in range(num_leases): - data = f.read(self.LEASE_SIZE) - if data: - yield LeaseInfo().from_immutable_data(data) + with open(self.home, 'rb') as f: + (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) + f.seek(self._lease_offset) + for i in range(num_leases): + data = f.read(self.LEASE_SIZE) + if data: + yield LeaseInfo().from_immutable_data(data) def add_lease(self, lease_info): - f = open(self.home, 'rb+') - num_leases = self._read_num_leases(f) - self._write_lease_record(f, num_leases, lease_info) - self._write_num_leases(f, num_leases+1) - f.close() + with open(self.home, 'rb+') as f: + num_leases = self._read_num_leases(f) + self._write_lease_record(f, num_leases, lease_info) + self._write_num_leases(f, num_leases+1) def renew_lease(self, renew_secret, new_expire_time): for i,lease in enumerate(self.get_leases()): @@ -146,9 +142,8 @@ class ShareFile(object): if new_expire_time > lease.expiration_time: # yes lease.expiration_time = new_expire_time - f = open(self.home, 'rb+') - self._write_lease_record(f, i, lease) - f.close() + with open(self.home, 'rb+') as f: + self._write_lease_record(f, i, lease) return raise IndexError("unable to renew non-existent lease") @@ -181,12 +176,11 @@ class ShareFile(object): # the same order as they were added, so that if we crash while # doing this, we won't lose any non-cancelled leases. leases = [l for l in leases if l] # remove the cancelled leases - f = open(self.home, 'rb+') - for i,lease in enumerate(leases): - self._write_lease_record(f, i, lease) - self._write_num_leases(f, len(leases)) - self._truncate_leases(f, len(leases)) - f.close() + with open(self.home, 'rb+') as f: + for i, lease in enumerate(leases): + self._write_lease_record(f, i, lease) + self._write_num_leases(f, len(leases)) + self._truncate_leases(f, len(leases)) space_freed = self.LEASE_SIZE * num_leases_removed if not len(leases): space_freed += os.stat(self.home)[stat.ST_SIZE] diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 7dfddd54b..0ccead0b9 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -332,13 +332,12 @@ class NoNetworkGrid(service.MultiService): if write_config: from twisted.internet import reactor _, port_endpoint = self.port_assigner.assign(reactor) - f = open(tahoe_cfg_path, "w") - f.write("[node]\n") - f.write("nickname = client-%d\n" % i) - f.write("web.port = {}\n".format(port_endpoint)) - f.write("[storage]\n") - f.write("enabled = false\n") - f.close() + with open(tahoe_cfg_path, "w") as f: + f.write("[node]\n") + f.write("nickname = client-%d\n" % i) + f.write("web.port = {}\n".format(port_endpoint)) + f.write("[storage]\n") + f.write("enabled = false\n") else: _assert(os.path.exists(tahoe_cfg_path), tahoe_cfg_path=tahoe_cfg_path) @@ -523,12 +522,14 @@ class GridTestMixin(object): def copy_shares(self, uri): shares = {} for (shnum, serverid, sharefile) in self.find_uri_shares(uri): - shares[sharefile] = open(sharefile, "rb").read() + with open(sharefile, "rb") as f: + shares[sharefile] = f.read() return shares def restore_all_shares(self, shares): for sharefile, data in shares.items(): - open(sharefile, "wb").write(data) + with open(sharefile, "wb") as f: + f.write(data) def delete_share(self, sharenum_and_serverid_and_sharefile): (shnum, serverid, sharefile) = sharenum_and_serverid_and_sharefile @@ -547,16 +548,20 @@ class GridTestMixin(object): def corrupt_share(self, sharenum_and_serverid_and_sharefile, corruptor_function): (shnum, serverid, sharefile) = sharenum_and_serverid_and_sharefile - sharedata = open(sharefile, "rb").read() + with open(sharefile, "rb") as f: + sharedata = f.read() corruptdata = corruptor_function(sharedata) - open(sharefile, "wb").write(corruptdata) + with open(sharefile, "wb") as f: + f.write(corruptdata) def corrupt_shares_numbered(self, uri, shnums, corruptor, debug=False): for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): if i_shnum in shnums: - sharedata = open(i_sharefile, "rb").read() + with open(i_sharefile, "rb") as f: + sharedata = f.read() corruptdata = corruptor(sharedata, debug=debug) - open(i_sharefile, "wb").write(corruptdata) + with open(i_sharefile, "wb") as f: + f.write(corruptdata) def corrupt_all_shares(self, uri, corruptor, debug=False): for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): From ca5e1ebe5208d0ec7e649a2eeb49f50d11fcffc9 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Dec 2019 03:19:06 -0700 Subject: [PATCH 0048/1054] look at correct mock-call object --- src/allmydata/test/test_connections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index ccec13336..3e2806dd0 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -114,7 +114,7 @@ class Tor(unittest.TestCase): ) tor_provider = create_tor_provider(reactor, config) h = tor_provider.get_tor_handler() - self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0])) + self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0][1][0])) self.assertIdentical(h, h1) def test_socksport_endpoint(self): @@ -128,7 +128,7 @@ class Tor(unittest.TestCase): ) tor_provider = create_tor_provider(reactor, config) h = tor_provider.get_tor_handler() - self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0])) + self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0][1][0])) self.assertIdentical(h, h1) def test_socksport_endpoint_otherhost(self): @@ -142,7 +142,7 @@ class Tor(unittest.TestCase): ) tor_provider = create_tor_provider(reactor, config) h = tor_provider.get_tor_handler() - self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0])) + self.assertTrue(IStreamClientEndpoint.providedBy(f.mock_calls[0][1][0])) self.assertIdentical(h, h1) def test_socksport_bad_endpoint(self): From a348d3d50700fa727121a60f8559b08215a29e93 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Dec 2019 03:20:01 -0700 Subject: [PATCH 0049/1054] relying on memory/gc bad --- src/allmydata/test/test_storage.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9f3aee9b8..f5a2b016f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1,4 +1,5 @@ import time, os.path, platform, stat, re, json, struct, shutil +import gc from twisted.trial import unittest @@ -553,8 +554,14 @@ class Server(unittest.TestCase): # we abandon the first set, so their provisional allocation should be # returned + + # XXX okay, so the weak-key-dictionary in storage-server is + # basically making an interface to the storage-server that is + # "whenever I drop my object AND the garbage-collector removes + # it, *then* that thing is no longer writing"..? ... yuuuuck. del already del writers + gc.collect() # for pypy's benefit self.failUnlessEqual(len(ss._active_writers), 1) # now we have a provisional allocation of 1001 bytes @@ -567,6 +574,7 @@ class Server(unittest.TestCase): del already2 del writers2 del bw + gc.collect() # for pypy's benefit self.failUnlessEqual(len(ss._active_writers), 0) # this also changes the amount reported as available by call_get_disk_stats From 5071ddf124169a2ec23c3b939f6aa1e04b7a7e7c Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Dec 2019 16:30:34 -0700 Subject: [PATCH 0050/1054] add more 'with' statements --- src/allmydata/test/test_system.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 2b7660705..2108ce9dd 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -543,16 +543,16 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.clients.append(c) c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) - f = open(os.path.join(basedirs[0],"private","helper.furl"), "r") - helper_furl = f.read() - f.close() + with open(os.path.join(basedirs[0],"private","helper.furl"), "r") as f: + helper_furl = f.read() + self.helper_furl = helper_furl if self.numclients >= 4: - f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+') - f.write( - "[client]\n" - "helper.furl = %s\n" % helper_furl) - f.close() + with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+') as f: + f.write( + "[client]\n" + "helper.furl = {}\n".format(helper_furl) + ) # this starts the rest of the clients for i in range(1, self.numclients): From 5175a6c6a011153793612ea1c22ebd425818755e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Dec 2019 16:31:46 -0700 Subject: [PATCH 0051/1054] asserts for systemexit --- src/allmydata/test/cli/test_start.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/cli/test_start.py b/src/allmydata/test/cli/test_start.py index b38ca7fd8..90fde3969 100644 --- a/src/allmydata/test/cli/test_start.py +++ b/src/allmydata/test/cli/test_start.py @@ -254,7 +254,11 @@ class RunTests(unittest.TestCase): ]) i, o, e = StringIO(), StringIO(), StringIO() - runner.dispatch(config, i, o, e) + d = runner.dispatch(config, i, o, e) + + def must_be_systemexit(f): + assert isinstance(f.value, SystemExit) + d.addErrback(must_be_systemexit) output = e.getvalue() # should print out the collected logs and an error-code @@ -266,10 +270,6 @@ class RunTests(unittest.TestCase): "Configuration error:", output, ) - # this is SystemExit(0) for some reason I can't understand, - # while running on the command-line, "echo $?" shows "1" on - # this same error (some config exception)... - errs = self.flushLoggedErrors(SystemExit) - self.assertEqual(1, len(errs)) # ensure reactor.stop was actually called self.assertEqual([None], stopped) + return d From d06127c720b31831c444fbba82e1b8e78640fbe9 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 18 Dec 2019 03:54:49 -0700 Subject: [PATCH 0052/1054] news --- newsfragments/1792.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1792.feature diff --git a/newsfragments/1792.feature b/newsfragments/1792.feature new file mode 100644 index 000000000..b2b839664 --- /dev/null +++ b/newsfragments/1792.feature @@ -0,0 +1 @@ +PyPy is now a supported platform. \ No newline at end of file From 4928d62d661d175a3511db08b799a52527f281b6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 18 Dec 2019 17:02:40 -0700 Subject: [PATCH 0053/1054] use set instead of WeakKeyDictionary --- src/allmydata/storage/server.py | 8 ++++---- src/allmydata/test/test_storage.py | 20 +++++++------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index eae6fd698..7741e0c18 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -1,4 +1,4 @@ -import os, re, weakref, struct, time +import os, re, struct, time import six from foolscap.api import Referenceable @@ -67,7 +67,7 @@ class StorageServer(service.MultiService, Referenceable): self.incomingdir = os.path.join(sharedir, 'incoming') self._clean_incomplete() fileutil.make_dirs(self.incomingdir) - self._active_writers = weakref.WeakKeyDictionary() + self._active_writers = set() log.msg("StorageServer created", facility="tahoe.storage") if reserved_space: @@ -302,7 +302,7 @@ class StorageServer(service.MultiService, Referenceable): if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw - self._active_writers[bw] = 1 + self._active_writers.add(bw) if limited: remaining_space -= max_space_per_bucket else: @@ -359,7 +359,7 @@ class StorageServer(service.MultiService, Referenceable): def bucket_writer_closed(self, bw, consumed_size): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) - del self._active_writers[bw] + self._active_writers.remove(bw) def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f5a2b016f..513d92978 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -541,27 +541,22 @@ class Server(unittest.TestCase): OVERHEAD = 3*4 LEASE_SIZE = 4+32+32+4 canary = FakeCanary(True) - already,writers = self.allocate(ss, "vid1", [0,1,2], 1000, canary) + already, writers = self.allocate(ss, "vid1", [0,1,2], 1000, canary) self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed self.failUnlessEqual(len(ss._active_writers), 3) # allocating 1001-byte shares only leaves room for one - already2,writers2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary) + already2, writers2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary) self.failUnlessEqual(len(writers2), 1) self.failUnlessEqual(len(ss._active_writers), 4) # we abandon the first set, so their provisional allocation should be # returned - # XXX okay, so the weak-key-dictionary in storage-server is - # basically making an interface to the storage-server that is - # "whenever I drop my object AND the garbage-collector removes - # it, *then* that thing is no longer writing"..? ... yuuuuck. - del already - del writers - gc.collect() # for pypy's benefit + for x in writers.values(): + x.remote_close() self.failUnlessEqual(len(ss._active_writers), 1) # now we have a provisional allocation of 1001 bytes @@ -574,7 +569,6 @@ class Server(unittest.TestCase): del already2 del writers2 del bw - gc.collect() # for pypy's benefit self.failUnlessEqual(len(ss._active_writers), 0) # this also changes the amount reported as available by call_get_disk_stats @@ -582,12 +576,12 @@ class Server(unittest.TestCase): # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and # 5000-1085=3915 free, therefore we can fit 39 100byte shares - already3,writers3 = self.allocate(ss,"vid3", range(100), 100, canary) + already3, writers3 = self.allocate(ss,"vid3", range(100), 100, canary) self.failUnlessEqual(len(writers3), 39) self.failUnlessEqual(len(ss._active_writers), 39) - del already3 - del writers3 + for x in writers3.values(): + x._disconnected() self.failUnlessEqual(len(ss._active_writers), 0) ss.disownServiceParent() del ss From 9bcc465f7603b06d058b0d6d3c3a0abed3528785 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 18 Dec 2019 17:52:57 -0700 Subject: [PATCH 0054/1054] unused import --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 513d92978..7e4eaa3b6 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1,5 +1,4 @@ import time, os.path, platform, stat, re, json, struct, shutil -import gc from twisted.trial import unittest From 4c3d0ea6cc87edc28f39e73766865245eb59d1d2 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 21 Dec 2019 00:03:38 -0700 Subject: [PATCH 0055/1054] use 'with open' for more file-opens --- src/allmydata/blacklist.py | 15 ++-- src/allmydata/control.py | 13 +-- src/allmydata/frontends/auth.py | 29 +++--- src/allmydata/scripts/common.py | 26 +++--- src/allmydata/stats.py | 10 +-- src/allmydata/storage/crawler.py | 10 +-- src/allmydata/storage/expirer.py | 16 ++-- src/allmydata/storage/mutable.py | 148 +++++++++++++++---------------- src/allmydata/storage/server.py | 22 +++-- src/allmydata/storage/shares.py | 5 +- src/allmydata/util/configutil.py | 10 +-- src/allmydata/version_checks.py | 24 ++--- 12 files changed, 155 insertions(+), 173 deletions(-) diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 874ff95ca..af1d185d0 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -34,13 +34,14 @@ class Blacklist(object): try: if self.last_mtime is None or current_mtime > self.last_mtime: self.entries.clear() - for line in open(self.blacklist_fn, "r").readlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - si_s, reason = line.split(None, 1) - si = base32.a2b(si_s) # must be valid base32 - self.entries[si] = reason + with open(self.blacklist_fn, "r") as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + si_s, reason = line.split(None, 1) + si = base32.a2b(si_s) # must be valid base32 + self.entries[si] = reason self.last_mtime = current_mtime except Exception as e: twisted_log.err(e, "unparseable blacklist file") diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 07802efba..55f6db4f9 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -19,12 +19,13 @@ def get_memory_usage(): "VmData") stats = {} try: - for line in open("/proc/self/status", "r").readlines(): - name, right = line.split(":",2) - if name in stat_names: - assert right.endswith(" kB\n") - right = right[:-4] - stats[name] = int(right) * 1024 + with open("/proc/self/status", "r") as f: + for line in f.readlines(): + name, right = line.split(":",2) + if name in stat_names: + assert right.endswith(" kB\n") + right = right[:-4] + stats[name] = int(right) * 1024 except: # Probably not on (a compatible version of) Linux stats['VmSize'] = 0 diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 49647bc60..ab56bf94d 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -31,20 +31,21 @@ class AccountFileChecker(object): self.passwords = {} self.pubkeys = {} self.rootcaps = {} - for line in open(abspath_expanduser_unicode(accountfile), "r"): - line = line.strip() - if line.startswith("#") or not line: - continue - name, passwd, rest = line.split(None, 2) - if passwd.startswith("ssh-"): - bits = rest.split() - keystring = " ".join([passwd] + bits[:-1]) - rootcap = bits[-1] - self.pubkeys[name] = keystring - else: - self.passwords[name] = passwd - rootcap = rest - self.rootcaps[name] = rootcap + with open(abspath_expanduser_unicode(accountfile), "r") as f: + for line in f.readlines(): + line = line.strip() + if line.startswith("#") or not line: + continue + name, passwd, rest = line.split(None, 2) + if passwd.startswith("ssh-"): + bits = rest.split() + keystring = " ".join([passwd] + bits[:-1]) + rootcap = bits[-1] + self.pubkeys[name] = keystring + else: + self.passwords[name] = passwd + rootcap = rest + self.rootcaps[name] = rootcap def _avatarId(self, username): return FTPAvatarID(username, self.rootcaps[username]) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index d3dde9a72..89c1f94d1 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -131,22 +131,22 @@ def get_aliases(nodedir): aliasfile = os.path.join(nodedir, "private", "aliases") rootfile = os.path.join(nodedir, "private", "root_dir.cap") try: - f = open(rootfile, "r") - rootcap = f.read().strip() - if rootcap: - aliases[DEFAULT_ALIAS] = rootcap + with open(rootfile, "r") as f: + rootcap = f.read().strip() + if rootcap: + aliases[DEFAULT_ALIAS] = rootcap except EnvironmentError: pass try: - f = codecs.open(aliasfile, "r", "utf-8") - for line in f.readlines(): - line = line.strip() - if line.startswith("#") or not line: - continue - name, cap = line.split(u":", 1) - # normalize it: remove http: prefix, urldecode - cap = cap.strip().encode('utf-8') - aliases[name] = cap + with codecs.open(aliasfile, "r", "utf-8") as f: + for line in f.readlines(): + line = line.strip() + if line.startswith("#") or not line: + continue + name, cap = line.split(u":", 1) + # normalize it: remove http: prefix, urldecode + cap = cap.strip().encode('utf-8') + aliases[name] = cap except EnvironmentError: pass return aliases diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 73e311fe3..34907b1fa 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -250,16 +250,15 @@ class JSONStatsGatherer(StdOutStatsGatherer): self.jsonfile = os.path.join(basedir, "stats.json") if os.path.exists(self.jsonfile): - f = open(self.jsonfile, 'rb') try: - self.gathered_stats = json.load(f) + with open(self.jsonfile, 'rb') as f: + self.gathered_stats = json.load(f) except Exception: print("Error while attempting to load stats file %s.\n" "You may need to restore this file from a backup," " or delete it if no backup is available.\n" % quote_local_unicode_path(self.jsonfile)) raise - f.close() else: self.gathered_stats = {} @@ -272,9 +271,8 @@ class JSONStatsGatherer(StdOutStatsGatherer): def dump_json(self): tmp = "%s.tmp" % (self.jsonfile,) - f = open(tmp, 'wb') - json.dump(self.gathered_stats, f) - f.close() + with open(tmp, 'wb') as f: + json.dump(self.gathered_stats, f) if os.path.exists(self.jsonfile): os.unlink(self.jsonfile) os.rename(tmp, self.jsonfile) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 438dd5e31..008619d64 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -191,9 +191,8 @@ class ShareCrawler(service.MultiService): # of the last bucket to be processed, or # None if we are sleeping between cycles try: - f = open(self.statefile, "rb") - state = pickle.load(f) - f.close() + with open(self.statefile, "rb") as f: + state = pickle.load(f) except Exception: state = {"version": 1, "last-cycle-finished": None, @@ -230,9 +229,8 @@ class ShareCrawler(service.MultiService): last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" - f = open(tmpfile, "wb") - pickle.dump(self.state, f) - f.close() + with open(tmpfile, "wb") as f: + pickle.dump(self.state, f) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index a284d4f74..a13c188bd 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -84,9 +84,8 @@ class LeaseCheckingCrawler(ShareCrawler): # initialize history if not os.path.exists(self.historyfile): history = {} # cyclenum -> dict - f = open(self.historyfile, "wb") - pickle.dump(history, f) - f.close() + with open(self.historyfile, "wb") as f: + pickle.dump(history, f) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() @@ -303,14 +302,14 @@ class LeaseCheckingCrawler(ShareCrawler): # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - history = pickle.load(open(self.historyfile, "rb")) + with open(self.historyfile, "rb") as f: + history = pickle.load(f) history[cycle] = h while len(history) > 10: oldcycles = sorted(history.keys()) del history[oldcycles[0]] - f = open(self.historyfile, "wb") - pickle.dump(history, f) - f.close() + with open(self.historyfile, "wb") as f: + pickle.dump(history, f) def get_state(self): """In addition to the crawler state described in @@ -379,7 +378,8 @@ class LeaseCheckingCrawler(ShareCrawler): progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - history = pickle.load(open(self.historyfile, "rb")) + with open(self.historyfile, "rb") as f: + history = pickle.load(f) state["history"] = history if not progress["cycle-in-progress"]: diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 37f773c0d..287ed8fb9 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -57,8 +57,8 @@ class MutableShareFile(object): self.home = filename if os.path.exists(self.home): # we don't cache anything, just check the magic - f = open(self.home, 'rb') - data = f.read(self.HEADER_SIZE) + with open(self.home, 'rb') as f: + data = f.read(self.HEADER_SIZE) (magic, write_enabler_nodeid, write_enabler, data_length, extra_least_offset) = \ @@ -80,17 +80,17 @@ class MutableShareFile(object): + data_length) assert extra_lease_offset == self.DATA_OFFSET # true at creation num_extra_leases = 0 - f = open(self.home, 'wb') - header = struct.pack(">32s20s32sQQ", - self.MAGIC, my_nodeid, write_enabler, - data_length, extra_lease_offset, - ) - leases = ("\x00"*self.LEASE_SIZE) * 4 - f.write(header + leases) - # data goes here, empty after creation - f.write(struct.pack(">L", num_extra_leases)) - # extra leases go here, none at creation - f.close() + with open(self.home, 'wb') as f: + header = struct.pack( + ">32s20s32sQQ", + self.MAGIC, my_nodeid, write_enabler, + data_length, extra_lease_offset, + ) + leases = ("\x00" * self.LEASE_SIZE) * 4 + f.write(header + leases) + # data goes here, empty after creation + f.write(struct.pack(">L", num_extra_leases)) + # extra leases go here, none at creation def unlink(self): os.unlink(self.home) @@ -261,10 +261,9 @@ class MutableShareFile(object): def get_leases(self): """Yields a LeaseInfo instance for all leases.""" - f = open(self.home, 'rb') - for i, lease in self._enumerate_leases(f): - yield lease - f.close() + with open(self.home, 'rb') as f: + for i, lease in self._enumerate_leases(f): + yield lease def _enumerate_leases(self, f): for i in range(self._get_num_lease_slots(f)): @@ -277,29 +276,26 @@ class MutableShareFile(object): def add_lease(self, lease_info): precondition(lease_info.owner_num != 0) # 0 means "no lease here" - f = open(self.home, 'rb+') - num_lease_slots = self._get_num_lease_slots(f) - empty_slot = self._get_first_empty_lease_slot(f) - if empty_slot is not None: - self._write_lease_record(f, empty_slot, lease_info) - else: - self._write_lease_record(f, num_lease_slots, lease_info) - f.close() + with open(self.home, 'rb+') as f: + num_lease_slots = self._get_num_lease_slots(f) + empty_slot = self._get_first_empty_lease_slot(f) + if empty_slot is not None: + self._write_lease_record(f, empty_slot, lease_info) + else: + self._write_lease_record(f, num_lease_slots, lease_info) def renew_lease(self, renew_secret, new_expire_time): accepting_nodeids = set() - f = open(self.home, 'rb+') - for (leasenum,lease) in self._enumerate_leases(f): - if timing_safe_compare(lease.renew_secret, renew_secret): - # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: - # yes - lease.expiration_time = new_expire_time - self._write_lease_record(f, leasenum, lease) - f.close() - return - accepting_nodeids.add(lease.nodeid) - f.close() + with open(self.home, 'rb+') as f: + for (leasenum,lease) in self._enumerate_leases(f): + if timing_safe_compare(lease.renew_secret, renew_secret): + # yup. See if we need to update the owner time. + if new_expire_time > lease.expiration_time: + # yes + lease.expiration_time = new_expire_time + self._write_lease_record(f, leasenum, lease) + return + accepting_nodeids.add(lease.nodeid) # Return the accepting_nodeids set, to give the client a chance to # update the leases on a share which has been migrated from its # original server to a new one. @@ -333,21 +329,21 @@ class MutableShareFile(object): cancel_secret="\x00"*32, expiration_time=0, nodeid="\x00"*20) - f = open(self.home, 'rb+') - for (leasenum,lease) in self._enumerate_leases(f): - accepting_nodeids.add(lease.nodeid) - if timing_safe_compare(lease.cancel_secret, cancel_secret): - self._write_lease_record(f, leasenum, blank_lease) - modified += 1 - else: - remaining += 1 - if modified: - freed_space = self._pack_leases(f) - f.close() - if not remaining: - freed_space += os.stat(self.home)[stat.ST_SIZE] - self.unlink() - return freed_space + with open(self.home, 'rb+') as f: + for (leasenum,lease) in self._enumerate_leases(f): + accepting_nodeids.add(lease.nodeid) + if timing_safe_compare(lease.cancel_secret, cancel_secret): + self._write_lease_record(f, leasenum, blank_lease) + modified += 1 + else: + remaining += 1 + if modified: + freed_space = self._pack_leases(f) + f.close() + if not remaining: + freed_space += os.stat(self.home)[stat.ST_SIZE] + self.unlink() + return freed_space msg = ("Unable to cancel non-existent lease. I have leases " "accepted by nodeids: ") @@ -372,10 +368,9 @@ class MutableShareFile(object): def readv(self, readv): datav = [] - f = open(self.home, 'rb') - for (offset, length) in readv: - datav.append(self._read_share_data(f, offset, length)) - f.close() + with open(self.home, 'rb') as f: + for (offset, length) in readv: + datav.append(self._read_share_data(f, offset, length)) return datav # def remote_get_length(self): @@ -385,10 +380,9 @@ class MutableShareFile(object): # return data_length def check_write_enabler(self, write_enabler, si_s): - f = open(self.home, 'rb+') - (real_write_enabler, write_enabler_nodeid) = \ - self._read_write_enabler_and_nodeid(f) - f.close() + with open(self.home, 'rb+') as f: + (real_write_enabler, write_enabler_nodeid) = \ + self._read_write_enabler_and_nodeid(f) # avoid a timing attack #if write_enabler != real_write_enabler: if not timing_safe_compare(write_enabler, real_write_enabler): @@ -405,27 +399,25 @@ class MutableShareFile(object): def check_testv(self, testv): test_good = True - f = open(self.home, 'rb+') - for (offset, length, operator, specimen) in testv: - data = self._read_share_data(f, offset, length) - if not testv_compare(data, operator, specimen): - test_good = False - break - f.close() + with open(self.home, 'rb+') as f: + for (offset, length, operator, specimen) in testv: + data = self._read_share_data(f, offset, length) + if not testv_compare(data, operator, specimen): + test_good = False + break return test_good def writev(self, datav, new_length): - f = open(self.home, 'rb+') - for (offset, data) in datav: - self._write_share_data(f, offset, data) - if new_length is not None: - cur_length = self._read_data_length(f) - if new_length < cur_length: - self._write_data_length(f, new_length) - # TODO: if we're going to shrink the share file when the - # share data has shrunk, then call - # self._change_container_size() here. - f.close() + with open(self.home, 'rb+') as f: + for (offset, data) in datav: + self._write_share_data(f, offset, data) + if new_length is not None: + cur_length = self._read_data_length(f) + if new_length < cur_length: + self._write_data_length(f, new_length) + # TODO: if we're going to shrink the share file when the + # share data has shrunk, then call + # self._change_container_size() here. def testv_compare(a, op, b): assert op in ("lt", "le", "eq", "ne", "ge", "gt") diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 7741e0c18..5cc4be95c 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -317,9 +317,8 @@ class StorageServer(service.MultiService, Referenceable): def _iter_share_files(self, storage_index): for shnum, filename in self._get_bucket_shares(storage_index): - f = open(filename, 'rb') - header = f.read(32) - f.close() + with open(filename, 'rb') as f: + header = f.read(32) if header[:32] == MutableShareFile.MAGIC: sf = MutableShareFile(filename, self) # note: if the share has been migrated, the renew_lease() @@ -682,15 +681,14 @@ class StorageServer(service.MultiService, Referenceable): # windows can't handle colons in the filename fn = os.path.join(self.corruption_advisory_dir, "%s--%s-%d" % (now, si_s, shnum)).replace(":","") - f = open(fn, "w") - f.write("report: Share Corruption\n") - f.write("type: %s\n" % share_type) - f.write("storage_index: %s\n" % si_s) - f.write("share_number: %d\n" % shnum) - f.write("\n") - f.write(reason) - f.write("\n") - f.close() + with open(fn, "w") as f: + f.write("report: Share Corruption\n") + f.write("type: %s\n" % share_type) + f.write("storage_index: %s\n" % si_s) + f.write("share_number: %d\n" % shnum) + f.write("\n") + f.write(reason) + f.write("\n") log.msg(format=("client claims corruption in (%(share_type)s) " + "%(si)s-%(shnum)d: %(reason)s"), share_type=share_type, si=si_s, shnum=shnum, reason=reason, diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index 558bddc19..bd94c0f3f 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -4,9 +4,8 @@ from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import ShareFile def get_share_file(filename): - f = open(filename, "rb") - prefix = f.read(32) - f.close() + with open(filename, "rb") as f: + prefix = f.read(32) if prefix == MutableShareFile.MAGIC: return MutableShareFile(filename) # otherwise assume it's immutable diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index d58bc4217..3699db35d 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -13,15 +13,12 @@ class UnknownConfigError(Exception): def get_config(tahoe_cfg): config = SafeConfigParser() - f = open(tahoe_cfg, "rb") - try: + with open(tahoe_cfg, "rb") as f: # Skip any initial Byte Order Mark. Since this is an ordinary file, we # don't need to handle incomplete reads, and can assume seekability. if f.read(3) != '\xEF\xBB\xBF': f.seek(0) config.readfp(f) - finally: - f.close() return config def set_config(config, section, option, value): @@ -31,11 +28,8 @@ def set_config(config, section, option, value): assert config.get(section, option) == value def write_config(tahoe_cfg, config): - f = open(tahoe_cfg, "wb") - try: + with open(tahoe_cfg, "wb") as f: config.write(f) - finally: - f.close() def validate_config(fname, cfg, valid_config): """ diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 9f084ff4b..aa42c2eee 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -235,18 +235,18 @@ def _get_linux_distro(): return (_distname, _version) try: - etclsbrel = open("/etc/lsb-release", "rU") - for line in etclsbrel: - m = _distributor_id_file_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - m = _release_file_re.search(line) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) + with open("/etc/lsb-release", "rU") as etclsbrel: + for line in etclsbrel: + m = _distributor_id_file_re.search(line) + if m: + _distname = m.group(1).strip() + if _distname and _version: + return (_distname, _version) + m = _release_file_re.search(line) + if m: + _version = m.group(1).strip() + if _distname and _version: + return (_distname, _version) except EnvironmentError: pass From 8965c77d6358a11ccc368b34d2708bd78464e0e8 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 21 Dec 2019 00:04:05 -0700 Subject: [PATCH 0056/1054] look for correct error-string --- src/allmydata/test/test_backupdb.py | 3 +-- src/allmydata/test/test_multi_introducers.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_backupdb.py b/src/allmydata/test/test_backupdb.py index b908292b2..a10c7514b 100644 --- a/src/allmydata/test/test_backupdb.py +++ b/src/allmydata/test/test_backupdb.py @@ -60,8 +60,7 @@ class BackupDB(unittest.TestCase): bdb = backupdb.get_backupdb(where, stderr_f) self.failUnlessEqual(bdb, None) stderr = stderr_f.getvalue() - self.failUnlessIn("Unable to create/open backupdb file %s" % (where,), stderr) - self.failUnlessIn("unable to open database file", stderr) + self.failUnlessIn("Could not open database", stderr) def writeto(self, filename, data): diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index 4c37cb5b0..2c1e933fa 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -159,7 +159,7 @@ class NoDefault(unittest.TestCase): yield create_client(self.basedir) self.assertEquals( str(ctx.exception), - "string indices must be integers", + "string index must be an integer, not str", ) @defer.inlineCallbacks From f8117320cb6b0748f8f1c066fa8b52430d17fc0b Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 21 Dec 2019 00:04:27 -0700 Subject: [PATCH 0057/1054] tests which rely on memory-reclamation need gc.collect() on pypy --- src/allmydata/test/test_util.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 9a576870a..ca05e4008 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -8,6 +8,7 @@ import six import hashlib import os, time, sys import yaml +import gc # support PyPy from six.moves import StringIO from datetime import timedelta @@ -1303,9 +1304,11 @@ class CacheDir(unittest.TestCase): a = cdm.get_file("a") b = cdm.get_file("b") c = cdm.get_file("c") - f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f - f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f - f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f + for x in {a, b, c}: + with open(x.get_filename(), "wb") as f: + f.write("hi") + del x + gc.collect() # for PyPy _failUnlessExists("a") _failUnlessExists("b") @@ -1318,6 +1321,7 @@ class CacheDir(unittest.TestCase): _failUnlessExists("c") del a + gc.collect() # for PyPy # this file won't be deleted yet, because it isn't old enough cdm.check() _failUnlessExists("a") @@ -1335,6 +1339,7 @@ class CacheDir(unittest.TestCase): cdm.old = 60*60 del b + gc.collect() # for PyPy cdm.check() _failIfExists("a") @@ -1348,6 +1353,7 @@ class CacheDir(unittest.TestCase): _failUnlessExists("b") _failUnlessExists("c") del b2 + gc.collect() # for PyPy ctr = [0] class EqButNotIs(object): @@ -1617,6 +1623,7 @@ class Pipeline(unittest.TestCase): self.calls[2][0].errback(ValueError("three-error")) del d1,d2,d3,d4 + gc.collect() # for PyPy class SampleError(Exception): pass From 29563b8f2dd809cda82628aad96d070864e90d8d Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 21 Dec 2019 18:01:42 -0700 Subject: [PATCH 0058/1054] assert proper errors --- src/allmydata/test/test_backupdb.py | 2 +- src/allmydata/test/test_multi_introducers.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_backupdb.py b/src/allmydata/test/test_backupdb.py index a10c7514b..03ab62e8f 100644 --- a/src/allmydata/test/test_backupdb.py +++ b/src/allmydata/test/test_backupdb.py @@ -60,7 +60,7 @@ class BackupDB(unittest.TestCase): bdb = backupdb.get_backupdb(where, stderr_f) self.failUnlessEqual(bdb, None) stderr = stderr_f.getvalue() - self.failUnlessIn("Could not open database", stderr) + self.failUnlessIn("unable to open database file", stderr) def writeto(self, filename, data): diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index 2c1e933fa..34e6e5d96 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -157,9 +157,9 @@ class NoDefault(unittest.TestCase): self.yaml_path.setContent(EQUALS_YAML) with self.assertRaises(TypeError) as ctx: yield create_client(self.basedir) - self.assertEquals( - str(ctx.exception), - "string index must be an integer, not str", + self.assertIsInstance( + ctx.exception, + TypeError, ) @defer.inlineCallbacks From bddd2cc9961d6c6a429c9f8ec9a81769725ce6af Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 21 Dec 2019 22:03:24 -0700 Subject: [PATCH 0059/1054] disable system tests on PyPy --- src/allmydata/test/test_system.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 2108ce9dd..66f460695 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -422,6 +422,9 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): + if 'pypy' in sys.version.lower(): + self.skip = "pypy can't run these, due to SSL errors (ee key too tiny)" + self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 13350d60c2905a79c313edd4dbd984ed2494dec2 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 22 Dec 2019 02:21:55 -0700 Subject: [PATCH 0060/1054] assertFailure instead of bespoke code --- src/allmydata/test/cli/test_start.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_start.py b/src/allmydata/test/cli/test_start.py index 90fde3969..42c70f024 100644 --- a/src/allmydata/test/cli/test_start.py +++ b/src/allmydata/test/cli/test_start.py @@ -256,9 +256,7 @@ class RunTests(unittest.TestCase): i, o, e = StringIO(), StringIO(), StringIO() d = runner.dispatch(config, i, o, e) - def must_be_systemexit(f): - assert isinstance(f.value, SystemExit) - d.addErrback(must_be_systemexit) + self.assertFailure(d, SystemExit) output = e.getvalue() # should print out the collected logs and an error-code From 00373fc21159ff55c47b208329fd75f1b5b61f25 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 30 Dec 2019 20:25:59 -0700 Subject: [PATCH 0061/1054] special-case pypy --- src/allmydata/test/test_backupdb.py | 8 ++++++-- src/allmydata/test/test_system.py | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_backupdb.py b/src/allmydata/test/test_backupdb.py index 03ab62e8f..858640d62 100644 --- a/src/allmydata/test/test_backupdb.py +++ b/src/allmydata/test/test_backupdb.py @@ -1,4 +1,4 @@ - +import sys import os.path, time from six.moves import cStringIO as StringIO from twisted.trial import unittest @@ -60,7 +60,11 @@ class BackupDB(unittest.TestCase): bdb = backupdb.get_backupdb(where, stderr_f) self.failUnlessEqual(bdb, None) stderr = stderr_f.getvalue() - self.failUnlessIn("unable to open database file", stderr) + # the error-message is different under PyPy ... not sure why? + if 'pypy' in sys.version.lower(): + self.failUnlessIn("Could not open database", stderr) + else: + self.failUnlessIn("unable to open database file", stderr) def writeto(self, filename, data): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 66f460695..3b8d2b614 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -421,10 +421,10 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): - def setUp(self): - if 'pypy' in sys.version.lower(): - self.skip = "pypy can't run these, due to SSL errors (ee key too tiny)" + if 'pypy' in sys.version.lower(): + skip = "pypy can't run these, due to SSL errors (ee key too tiny)" + def setUp(self): self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 1e039d8364e3a74a43af613f5e931e0689d90005 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Jan 2020 18:59:40 -0700 Subject: [PATCH 0062/1054] certs big enough to make pypy happy --- src/allmydata/test/test_system.py | 809 +++++++++++++++++++----------- 1 file changed, 517 insertions(+), 292 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 3b8d2b614..b3b82582e 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -53,333 +53,561 @@ enough to not fit inside a LIT uri. # overhead of key generation SYSTEM_TEST_CERTS = [ """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxHCWajrR -2h/iurw8k93m8WUdE3xypJiiAITw7GkKlKbCLD+dEce2MXwVVYca0n/MZZsj89Cu -Ko0lLjksMseoSDoj98iEmVpaY5mc2ntpQ+FXdoEmPP234XRWEg2HQ+EaK6+WkGQg -DDXQvFJCVCQk/n1MdAwZZ6vqf2ITzSuD44kCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQBn6qPKGdFjWJy7sOOTUFfm/THhHQqAh1pBDLkjR+OtzuobCoP8n8J1LNG3Yxds -Jj7NWQL7X5TfOlfoi7e9jK0ujGgWh3yYU6PnHzJLkDiDT3LCSywQuGXCjh0tOStS -2gaCmmAK2cfxSStKzNcewl2Zs8wHMygq8TLFoZ6ozN1+xQ== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV +z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG +bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv +Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88 +LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg +FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r +U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW +r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5 +tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE +RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0 +i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl +OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES +cEYA/wmk -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDEcJZqOtHaH+K6vDyT3ebxZR0TfHKkmKIAhPDsaQqUpsIsP50R -x7YxfBVVhxrSf8xlmyPz0K4qjSUuOSwyx6hIOiP3yISZWlpjmZzae2lD4Vd2gSY8 -/bfhdFYSDYdD4Rorr5aQZCAMNdC8UkJUJCT+fUx0DBlnq+p/YhPNK4PjiQIDAQAB -AoGAZyDMdrymiyMOPwavrtlicvyohSBid3MCKc+hRBvpSB0790r2RO1aAySndp1V -QYmCXx1RhKDbrs8m49t0Dryu5T+sQrFl0E3usAP3vvXWeh4jwJ9GyiRWy4xOEuEQ -3ewjbEItHqA/bRJF0TNtbOmZTDC7v9FRPf2bTAyFfTZep5kCQQD33q1RA8WUYtmQ -IArgHqt69i421lpXlOgqotFHwTx4FiGgVzDQCDuXU6txB9EeKRM340poissav/n6 -bkLZ7/VDAkEAyuIPkeI59sE5NnmW+N47NbCfdM1Smy1YxZpv942EmP9Veub5N0dw -iK5bLAgEguUIjpTsh3BRmsE9Xd+ItmnRQwJBAMZhbg19G1EbnE0BmDKv2UbcaThy -bnPSNc6J6T2opqDl9ZvCrMqTDD6dNIWOYAvni/4a556sFsoeBBAu10peBskCQE6S -cB86cuJagLLVMh/dySaI6ahNoFFSpY+ZuQUxfInYUR2Q+DFtbGqyw8JwtHaRBthZ -WqU1XZVGg2KooISsxIsCQQD1PS7//xHLumBb0jnpL7n6W8gmiTyzblT+0otaCisP -fN6rTlwV1o8VsOUAz0rmKO5RArCbkmb01WtMgPCDBYkk ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5 +yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ +MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp +06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB +unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r +G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN +OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc +j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu +0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy +BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9 +48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp +g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC +NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41 +nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL +5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0 +hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ +wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4 +3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F +9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9 +PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61 +7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I +ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b +xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3 +rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K +iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune +yfusXLtCgajpreoVIa0xWdQ= +-----END PRIVATE KEY----- """, # 0 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDVaFw0wOTA3MjUyMjQyMDVaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAs9CALdmW -kJ6r0KPSLdGCA8rzQKxWayrMckT22ZtbRv3aw6VA96dWclpY+T2maV0LrAzmMSL8 -n61ydJHM33iYDOyWbwHWN45XCjY/e20PL54XUl/DmbBHEhQVQLIfCldcRcnWEfoO -iOhDJfWpDO1dmP/aOYLdkZCZvBtPAfyUqRcCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQAN9eaCREkzzk4yPIaWYkWHg3Igs1vnOR/iDw3OjyxO/xJFP2lkA2WtrwL2RTRq -dxA8gwdPyrWgdiZElwZH8mzTJ4OdUXLSMclLOg9kvH6gtSvhLztfEDwDP1wRhikh -OeWWu2GIC+uqFCI1ftoGgU+aIa6yrHswf66rrQvBSSvJPQ== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApDzW +4ZBeK9w4xpRaed6lXzeCO0Xmr3f0ynbueSdiZ89FWoAMgK+SiBIOViYV6hfm0Wah +lemSNzFGx5LvDSg2uwSqEP23DeM9O/SQPgIAiLeeEsYZJcgg2jz92YfFEaahsGdI +6qSP4XI2/5dgKRpPOYDGyw6R5PQR6w22Xq1WD1jBvImk/k09I9jHRn40pYbaJzbg +U2aIjvOruo2kqe4f6iDqE0piYimAZJUvemu1UoyV5NG590hGkDuWsMD77+d2FxCj +9Nzb+iuuG3ksnanHPyXi1hQmzp5OmzVWaevCHinNjWgsuSuLGO9H2SLf3wwp2UCs +EpKtzoKrnZdEg/anNwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQChxtr67o1aZZMJ +A6gESPtFjZLw6wG0j50JsrWKLvoXVts1ToJ9u2nx01aFKjBwb4Yg+vdJfDgIIAEm +jS56h6H2DfJlkTWHmi8Vx1wuusWnrNwYMI53tdlRIpD2+Ne7yeoLQZcVN2wuPmxD +Mbksg4AI4csmbkU/NPX5DtMy4EzM/pFvIcxNIVRUMVTFzn5zxhKfhyPqrMI4fxw1 +UhUbEKO+QgIqTNp/dZ0lTbFs5HJQn6yirWyyvQKBPmaaK+pKd0RST/T38OU2oJ/J +LojRs7ugCJ+bxJqegmQrdcVqZZGbpYeK4O/5eIn8KOlgh0nUza1MyjJJemgBBWf7 +HoXB8Fge -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQCz0IAt2ZaQnqvQo9It0YIDyvNArFZrKsxyRPbZm1tG/drDpUD3 -p1ZyWlj5PaZpXQusDOYxIvyfrXJ0kczfeJgM7JZvAdY3jlcKNj97bQ8vnhdSX8OZ -sEcSFBVAsh8KV1xFydYR+g6I6EMl9akM7V2Y/9o5gt2RkJm8G08B/JSpFwIDAQAB -AoGBAIUy5zCPpSP+FeJY6CG+t6Pdm/IFd4KtUoM3KPCrT6M3+uzApm6Ny9Crsor2 -qyYTocjSSVaOxzn1fvpw4qWLrH1veUf8ozMs8Z0VuPHD1GYUGjOXaBPXb5o1fQL9 -h7pS5/HrDDPN6wwDNTsxRf/fP58CnfwQUhwdoxcx8TnVmDQxAkEA6N3jBXt/Lh0z -UbXHhv3QBOcqLZA2I4tY7wQzvUvKvVmCJoW1tfhBdYQWeQv0jzjL5PzrrNY8hC4l -8+sFM3h5TwJBAMWtbFIEZfRSG1JhHK3evYHDTZnr/j+CdoWuhzP5RkjkIKsiLEH7 -2ZhA7CdFQLZF14oXy+g1uVCzzfB2WELtUbkCQQDKrb1XWzrBlzbAipfkXWs9qTmj -uJ32Z+V6+0xRGPOXxJ0sDDqw7CeFMfchWg98zLFiV+SEZV78qPHtkAPR3ayvAkB+ -hUMhM4N13t9x2IoclsXAOhp++9bdG0l0woHyuAdOPATUw6iECwf4NQVxFRgYEZek -4Ro3Y7taddrHn1dabr6xAkAic47OoLOROYLpljmJJO0eRe3Z5IFe+0D2LfhAW3LQ -JU+oGq5pCjfnoaDElRRZn0+GmunnWeQEYKoflTi/lI9d ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCkPNbhkF4r3DjG +lFp53qVfN4I7Reavd/TKdu55J2Jnz0VagAyAr5KIEg5WJhXqF+bRZqGV6ZI3MUbH +ku8NKDa7BKoQ/bcN4z079JA+AgCIt54SxhklyCDaPP3Zh8URpqGwZ0jqpI/hcjb/ +l2ApGk85gMbLDpHk9BHrDbZerVYPWMG8iaT+TT0j2MdGfjSlhtonNuBTZoiO86u6 +jaSp7h/qIOoTSmJiKYBklS96a7VSjJXk0bn3SEaQO5awwPvv53YXEKP03Nv6K64b +eSydqcc/JeLWFCbOnk6bNVZp68IeKc2NaCy5K4sY70fZIt/fDCnZQKwSkq3Ogqud +l0SD9qc3AgMBAAECggEBAIu55uaIOFYASZ1IYaEFNpRHWVisI5Js76nAfSo9w46l +3E8eWYSx2mxBUEkipco/A3RraFVuHaMvHRR1gUMkT0vUsAs8jxwVk+cKLh1S/rlR +3f4C4yotlSWWdjE3PQXDShQWCwb1ciNPVFMmqfzOEVDOqlHe12h97TCYverWdT0f +3LZICLQsZd1WPKnPNXrsRRDCBuRLapdg+M0oJ+y6IiCdm+qM7Qvaoef6hlvm5ECz +LCM92db5BKTuPOQXMx2J8mjaBgU3aHxRV08IFgs7mI6q0t0FM7LlytIAJq1Hg5QU +36zDKo8tblkPijWZWlqlZCnlarrd3Ar/BiLEiuOGDMECgYEA1GOp0KHy0mbJeN13 ++TDsgP7zhmqFcuJREu2xziNJPK2S06NfGYE8vuVqBGzBroLTQ3dK7rOJs9C6IjCE +mH7ZeHzfcKohpZnl443vHMSpgdh/bXTEO1aQZNbJ2hLYs8ie/VqqHR0u6YtpUqZL +LgaUA0U8GnlsO55B8kyCelckmDkCgYEAxfYQMPEEzg1tg2neqEfyoeY0qQTEJTeh +CPMztowSJpIyF1rQH6TaG0ZchkiAkw3W58RVDfvK72TuVlC5Kz00C2/uPnrqm0dX +iMPeML5rFlG3VGCrSTnAPI+az6P65q8zodqcTtA8xoxgPOlc/lINOxiTEMxLyeGF +8GyP+sCM2u8CgYEAvMBR05OJnEky9hJEpBZBqSZrQGL8dCwDh0HtCdi8JovPd/yx +8JW1aaWywXnx6uhjXoru8hJm54IxWV8rB+d716OKY7MfMfACqWejQDratgW0wY7L +MjztGGD2hLLJGYXLHjfsBPHBllaKZKRbHe1Er19hWdndQWKVEwPB1X4KjKkCgYEA +nWHmN3K2djbYtRyLR1CEBtDlVuaSJmCWp23q1BuCJqYeKtEpG69NM1f6IUws5Dyh +eXtuf4KKMU8V6QueW1D6OomPaJ8CO9c5MWM/F5ObwY/P58Y/ByVhvwQQeToONC5g +JzKNCF+nodZigKqrIwoKuMvtx/IT4vloKd+1jA5fLYMCgYBoT3HLCyATVdDSt1TZ +SbEDoLSYt23KRjQV93+INP949dYCagtgh/kTzxBopw5FljISLfdYizIRo2AzhhfP +WWpILlnt19kD+sNirJVqxJacfEZsu5baWTedI/yrCuVsAs/s3/EEY6q0Qywknxtp +Fwh1/8y5t14ib5fxOVhi8X1nEA== +-----END PRIVATE KEY----- """, # 1 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsxG7LTrz -DF+9wegOR/BRJhjSumPUbYQnNAUKtPraFsGjAJILP44AHdnHt1MONLgTeX1ynapo -q6O/q5cdKtBB7uEh7FpkLCCwpZt/m0y79cynn8AmWoQVgl8oS0567UmPeJnTzFPv -dmT5dlaQALeX5YGceAsEvhmAsdOMttaor38CAwEAATANBgkqhkiG9w0BAQQFAAOB -gQA345rxotfvh2kfgrmRzAyGewVBV4r23Go30GSZir8X2GoH3qKNwO4SekAohuSw -AiXzLUbwIdSRSqaLFxSC7Duqc9eIeFDAWjeEmpfFLBNiw3K8SLA00QrHCUXnECTD -b/Kk6OGuvPOiuuONVjEuEcRdCH3/Li30D0AhJaMynjhQJQ== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwMTn +hXnpKHGAir3WYbOxefVrMA07OZNAsNa29nBwLA+NVIJNUFgquibMj7QYo8+M45oY +6LKr4yRcBryZVvyxfdr92xp8+kLeVApk2WLjkdBTRagHh9qdrY0hQmagCBN6/hLG +Xug8VksQUdhX3vu6ZyMvTLfKRkDOMRVkRGRGg/dOcvom7zpqMCGYenMG2FStr6UV +3s3dlCSZZTdTX5Uoq6yfUUJE3nITGKjpnpJKqIs3PWCIxdj7INIcjJKvIdUcavIV +2hEhh60A8ltmtdpQAXVBE+U7aZgS1fGAWS2A0a3UwuP2pkQp6OyKCUVHpZatbl9F +ahDN2QBzegv/rdJ1zwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAl4OQZ+FB9ZSUv +FL/KwLNt+ONU8Sve/xiX+8vKAvgKm2FrjyK+AZPwibnu+FSt2G4ndZBx4Wvpe5V+ +gCsbzSXlh9cDn2SRXyprt2l/8Fj4eUMaThmLKOK200/N/s2SpmBtnuflBrhNaJpw +DEi2KEPuXsgvkuVzXN06j75cUHwn5LeWDAh0RalkVuGbEWBoFx9Hq8WECdlCy0YS +y09+yO01qz70y88C2rPThKw8kP4bX8aFZbvsnRHsLu/8nEQNlrELcfBarPVHjJ/9 +imxOdymJkV152V58voiXP/PwXhynctQbF7e+0UZ+XEGdbAbZA0BMl7z+b09Z+jF2 +afm4mVox -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQCzEbstOvMMX73B6A5H8FEmGNK6Y9RthCc0BQq0+toWwaMAkgs/ -jgAd2ce3Uw40uBN5fXKdqmiro7+rlx0q0EHu4SHsWmQsILClm3+bTLv1zKefwCZa -hBWCXyhLTnrtSY94mdPMU+92ZPl2VpAAt5flgZx4CwS+GYCx04y21qivfwIDAQAB -AoGBAIlhFg/aRPL+VM9539LzHN60dp8GzceDdqwjHhbAySZiQlLCuJx2rcI4/U65 -CpIJku9G/fLV9N2RkA/trDPXeGyqCTJfnNzyZcvvMscRMFqSGyc21Y0a+GS8bIxt -1R2B18epSVMsWSWWMypeEgsfv29LV7oSWG8UKaqQ9+0h63DhAkEA4i2L/rori/Fb -wpIBfA+xbXL/GmWR7xPW+3nG3LdLQpVzxz4rIsmtO9hIXzvYpcufQbwgVACyMmRf -TMABeSDM7wJBAMquEdTaVXjGfH0EJ7z95Ys2rYTiCXjBfyEOi6RXXReqV9SXNKlN -aKsO22zYecpkAjY1EdUdXWP/mNVEybjpZnECQQCcuh0JPS5RwcTo9c2rjyBOjGIz -g3B1b5UIG2FurmCrWe6pgO3ZJFEzZ/L2cvz0Hj5UCa2JKBZTDvRutZoPumfnAkAb -nSW+y1Rz1Q8m9Ub4v9rjYbq4bRd/RVWtyk6KQIDldYbr5wH8wxgsniSVKtVFFuUa -P5bDY3HS6wMGo42cTOhxAkAcdweQSQ3j7mfc5vh71HeAC1v/VAKGehGOUdeEIQNl -Sb2WuzpZkbfsrVzW6MdlgY6eE7ufRswhDPLWPC8MP0d1 ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAxOeFeekocYCK +vdZhs7F59WswDTs5k0Cw1rb2cHAsD41Ugk1QWCq6JsyPtBijz4zjmhjosqvjJFwG +vJlW/LF92v3bGnz6Qt5UCmTZYuOR0FNFqAeH2p2tjSFCZqAIE3r+EsZe6DxWSxBR +2Ffe+7pnIy9Mt8pGQM4xFWREZEaD905y+ibvOmowIZh6cwbYVK2vpRXezd2UJJll +N1NflSirrJ9RQkTechMYqOmekkqoizc9YIjF2Psg0hyMkq8h1Rxq8hXaESGHrQDy +W2a12lABdUET5TtpmBLV8YBZLYDRrdTC4/amRCno7IoJRUellq1uX0VqEM3ZAHN6 +C/+t0nXPAgMBAAECggEAF+2ZK4lZdsq4AQDVhqUuh4v+NSW/T0NHCWxto6OLWPzJ +N09BV5LKIvdD9yaM1HCj9XCgXOooyfYuciuhARo20f+H+VWNY+c+/8GWiSFsTCJG +4+Oao7NwVSWqljp07Ou2Hamo9AjxzGhe6znmlmg62CiW63f45MWQkqksHA0yb5jg +/onJ2//I+OI+aTKNfjt1G6h2x7oxeGTU1jJ0Hb2xSh+Mpqx9NDfb/KZyOndhSG5N +xRVosQ6uV+9mqHxTTwTZurTG31uhZzarkMuqxhcHS94ub7berEc/OlqvbyMKNZ3A +lzuvq0NBZhEUhAVgORAIS17r/q2BvyG4u5LFbG2p0QKBgQDeyyOl+A7xc4lPE2OL +Z3KHJPP4RuUnHnWFC+bNdr5Ag8K7jcjZIcasyUom9rOR0Fpuw9wmXpp3+6fyp9bJ +y6Bi5VioR0ZFP5X+nXxIN3yvgypu6AZvkhHrEFer+heGHxPlbwNKCKMbPzDZPBTZ +vlC7g7xUUcpNmGhrOKr3Qq5FlwKBgQDdgCmRvsHUyzicn8TI3IJBAOcaQG0Yr/R2 +FzBqNfHHx7fUZlJfKJsnu9R9VRZmBi4B7MA2xcvz4QrdZWEtY8uoYp8TAGILfW1u +CP4ZHrzfDo/67Uzk2uTMTd0+JOqSm/HiVNguRPvC8EWBoFls+h129GKThMvKR1hP +1oarfAGIiQKBgQCIMAq5gHm59JMhqEt4QqMKo3cS9FtNX1wdGRpbzFMd4q0dstzs +ha4Jnv3Z9YHtBzzQap9fQQMRht6yARDVx8hhy6o3K2J0IBtTSfdXubtZGkfNBb4x +Y0vaseG1uam5jbO+0u5iygbSN/1nPUfNln2JMkzkCh8s8ZYavMgdX0BiPwKBgChR +QL/Hog5yoy5XIoGRKaBdYrNzkKgStwObuvNKOGUt5DckHNA3Wu6DkOzzRO1zKIKv +LlmJ7VLJ3qln36VcaeCPevcBddczkGyb9GxsHOLZCroY4YsykLzjW2cJXy0qd3/E +A8mAQvc7ttsebciZSi2x1BOX82QxUlDN8ptaKglJAoGBAMnLN1TQB0xtWYDPGcGV +2IvgX7OTRRlMVrTvIOvP5Julux9z1r0x0cesl/jaXupsBUlLLicPyBMSBJrXlr24 +mrgkodk4TdqO1VtBCZBqak97DHVezstMrbpCGlUD5jBnsHVRLERvS09QlGhqMeNL +jpNQbWH9VhutzbvpYquKrhvK +-----END PRIVATE KEY----- """, # 2 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxnH+pbOS -qlJlsHpKUQtV0oN1Mv+ESG+yUDxStFFGjkJv/UIRzpxqFqY/6nJ3D03kZsDdcXyi -CfV9hPYQaVNMn6z+puPmIagfBQ0aOyuI+nUhCttZIYD9071BjW5bCMX5NZWL/CZm -E0HdAZ77H6UrRckJ7VR8wAFpihBxD5WliZcCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQAwXqY1Sjvp9JSTHKklu7s0T6YmH/BKSXrHpS2xO69svK+ze5/+5td3jPn4Qe50 -xwRNZSFmSLuJLfCO32QJSJTB7Vs5D3dNTZ2i8umsaodm97t8hit7L75nXRGHKH// -xDVWAFB9sSgCQyPMRkL4wB4YSfRhoSKVwMvaz+XRZDUU0A== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAypqi +YTni3s60Uo8vgGcFvjWWkB5CD9Fx9pW/2KcxRJ/u137Y+BG8qWMA4lgII3ZIuvo4 +6rLDiXnAnDZqUtrvZ90O/gH6RyQqX3AI4EwPvCnRIIe0okRcxnxYBL/LfBY54xuv +46JRYZP4c9IImqQH9QVo2/egtEzcpbmT/mfhpf6NGQWC3Xps2BqDT2SV/DrX/wPA +8P1atE1AxNp8ENxK/cjFAteEyDZOsDSa757ZHKAdM7L8rZ1Fd2xAA1Dq7IyYpTNE +IX72xytWxllcNvSUPLT+oicsSZBadc/p3moc3tR/rNdgrHKybedadru/f9Gwpa+v +0sllZlEcVPSYddAzWwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCmk60Nj5FPvemx +DSSQjJPyJoIDpTxQ4luSzIq4hPwlUXw7dqrvHyCWgn2YVe9xZsGrT/+n376ecmgu +sw4s4qVhR9bzKkTMewjC2wUooTA5v9HYsNWZy3Ah7hHPbDHlMADYobjB5/XolNUP +bCM9xALEdM9DxpC4vjUZexlRKmjww9QKE22jIM+bqsK0zqDSq+zHpfHNGGcS3vva +OvI6FPc1fAr3pZpVzevMSN2zufIJwjL4FT5/uzwOCaSCwgR1ztD5CSbQLTLlwIsX +S7h2WF9078XumeRjKejdjEjyH4abKRq8+5LVLcjKEpg7OvktuRpPoGPCEToaAzuv +h+RSQwwY -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDGcf6ls5KqUmWwekpRC1XSg3Uy/4RIb7JQPFK0UUaOQm/9QhHO -nGoWpj/qcncPTeRmwN1xfKIJ9X2E9hBpU0yfrP6m4+YhqB8FDRo7K4j6dSEK21kh -gP3TvUGNblsIxfk1lYv8JmYTQd0BnvsfpStFyQntVHzAAWmKEHEPlaWJlwIDAQAB -AoGAdHNMlXwtItm7ZrY8ihZ2xFP0IHsk60TwhHkBp2LSXoTKJvnwbSgIcUYZ18BX -8Zkp4MpoqEIU7HcssyuaMdR572huV2w0D/2gYJQLQ5JapaR3hMox3YG4wjXasN1U -1iZt7JkhKlOy+ElL5T9mKTE1jDsX2RAv4WALzMpYFo7vs4ECQQDxqrPaqRQ5uYS/ -ejmIk05nM3Q1zmoLtMDrfRqrjBhaf/W3hqGihiqN2kL3PIIYcxSRWiyNlYXjElsR -2sllBTe3AkEA0jcMHVThwKt1+Ce5VcE7N6hFfbsgISTjfJ+Q3K2NkvJkmtE8ZRX5 -XprssnPN8owkfF5yuKbcSZL3uvaaSGN9IQJAfTVnN9wwOXQwHhDSbDt9/KRBCnum -n+gHqDrKLaVJHOJ9SZf8eLswoww5c+UqtkYxmtlwie61Tp+9BXQosilQ4wJBAIZ1 -XVNZmriBM4jR59L5MOZtxF0ilu98R+HLsn3kqLyIPF9mXCoQPxwLHkEan213xFKk -mt6PJDIPRlOZLqAEuuECQFQMCrn0VUwPg8E40pxMwgMETvVflPs/oZK1Iu+b7+WY -vBptAyhMu31fHQFnJpiUOyHqSZnOZyEn1Qu2lszNvUg= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDKmqJhOeLezrRS +jy+AZwW+NZaQHkIP0XH2lb/YpzFEn+7Xftj4EbypYwDiWAgjdki6+jjqssOJecCc +NmpS2u9n3Q7+AfpHJCpfcAjgTA+8KdEgh7SiRFzGfFgEv8t8FjnjG6/jolFhk/hz +0giapAf1BWjb96C0TNyluZP+Z+Gl/o0ZBYLdemzYGoNPZJX8Otf/A8Dw/Vq0TUDE +2nwQ3Er9yMUC14TINk6wNJrvntkcoB0zsvytnUV3bEADUOrsjJilM0QhfvbHK1bG +WVw29JQ8tP6iJyxJkFp1z+neahze1H+s12CscrJt51p2u79/0bClr6/SyWVmURxU +9Jh10DNbAgMBAAECggEBALv7Q+Rf+C7wrQDZF6LUc9CrGfq4CGVy2IGJKgqT/jOF +DO9nI1rv4hNr55sbQNneWtcZaYvht2mrzNlj57zepDjDM7DcFuLBHIuWgLXT/NmC +FyZOo3vXYBlNr8EgT2XfnXAp9UWJCmc2CtUzsIYC4dsmXMeTd8kyc5tUl4r5ybTf +1g+RTck/IGgqdfzpuTsNl79FW2rP9z111Py6dbqgQzhuSAune9dnLFvZst8dyL8j +FStETMxBM6jrCF1UcKXzG7trDHiCdzJ8WUhx6opN/8OasQGndwpXto6FZuBy/AVP +4kVQNpUXImYcLEpva0MqGRHg+YN+c84C71CMchnF4aECgYEA7J2go4CkCcZNKCy5 +R5XVCqNFYRHjekR+UwH8cnCa7pMKKfP+lTCiBrO2q8zwWwknRMyuycS5g/xbSpg1 +L6hi92CV1YQy1/JhlQedekjejNTTuLOPKf78AFNSfc7axDnes2v4Bvcdp9gsbUIO +10cXh0tOSLE7P9y+yC86KQkFAPECgYEA2zO0M2nvbPHv2jjtymY3pflYm0HzhM/T +kPtue3GxOgbEPsHffBGssShBTE3yCOX3aAONXJucMrSAPL9iwUfgfGx6ADdkwBsA +OjDlkxvTbP/9trE6/lsSPtGpWRdJNHqXN4Hx7gXJizRwG7Ym+oHvIIh53aIjdFoE +HLQLpxObuQsCgYAuMQ99G83qQpYpc6GwAeYXL4yJyK453kky9z5LMQRt8rKXQhS/ +F0FqQYc1vsplW0IZQkQVC5yT0Z4Yz+ICLcM0O9zEVAyA78ZxC42Io9UedSXn9tXK +Awc7IQkHmmxGxm1dZYSEB5X4gFEb+zted3h2ZxMfScohS3zLI70c6a/aYQKBgQCU +phRuxUkrTUpFZ1PCbN0R/ezbpLbaewFTEV7T8b6oxgvxLxI6FdZRcSYO89DNvf2w +GLCVe6VKMWPBTlxPDEostndpjCcTq3vU+nHE+BrBkTvh14BVGzddSFsaYpMvNm8z +ojiJHH2XnCDmefkm6lRacJKL/Tcj4SNmv6YjUEXLDwKBgF8WV9lzez3d/X5dphLy +2S7osRegH99iFanw0v5VK2HqDcYO9A7AD31D9nwX46QVYfgEwa6cHtVCZbpLeJpw +qXnYXe/hUU3yn5ipdNJ0Dm/ZhJPDD8TeqhnRRhxbZmsXs8EzfwB2tcUbASvjb3qA +vAaPlOSU1wXqhAsG9aVs8gtL +-----END PRIVATE KEY----- """, # 3 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnjiOwipn -jigDuNMfNG/tBJhPwYUHhSbQdvrTubhsxw1oOq5XpNqUwRtC8hktOKM3hghyqExP -62EOi0aJBkRhtwtPSLBCINptArZLfkog/nTIqVv4eLEzJ19nTi/llHHWKcgA6XTI -sU/snUhGlySA3RpETvXqIJTauQRZz0kToSUCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQCQ+u/CsX5WC5m0cLrpyIS6qZa62lrB3mj9H1aIQhisT5kRsMz3FJ1aOaS8zPRz -w0jhyRmamCcSsWf5WK539iOtsXbKMdAyjNtkQO3g+fnsLgmznAjjst24jfr+XU59 -0amiy1U6TY93gtEBZHtiLldPdUMsTuFbBlqbcMBQ50x9rA== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzUqQ +M08E7F2ZE99bFHvpsR6LmgIJOOoGMXacTcEUhRF63E6+730FjxER2a30synv9GGS +3G9FstUmfhyimufkbTumri8Novw5CWZQLiE1rmMBI5nPcR2wAzy9z2odR6bfAwms +yyc3IPYg1BEDBPZl0LCQrQRRU/rVOrbCf7IMq+ATazmBg01gXMzq2M953ieorkQX +MsHVR/kyW0Q0yzhYF1OtIqbXxrdiZ+laTLWNqivj/FdegiWPCf8OcqpcpbgEjlDW +gBcC/vre+0E+16nfUV8xHL5jseJMJqfT508OtHxAzp+2D7b54NvYNIvbOAP+F9gj +aXy5mOvjXclK+hNmDwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAjZzTFKG7uoXxm +BPHfQvsKHIB/Cx9zMKj6pLwJzCPHQBzKOMoUen09oq+fb77RM7WvdX0pvFgEXaJW +q/ImooRMo+paf8GOZAuPwdafb2/OGdHZGZ2Cbo/ICGo1wGDCdMvbxTxrDNq1Yae+ +m+2epN2pXAO1rlc7ktRkojM/qi3zXtbLjTs3IoPDXWhYPHdI1ThkneRmvxpzB1rW +2SBqj2snvyI+/3k3RHmldcdOrTlgWQ9hq05jWR8IVtRUFFVn9A+yQC3gnnLIUhwP +HJWwTIPuYW25TuxFxYZXIbnAiluZL0UIjd3IAwxaafvB6uhI7v0K789DKj2vRUkY +E8ptxZH4 -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCeOI7CKmeOKAO40x80b+0EmE/BhQeFJtB2+tO5uGzHDWg6rlek -2pTBG0LyGS04ozeGCHKoTE/rYQ6LRokGRGG3C09IsEIg2m0Ctkt+SiD+dMipW/h4 -sTMnX2dOL+WUcdYpyADpdMixT+ydSEaXJIDdGkRO9eoglNq5BFnPSROhJQIDAQAB -AoGAAPrst3s3xQOucjismtCOsVaYN+SxFTwWUoZfRWlFEz6cBLELzfOktEWM9p79 -TrqEH4px22UNobGqO2amdql5yXwEFVhYQkRB8uDA8uVaqpL8NLWTGPRXxZ2DSU+n -7/FLf/TWT3ti/ZtXaPVRj6E2/Mq9AVEVOjUYzkNjM02OxcECQQDKEqmPbdZq2URU -7RbUxkq5aTp8nzAgbpUsgBGQ9PDAymhj60BDEP0q28Ssa7tU70pRnQ3AZs9txgmL -kK2g97FNAkEAyHH9cIb6qXOAJPIr/xamFGr5uuYw9TJPz/hfVkVimW/aZnBB+e6Q -oALJBDKJWeYPzdNbouJYg8MeU0qWdZ5DOQJADUk+1sxc/bd9U6wnBSRog1pU2x7I -VkmPC1b8ULCaJ8LnLDKqjf5O9wNuIfwPXB1DoKwX3F+mIcyUkhWYJO5EPQJAUj5D -KMqZSrGzYHVlC/M1Daee88rDR7fu+3wDUhiCDkbQq7tftrbl7GF4LRq3NIWq8l7I -eJq6isWiSbaO6Y+YMQJBAJFBpVhlY5Px2BX5+Hsfq6dSP3sVVc0eHkdsoZFFxq37 -fksL/q2vlPczvBihgcxt+UzW/UrNkelOuX3i57PDvFs= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDNSpAzTwTsXZkT +31sUe+mxHouaAgk46gYxdpxNwRSFEXrcTr7vfQWPERHZrfSzKe/0YZLcb0Wy1SZ+ +HKKa5+RtO6auLw2i/DkJZlAuITWuYwEjmc9xHbADPL3Pah1Hpt8DCazLJzcg9iDU +EQME9mXQsJCtBFFT+tU6tsJ/sgyr4BNrOYGDTWBczOrYz3neJ6iuRBcywdVH+TJb +RDTLOFgXU60iptfGt2Jn6VpMtY2qK+P8V16CJY8J/w5yqlyluASOUNaAFwL++t77 +QT7Xqd9RXzEcvmOx4kwmp9PnTw60fEDOn7YPtvng29g0i9s4A/4X2CNpfLmY6+Nd +yUr6E2YPAgMBAAECggEBAIiL6uQl0AmDrBj6vHMghGzp+0MBza6MgngOA6L4JTTp +ToYQ3pEe4D6rxOq7+QHeiBtNd0ilvn9XpVXGqCVOzrIVNiWvaGubRjjJU9WLA1Ct +y4kpekAr1fIhScMXOsh45ub3XXZ27AVBkM5dTlvTpB8uAd0C/TFVqtR10WLsQ99h +Zm9Jczgs/6InYTssnAaqdeCLAf1LbmO4zwFsJfJOeSGGT6WBwlpHwMAgPhg8OLEu +kVWG7BEJ0hxcODk/es/vce9SN7BSyIzNY+qHcGtsrx/o0eO2Av/Z7ltV4Sz6UN1K +0y0OTiDyT/l62U2OugSN3wQ4xPTwlrWl7ZUHJmvpEaECgYEA+w2JoB2i1OV2JTPl +Y0TKSKcZYdwn7Nwh4fxMAJNJ8UbpPqrZEo37nxqlWNJrY/jKX3wHVk4ESSTaxXgF +UY7yKT0gRuD9+vE0gCbUmJQJTwbceNJUu4XrJ6SBtf72WgmphL+MtyKdwV8XltVl +Yp0hkswGmxl+5+Js6Crh7WznPl8CgYEA0VYtKs2YaSmT1zraY6Fv3AIQZq012vdA +7nVxmQ6jKDdc401OWARmiv0PrZaVNiEJ1YV8KxaPrKTfwhWqxNegmEBgA1FZ66NN +SAm8P9OCbt8alEaVkcATveXTeOCvfpZUO3sqZdDOiYLiLCsokHblkcenK85n0yT6 +CzhTbvzDllECgYEAu9mfVy2Vv5OK2b+BLsw0SDSwa2cegL8eo0fzXqLXOzCCKqAQ +GTAgTSbU/idEr+NjGhtmKg/qaQioogVyhVpenLjeQ+rqYDDHxfRIM3rhlD5gDg/j +0wUbtegEHrgOgcSlEW16zzWZsS2EKxq16BoHGx6K+tcS/FOShg5ASzWnuiUCgYEA +sMz+0tLX8aG7CqHbRyBW8FMR9RY/kRMY1Q1+Bw40wMeZfSSSkYYN8T9wWWT/2rqm +qp7V0zJ34BFUJoDUPPH84fok3Uh9EKZYpAoM4z9JP0jREwBWXMYEJnOQWtwxfFGN +DLumgF2Nwtg3G6TL2s+AbtJYH4hxagQl5woIdYmnyzECgYEAsLASpou16A3uXG5J ++5ZgF2appS9Yfrqfh6TKywMsGG/JuiH3djdYhbJFIRGeHIIDb4XEXOHrg/SFflas +If0IjFRh9WCvQxnoRha3/pKRSc3OEka1MR/ZREK/d/LQEPmsRJVzY6ABKqmPAMDD +5CnG6Hz/rP87BiEKd1+3PGp8GCw= +-----END PRIVATE KEY----- """, # 4 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQyMDZaFw0wOTA3MjUyMjQyMDZaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsCQuudDF -zgmY5tDpT0TkUo8fpJ5JcvgCkLFpSDD8REpXhLFkHWhTmTj3CAxfv4lA3sQzHZxe -4S9YCb5c/VTbFEdgwc/wlxMmJiz2jYghdmWPBb8pBEk31YihIhC+u4kex6gJBH5y -ixiZ3PPRRMaOBBo+ZfM50XIyWbFOOM/7FwcCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQB4cFURaiiUx6n8eS4j4Vxrii5PtsaNEI4acANFSYknGd0xTP4vnmoivNmo5fWE -Q4hYtGezNu4a9MnNhcQmI20KzXmvhLJtkwWCgGOVJtMem8hDWXSALV1Ih8hmVkGS -CI1elfr9eyguunGp9eMMQfKhWH52WHFA0NYa0Kpv5BY33A== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0sap +75YbbkEL85LFava3FrO1jpgVteQ4NGxxy1Nu9w2hPfMMeCPWjB8UfAwFk+LVPyvW +LAXd1zWL5rGpQ2ytIVQlTraR5EnALA1sMcQYbFz1ISPTYB031bEN/Ch8JWYwCG5A +X2H4D6BC7NgT6YyWDt8vxQnqAisPHQ/OK4ABD15CwkTyPimek2/ufYN2dapg1xhG +IUD96gqetJv9bu0r869s688kADIComsYG+8KKfFN67S3rSHMIpZPuGTtoHGnVO89 +XBm0vNe0UxQkJEGJzZPn0tdec0LTC4GNtTaz5JuCjx/VsJBqrnTnHHjx0wFz8pff +afCimRwA+LCopxPE1QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBOkAnpBb3nY+dG +mKCjiLqSsuEPqpNiBYR+ue/8aVDnOKLKqAyQuyRZttQ7bPpKHaw7pwyCZH8iHnt6 +pMCLCftNSlV2Fa8msRmuf5AiGjUvR1M8VtHWNYE8pedWrJqUgBhF/405B99yd8CT +kQJXKF18LObj7YKNsWRoMkVgqlQzWDMEqbfmy9MhuLx2EZPsTB1L0BHNGGDVBd9o +cpPLUixcc12u+RPMKq8x3KgwsnUf5vX/pCnoGcCy4JahWdDgcZlf0hUKGT7PUem5 +CWW8SMeqSWQX9XpE5Qlm1+W/QXdDXLbbHqDtvBeUy3iFQe3C9RSkp0qdutxkAlFk +f5QHXfJ7 -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQCwJC650MXOCZjm0OlPRORSjx+knkly+AKQsWlIMPxESleEsWQd -aFOZOPcIDF+/iUDexDMdnF7hL1gJvlz9VNsUR2DBz/CXEyYmLPaNiCF2ZY8FvykE -STfViKEiEL67iR7HqAkEfnKLGJnc89FExo4EGj5l8znRcjJZsU44z/sXBwIDAQAB -AoGABA7xXKqoxBSIh1js5zypHhXaHsre2l1Igdj0mgs25MPpvE7yBZNvyan8Vx0h -36Hj8r4Gh3og3YNfvem67sNTwNwONY0ep+Xho/3vG0jFATGduSXdcT04DusgZNqg -UJqW75cqxrD6o/nya5wUoN9NL5pcd5AgVMdOYvJGbrwQuaECQQDiCs/5dsUkUkeC -Tlur1wh0wJpW4Y2ctO3ncRdnAoAA9y8dELHXMqwKE4HtlyzHY7Bxds/BDh373EVK -rsdl+v9JAkEAx3xTmsOQvWa1tf/O30sdItVpGogKDvYqkLCNthUzPaL85BWB03E2 -xunHcVVlqAOE5tFuw0/UEyEkOaGlNTJTzwJAPIVel9FoCUiKYuYt/z1swy3KZRaw -/tMmm4AZHvh5Y0jLcYHFy/OCQpRkhkOitqQHWunPyEXKW2PnnY5cTv68GQJAHG7H -B88KCUTjb25nkQIGxBlA4swzCtDhXkAb4rEA3a8mdmfuWjHPyeg2ShwO4jSmM7P0 -Iph1NMjLff9hKcTjlwJARpItOFkYEdtSODC7FMm7KRKQnNB27gFAizsOYWD4D2b7 -w1FTEZ/kSA9wSNhyNGt7dgUo6zFhm2u973HBCUb3dg== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSxqnvlhtuQQvz +ksVq9rcWs7WOmBW15Dg0bHHLU273DaE98wx4I9aMHxR8DAWT4tU/K9YsBd3XNYvm +salDbK0hVCVOtpHkScAsDWwxxBhsXPUhI9NgHTfVsQ38KHwlZjAIbkBfYfgPoELs +2BPpjJYO3y/FCeoCKw8dD84rgAEPXkLCRPI+KZ6Tb+59g3Z1qmDXGEYhQP3qCp60 +m/1u7Svzr2zrzyQAMgKiaxgb7wop8U3rtLetIcwilk+4ZO2gcadU7z1cGbS817RT +FCQkQYnNk+fS115zQtMLgY21NrPkm4KPH9WwkGqudOccePHTAXPyl99p8KKZHAD4 +sKinE8TVAgMBAAECggEALU5EotoqJUXYEtAenUJQ0pFoWjE4oXNf3Wzd/O1/MZ19 +ZjqDGKPjbxUTKyLOZB5i5gQ/MhFEwQiifMD9eB+5CyvyJPw7Wc28f/uWoQ/cjBZj +Hm979PHy2X0IW4Y8QTG462b/cUE2t+0j1ZMQnKf6bVHuC7V41mR5CC8oitMl5y5g +34yJmWXlIA0ep/WotLMqvil6DnSM/2V8Ch4SxjnzPpjbe4Kj+woucGNr4UKstZER +8iuHTsR64LjoGktRnnMwZxGZQI7EC428zsliInuWMdXe//w2chLdkirqpSrIQwSZ +3jNWStqBXGYaRg5Z1ilBvHtXxkzDzbAlzRBzqfEwwQKBgQDqYdMRrzHJaXWLdsyU +6jAuNX9tLh7PcicjP93SbPujS6mWcNb+D/au+VhWD+dZQDPRZttXck7wvKY1lw1V +MK0TYI7ydf8h3DFx3Mi6ZD4JVSU1MH233C3fv/FHenDoOvMXXRjUZxaRmuzFJvzt +6QlKIfSvwT+1wrOACNfteXfZUQKBgQDmN3Uuk01qvsETPwtWBp5RNcYhS/zGEQ7o +Q4K+teU453r1v8BGsQrCqulIZ3clMkDru2UroeKn1pzyVAS2AgajgXzfXh3VeZh1 +vHTLP91BBYZTTWggalEN4aAkf9bxX/hA+9Bw/dzZcQW2aNV7WrYuCSvp3SDCMina +anQq/PaSRQKBgHjw23HfnegZI89AENaydQQTFNqolrtiYvGcbgC7vakITMzVEwrr +/9VP0pYuBKmYKGTgF0RrNnKgVX+HnxibUmOSSpCv9GNrdJQVYfpT6XL1XYqxp91s +nrs7FuxUMNiUOoWOw1Yuj4W4lH4y3QaCXgnDtbfPFunaOrdRWOIv8HjRAoGAV3NT +mSitbNIfR69YIAqNky3JIJbb42VRc1tJzCYOd+o+pCF96ZyRCNehnDZpZQDM9n8N +9GAfWEBHCCpwS69DVFL422TGEnSJPJglCZwt8OgnWXd7CW05cvt1OMgzHyekhxLg +4Dse7J5pXBxAlAYmVCB5xPGR4xLpISX1EOtcwr0CgYEA5rA2IUfjZYb4mvFHMKyM +xWZuV9mnl3kg0ULttPeOl3ppwjgRbWpyNgOXl8nVMYzxwT/A+xCPA18P0EcgNAWc +frJqQYg3NMf+f0K1wSaswUSLEVrQOj25OZJNpb21JEiNfEd5DinVVj4BtVc6KSpS +kvjbn2WhEUatc3lPL3V0Fkw= +-----END PRIVATE KEY----- """, # 5 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvhTRj1dA -NOfse/UBeTfMekZKxZHsNPr+qBYaveWAHDded/BMyMgaMV2n6HQdiDaRjJkzjHCF -3xBtpIJeEGUqfrF0ob8BIZXy3qk68eX/0CVUbgmjSBN44ahlo63NshyXmZtEAkRV -VE/+cRKw3N2wtuTed5xwfNcL6dg4KTOEYEkCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQCN+CLuVwLeWjSdVbdizYyrOVckqtwiIHG9BbGMlcIdm0qpvD7V7/sN2csk5LaT -BNiHi1t5628/4UHqqodYmFw8ri8ItFwB+MmTJi11CX6dIP9OUhS0qO8Z/BKtot7H -j04oNwl+WqZZfHIYwTIEL0HBn60nOvCQPDtnWG2BhpUxMA== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExM1oXDTIxMDEwMTAxNTExM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1c5y +S9IZHF9MIuwdafzhMkgP37I3RVpHEbpnPwnLFqSWelS5m2eDkwWd5SkfGjrmQ5q0 +PEpqLlh3zHGw9yQjnHS3CCS1PwQ1kmwvpIK3HM5y8GM7ry1zkam8ZR4iX6Y7VG9g +9mhiVVFoVhe1gHeiC/3Mp6XeNuEiD0buM+8qZx9B21I+iwzy4wva7Gw0fJeq9G1c +lq2rhpD1LlIEodimWOi7lOEkNmUiO1SvpdrGdxUDpTgbdg6r5pCGjOXLd74tAQHP +P/LuqRNJDXtwvHtLIVQnW6wjjy4oiWZ8DXOdc9SkepwQLIF5Wh8O7MzF5hrd6Cvw +SOD3EEsJbyycAob6RwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBDNcbKVUyGOAVm +k3iVuzkkkymlTAMm/gsIs6loLJrkSqNg160FdVKJoZFjQtqoqLgLrntdCJ377nZ9 +1i+yzbZsA4DA7nxj0IEdnd7rRYgGLspGqWeKSTROATeT4faLTXenecm0v2Rpxqc7 +dSyeZJXOd2OoUu+Q64hzXCDXC6LNM+xZufxV9qv+8d+CipV6idSQZaUWSVuqFCwD +PT0R4eWfkMMaM8QqtNot/hVCEaKT+9rG0mbpRe/b/qBy5SR0u+XgGEEIV+33L59T +FXY+DpI1Dpt/bJFoUrfj6XohxdTdqYVCn1F8in98TsRcFHyH1xlkS3Y0RIiznc1C +BwAoGZ4B -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQC+FNGPV0A05+x79QF5N8x6RkrFkew0+v6oFhq95YAcN1538EzI -yBoxXafodB2INpGMmTOMcIXfEG2kgl4QZSp+sXShvwEhlfLeqTrx5f/QJVRuCaNI -E3jhqGWjrc2yHJeZm0QCRFVUT/5xErDc3bC25N53nHB81wvp2DgpM4RgSQIDAQAB -AoGALl2BqIdN4Bnac3oV++2CcSkIQB0SEvJOf820hDGhCEDxSCxTbn5w9S21MVxx -f7Jf2n3cNxuTbA/jzscGDtW+gXCs+WAbAr5aOqHLUPGEobhKQrQT2hrxQHyv3UFp -0tIl9eXFknOyVAaUJ3athK5tyjSiCZQQHLGzeLaDSKVAPqECQQD1GK7DkTcLaSvw -hoTJ3dBK3JoKT2HHLitfEE0QV58mkqFMjofpe+nyeKWvEb/oB4WBp/cfTvtf7DJK -zl1OSf11AkEAxomWmJeub0xpqksCmnVI1Jt1mvmcE4xpIcXq8sxzLHRc2QOv0kTw -IcFl4QcN6EQBmE+8kl7Tx8SPAVKfJMoZBQJAGsUFYYrczjxAdlba7glyFJsfn/yn -m0+poQpwwFYxpc7iGzB+G7xTAw62WfbAVSFtLYog7aR8xC9SFuWPP1vJeQJBAILo -xBj3ovgWTXIRJbVM8mnl28UFI0msgsHXK9VOw/6i93nMuYkPFbtcN14KdbwZ42dX -5EIrLr+BNr4riW4LqDUCQQCbsEEpTmj3upKUOONPt+6CH/OOMjazUzYHZ/3ORHGp -Q3Wt+I4IrR/OsiACSIQAhS4kBfk/LGggnj56DrWt+oBl ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDVznJL0hkcX0wi +7B1p/OEySA/fsjdFWkcRumc/CcsWpJZ6VLmbZ4OTBZ3lKR8aOuZDmrQ8SmouWHfM +cbD3JCOcdLcIJLU/BDWSbC+kgrccznLwYzuvLXORqbxlHiJfpjtUb2D2aGJVUWhW +F7WAd6IL/cynpd424SIPRu4z7ypnH0HbUj6LDPLjC9rsbDR8l6r0bVyWrauGkPUu +UgSh2KZY6LuU4SQ2ZSI7VK+l2sZ3FQOlOBt2DqvmkIaM5ct3vi0BAc8/8u6pE0kN +e3C8e0shVCdbrCOPLiiJZnwNc51z1KR6nBAsgXlaHw7szMXmGt3oK/BI4PcQSwlv +LJwChvpHAgMBAAECggEBAK0KLeUBgIM++Y7WDCRInzYjrn08bpE5tIU7mO4jDfQg +dw1A3wtQZuOpyxW6B0siWlRis/aLv44M2cBkT3ZmEFBDAhOcKfh7fqQn3RNHG847 +pDi8B4UKwxskBa7NCcLh9eirUA19hABLJ6dt/t6fdE5CNc2FZ+iAoyE8JfNwYKAd +6Fa3HqUBPNWt8ryj4ftgpMNBdfmLugEM4N20SXJA28hOq2lUcwNKQQ1xQrovl0ig +iMbMWytV4gUPKC9Wra66OYIkk/K8teiUNIYA4JwAUVTs1NEWoyfwUTz1onutCkMl +5vY7JAqRoDWoSUX6FI+IHUdyqPAMdOMhC37gjrxoo2ECgYEA7trDMu6xsOwEckDh +iz148kejMlnTTuCNetOFBw3njFgxISx0PrDLWmJmnHMxPv9AAjXYb2+UCCm3fj6Q +OB8o4ZJm0n504qbFHcb2aI22U5hZ99ERvqx8WBnJ2RarIBmg06y0ktxq8gFR2qxF +0hWAOcDn1DWQ8QI0XBiFFcJTGtcCgYEA5SdlIXRnVZDKi5YufMAORG9i74dXUi0Y +02UoVxJ+q8VFu+TT8wrC5UQehG3gX+79Cz7hthhDqOSCv6zTyE4Evb6vf9OLgnVe +E5iLF033zCxLSS9MgiZ+jTO+wK3RsapXDtGcSEk2P82Pj5seNf4Ei1GNCRlm1DbX +71wlikprHhECgYABqmLcExAIJM0vIsav2uDiB5/atQelMCmsZpcx4mXv85l8GrxA +x6jTW4ZNpvv77Xm7yjZVKJkGqYvPBI6q5YS6dfPjmeAkyHbtazrCpeJUmOZftQSD +qN5BGwTuT5sn4SXe9ABaWdEhGONCPBtMiLvZK0AymaEGHTbSQZWD/lPoBwKBgGhk +qg2zmd/BNoSgxkzOsbE7jTbR0VX+dXDYhKgmJM7b8AjJFkWCgYcwoTZzV+RcW6rj +2q+6HhizAV2QvmpiIIbQd+Mj3EpybYk/1R2ox1qcUy/j/FbOcpihGiVtCjqF/2Mg +2rGTqMMoQl6JrBmsvyU44adjixTiZz0EHZYCkQoBAoGBAMRdmoR4mgIIWFPgSNDM +ISLJxKvSFPYDLyAepLfo38NzKfPB/XuZrcOoMEWRBnLl6dNN0msuzXnPRcn1gc1t +TG7db+hivAyUoRkIW3dB8pRj9dDUqO9OohjKsJxJaQCyH5vPkQFSLbTIgWrHhU+3 +oSPiK/YngDV1AOmPDH7i62po +-----END PRIVATE KEY----- """, #6 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtKhx6sEA -jn6HWc6T2klwlPn0quyHtATIw8V3ezP46v6g2rRS7dTywo4GTP4vX58l+sC9z9Je -qhQ1rWSwMK4FmnDMZCu7AVO7oMIXpXdSz7l0bgCnNjvbpkA2pOfbB1Z8oj8iebff -J33ID5DdkmCzqYVtKpII1o/5z7Jo292JYy8CAwEAATANBgkqhkiG9w0BAQQFAAOB -gQA0PYMA07wo9kEH4fv9TCfo+zz42Px6lUxrQBPxBvDiGYhk2kME/wX0IcoZPKTV -WyBGmDAYWvFaHWbrbbTOfzlLWfYrDD913hCi9cO8iF8oBqRjIlkKcxAoe7vVg5Az -ydVcrY+zqULJovWwyNmH1QNIQfMat0rj7fylwjiS1y/YsA== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAojGu +fQaTVT9DJWJ/zogGfrryEJXYVy9c441O5MrLlRx7nCIWIUs2NEhHDJdqJjYOTdmk +K98VhdMpDPZwxjgvvZrh43lStBRIW3zZxv747rSl2VtpSqD/6UNWJe5u4SR7oga4 +JfITOKHg/+ASxnOxp/iu6oT6jBL6T7KSPh6Rf2+it2rsjhktRreFDJ2hyroNq1w4 +ZVNCcNPgUIyos8u9RQKAWRNchFh0p0FCS9xNrn3e+yHnt+p6mOOF2gMzfXT/M2hq +KQNmc5D3yNoH2smWoz7F3XsRjIB1Ie4VWoRRaGEy7RwcwiDfcaemD0rQug6iqH7N +oomF6f3R4DyvVVLUkQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB/8SX6qyKsOyex +v3wubgN3FPyU9PqMfEzrFM6X5sax0VMVbSnekZrrXpdnXYV+3FBu2GLLQc900ojj +vKD+409JIriTcwdFGdLrQPTCRWkEOae8TlXpTxuNqJfCPVNxFN0znoat1bSRsX1U +K0mfEETQ3ARwlTkrF9CM+jkU3k/pnc9MoCLif8P7OAF38AmIbuTUG6Gpzy8RytJn +m5AiA3sds5R0rpGUu8mFeBpT6jIA1QF2g+QNHKOQcfJdCdfqTjKw5y34hjFqbWG9 +RxWGeGNZkhC/jADCt+m+R6+hlyboLuIcVp8NJw6CGbr1+k136z/Dj+Fdhm6FzF7B +qULeRQJ+ -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQC0qHHqwQCOfodZzpPaSXCU+fSq7Ie0BMjDxXd7M/jq/qDatFLt -1PLCjgZM/i9fnyX6wL3P0l6qFDWtZLAwrgWacMxkK7sBU7ugwheld1LPuXRuAKc2 -O9umQDak59sHVnyiPyJ5t98nfcgPkN2SYLOphW0qkgjWj/nPsmjb3YljLwIDAQAB -AoGAU4CYRv22mCZ7wVLunDLdyr5ODMMPZnHfqj2XoGbBYz0WdIBs5GlNXAfxeZzz -oKsbDvAPzANcphh5RxAHMDj/dT8rZOez+eJrs1GEV+crl1T9p83iUkAuOJFtgUgf -TtQBL9vHaj7DfvCEXcBPmN/teDFmAAOyUNbtuhTkRa3PbuECQQDwaqZ45Kr0natH -V312dqlf9ms8I6e873pAu+RvA3BAWczk65eGcRjEBxVpTvNEcYKFrV8O5ZYtolrr -VJl97AfdAkEAwF4w4KJ32fLPVoPnrYlgLw86NejMpAkixblm8cn51avPQmwbtahb -BZUuca22IpgDpjeEk5SpEMixKe/UjzxMewJBALy4q2cY8U3F+u6sshLtAPYQZIs3 -3fNE9W2dUKsIQvRwyZMlkLN7UhqHCPq6e+HNTM0MlCMIfAPkf4Rdy4N6ZY0CQCKE -BAMaQ6TwgzFDw5sIjiCDe+9WUPmRxhJyHL1/fvtOs4Z4fVRP290ZklbFU2vLmMQH -LBuKzfb7+4XJyXrV1+cCQBqfPFQQZLr5UgccABYQ2jnWVbJPISJ5h2b0cwXt+pz/ -8ODEYLjqWr9K8dtbgwdpzwbkaGhQYpyvsguMvNPMohs= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiMa59BpNVP0Ml +Yn/OiAZ+uvIQldhXL1zjjU7kysuVHHucIhYhSzY0SEcMl2omNg5N2aQr3xWF0ykM +9nDGOC+9muHjeVK0FEhbfNnG/vjutKXZW2lKoP/pQ1Yl7m7hJHuiBrgl8hM4oeD/ +4BLGc7Gn+K7qhPqMEvpPspI+HpF/b6K3auyOGS1Gt4UMnaHKug2rXDhlU0Jw0+BQ +jKizy71FAoBZE1yEWHSnQUJL3E2ufd77Iee36nqY44XaAzN9dP8zaGopA2ZzkPfI +2gfayZajPsXdexGMgHUh7hVahFFoYTLtHBzCIN9xp6YPStC6DqKofs2iiYXp/dHg +PK9VUtSRAgMBAAECggEANjn0A3rqUUr4UQxwfIV/3mj0O1VN4kBEhxOcd+PRUsYW +EapXycPSmII9ttj8tU/HUoHcYIqSMI7bn6jZJXxtga/BrALJAsnxMx031k8yvOQK +uvPT7Q6M4NkReVcRHRbMeuxSLuWTRZDhn8qznEPb9rOvD1tsRN6nb3PdbwVbUcZh +2F6JDrTyI/Df6nrYQAWOEe2ay7tzgrNYE4vh+DW7oVmyHRgFYA+DIG5Q+7OVWeW5 +bwYYPKlo4/B0L+GfMKfMVZ+5TvFWAK0YD1e/CW1Gv+i/8dWm4O7UNGg5mTnrIcy1 +g5wkKbyea02/np2B/XBsSWXDl6rTDHL7ay0rH2hjEQKBgQDMKSm3miQTIcL/F2kG +ieapmRtSc7cedP967IwUfjz4+pxPa4LiU47OCGp1bmUTuJAItyQyu/5O3uLpAriD +PTU+oVlhqt+lI6+SJ4SIYw01/iWI3EF2STwXVnohWG1EgzuFM/EqoB+mrodNONfG +UmP58vI9Is8fdugXgpTz4Yq9pQKBgQDLYJoyMVrYTvUn5oWft8ptsWZn6JZXt5Bd +aXh+YhNmtCrSORL3XjcH4yjlcn7X8Op33WQTbPo7QAJ1CumJzAI88BZ/8za638xb +nLueviZApCt0bNMEEdxDffxHFc5TyHE+obMKFfApbCnD0ggO6lrZ8jK9prArLOCp +mRU9SSRffQKBgAjoBszeqZI4F9SfBdLmMyzU5A89wxBOFFMdfKLsOua1sBn627PZ +51Hvpg1HaptoosfujWK1NsvkB0wY9UmsYuU/jrGnDaibnO4oUSzN/WaMlsCYszZg +zYFLIXrQ67tgajlOYcf1Qkw4MujYgPlC4N+njI/EM/rwagGUjcDx5uaNAoGASyqz +EuYG63eTSGH89SEaohw0+yaNmnHv23aF4EAjZ4wjX3tUtTSPJk0g6ly84Nbb8d1T +hZJ7kbaAsf2Mfy91jEw4JKYhjkP05c8x0OP6g12p6efmvdRUEmXX/fXjQjgNEtb0 +sz+UedrOPN+9trWLSo4njsyyw+JcTpKTtQj5dokCgYEAg9Y3msg+GvR5t/rPVlKd +keZkrAp0xBJZgqG7CTPXWS1FjwbAPo7x4ZOwtsgjCuE52lar4j+r2Il+CDYeLfxN +h/Jfn6S9ThUh+B1PMvKMMnJUahg8cVL8uQuBcbAy8HPRK78WO2BTnje44wFAJwTc +0liuYqVxZIRlFLRl8nGqog8= +-----END PRIVATE KEY----- """, #7 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAnBfNHycn -5RnYzDN4EWTk2q1BBxA6ZYtlG1WPkj5iKeaYKzUk58zBL7mNOA0ucq+yTwh9C4IC -EutWPaKBSKY5XI+Rdebh+Efq+urtOLgfJHlfcCraEx7hYN+tqqMVgEgnO/MqIsn1 -I1Fvnp89mSYbQ9tmvhSH4Hm+nbeK6iL2tIsCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQBt9zxfsKWoyyV764rRb6XThuTDMNSDaVofqePEWjudAbDu6tp0pHcrL0XpIrnT -3iPgD47pdlwQNbGJ7xXwZu2QTOq+Lv62E6PCL8FljDVoYqR3WwJFFUigNvBT2Zzu -Pxx7KUfOlm/M4XUSMu31sNJ0kQniBwpkW43YmHVNFb/R7g== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu9oO +cFlNukUcLfFrfkEaUiilcHLmn5OokQbj95CGd2ehQCCVwrkunYLBisthRaancFFb +/yM998B0IUsKTsoLi5DAN3/SkSm6GiQIGO05E4eBPljwJ61QQMxh8+1TwQ9HTun1 +ZE1lhVN1aRmI9VsbyTQLjXh9OFNLSJEKb29mXsgzYwYwNOvo+idzXpy4bMyNoGxY +Y+s2FIKehNHHCv4ravDn8rf6DtDOvyN4d0/QyNws9FpAZMXmLwtBJ9exOqKFW43w +97NxgdNiTFyttrTKTi0b+9v3GVdcEZw5b2RMIKi6ZzPof6/0OlThK6C3xzFK3Bp4 +PMjTfXw5yyRGVBnZZwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA4Ms6LqzMu757z +bxISiErRls6fcnq0fpSmiPNHNKM7YwG9KHYwPT6A0UMt30zDwNOXCQBI19caGeeO +MLPWa7Gcqm2XZB2jQwvLRPeFSy9fm6RzJFeyhrh/uFEwUetwYmi/cqeIFDRDBQKn +bOaXkBk0AaSmI5nRYfuqpMMjaKOFIFcoADw4l9wWhv6DmnrqANzIdsvoSXi5m8RL +FcZQDZyHFlHh3P3tLkmQ7ErM2/JDwWWPEEJMlDm/q47FTOQSXZksTI3WRqbbKVv3 +iQlJjpgi9yAuxZwoM3M4975iWH4LCZVMCSqmKCBt1h9wv4LxqX/3kfZhRdy1gG+j +41NOSwJ/ -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQCcF80fJyflGdjMM3gRZOTarUEHEDpli2UbVY+SPmIp5pgrNSTn -zMEvuY04DS5yr7JPCH0LggIS61Y9ooFIpjlcj5F15uH4R+r66u04uB8keV9wKtoT -HuFg362qoxWASCc78yoiyfUjUW+enz2ZJhtD22a+FIfgeb6dt4rqIva0iwIDAQAB -AoGBAIHstcnWd7iUeQYPWUNxLaRvTY8pjNH04yWLZEOgNWkXDVX5mExw++RTmB4t -qpm/cLWkJSEtB7jjthb7ao0j/t2ljqfr6kAbClDv3zByAEDhOu8xB/5ne6Ioo+k2 -dygC+GcVcobhv8qRU+z0fpeXSP8yS1bQQHOaa17bSGsncvHRAkEAzwsn8jBTOqaW -6Iymvr7Aql++LiwEBrqMMRVyBZlkux4hiKa2P7XXEL6/mOPR0aI2LuCqE2COrO7R -0wAFZ54bjwJBAMEAe6cs0zI3p3STHwA3LoSZB81lzLhGUnYBvOq1yoDSlJCOYpld -YM1y3eC0vwiOnEu3GG1bhkW+h6Kx0I/qyUUCQBiH9NqwORxI4rZ4+8S76y4EnA7y -biOx9KxYIyNgslutTUHYpt1TmUDFqQPfclvJQWw6eExFc4Iv5bJ/XSSSyicCQGyY -5PrwEfYTsrm5fpwUcKxTnzxHp6WYjBWybKZ0m/lYhBfCxmAdVrbDh21Exqj99Zv0 -7l26PhdIWfGFtCEGrzECQQCtPyXa3ostSceR7zEKxyn9QBCNXKARfNNTBja6+VRE -qDC6jLqzu/SoOYaqa13QzCsttO2iZk8Ygfy3Yz0n37GE ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC72g5wWU26RRwt +8Wt+QRpSKKVwcuafk6iRBuP3kIZ3Z6FAIJXCuS6dgsGKy2FFpqdwUVv/Iz33wHQh +SwpOyguLkMA3f9KRKboaJAgY7TkTh4E+WPAnrVBAzGHz7VPBD0dO6fVkTWWFU3Vp +GYj1WxvJNAuNeH04U0tIkQpvb2ZeyDNjBjA06+j6J3NenLhszI2gbFhj6zYUgp6E +0ccK/itq8Ofyt/oO0M6/I3h3T9DI3Cz0WkBkxeYvC0En17E6ooVbjfD3s3GB02JM +XK22tMpOLRv72/cZV1wRnDlvZEwgqLpnM+h/r/Q6VOEroLfHMUrcGng8yNN9fDnL +JEZUGdlnAgMBAAECggEALlZdlW0R9U6y4spYf65Dddy84n4VUWu0+wE+HoUyBiYz +6oOfLYdMbmIgp8H/XpT7XINVNBxXXtPEUaoXAtRoAKdWItqO8Gvgki4tKSjrGVwl +j2GU69SepT1FNExoiojgSCEB/RnyXu71WVWJKSyuL/V8nAsKqGgze9T7Q/2wvNQt +SQqLxZlrWF0P8WqaAiSrHV4GnDrdeF+k1KBo2+pSaDNv6cNwOyVG8EII9tqhF8kj +6nD6846ish6OqmlSisaSGopJZL1DCQzszFMxKd2+iBDY7Kn6hVIhRaNnaZUFhpKM +dNh6hBqOycMepAp0sz5pdo+fxpifkoR/cPWgyC3QkQKBgQDixe9VsiZ7u2joxF/9 +JcAExKhqE28OUmIwt6/j+uzYShxN6Oo9FUo3ICtAPCCFsjhvb3Qum7FspmxrqtNy +fzclibZJPO8ey2PzqaiOfiVfgJmNSvoCOdgM4OqFLtRO6eSTzhJeI4VPrPcq/5la +0FuOi1WZs/Au9llqLqGSDH3UAwKBgQDUD/bSJbOk5SvNjFtFm0ClVJr66mJ5e4uN +4VGv8KGFAJ+ExIxujAukfKdwLjS1wEy2RePcshfT8Y9FVh/Q1KzzrQi3Gwmfq1G6 +Dpu2HlJpaZl+9T81x2KS8GP3QNczWMe2nh7Lj+6st+b4F+6FYbVTFnHaae27sXrD +XPX15+uxzQKBgGy+pBWBF4kwBo/QU4NuTdU7hNNRPGkuwl1ASH1Xv6m8aDRII8Nk +6TDkITltW98g5oUxehI7oOpMKCO9SCZYsNY0YpBeQwCOYgDfc6/Y+A0C+x9RO/BD +UsJiPLPfD/pDmNPz9sTj3bKma+RXq29sCOujD0pkiiHLCnerotkJWnGHAoGAAkCJ +JoIv/jhQ1sX+0iZr8VWMr819bjzZppAWBgBQNtFi4E4WD7Z9CSopvQ9AkA2SwvzL +BrT9e8q88sePXvBjRdM4nHk1CPUQ0SEGllCMH4J3ltmT6kZLzbOv3BhcMLdop4/W +U+MbbcomMcxPRCtdeZxraR5m3+9qlliOZCYqYqECgYA5eLdxgyHxCS33QGFHRvXI +TLAHIrr7wK1xwgkmZlLzYSQ8Oqh1UEbgoMt4ulRczP2g7TCfvANw2Sw0H2Q5a6Fj +cnwVcXJ38DLg0GCPMwzE8dK7d8tKtV6kGiKy+KFvoKChPjE6uxhKKmCJaSwtQEPS +vsjX3iiIgUQPsSz8RrNFfQ== +-----END PRIVATE KEY----- """, #8 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA4mnLf+x0 -CWKDKP5PLZ87t2ReSDE/J5QoI5VhE0bXaahdhPrQTC2wvOpT+N9nzEpI9ASh/ejV -kYGlc03nNKRL7zyVM1UyGduEwsRssFMqfyJhI1p+VmxDMWNplex7mIAheAdskPj3 -pwi2CP4VIMjOj368AXvXItPzeCfAhYhEVaMCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQAEzmwq5JFI5Z0dX20m9rq7NKgwRyAH3h5aE8bdjO8nEc69qscfDRx79Lws3kK8 -A0LG0DhxKB8cTNu3u+jy81tjcC4pLNQ5IKap9ksmP7RtIHfTA55G8M3fPl2ZgDYQ -ZzsWAZvTNXd/eme0SgOzD10rfntA6ZIgJTWHx3E0RkdwKw== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DNu +CKhhl6wCbgoCkFemwJh3ATbAjhInHpvQWIFDfSK1USElCKxqosIxiBQCx3Zs2d/U +GeIA7QAM2atNdXaateacEaKMmGE9LEtO0Dg5lmT43WzmGkG9NmCwK3JjAekc5S9d +HKNtEQo7o8RKfj81zlDSq2kzliy98cimk24VBBGkS2Cn7Vy/mxMCqWjQazTXbpoS +lXw6LiY5wFXQmXOB5GTSHvqyCtBQbOSSbJB77z/fm7bufTDObufTbJIq53WPt00Y +f+JNnzkX1X0MaBCUztoZwoMaExWucMe/7xsQ46hDn6KB4b0lZk+gsK45QHxvPE1R +72+ZkkIrGS/ljIKahQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQDib1653CneSmy2 +gYzGeMlrI05Jqo3JuHNMQHzAjIrb4ee57VA4PTQa1ygGol/hVv6eTvZr3p2ospDS +5Kfwj1HLO4jSMX1Bnm1FG0naQogz2CD3xfYjbYOVRhAxpld1MNyRveIOhDRARY7N +XNAaNPZ1ALrwbENSYArr18xDzgGWe/dgyRCEpCFIsztiA+7jGvrmAZgceIE8K3h3 +fkvNmXBH58ZHAGTiyRriBZqS+DXrBrQOztXSJwFnOZnRt6/efeBupt8j5hxVpBLW +vtjpBc23uUcbbHOY2AW2Bf+vIr4/LmJ/MheKV+maa2990vmC93tvWlFfc74mgUkW +HJfXDmR6 -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDiact/7HQJYoMo/k8tnzu3ZF5IMT8nlCgjlWETRtdpqF2E+tBM -LbC86lP432fMSkj0BKH96NWRgaVzTec0pEvvPJUzVTIZ24TCxGywUyp/ImEjWn5W -bEMxY2mV7HuYgCF4B2yQ+PenCLYI/hUgyM6PfrwBe9ci0/N4J8CFiERVowIDAQAB -AoGAQYTl+8XcKl8Un4dAOG6M5FwqIHAH25c3Klzu85obehrbvUCriG/sZi7VT/6u -VeLlS6APlJ+NNgczbrOLhaNJyYzjICSt8BI96PldFUzCEkVlgE+29pO7RNoZmDYB -dSGyIDrWdVYfdzpir6kC0KDcrpA16Sc+/bK6Q8ALLRpC7QECQQD7F7fhIQ03CKSk -lS4mgDuBQrB/52jXgBumtjp71ANNeaWR6+06KDPTLysM+olsh97Q7YOGORbrBnBg -Y2HPnOgjAkEA5taZaMfdFa8V1SPcX7mgCLykYIujqss0AmauZN/24oLdNE8HtTBF -OLaxE6PnQ0JWfx9KGIy3E0V3aFk5FWb0gQJBAO4KFEaXgOG1jfCBhNj3JHJseMso -5Nm4F366r0MJQYBHXNGzqphB2K/Svat2MKX1QSUspk2u/a0d05dtYCLki6UCQHWS -sChyQ+UbfF9HGKOZBC3vBzo1ZXNEdIUUj5bJjBHq3YgbCK38nAU66A482TmkvDGb -Wj4OzeB+7Ua0yyJfggECQQDVlAa8HqdAcrbEwI/YfPydFsavBJ0KtcIGK2owQ+dk -dhlDnpXDud/AtX4Ft2LaquQ15fteRrYjjwI9SFGytjtp ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkM24IqGGXrAJu +CgKQV6bAmHcBNsCOEicem9BYgUN9IrVRISUIrGqiwjGIFALHdmzZ39QZ4gDtAAzZ +q011dpq15pwRooyYYT0sS07QODmWZPjdbOYaQb02YLArcmMB6RzlL10co20RCjuj +xEp+PzXOUNKraTOWLL3xyKaTbhUEEaRLYKftXL+bEwKpaNBrNNdumhKVfDouJjnA +VdCZc4HkZNIe+rIK0FBs5JJskHvvP9+btu59MM5u59NskirndY+3TRh/4k2fORfV +fQxoEJTO2hnCgxoTFa5wx7/vGxDjqEOfooHhvSVmT6CwrjlAfG88TVHvb5mSQisZ +L+WMgpqFAgMBAAECggEABTdPuo7uvCLIY2+DI319aEWT4sk3mYe8sSxqlLtPqZqT +fmk9iXc3cMTzkOK0NY71af19waGy17f6kzchLCAr5SCCTLzkbc87MLn/8S530oI4 +VgdZMxxxkL6hCD0zGiYT7QEqJa9unMcZGeMwuLYFKtQaHKTo8vPO26n0dMY9YLxj +cNUxsKLcKk8dbfKKt4B4fZgB7nU0BG9YbKYZ3iZ7/3mG+6jA6u+VYc/WHYQjTmpL +oLFN7NOe3R7jIx/kJ1OqNWqsFoLpyiiWd1Mr0l3EdD1kCudptMgD8hd++nx2Yk2w +K4+CpOVIN/eCxDDaAOJgYjCtOayVwUkDAxRRt9VnAQKBgQD5s1j6RJtBNTlChVxS +W3WpcG4q8933AiUY/Chx0YTyopOiTi7AGUaA8AOLFBcO2npa+vzC+lvuOyrgOtVW +sD10H2v5jNKlbeBp+Q9rux2LAyp4TvzdXWKhVyZrdtITF0hn6vEYNp7MtyWRFb1O +3Ie5HQBPHtzllFOMynacjOdjpQKBgQDp9TrbfOmwGWmwPKmaFKuy8BKxjJM+ct0X +4Xs1uSy9Z9Y8QlDNbNaooI8DA1NY0jDVHwemiGC4bYsBNKNRcbI0s2nr0hQMft42 +P/NpugHv0YXiVz+5bfim4woTiHHbfREqchlIGo3ryClAiDU9fYZwTOtb9jPIhX3G +9v+OsoMlYQKBgQDJUQW90S5zJlwh+69xXvfAQjswOimNCpeqSzK4gTn0/YqV4v7i +Nf6X2eqhaPMmMJNRYuYCtSMFMYLiAc0a9UC2rNa6/gSfB7VU+06phtTMzSKimNxa +BP6OIduB7Ox2I+Fmlw8GfJMPbeHF1YcpW7e5UV58a9+g4TNzYZC7qwarWQKBgQCA +FFaCbmHonCD18F/REFvm+/Lf7Ft3pp5PQouXH6bUkhIArzVZIKpramqgdaOdToSZ +SAGCM8rvbFja8hwurBWpMEdeaIW9SX8RJ/Vz/fateYDYJnemZgPoKQcNJnded5t8 +Jzab+J2VZODgiTDMVvnQZOu8To6OyjXPRM0nK6cMQQKBgQDyX44PHRRhEXDgJFLU +qp2ODL54Qadc/thp2m+JmAvqmCCLwuYlGpRKVkLLuZW9W6RlVqarOC3VD3wX5PRZ +IsyCGLi+Jbrv9JIrYUXE80xNeQVNhrrf02OW0KHbqGxRaNOmp1THPw98VUGR2J/q +YAp6XUXU7LEBUrowye+Ty2o7Lg== +-----END PRIVATE KEY----- """, #9 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAueLfowPT -kXXtHeU2FZSz2mJhHmjqeyI1oMoyyggonccx65vMxaRfljnz2dOjVVYpCOn/LrdP -wVxHO8KNDsmQeWPRjnnBa2dFqqOnp/8gEJFJBW7K/gI9se6o+xe9QIWBq6d/fKVR -BURJe5TycLogzZuxQn1xHHILa3XleYuHAbMCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQBEC1lfC3XK0galQC96B7faLpnQmhn5lX2FUUoFIQQtBTetoE+gTqnLSOIZcOK4 -pkT3YvxUvgOV0LOLClryo2IknMMGWRSAcXtVUBBLRHVTSSuVUyyLr5kdRU7B4E+l -OU0j8Md/dzlkm//K1bzLyUaPq204ofH8su2IEX4b3IGmAQ== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1k2R +PWYihftppo3CoxeseFwgg7guxZVkP7aAur5uBzSeAB7sBG1G2bRrwMX71S4xPwot +zYiEoxUrTStUqEKjL2aozfHsXnHZ7kwwUgZFDZUg+ve2tZDA3HCUr4tLYKlyFqpx +2nCouc45MjQ4wAxRl4rQxIUG2uSTzvP+xXtjoJYMIEEyCpcsRXfqfVkEUe9nrPsF +0Ibzk7Cyt75HDI4uEzBuHux0DYuGy6R02jz/vf/dIZ4WepjSY06xpblTHZgieDRX +fU2+YOcvb0eDHyA8Q5p8ropK71MNIP5+kffFd90SVr4EkCA8S+cd6FdKQasRr+jF +9MUhMS4ObvlrYTG+hwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCy62MZ3+59/VpX +c9Hsmb4/BMWt0irLJit4w4SkuYKGFLCMKZI4LN4pEkXaiE1eqF2DNS1qOvl5luty +Zz4oggrqilwuFeH98o9Zeg9SYnouuypORVP/3DPbJF/jiQg5J8kJb1sy+BjRiT8I +5X6/cCBYT+MljFz5tpqWOtWTgA30e1BV8JFj8F4dgUcWsAVT/I4l9zgMLUnhcO6E +wLtEE0I6aT1RHJB28ndwJzj4La98Oirw7LAEAWbExWYB90ypLaGY+JVJe3f5fijC +fJpQ2mbs4syXDmb5bU2C2pGPTKZPcyx15iQrq1uHInD0facOw+pmllAFxuG96lA1 ++o2VzKwP -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQC54t+jA9ORde0d5TYVlLPaYmEeaOp7IjWgyjLKCCidxzHrm8zF -pF+WOfPZ06NVVikI6f8ut0/BXEc7wo0OyZB5Y9GOecFrZ0Wqo6en/yAQkUkFbsr+ -Aj2x7qj7F71AhYGrp398pVEFREl7lPJwuiDNm7FCfXEccgtrdeV5i4cBswIDAQAB -AoGAO4PnJHNaLs16AMNdgKVevEIZZDolMQ1v7C4w+ryH/JRFaHE2q+UH8bpWV9zK -A82VT9RTrqpkb71S1VBiB2UDyz263XdAI/N2HcIVMmfKb72oV4gCI1KOv4DfFwZv -tVVcIdVEDBOZ2TgqK4opGOgWMDqgIAl2z3PbsIoNylZHEJECQQDtQeJFhEJGH4Qz -BGpdND0j2nnnJyhOFHJqikJNdul3uBwmxTK8FPEUUH/rtpyUan3VMOyDx3kX4OQg -GDNSb32rAkEAyJIZIJ0EMRHVedyWsfqR0zTGKRQ+qsc3sCfyUhFksWms9jsSS0DT -tVeTdC3F6EIAdpKOGhSyfBTU4jxwbFc0GQJADI4L9znEeAl66Wg2aLA2/Aq3oK/F -xjv2wgSG9apxOFCZzMNqp+FD0Jth6YtEReZMuldYbLDFi6nu6HPfY2Fa+QJAdpm1 -lAxk6yMxiZK/5VRWoH6HYske2Vtd+aNVbePtF992ME/z3F3kEkpL3hom+dT1cyfs -MU3l0Ot8ip7Ul6vlGQJAegNzpcfl2GFSdWQMxQ+nN3woKnPqpR1M3jgnqvo7L4Xe -JW3vRxvfdrUuzdlvZ/Pbsu/vOd+cuIa4h0yD5q3N+g== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWTZE9ZiKF+2mm +jcKjF6x4XCCDuC7FlWQ/toC6vm4HNJ4AHuwEbUbZtGvAxfvVLjE/Ci3NiISjFStN +K1SoQqMvZqjN8execdnuTDBSBkUNlSD697a1kMDccJSvi0tgqXIWqnHacKi5zjky +NDjADFGXitDEhQba5JPO8/7Fe2OglgwgQTIKlyxFd+p9WQRR72es+wXQhvOTsLK3 +vkcMji4TMG4e7HQNi4bLpHTaPP+9/90hnhZ6mNJjTrGluVMdmCJ4NFd9Tb5g5y9v +R4MfIDxDmnyuikrvUw0g/n6R98V33RJWvgSQIDxL5x3oV0pBqxGv6MX0xSExLg5u ++WthMb6HAgMBAAECggEAeCyRSNwQeg/NZD/UqP6qkegft52+ZMBssinWsGH/c3z3 +KVwtwCHDfGvnjPe5TAeWSCKeIsbukkFZwfGNjLmppvgrqymCAkhYDICfDDBF4uMA +1pu40sJ01Gkxh+tV/sOmnb1BEVzh0Sgq/NM6C8ActR18CugKOw+5L3G2KeoSqUbT +2hcPUsnik10KwqW737GQW4LtEQEr/iRmQkxI3+HBzvPWjFZzjOcpUph+FW5TXtaU +T26mt1j+FjbdvvhCuRMY/VZBJ5h1RKU95r57F1AjW/C0RRJ8FxR1CeSy4IlmQBrh +6wAa3Tdm0k/n4ZspC9bF5eVTJEtb0AohiYZrIa8MuQKBgQD8yjCLYa41H304odCx +NwPRJcmlIk5YGxPrhHAT9GEgU6n/no7YMVx1L7fNLcMjAyx54jauEU7J19Aki7eV +SIdU9TwqmkOAFfM6TOEJZiOi66gABOxeK2yDyfmR6Apaw3caku4O058t4KVwHSCB +DanYCMzxCBqS9jUTTyAh0fMg6wKBgQDZBkIukg3FKPor5LzkUXIKnNHYPfHbERHw +piWS6GZwqhuWNlOCWxiBR4rEUU/RbFQZw/FCi5OuAk2lBC0LBmC0/Sz4/+xDdCbv +uNhMOTRcy9nFVpmpIWCx4N/KmXHEuFxli/JNXux7iki74AVC9VPrAt/kCvwf06Df +oDb8ljdR1QKBgQChVOD6c5Lc8IXYeN1Z3IShHH6+11AsxstFyjZFZff+y6Z5L1Z2 +/7nESHoDhqs9Uy81cnv3R7CC/Ssnx8uYiLtmK0UE44Mk4d1jXeFZQEiKF+AWcw3v +Y8NTsLmItxC0sH75BMDN0Z2LiA3Nqaku8+trpuI1Cjj7hgqFkkAtlXKXlQKBgBMb +c/Q5s7CqHOyEZQUNDqdUiz0opwSMijHPzvsSLwK4V1lwSwXtE0k+jT8fkZF0oirq +j3E2bLgjR8bBiV2xIA6PQ8hgb+K4dT0h3xlG6A9Le07egwTbBXJjxBBIVjXlrWzb +V2fsdZGi6ShxXsU4aD0GscOYG/6JWV6W8oBmkVRJAoGAepIZ+OYmFjb7uxdh4EtP +hluEtx5bLOLuo6c0S149omUXUhbsuyzTZS6Ip9ySDMnK3954c4Q4WJ4yQKixQNVq +78aDfy4hP/8TE/Q9CRddUof2P33PJMhVNqzTRYMpqV+zxifvtw3hoDTLKHTQxCR2 +M1+O4VvokU5pBqUpGXiMDfs= +-----END PRIVATE KEY----- """, #10 """-----BEGIN CERTIFICATE----- -MIIBnjCCAQcCAgCEMA0GCSqGSIb3DQEBBAUAMBcxFTATBgNVBAMUDG5ld3BiX3Ro -aW5neTAeFw0wODA3MjUyMjQ3NThaFw0wOTA3MjUyMjQ3NThaMBcxFTATBgNVBAMU -DG5ld3BiX3RoaW5neTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAruBhwk+J -XdlwfKXXN8K+43JyEYCV7Fp7ZiES4t4AEJuQuBqJVMxpzeZzu2t/vVb59ThaxxtY -NGD3Xy6Og5dTv//ztWng8P7HwwvfbrUICU6zo6JAhg7kfaNa116krCYOkC/cdJWt -o5W+zsDmI1jUVGH0D73h29atc1gn6wLpAsMCAwEAATANBgkqhkiG9w0BAQQFAAOB -gQAEJ/ITGJ9lK/rk0yHcenW8SHsaSTlZMuJ4yEiIgrJ2t71Rd6mtCC/ljx9USvvK -bF500whTiZlnWgKi02boBEKa44z/DytF6pljeNPefBQSqZyUByGEb/8Mn58Idyls -q4/d9iKXMPvbpQdcesOzgOffFZevLQSWyPRaIdYBOOiYUA== +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnbCU +M37hG7zrCyyJEI6pZmOomnI+CozbP5KAhWSV5y7R5H6lcAEG2UDV+lCUxHT2ufOa +i1H16bXyBt7VoMTHIH50S58NUCUEXcuRWVR16tr8CzcTHQAkfIrmhY2XffPilX7h +aw35UkoVmXcqSDNNJD6jmvWexvmbhzVWW8Vt5Pivet2/leVuqPXB54/alSbkC74m +x6X5XKQc6eyPsb1xvNBuiSpFzdqbEn7lUwj6jFTkh9tlixgmgx+J0XoQXbawyrAg +rcIQcse/Ww+KBA1KSccFze+XBTbIull4boYhbJqkb6DW5bY7/me2nNxE9DRGwq+S +kBsKq3YKeCf8LEhfqQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAD+tWGFhINYsWT +ibKWlCGgBc5uB7611cLCevx1yAL6SaOECVCQXzaaXIaETSbyY03UO2yBy3Pl10FV +GYXLrAWTFZsNVJm55XIibTNw1UBPNwdIoCSzAYuOgMF0GHhTTQU0hNYWstOnnE2T +6lSAZQZFkaW4ZKs6sUp42Em9Bu99PehyIgnw14qb9NPg5qKdi2GAvkImZCrGpMdK +OF31U7Ob0XQ0lxykcNgG4LlUACd+QxLfNpmLBZUGfikexYa1VqBFm3oAvTt8ybNQ +qr7AKXDFnW75aCBaMpQWzrstA7yYZ3D9XCd5ZNf6d08lGM/oerDAIGnZOZPJgs5U +FaWPHdS9 -----END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQCu4GHCT4ld2XB8pdc3wr7jcnIRgJXsWntmIRLi3gAQm5C4GolU -zGnN5nO7a3+9Vvn1OFrHG1g0YPdfLo6Dl1O///O1aeDw/sfDC99utQgJTrOjokCG -DuR9o1rXXqSsJg6QL9x0la2jlb7OwOYjWNRUYfQPveHb1q1zWCfrAukCwwIDAQAB -AoGAcZAXC/dYrlBpIxkTRQu7qLqGZuVI9t7fabgqqpceFargdR4Odrn0L5jrKRer -MYrM8bjyAoC4a/NYUUBLnhrkcCQWO9q5fSQuFKFVWHY53SM63Qdqk8Y9Fmy/h/4c -UtwZ5BWkUWItvnTMgb9bFcvSiIhEcNQauypnMpgNknopu7kCQQDlSQT10LkX2IGT -bTUhPcManx92gucaKsPONKq2mP+1sIciThevRTZWZsxyIuoBBY43NcKKi8NlZCtj -hhSbtzYdAkEAw0B93CXfso8g2QIMj/HJJz/wNTLtg+rriXp6jh5HWe6lKWRVrce+ -1w8Qz6OI/ZP6xuQ9HNeZxJ/W6rZPW6BGXwJAHcTuRPA1p/fvUvHh7Q/0zfcNAbkb -QlV9GL/TzmNtB+0EjpqvDo2g8XTlZIhN85YCEf8D5DMjSn3H+GMHN/SArQJBAJlW -MIGPjNoh5V4Hae4xqBOW9wIQeM880rUo5s5toQNTk4mqLk9Hquwh/MXUXGUora08 -2XGpMC1midXSTwhaGmkCQQCdivptFEYl33PrVbxY9nzHynpp4Mi89vQF0cjCmaYY -N8L+bvLd4BU9g6hRS8b59lQ6GNjryx2bUnCVtLcey4Jd ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdsJQzfuEbvOsL +LIkQjqlmY6iacj4KjNs/koCFZJXnLtHkfqVwAQbZQNX6UJTEdPa585qLUfXptfIG +3tWgxMcgfnRLnw1QJQRdy5FZVHXq2vwLNxMdACR8iuaFjZd98+KVfuFrDflSShWZ +dypIM00kPqOa9Z7G+ZuHNVZbxW3k+K963b+V5W6o9cHnj9qVJuQLvibHpflcpBzp +7I+xvXG80G6JKkXN2psSfuVTCPqMVOSH22WLGCaDH4nRehBdtrDKsCCtwhByx79b +D4oEDUpJxwXN75cFNsi6WXhuhiFsmqRvoNbltjv+Z7ac3ET0NEbCr5KQGwqrdgp4 +J/wsSF+pAgMBAAECggEAPSu1ofBTRN5ZU4FYPlsJLdX1Hsy4coFHv/aF8rkdSYwp +EflrFfLgBEEZgLvnqfoxh9sPFYKa4amaFL42ouIS2PEVDgzKLk/dzMDeRof0IkIG +yhb4TCS1ArcjS6WsociNGi8ZJN1L3Xctv9WxSkbUYv4Fm2Qyzr8fbSjssjb5NXwD +K11fsj6Pfy/mQrI0TSTlzWC7ARIlCMTWQ8G8zEU6bMFIG6DMjt2J4VgYVXUKetZA +VPuS+pwkH2obQe6FLRuiNxH4GitVAASYPea6foER4AggBMRp8q8F6+WssjoyEORb +0sJxmnxhznoTRMCuTsUj6XMgmOTOnA3lQXsIB0DYcQKBgQDO6mMRVVFWzgkE9Q5/ +36n06KvGYF9TCRDL9vRC8kCqcGd1Hy6jRj0D8049KUHaN74pfWg6gsQjPkKzwKnC +vxNl72tVvLqm7Fo531BGfKK/46ZvxeWMMraNW4+9LhwMPu2LN5OEdwwCgyaURpxh +ktCp+RrGjz08Kn82X1jJPdwxDQKBgQDDGMvZ7ZUDGq5+RJkmHJ58lQtiaMZclmYV +R9YwOxJV6ino3EYrGOtUkqiemgAACdMWE/JMJlB1/JINawJwUsZ2XDp/9jNLPgLc +gphCmagaO34U/YMaJbJIK2gkCX7p8EcD+x45qWa0bEMPW38QfN/qQdUPjNmpuIiI +Zleyl1TqDQKBgQCvIoat0ighsAzETGN0aqzhJdrW8xVcJA06hpFi5MdFPBTldno0 +KqxUXqj3badWe94SIhqJg8teBUHSAZ3uv2o82nRgQnk99km8OD8rGi1q+9YRP1C2 +5OnNJhW4y4FkABNxxZ2v/k+FBNsvn8CXefvyEm3OaMks1s+MBxIQa7KnNQKBgFwX +HUo+GiN/+bPCf6P8yFa4J8qI+HEF0SPkZ9cWWx5QzP2M1FZNie++1nce7DcYbBo0 +yh9lyn8W/H328AzDFckS2c5DEY1HtSQPRP3S+AWB5Y7U54h1GMV2L88q6ExWzb60 +T10aeE9b9v+NydmniC5UatTPQIMbht8Tp/u18TAVAoGBAJphAfiqWWV2M5aBCSXq +WxLZ71AJ0PZBmRa/9iwtccwXQpMcW6wHK3YSQxci+sB97TElRa3/onlVSpohrUtg +VCvCwfSHX1LmrfWNSkoJZwCQt+YYuMqW86K0tzLzI1EMjIH9LgQvB6RR26PZQs+E +jr1ZvRc+wPTq6sxCF1h9ZAfN +-----END PRIVATE KEY----- """, #11 ] # To disable the pre-computed tub certs, uncomment this line. -#SYSTEM_TEST_CERTS = [] +# SYSTEM_TEST_CERTS = [] def flush_but_dont_ignore(res): d = flushEventualQueue() @@ -421,9 +649,6 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): - if 'pypy' in sys.version.lower(): - skip = "pypy can't run these, due to SSL errors (ee key too tiny)" - def setUp(self): self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() From b54dc1e5d53243c59a254068bb597702feafad96 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 May 2020 13:56:59 -0600 Subject: [PATCH 0063/1054] whitespace (1 line per import) --- src/allmydata/test/test_storage.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 7e4eaa3b6..baf6c6d4b 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1,4 +1,11 @@ -import time, os.path, platform, stat, re, json, struct, shutil +import time +import os.path +import platform +import stat +import re +import json +import struct +import shutil from twisted.trial import unittest From d4f29e7d6c5454f49e9206893638383892f5e69c Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 May 2020 13:57:08 -0600 Subject: [PATCH 0064/1054] restore old code, run the GC --- src/allmydata/test/test_storage.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index baf6c6d4b..73c183a38 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -6,6 +6,7 @@ import re import json import struct import shutil +import gc from twisted.trial import unittest @@ -561,8 +562,10 @@ class Server(unittest.TestCase): # we abandon the first set, so their provisional allocation should be # returned - for x in writers.values(): - x.remote_close() + del already + del writers + gc.collect() + self.failUnlessEqual(len(ss._active_writers), 1) # now we have a provisional allocation of 1001 bytes @@ -586,8 +589,10 @@ class Server(unittest.TestCase): self.failUnlessEqual(len(writers3), 39) self.failUnlessEqual(len(ss._active_writers), 39) - for x in writers3.values(): - x._disconnected() + del already3 + del writers3 + gc.collect() + self.failUnlessEqual(len(ss._active_writers), 0) ss.disownServiceParent() del ss From 6eade15b4edd59764b0a5a3c13d4b26a08d764d1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 May 2020 14:49:16 -0600 Subject: [PATCH 0065/1054] put WeakKeyDictionary use back --- src/allmydata/storage/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 5cc4be95c..26823957e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -1,4 +1,5 @@ import os, re, struct, time +import weakref import six from foolscap.api import Referenceable @@ -67,7 +68,7 @@ class StorageServer(service.MultiService, Referenceable): self.incomingdir = os.path.join(sharedir, 'incoming') self._clean_incomplete() fileutil.make_dirs(self.incomingdir) - self._active_writers = set() + self._active_writers = weakref.WeakKeyDictionary() log.msg("StorageServer created", facility="tahoe.storage") if reserved_space: @@ -302,7 +303,7 @@ class StorageServer(service.MultiService, Referenceable): if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw - self._active_writers.add(bw) + self._active_writers[bw] = 1 if limited: remaining_space -= max_space_per_bucket else: @@ -358,7 +359,7 @@ class StorageServer(service.MultiService, Referenceable): def bucket_writer_closed(self, bw, consumed_size): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) - self._active_writers.remove(bw) + del self._active_writers[bw] def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold From 1602c6625006a7b8f2ad439fbe1f5e1b32898f76 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 5 May 2020 17:38:08 -0400 Subject: [PATCH 0066/1054] Render map update status using twisted.web.template Fixes: ticket:3291 --- newsfragments/3291.minor | 0 src/allmydata/web/map-update-status.xhtml | 50 ++++---- src/allmydata/web/status.py | 136 ++++++++++++++-------- 3 files changed, 113 insertions(+), 73 deletions(-) create mode 100644 newsfragments/3291.minor diff --git a/newsfragments/3291.minor b/newsfragments/3291.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/web/map-update-status.xhtml b/src/allmydata/web/map-update-status.xhtml index 25b6a28e0..603982623 100644 --- a/src/allmydata/web/map-update-status.xhtml +++ b/src/allmydata/web/map-update-status.xhtml @@ -1,35 +1,39 @@ - + Tahoe-LAFS - Mutable File Servermap Update Status + -

        Mutable File Servermap Update Status

        +

        Mutable File Servermap Update Status

        -
          -
        • Started:
          • -
        • Finished:
          • -
        • Storage Index:
          • -
        • Helper?:
          • -
        • Progress:
          • -
        • Status:
          • -
        +
          +
        • Started:
          • +
        • Finished:
          • +
        • Storage Index:
          • +
        • Helper?:
          • +
        • Progress:
          • +
        • Status:
          • +
        -

        Update Results

        -
          -
        • -
        • Total:
          • -
            -
          • Initial Queries:
            • -
          • -
          • Cumulative Verify:
            • -
          -
        • -
        +

        Update Results

        -
        Return to the Welcome Page
        +
          +
        • +
        • Total:
          • +
            +
          • Initial Queries:
            • +
          • +
          • Cumulative Verify:
            • +
          +
        • +
        - + +
        Return to the Welcome Page
        + + + diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index bc0d69d77..353022174 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -9,7 +9,7 @@ from twisted.web.template import ( XMLFile, renderer, renderElement, - tags + tags, ) from nevow import rend, tags as T from allmydata.util import base32, idlib @@ -868,73 +868,109 @@ class PublishStatusPage(rend.Page, RateAndTimeMixin): l[T.li["[%s]: %s" % (server.get_name(), times_s)]] return T.li["Per-Server Response Times: ", l] -class MapupdateStatusPage(rend.Page, RateAndTimeMixin): - docFactory = getxmlfile("map-update-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.update_status = data +class MapupdateStatusPage(MultiFormatResource): - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + def __init__(self, update_status): + super(MapupdateStatusPage, self).__init__() + self._update_status = update_status - def render_finished(self, ctx, data): - when = data.get_finished() + def render_HTML(self, req): + elem = MapupdateStatusElement(self._update_status); + return renderElement(req, elem) + + +class MapupdateStatusElement(RateAndTimeMixin, Element): + + loader = XMLFile(FilePath(__file__).sibling("map-update-status.xhtml")) + + def __init__(self, update_status): + super(MapupdateStatusElement, self).__init__() + self._update_status = update_status + + @renderer + def started(self, req, tag): + started_s = render_time(self._update_status.get_started()) + return tag(started_s) + + @renderer + def finished(self, req, tag): + when = self._update_status.get_finished() if not when: - return "not yet" - started_s = render_time(data.get_finished()) - return started_s + return tag("not yet") + started_s = render_time(self._update_status.get_finished()) + return tag(started_s) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._update_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(si_s) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._update_status.using_helper()]) - def render_progress(self, ctx, data): - progress = data.get_progress() + @renderer + def progress(self, req, tag): + progress = self._update_status.get_progress() # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_status(self, ctx, data): - return data.get_status() + @renderer + def status(self, req, tag): + return tag(self._update_status.get_status()) - def render_problems(self, ctx, data): - problems = data.problems + @renderer + def problems(self, req, tag): + problems = self._update_status.problems if not problems: - return "" - l = T.ul() + return tag("") + l = tags.ul() for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]] - return ctx.tag["Server Problems:", l] + l(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) + return tag("Server Problems:", l) - def render_privkey_from(self, ctx, data): - server = data.get_privkey_from() + @renderer + def privkey_from(self, req, tag): + server = self._update_status.get_privkey_from() if server: - return ctx.tag["Got privkey from: [%s]" % server.get_name()] + return tag("Got privkey from: [%s]" % server.get_name()) else: - return "" + return tag("") - def data_time_total(self, ctx, data): - return self.update_status.timings.get("total") + # Helper to query update status timings. + # + # Querying `update_status.timings` can yield `None` or a numeric + # value, but twisted.web has trouble flattening the element tree + # when a node contains numeric values. Stringifying them helps. + def _get_update_status_timing(self, name, tag): + res = self._update_status.timings.get(name) + if not res: + return tag("") + return tag(str(res)) - def data_time_initial_queries(self, ctx, data): - return self.update_status.timings.get("initial_queries") + @renderer + def time_total(self, req, tag): + return self._get_update_status_timing("total", tag) - def data_time_cumulative_verify(self, ctx, data): - return self.update_status.timings.get("cumulative_verify") + @renderer + def time_initial_queries(self, req, tag): + return self._get_update_status_timing("initial_queries", tag) - def render_server_timings(self, ctx, data): - per_server = self.update_status.timings.get("per_server") + @renderer + def time_cumulative_verify(self, req, tag): + return self._get_update_status_timing("cumulative_verify", tag) + + @renderer + def server_timings(self, req, tag): + per_server = self._update_status.timings.get("per_server") if not per_server: - return "" - l = T.ul() + return tag("") + l = tags.ul() for server in sorted(per_server.keys(), key=lambda s: s.get_name()): times = [] for op,started,t in per_server[server]: @@ -943,14 +979,14 @@ class MapupdateStatusPage(rend.Page, RateAndTimeMixin): # self.render_time(None, started - self.update_status.get_started()), # self.render_time(None,t))) if op == "query": - times.append( self.render_time(None, t) ) + times.append(self.render_time(None, t)) elif op == "late": - times.append( "late(" + self.render_time(None, t) + ")" ) + times.append("late(" + self.render_time(None, t) + ")") else: - times.append( "privkey(" + self.render_time(None, t) + ")" ) + times.append("privkey(" + self.render_time(None, t) + ")") times_s = ", ".join(times) - l[T.li["[%s]: %s" % (server.get_name(), times_s)]] - return T.li["Per-Server Response Times: ", l] + l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + return tags.li("Per-Server Response Times: ", l) def marshal_json(s): From 86871363ea61609f18061f03c3d993ec0577efa9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 08:44:31 -0400 Subject: [PATCH 0067/1054] Add notes to MapupdateStatusPage --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 353022174..cd23e9ff5 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -869,9 +869,13 @@ class PublishStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Response Times: ", l] +# Renders "/status/mapupdate-%d" class MapupdateStatusPage(MultiFormatResource): def __init__(self, update_status): + """ + :update_status servermap.UpdateStatus: server map stats provider. + """ super(MapupdateStatusPage, self).__init__() self._update_status = update_status From 8a71567137249d7310d436adaeece97ab0c2bc7f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 09:07:32 -0400 Subject: [PATCH 0068/1054] Avoid use of RateAndTimeMixin in MapupdateStatusPage --- src/allmydata/web/status.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index cd23e9ff5..e8becea98 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -884,7 +884,7 @@ class MapupdateStatusPage(MultiFormatResource): return renderElement(req, elem) -class MapupdateStatusElement(RateAndTimeMixin, Element): +class MapupdateStatusElement(Element): loader = XMLFile(FilePath(__file__).sibling("map-update-status.xhtml")) @@ -983,11 +983,11 @@ class MapupdateStatusElement(RateAndTimeMixin, Element): # self.render_time(None, started - self.update_status.get_started()), # self.render_time(None,t))) if op == "query": - times.append(self.render_time(None, t)) + times.append(abbreviate_time(t)) elif op == "late": - times.append("late(" + self.render_time(None, t) + ")") + times.append("late(" + abbreviate_time(t) + ")") else: - times.append("privkey(" + self.render_time(None, t) + ")") + times.append("privkey(" + abbreviate_time(t) + ")") times_s = ", ".join(times) l(tags.li("[%s]: %s" % (server.get_name(), times_s))) return tags.li("Per-Server Response Times: ", l) From bbbca6c0001eed27f3d5300086031184126f362b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 10:41:58 -0400 Subject: [PATCH 0069/1054] Drop a newline Party for consistency, partly in order to trigger a CircleCI run. --- src/allmydata/web/map-update-status.xhtml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/map-update-status.xhtml b/src/allmydata/web/map-update-status.xhtml index 603982623..3bfd53dd1 100644 --- a/src/allmydata/web/map-update-status.xhtml +++ b/src/allmydata/web/map-update-status.xhtml @@ -32,7 +32,6 @@
      -
      Return to the Welcome Page
      From 0c6889274da9fb5f1eb2538bac7565cb67366045 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 13:10:31 -0400 Subject: [PATCH 0070/1054] Render problems only when there's a problem list --- src/allmydata/web/map-update-status.xhtml | 2 +- src/allmydata/web/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/map-update-status.xhtml b/src/allmydata/web/map-update-status.xhtml index 3bfd53dd1..3b963b11d 100644 --- a/src/allmydata/web/map-update-status.xhtml +++ b/src/allmydata/web/map-update-status.xhtml @@ -22,7 +22,7 @@

      Update Results

        -
      • +
      • Total:
        • Initial Queries:
          • diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index e8becea98..4c79e2b25 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -931,7 +931,7 @@ class MapupdateStatusElement(Element): def problems(self, req, tag): problems = self._update_status.problems if not problems: - return tag("") + return tag l = tags.ul() for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) From 008812d6c8747bbda753bbf69ab24e4b273817f3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 13:11:53 -0400 Subject: [PATCH 0071/1054] Render abbreviated time in map update results --- src/allmydata/web/status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4c79e2b25..4755ce445 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -944,7 +944,7 @@ class MapupdateStatusElement(Element): if server: return tag("Got privkey from: [%s]" % server.get_name()) else: - return tag("") + return tag # Helper to query update status timings. # @@ -954,8 +954,8 @@ class MapupdateStatusElement(Element): def _get_update_status_timing(self, name, tag): res = self._update_status.timings.get(name) if not res: - return tag("") - return tag(str(res)) + return tag("0") + return tag(abbreviate_time(res)) @renderer def time_total(self, req, tag): From 13bbda15291ea120a875b11f37055420c6913af7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 13:21:26 -0400 Subject: [PATCH 0072/1054] Render privkey only when there's a privkey --- src/allmydata/web/map-update-status.xhtml | 2 +- src/allmydata/web/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/map-update-status.xhtml b/src/allmydata/web/map-update-status.xhtml index 3b963b11d..445c0b470 100644 --- a/src/allmydata/web/map-update-status.xhtml +++ b/src/allmydata/web/map-update-status.xhtml @@ -26,7 +26,7 @@
        • Total:
          • Initial Queries:
            • -
          • +
          • Cumulative Verify:
        • diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4755ce445..eaf0f726e 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -942,7 +942,7 @@ class MapupdateStatusElement(Element): def privkey_from(self, req, tag): server = self._update_status.get_privkey_from() if server: - return tag("Got privkey from: [%s]" % server.get_name()) + return tag(tags.li("Got privkey from: [%s]" % server.get_name())) else: return tag From 1549e7eb18600832a3a7699f7fdbf9e975133f4b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 15:15:19 -0600 Subject: [PATCH 0073/1054] simplify (for line in f.readlines() -> for line in f) --- src/allmydata/blacklist.py | 2 +- src/allmydata/control.py | 2 +- src/allmydata/frontends/auth.py | 2 +- src/allmydata/scripts/common.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index af1d185d0..89ee81a96 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -35,7 +35,7 @@ class Blacklist(object): if self.last_mtime is None or current_mtime > self.last_mtime: self.entries.clear() with open(self.blacklist_fn, "r") as f: - for line in f.readlines(): + for line in f: line = line.strip() if not line or line.startswith("#"): continue diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 55f6db4f9..b4603b2ac 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -20,7 +20,7 @@ def get_memory_usage(): stats = {} try: with open("/proc/self/status", "r") as f: - for line in f.readlines(): + for line in f: name, right = line.split(":",2) if name in stat_names: assert right.endswith(" kB\n") diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index ab56bf94d..1bd481321 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -32,7 +32,7 @@ class AccountFileChecker(object): self.pubkeys = {} self.rootcaps = {} with open(abspath_expanduser_unicode(accountfile), "r") as f: - for line in f.readlines(): + for line in f: line = line.strip() if line.startswith("#") or not line: continue diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 89c1f94d1..91a3882fb 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -139,7 +139,7 @@ def get_aliases(nodedir): pass try: with codecs.open(aliasfile, "r", "utf-8") as f: - for line in f.readlines(): + for line in f: line = line.strip() if line.startswith("#") or not line: continue From 2418e90bf0d4e462f53cdd74e7a8dc2fb7d85efd Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 15:22:22 -0600 Subject: [PATCH 0074/1054] CacheDirectoryManager is unused --- src/allmydata/test/test_util.py | 70 +-------------------------------- src/allmydata/util/cachedir.py | 43 -------------------- 2 files changed, 1 insertion(+), 112 deletions(-) delete mode 100644 src/allmydata/util/cachedir.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index ca05e4008..3bddc8bcc 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -19,7 +19,7 @@ from twisted.python import log from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil, abbreviate -from allmydata.util import limiter, time_format, pollmixin, cachedir +from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans @@ -1286,74 +1286,6 @@ class TimeFormat(unittest.TestCase, TimezoneMixin): self.failUnlessEqual( time_format.format_delta(time_1decimal, time_1d21h46m49s_delta), '1d 21h 46m 48s') -class CacheDir(unittest.TestCase): - def test_basic(self): - basedir = "test_util/CacheDir/test_basic" - - def _failIfExists(name): - absfn = os.path.join(basedir, name) - self.failIf(os.path.exists(absfn), - "%s exists but it shouldn't" % absfn) - - def _failUnlessExists(name): - absfn = os.path.join(basedir, name) - self.failUnless(os.path.exists(absfn), - "%s doesn't exist but it should" % absfn) - - cdm = cachedir.CacheDirectoryManager(basedir) - a = cdm.get_file("a") - b = cdm.get_file("b") - c = cdm.get_file("c") - for x in {a, b, c}: - with open(x.get_filename(), "wb") as f: - f.write("hi") - del x - gc.collect() # for PyPy - - _failUnlessExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - - cdm.check() - - _failUnlessExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - - del a - gc.collect() # for PyPy - # this file won't be deleted yet, because it isn't old enough - cdm.check() - _failUnlessExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - - # we change the definition of "old" to make everything old - cdm.old = -10 - - cdm.check() - _failIfExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - - cdm.old = 60*60 - - del b - gc.collect() # for PyPy - - cdm.check() - _failIfExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - - b2 = cdm.get_file("b") - - cdm.check() - _failIfExists("a") - _failUnlessExists("b") - _failUnlessExists("c") - del b2 - gc.collect() # for PyPy ctr = [0] class EqButNotIs(object): diff --git a/src/allmydata/util/cachedir.py b/src/allmydata/util/cachedir.py deleted file mode 100644 index 118a8a7eb..000000000 --- a/src/allmydata/util/cachedir.py +++ /dev/null @@ -1,43 +0,0 @@ - -import os.path, stat, weakref, time -from twisted.application import service, internet -from allmydata.util import fileutil - -HOUR = 60*60 - -class CacheDirectoryManager(service.MultiService): - def __init__(self, basedir, pollinterval=1*HOUR, old=1*HOUR): - service.MultiService.__init__(self) - self.basedir = basedir - fileutil.make_dirs(basedir) - self.old = old - self.files = weakref.WeakValueDictionary() - - t = internet.TimerService(pollinterval, self.check) - t.setServiceParent(self) - - def get_file(self, key): - assert isinstance(key, str) # used as filename - absfn = os.path.join(self.basedir, key) - if os.path.exists(absfn): - os.utime(absfn, None) - cf = CacheFile(absfn) - self.files[key] = cf - return cf - - def check(self): - now = time.time() - for fn in os.listdir(self.basedir): - if fn in self.files: - continue - absfn = os.path.join(self.basedir, fn) - mtime = os.stat(absfn)[stat.ST_MTIME] - if now - mtime > self.old: - os.remove(absfn) - -class CacheFile(object): - def __init__(self, absfn): - self.filename = absfn - - def get_filename(self): - return self.filename From 825bc91256fd12ce16df21cb968f5670e6acef1b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:17:23 -0600 Subject: [PATCH 0075/1054] mark PyPy as required in circle-ci --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61ed12a5d..d0f596fc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,7 +207,6 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage" - ALLOWED_FAILURE: "yes" c-locale: From d2375e4fe20d1a1b8cb6e67b71083869f36092bd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 6 May 2020 22:42:32 -0400 Subject: [PATCH 0076/1054] Reformat retrieve status page template --- src/allmydata/web/retrieve-status.xhtml | 63 +++++++++++++------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/src/allmydata/web/retrieve-status.xhtml b/src/allmydata/web/retrieve-status.xhtml index f586429f5..0b060d38b 100644 --- a/src/allmydata/web/retrieve-status.xhtml +++ b/src/allmydata/web/retrieve-status.xhtml @@ -1,43 +1,48 @@ + Tahoe-LAFS - Mutable File Retrieve Status + -

          Mutable File Retrieve Status

          +

          Mutable File Retrieve Status

          -
            -
          • Started:
            • -
          • Storage Index:
            • -
          • Helper?:
            • -
          • Current Size:
            • -
          • Progress:
            • -
          • Status:
            • -
          - -

          Retrieve Results

          -
            -
          • -
          • -
          • Timings:
            • -
              -
            • Total: - ()
                • -
              • Fetching: - ()
                • -
              • Decoding: - ()
                • -
              • Decrypting: - ()
                • +
              • Started:
                • +
              • Storage Index:
                • +
              • Helper?:
                • +
              • Current Size:
                • +
              • Progress:
                • +
              • Status:
              -
            • -
            -
          -
          Return to the Welcome Page
          +

          Retrieve Results

          - +
            +
          • +
          • +
          • Timings:
            • +
              +
            • Total: + ()
              • +
                +
              • Fetching: + ()
                • +
              • Decoding: + ()
                • +
              • Decrypting: + ()
                • +
              +
            • +
            +
          + +
          Return to the Welcome Page
          + + + + From 5fa7c4c446a228e68be28b560b4ebbcfc69d57cd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 09:32:01 -0400 Subject: [PATCH 0077/1054] Update markup in retrieve status template page --- src/allmydata/web/retrieve-status.xhtml | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/allmydata/web/retrieve-status.xhtml b/src/allmydata/web/retrieve-status.xhtml index 0b060d38b..04e08af87 100644 --- a/src/allmydata/web/retrieve-status.xhtml +++ b/src/allmydata/web/retrieve-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Mutable File Retrieve Status @@ -12,32 +12,32 @@

          Mutable File Retrieve Status

            -
          • Started:
            • -
          • Storage Index:
            • -
          • Helper?:
            • -
          • Current Size:
            • -
          • Progress:
            • -
          • Status:
            • +
          • Started:
            • +
          • Storage Index:
            • +
          • Helper?:
            • +
          • Current Size:
            • +
          • Progress:
            • +
          • Status:

          Retrieve Results

            -
          • -
          • +
          • +
          • Timings:
              • -
            • Total: - ()
              • +
            • Total: + ()
                • -
              • Fetching: - ()
                • -
              • Decoding: - ()
                • -
              • Decrypting: - ()
                • +
              • Fetching: + ()
                • +
              • Decoding: + ()
                • +
              • Decrypting: + ()
              -
            • +
          From de4130274374cb5ce9d8f00ba60c01f1b2e6ed3e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 09:34:42 -0400 Subject: [PATCH 0078/1054] Update RetrieveStatusPage to use twisted renderers --- src/allmydata/web/status.py | 144 ++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 55 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index bc0d69d77..60e2a0a70 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -673,90 +673,124 @@ class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): def render_status(self, ctx, data): return data.get_status() -class RetrieveStatusPage(rend.Page, RateAndTimeMixin): - docFactory = getxmlfile("retrieve-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.retrieve_status = data +class RetrieveStatusPage(MultiFormatResource): - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + def __init__(self, retrieve_status): + super(RetrieveStatusPage, self).__init__() + self._retrieve_status = retrieve_status - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + def render_HTML(self, req): + elem = RetrieveStatusElement(self._retrieve_status) + return renderElement(req, elem) + + +class RetrieveStatusElement(Element, RateAndTimeMixin): + + loader = XMLFile(FilePath(__file__).sibling("retrieve-status.xhtml")) + + def __init__(self, retrieve_status): + super(RetrieveStatusElement, self).__init__() + self._retrieve_status = retrieve_status + + @renderer + def started(self, req, tag): + started_s = render_time(self._retrieve_status.get_started()) + return tag(started_s) + + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._retrieve_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(si_s) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._retrieve_status.using_helper()]) - def render_current_size(self, ctx, data): - size = data.get_size() + @renderer + def current_size(self, req, tag): + size = self._retrieve_status.get_size() if size is None: size = "(unknown)" - return size + return tag(size) - def render_progress(self, ctx, data): - progress = data.get_progress() + @renderer + def progress(self, req, tag): + progress = self._retrieve_status.get_progress() # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_status(self, ctx, data): - return data.get_status() + @renderer + def status(self, req, tag): + return tag(self._retrieve_status.get_status()) - def render_encoding(self, ctx, data): - k, n = data.get_encoding() - return ctx.tag["Encoding: %s of %s" % (k, n)] + @renderer + def encoding(self, req, tag): + k, n = self._retrieve_status.get_encoding() + return tag("Encoding: %s of %s" % (k, n)) - def render_problems(self, ctx, data): - problems = data.get_problems() + @renderer + def problems(self, req, tag): + problems = self._retrieve_status.get_problems() if not problems: return "" - l = T.ul() + l = tags.ul() for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]] - return ctx.tag["Server Problems:", l] + l(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) + return tag("Server Problems:", l) - def _get_rate(self, data, name): - file_size = self.retrieve_status.get_size() - duration = self.retrieve_status.timings.get(name) + def _get_rate(self, name): + file_size = self._retrieve_status.get_size() + duration = self._retrieve_status.timings.get(name) return compute_rate(file_size, duration) - def data_time_total(self, ctx, data): - return self.retrieve_status.timings.get("total") - def data_rate_total(self, ctx, data): - return self._get_rate(data, "total") + @renderer + def time_total(self, req, tag): + return tag(self._retrieve_status.timings.get("total")) - def data_time_fetch(self, ctx, data): - return self.retrieve_status.timings.get("fetch") - def data_rate_fetch(self, ctx, data): - return self._get_rate(data, "fetch") + @renderer + def rate_total(self, req, tag): + return tag(self._get_rate("total")) - def data_time_decode(self, ctx, data): - return self.retrieve_status.timings.get("decode") - def data_rate_decode(self, ctx, data): - return self._get_rate(data, "decode") + @renderer + def time_fetch(self, req, tag): + return tag(self._retrieve_status.timings.get("fetch")) - def data_time_decrypt(self, ctx, data): - return self.retrieve_status.timings.get("decrypt") - def data_rate_decrypt(self, ctx, data): - return self._get_rate(data, "decrypt") + @renderer + def rate_fetch(self, req, tag): + return tag(self._get_rate("fetch")) - def render_server_timings(self, ctx, data): - per_server = self.retrieve_status.timings.get("fetch_per_server") + @renderer + def time_decode(self, req, tag): + return tag(self._retrieve_status.timings.get("decode")) + + @renderer + def rate_decode(self, req, tag): + return tag(self._get_rate("decode")) + + @renderer + def time_decrypt(self, req, tag): + return tag(self._retrieve_status.timings.get("decrypt")) + + @renderer + def rate_decrypt(self, req, tag): + return tag(self._get_rate("decrypt")) + + @renderer + def server_timings(self, req, tag): + per_server = self._retrieve_status.timings.get("fetch_per_server") if not per_server: - return "" - l = T.ul() + return tag("") + l = tags.ul() for server in sorted(per_server.keys(), key=lambda s: s.get_name()): times_s = ", ".join([self.render_time(None, t) for t in per_server[server]]) - l[T.li["[%s]: %s" % (server.get_name(), times_s)]] - return T.li["Per-Server Fetch Response Times: ", l] + l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + return tags.li("Per-Server Fetch Response Times: ", l) class PublishStatusPage(rend.Page, RateAndTimeMixin): From a544e8a55f09cacaf4974d2efd9ecfd1e8e7d4f1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 10:55:08 -0400 Subject: [PATCH 0079/1054] Stringify tag parameters --- src/allmydata/web/status.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 60e2a0a70..89168c7ec 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -712,7 +712,7 @@ class RetrieveStatusElement(Element, RateAndTimeMixin): @renderer def current_size(self, req, tag): - size = self._retrieve_status.get_size() + size = str(self._retrieve_status.get_size()) if size is None: size = "(unknown)" return tag(size) @@ -750,35 +750,35 @@ class RetrieveStatusElement(Element, RateAndTimeMixin): @renderer def time_total(self, req, tag): - return tag(self._retrieve_status.timings.get("total")) + return tag(str(self._retrieve_status.timings.get("total"))) @renderer def rate_total(self, req, tag): - return tag(self._get_rate("total")) + return tag(str(self._get_rate("total"))) @renderer def time_fetch(self, req, tag): - return tag(self._retrieve_status.timings.get("fetch")) + return tag(str(self._retrieve_status.timings.get("fetch"))) @renderer def rate_fetch(self, req, tag): - return tag(self._get_rate("fetch")) + return tag(str(self._get_rate("fetch"))) @renderer def time_decode(self, req, tag): - return tag(self._retrieve_status.timings.get("decode")) + return tag(str(self._retrieve_status.timings.get("decode"))) @renderer def rate_decode(self, req, tag): - return tag(self._get_rate("decode")) + return tag(str(self._get_rate("decode"))) @renderer def time_decrypt(self, req, tag): - return tag(self._retrieve_status.timings.get("decrypt")) + return tag(str(self._retrieve_status.timings.get("decrypt"))) @renderer def rate_decrypt(self, req, tag): - return tag(self._get_rate("decrypt")) + return tag(str(self._get_rate("decrypt"))) @renderer def server_timings(self, req, tag): From 07f14a19f81db85aa1a66d2c39a3350036dc8461 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 11:15:36 -0400 Subject: [PATCH 0080/1054] Drop RateAndTimeMixin from RetrieveStatusElement --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 89168c7ec..3d9175ff1 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -685,7 +685,7 @@ class RetrieveStatusPage(MultiFormatResource): return renderElement(req, elem) -class RetrieveStatusElement(Element, RateAndTimeMixin): +class RetrieveStatusElement(Element): loader = XMLFile(FilePath(__file__).sibling("retrieve-status.xhtml")) @@ -787,7 +787,7 @@ class RetrieveStatusElement(Element, RateAndTimeMixin): return tag("") l = tags.ul() for server in sorted(per_server.keys(), key=lambda s: s.get_name()): - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) l(tags.li("[%s]: %s" % (server.get_name(), times_s))) return tags.li("Per-Server Fetch Response Times: ", l) From 3aaa87d2ba58620290f9f448d4efde4ea8cf5a3c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 13:15:05 -0400 Subject: [PATCH 0081/1054] Add notes to RetrieveStatusPage --- src/allmydata/web/status.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3d9175ff1..8a423e69f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -674,9 +674,14 @@ class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): return data.get_status() + +# Renders "/status/retrieve-%d". class RetrieveStatusPage(MultiFormatResource): def __init__(self, retrieve_status): + """ + :retrieve_status retrieve.RetrieveStatus: stats provider. + """ super(RetrieveStatusPage, self).__init__() self._retrieve_status = retrieve_status From a39f6d36f4d95b09998fe57cc389f2c6a7da7056 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 7 May 2020 16:40:55 -0400 Subject: [PATCH 0082/1054] Add newsfragment --- newsfragments/3290.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3290.minor diff --git a/newsfragments/3290.minor b/newsfragments/3290.minor new file mode 100644 index 000000000..e69de29bb From 65cd0e8ecd22f4227f30019fb9e7d11e2f7eeee7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 10:38:15 -0400 Subject: [PATCH 0083/1054] Reformat publish status template page --- src/allmydata/web/publish-status.xhtml | 79 ++++++++++++++------------ 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/src/allmydata/web/publish-status.xhtml b/src/allmydata/web/publish-status.xhtml index 4204de8e2..2c676493f 100644 --- a/src/allmydata/web/publish-status.xhtml +++ b/src/allmydata/web/publish-status.xhtml @@ -1,51 +1,56 @@ + Tahoe-LAFS - Mutable File Publish Status + -

          Mutable File Publish Status

          +

          Mutable File Publish Status

          -
            -
          • Started:
            • -
          • Storage Index:
            • -
          • Helper?:
            • -
          • Current Size:
            • -
          • Progress:
            • -
          • Status:
            • -
          - -

          Publish Results

          -
            -
          • -
          • -
          • -
          • Timings:
            • -
              -
            • Total: - ()
                • -
              • Setup:
                • -
              • Encrypting: - ()
                • -
              • Encoding: - ()
                • -
              • Packing Shares: - () -
                  -
                • RSA Signature:
                  • -
                • - -
              • Pushing: - ()
                • +
              • Started:
                • +
              • Storage Index:
                • +
              • Helper?:
                • +
              • Current Size:
                • +
              • Progress:
                • +
              • Status:
              -
            • -
            -
          -
          Return to the Welcome Page
          +

          Publish Results

          - +
            +
          • +
          • +
          • +
          • Timings:
            • +
              +
            • Total: + ()
              • +
                +
              • Setup:
                • +
              • Encrypting: + ()
                • +
              • Encoding: + ()
                • +
              • Packing Shares: + () +
                  +
                • RSA Signature:
                  • +
                • + +
              • Pushing: + ()
                • +
              +
            • +
            +
          + +
          Return to the Welcome Page
          + + + + From 89c42100907956c3c6189c3905a133274db84cd3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 10:40:48 -0400 Subject: [PATCH 0084/1054] Use twisted tags in publish status template page --- src/allmydata/web/publish-status.xhtml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/publish-status.xhtml b/src/allmydata/web/publish-status.xhtml index 2c676493f..49f5bb08c 100644 --- a/src/allmydata/web/publish-status.xhtml +++ b/src/allmydata/web/publish-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Mutable File Publish Status @@ -12,40 +12,40 @@

          Mutable File Publish Status

            -
          • Started:
            • -
          • Storage Index:
            • -
          • Helper?:
            • -
          • Current Size:
            • -
          • Progress:
            • -
          • Status:
            • +
          • Started:
            • +
          • Storage Index:
            • +
          • Helper?:
            • +
          • Current Size:
            • +
          • Progress:
            • +
          • Status:

          Publish Results

            -
          • -
          • -
          • +
          • +
          • +
          • Timings:
              • -
            • Total: - ()
              • +
            • Total: + ()
                • -
              • Setup:
                • -
              • Encrypting: - ()
                • -
              • Encoding: - ()
                • -
              • Packing Shares: - () +
              • Setup:
                • +
              • Encrypting: + ()
                • +
              • Encoding: + ()
                • +
              • Packing Shares: + ()
                  -
                • RSA Signature:
                  • +
                • RSA Signature:
                • -
              • Pushing: - ()
                • +
              • Pushing: + ()
              -
            • +
          From fdf6449ba8bcff3d1da1283fcd4d12ccf57f4aff Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 11:24:56 -0400 Subject: [PATCH 0085/1054] Use MultiFormatResource to implement PublishStatusPage --- src/allmydata/web/status.py | 190 ++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 72 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index bc0d69d77..225f30193 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -759,114 +759,160 @@ class RetrieveStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Fetch Response Times: ", l] -class PublishStatusPage(rend.Page, RateAndTimeMixin): +class PublishStatusPage(MultiFormatResource): + docFactory = getxmlfile("publish-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.publish_status = data + def __init__(self, publish_status): + super(PublishStatusPage, self).__init__() + self._publish_status = publish_status - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + def render_HTML(self, req): + elem = PublishStatusElement(self._publish_status); + return renderElement(req, elem) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + +class PublishStatusElement(Element, RateAndTimeMixin): + + loader = XMLFile(FilePath(__file__).sibling("publish-status.xhtml")) + + def __init__(self, publish_status): + super(PublishStatusElement, self).__init__() + self._publish_status = publish_status + + @renderer + def started(self, req, tag): + started_s = render_time(self._publish_status.get_started()) + return tag(started_s) + + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._publish_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(str(si_s)) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._publish_status.using_helper()]) - def render_current_size(self, ctx, data): - size = data.get_size() + @renderer + def current_size(self, req, tag): + size = self._publish_status.get_size() if size is None: size = "(unknown)" - return size + return tag(str(size)) - def render_progress(self, ctx, data): - progress = data.get_progress() + @renderer + def progress(self, req, tag): + progress = self._publish_status.get_progress() # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_status(self, ctx, data): - return data.get_status() + @renderer + def status(self, req, tag): + return tag(self._publish_status.get_status()) - def render_encoding(self, ctx, data): - k, n = data.get_encoding() - return ctx.tag["Encoding: %s of %s" % (k, n)] + @renderer + def encoding(self, req, tag): + k, n = self._publish_status.get_encoding() + return tag("Encoding: %s of %s" % (k, n)) - def render_sharemap(self, ctx, data): - servermap = data.get_servermap() + @renderer + def sharemap(self, req, tag): + servermap = self._publish_status.get_servermap() if servermap is None: - return ctx.tag["None"] - l = T.ul() + return tag("None") + l = tags.ul() sharemap = servermap.make_sharemap() for shnum in sorted(sharemap.keys()): - l[T.li["%d -> Placed on " % shnum, - ", ".join(["[%s]" % server.get_name() - for server in sharemap[shnum]])]] - return ctx.tag["Sharemap:", l] + l(tags.li("%d -> Placed on " % shnum, + ", ".join(["[%s]" % server.get_name() + for server in sharemap[shnum]]))) + return tag("Sharemap:", l) - def render_problems(self, ctx, data): - problems = data.get_problems() + @renderer + def problems(self, req, tag): + problems = self._publish_status.get_problems() if not problems: - return "" - l = T.ul() + return tag() + l = tags.ul() # XXX: is this exercised? I don't think PublishStatus.problems is # ever populated for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]] - return ctx.tag["Server Problems:", l] + l(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) + return tag("Server Problems:", l) - def _get_rate(self, data, name): - file_size = self.publish_status.get_size() - duration = self.publish_status.timings.get(name) - return compute_rate(file_size, duration) + def _get_rate(self, name): + file_size = self._publish_status.get_size() + duration = self._publish_status.timings.get(name) + return str(compute_rate(file_size, duration)) - def data_time_total(self, ctx, data): - return self.publish_status.timings.get("total") - def data_rate_total(self, ctx, data): - return self._get_rate(data, "total") + def _get_time(self, name): + return str(self._publish_status.timings.get(name)) - def data_time_setup(self, ctx, data): - return self.publish_status.timings.get("setup") + @renderer + def time_total(self, req, tag): + return tag(self._get_time("total")) - def data_time_encrypt(self, ctx, data): - return self.publish_status.timings.get("encrypt") - def data_rate_encrypt(self, ctx, data): - return self._get_rate(data, "encrypt") + @renderer + def rate_total(self, req, tag): + return tag(self._get_rate("total")) - def data_time_encode(self, ctx, data): - return self.publish_status.timings.get("encode") - def data_rate_encode(self, ctx, data): - return self._get_rate(data, "encode") + @renderer + def time_setup(self, req, tag): + return tag(self._get_time("setup")) - def data_time_pack(self, ctx, data): - return self.publish_status.timings.get("pack") - def data_rate_pack(self, ctx, data): - return self._get_rate(data, "pack") - def data_time_sign(self, ctx, data): - return self.publish_status.timings.get("sign") + @renderer + def time_encrypt(self, req, tag): + return tag(self._get_time("encrypt")) - def data_time_push(self, ctx, data): - return self.publish_status.timings.get("push") - def data_rate_push(self, ctx, data): - return self._get_rate(data, "push") + @renderer + def rate_encrypt(self, req, tag): + return tag(self._get_rate("encrypt")) - def render_server_timings(self, ctx, data): - per_server = self.publish_status.timings.get("send_per_server") + @renderer + def time_encode(self, req, tag): + return tag(self._get_time("encode")) + + @renderer + def rate_encode(self, req, tag): + return tag(self._get_rate("encode")) + + @renderer + def time_pack(self, req, tag): + return tag(self._get_time("pack")) + + @renderer + def rate_pack(self, req, tag): + return tag(self._get_rate("pack")) + + @renderer + def time_sign(self, req, tag): + return tag(self._get_time("sign")) + + @renderer + def time_push(self, req, tag): + return tag(self._get_time("push")) + + @renderer + def rate_push(self, req, tag): + return self._get_rate("push") + + @renderer + def server_timings(self, req, tag): + per_server = self._publish_status.timings.get("send_per_server") if not per_server: - return "" - l = T.ul() + return tag() + l = tags.ul() for server in sorted(per_server.keys(), key=lambda s: s.get_name()): times_s = ", ".join([self.render_time(None, t) for t in per_server[server]]) - l[T.li["[%s]: %s" % (server.get_name(), times_s)]] - return T.li["Per-Server Response Times: ", l] + l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + return tags.li("Per-Server Response Times: ", l) + class MapupdateStatusPage(rend.Page, RateAndTimeMixin): docFactory = getxmlfile("map-update-status.xhtml") From 0fba615afb192c04e23e47e821abc3e58b57d665 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 11:28:01 -0400 Subject: [PATCH 0086/1054] Remove RateAndTimeMixin from PublishStatusPage --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 225f30193..07585435a 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -772,7 +772,7 @@ class PublishStatusPage(MultiFormatResource): return renderElement(req, elem) -class PublishStatusElement(Element, RateAndTimeMixin): +class PublishStatusElement(Element): loader = XMLFile(FilePath(__file__).sibling("publish-status.xhtml")) @@ -908,7 +908,7 @@ class PublishStatusElement(Element, RateAndTimeMixin): return tag() l = tags.ul() for server in sorted(per_server.keys(), key=lambda s: s.get_name()): - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) l(tags.li("[%s]: %s" % (server.get_name(), times_s))) return tags.li("Per-Server Response Times: ", l) From 9e200fc0141615de83f33b681b8644fbe59f7a9d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 11:34:57 -0400 Subject: [PATCH 0087/1054] Render publish problems only when there are problems --- src/allmydata/web/publish-status.xhtml | 2 +- src/allmydata/web/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/publish-status.xhtml b/src/allmydata/web/publish-status.xhtml index 49f5bb08c..1949230c2 100644 --- a/src/allmydata/web/publish-status.xhtml +++ b/src/allmydata/web/publish-status.xhtml @@ -24,7 +24,7 @@
          • -
          • +
          • Timings:
              • diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 07585435a..70edbc4c9 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -843,7 +843,7 @@ class PublishStatusElement(Element): for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) l(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) - return tag("Server Problems:", l) + return tag(tags.li("Server Problems:", l)) def _get_rate(self, name): file_size = self._publish_status.get_size() From a4bc31a13798c66f0ef082355f14d8eefdffcf6d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 11:42:30 -0400 Subject: [PATCH 0088/1054] Add newsfragment --- newsfragments/3289.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3289.minor diff --git a/newsfragments/3289.minor b/newsfragments/3289.minor new file mode 100644 index 000000000..e69de29bb From 0ae045ea166651c122f92e7c169e501f339845a4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 12:47:48 -0400 Subject: [PATCH 0089/1054] Add comments to PublishStatusPage --- src/allmydata/web/status.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 70edbc4c9..7e05a2677 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -759,11 +759,13 @@ class RetrieveStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Fetch Response Times: ", l] +# Renders "status/publish-%n" class PublishStatusPage(MultiFormatResource): - docFactory = getxmlfile("publish-status.xhtml") - def __init__(self, publish_status): + """ + :publish_status mutable.publish.PublishStatus: publish stats provider. + """ super(PublishStatusPage, self).__init__() self._publish_status = publish_status From 3fad9a127c5ec4b1d13f0f3aca53072247a45345 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 12 May 2020 07:40:22 -0400 Subject: [PATCH 0090/1054] Remove web.common.RenderMixin Fixes: ticket:3308 --- newsfragments/3308.minor | 0 src/allmydata/web/common.py | 36 ------------------------------------ 2 files changed, 36 deletions(-) create mode 100644 newsfragments/3308.minor diff --git a/newsfragments/3308.minor b/newsfragments/3308.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index e7b9f8ed1..a930fd2b1 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -365,42 +365,6 @@ class NeedOperationHandleError(WebError): pass -# XXX should be phased out by the nevow -> twisted.web port (that is, -# this whole class should have no users and can be deleted once the -# port away from nevow is complete) -class RenderMixin(object): - - def renderHTTP(self, ctx): - request = IRequest(ctx) - - # if we were using regular twisted.web Resources (and the regular - # twisted.web.server.Request object) then we could implement - # render_PUT and render_GET. But Nevow's request handler - # (NevowRequest.gotPageContext) goes directly to renderHTTP. Copy - # some code from the Resource.render method that Nevow bypasses, to - # do the same thing. - m = getattr(self, 'render_' + request.method, None) - if not m: - raise server.UnsupportedMethod(getattr(self, 'allowedMethods', ())) - return m(ctx) - - def render_OPTIONS(self, ctx): - """ - Handle HTTP OPTIONS request by adding appropriate headers. - - :param ctx: client transaction from which request is extracted - :returns: str (empty) - """ - req = IRequest(ctx) - req.setHeader("server", "Tahoe-LAFS gateway") - methods = ', '.join([m[7:] for m in dir(self) if m.startswith('render_')]) - req.setHeader("allow", methods) - req.setHeader("public", methods) - req.setHeader("compliance", "rfc=2068, rfc=2616") - req.setHeader("content-length", 0) - return "" - - class MultiFormatPage(Page): """ ```MultiFormatPage`` is a ``rend.Page`` that can be rendered in a number From b1d7e9967135f488990c192559e65fd3f38aec3d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 12 May 2020 18:15:17 -0400 Subject: [PATCH 0091/1054] Try running PyInstaller-generated binary in GitHub Actions --- .github/workflows/ci.yml | 4 ++++ newsfragments/3309.minor | 0 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3309.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d41a78b4..cbb03eb2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,3 +170,7 @@ jobs: - name: Run "tox -e pyinstaller" run: tox -e pyinstaller + + # This step is to ensure there are no packaging/import errors. + - name: Test PyInstaller executable + run: dist/Tahoe-LAFS/tahoe --version diff --git a/newsfragments/3309.minor b/newsfragments/3309.minor new file mode 100644 index 000000000..e69de29bb From 86184e9aa9dc475521e54baaed0bca92fd4eb1fe Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 12 May 2020 18:28:42 -0400 Subject: [PATCH 0092/1054] Do not install codecov in GitHub Actions packaging tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbb03eb2b..fce49cdb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools + pip install --upgrade tox setuptools pip list - name: Display tool versions From f3d37e52dbd20ebaf85ac7fb2d73fbd54f5fad30 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 12 May 2020 18:37:50 -0400 Subject: [PATCH 0093/1054] Upload PyInstaller package as a GitHub Actions artifact --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fce49cdb8..e95686eb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,3 +174,9 @@ jobs: # This step is to ensure there are no packaging/import errors. - name: Test PyInstaller executable run: dist/Tahoe-LAFS/tahoe --version + + - name: Upload PyInstaller package + uses: actions/upload-artifact@v2 + with: + name: Tahoe-LAFS-${{ matrix.os }}-Python-${{ matrix.python-version }} + path: dist/Tahoe-LAFS-*-*.* From 4b1ff9546c313fbf1e734ba6bb0350fcf774fdcd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 12:06:11 -0400 Subject: [PATCH 0094/1054] Do not install setuptools in pyinstaller step This is unnecessary, and likely was the result of copy-pasting. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e95686eb6..2ded8f418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox setuptools + pip install --upgrade tox pip list - name: Display tool versions From c18488872ceb00ae7731d4e22bed87fdda54d2ed Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 16:47:51 -0400 Subject: [PATCH 0095/1054] Make comment on MapupdateStatusPage a docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index eaf0f726e..4232a504a 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -869,8 +869,8 @@ class PublishStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Response Times: ", l] -# Renders "/status/mapupdate-%d" class MapupdateStatusPage(MultiFormatResource): + """Renders "/status/mapupdate-%d.""" def __init__(self, update_status): """ From 8e17d203cf46c6f2a200b6641ed616cf17bb98aa Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 16:59:52 -0400 Subject: [PATCH 0096/1054] Add missing "param:" to docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4232a504a..5b57aa239 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -874,7 +874,7 @@ class MapupdateStatusPage(MultiFormatResource): def __init__(self, update_status): """ - :update_status servermap.UpdateStatus: server map stats provider. + :param update_status servermap.UpdateStatus: server map stats provider. """ super(MapupdateStatusPage, self).__init__() self._update_status = update_status From 62760c09975566236a847656fdff565854ac1217 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:07:09 -0400 Subject: [PATCH 0097/1054] Make RetrieveStatusPage comment a docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 8a423e69f..f925ad961 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -675,8 +675,8 @@ class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): -# Renders "/status/retrieve-%d". class RetrieveStatusPage(MultiFormatResource): + """Renders /status/retrieve-%d.""" def __init__(self, retrieve_status): """ From 4af2e4b4df300f6d0ad7b2950e9249ae340efb14 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:08:26 -0400 Subject: [PATCH 0098/1054] Drop an unbalanced quote --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 5b57aa239..d26079345 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -870,7 +870,7 @@ class PublishStatusPage(rend.Page, RateAndTimeMixin): class MapupdateStatusPage(MultiFormatResource): - """Renders "/status/mapupdate-%d.""" + """Renders /status/mapupdate-%d.""" def __init__(self, update_status): """ From 812f03934ab5f333c812bd6f3887e5795a7e96d9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:09:53 -0400 Subject: [PATCH 0099/1054] Add missing "param:" to docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index f925ad961..3931dc633 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -680,7 +680,7 @@ class RetrieveStatusPage(MultiFormatResource): def __init__(self, retrieve_status): """ - :retrieve_status retrieve.RetrieveStatus: stats provider. + :param retrieve_status retrieve.RetrieveStatus: stats provider. """ super(RetrieveStatusPage, self).__init__() self._retrieve_status = retrieve_status From 5aa0dd4085e07a1a316a69fcaddc0d62e38cf9d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:12:08 -0400 Subject: [PATCH 0100/1054] Rename a variable for readability Noticed in code review: `l` can be confused with `1` for many typefaces. --- src/allmydata/web/status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3931dc633..0dc35b231 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -742,11 +742,11 @@ class RetrieveStatusElement(Element): problems = self._retrieve_status.get_problems() if not problems: return "" - l = tags.ul() + ul = tags.ul() for peerid in sorted(problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - l(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) - return tag("Server Problems:", l) + ul(tags.li("[%s]: %s" % (peerid_s, problems[peerid]))) + return tag("Server Problems:", ul) def _get_rate(self, name): file_size = self._retrieve_status.get_size() From 2e0e210c14fd533bf5fa477e3a385b5876a53c5a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:43:55 -0400 Subject: [PATCH 0101/1054] Make PublishStatusPage comment a docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7e05a2677..203b73fd6 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -759,8 +759,8 @@ class RetrieveStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Fetch Response Times: ", l] -# Renders "status/publish-%n" class PublishStatusPage(MultiFormatResource): + """Renders status/publish-%d.""" def __init__(self, publish_status): """ From ebe80221d59208620c4c2ddcf397a89069ed2a3f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 17:44:50 -0400 Subject: [PATCH 0102/1054] Add missing "param:" to init method docstring --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 203b73fd6..68c5d5341 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -764,7 +764,7 @@ class PublishStatusPage(MultiFormatResource): def __init__(self, publish_status): """ - :publish_status mutable.publish.PublishStatus: publish stats provider. + :param publish_status mutable.publish.PublishStatus: stats provider. """ super(PublishStatusPage, self).__init__() self._publish_status = publish_status From 9d5d3389938159cf1c43fb60cf90bc2569ab0206 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 19:04:10 -0400 Subject: [PATCH 0103/1054] Correct docstring to ":param " format --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0dc35b231..d62edb3d2 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -680,7 +680,7 @@ class RetrieveStatusPage(MultiFormatResource): def __init__(self, retrieve_status): """ - :param retrieve_status retrieve.RetrieveStatus: stats provider. + :param retrieve.RetrieveStatus retrieve_status: stats provider. """ super(RetrieveStatusPage, self).__init__() self._retrieve_status = retrieve_status From 79a44eac33ded8088ec58840fb8e87a54dfc7c5f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 13 May 2020 19:29:41 -0400 Subject: [PATCH 0104/1054] Correct docstring to ":param " format --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 68c5d5341..0217978d3 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -764,7 +764,7 @@ class PublishStatusPage(MultiFormatResource): def __init__(self, publish_status): """ - :param publish_status mutable.publish.PublishStatus: stats provider. + :param mutable.publish.PublishStatus publish_status: stats provider. """ super(PublishStatusPage, self).__init__() self._publish_status = publish_status From a203f9c8d837fb569ac3ccf7d1cd3ea70142a10e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 13:14:10 -0400 Subject: [PATCH 0105/1054] Reformat download status page template --- src/allmydata/web/download-status.xhtml | 89 +++++++++++++------------ 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/src/allmydata/web/download-status.xhtml b/src/allmydata/web/download-status.xhtml index e5794ca79..e47f1f1f8 100644 --- a/src/allmydata/web/download-status.xhtml +++ b/src/allmydata/web/download-status.xhtml @@ -1,58 +1,63 @@ + Tahoe-LAFS - File Download Status + -

              File Download Status

              +

              File Download Status

              -
                -
              • Started:
                • -
              • Storage Index:
                • -
              • Helper?:
                • -
              • Total Size:
                • -
              • Progress:
                • -
              • Status:
                • -
              - -
              - -
              -

              Download Results

              -
                -
              • -
              • Servermap:
                • -
              • -
              • Timings:
                  • -
                • File Size: bytes
                  • -
                • Total: - ()
                  • -
                    -
                  • Peer Selection:
                    • -
                  • UEB Fetch:
                    • -
                  • Hashtree Fetch:
                    • -
                  • Segment Fetch: - ()
                    • -
                      -
                    • Cumulative Fetching: - ()
                      • -
                    • Cumulative Decoding: - ()
                      • -
                    • Cumulative Decrypting: - ()
                      • -
                    -
                  • Paused by client:
                    • -
                  -
                • +
                • Started:
                  • +
                • Storage Index:
                  • +
                • Helper?:
                  • +
                • Total Size:
                  • +
                • Progress:
                  • +
                • Status:
                -
              -
              -
              Return to the Welcome Page
              +
              + +
              + +

              Download Results

              + +
                +
              • +
              • Servermap:
                • +
              • +
              • Timings:
                • +
                  +
                • File Size: bytes
                  • +
                • Total: + ()
                  • +
                    +
                  • Peer Selection:
                    • +
                  • UEB Fetch:
                    • +
                  • Hashtree Fetch:
                    • +
                  • Segment Fetch: + ()
                    • +
                      +
                    • Cumulative Fetching: + ()
                      • +
                    • Cumulative Decoding: + ()
                      • +
                    • Cumulative Decrypting: + ()
                      • +
                    +
                  • Paused by client:
                    • +
                  +
                • +
                +
              +
              + +
              Return to the Welcome Page
              + From 8a632b666826651a749699883ece477c9e2e025e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 13:20:52 -0400 Subject: [PATCH 0106/1054] Use twisted tags in download status page template --- src/allmydata/web/download-status.xhtml | 56 ++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/allmydata/web/download-status.xhtml b/src/allmydata/web/download-status.xhtml index e47f1f1f8..93bd08408 100644 --- a/src/allmydata/web/download-status.xhtml +++ b/src/allmydata/web/download-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Download Status @@ -12,46 +12,46 @@

              File Download Status

                -
              • Started:
                • -
              • Storage Index:
                • -
              • Helper?:
                • -
              • Total Size:
                • -
              • Progress:
                • -
              • Status:
                • +
              • Started:
                • +
              • Storage Index:
                • +
              • Helper?:
                • +
              • Total Size:
                • +
              • Progress:
                • +
              • Status:
              -
              +
              -
              +

              Download Results

                -
              • -
              • Servermap:
                • -
              • +
              • +
              • Servermap:
                • +
              • Timings:
                  • -
                • File Size: bytes
                  • -
                • Total: - ()
                  • +
                • File Size: bytes
                  • +
                • Total: + ()
                    • -
                  • Peer Selection:
                    • -
                  • UEB Fetch:
                    • -
                  • Hashtree Fetch:
                    • -
                  • Segment Fetch: - ()
                    • +
                  • Peer Selection:
                    • +
                  • UEB Fetch:
                    • +
                  • Hashtree Fetch:
                    • +
                  • Segment Fetch: + ()
                      • -
                    • Cumulative Fetching: - ()
                      • -
                    • Cumulative Decoding: - ()
                      • -
                    • Cumulative Decrypting: - ()
                      • +
                    • Cumulative Fetching: + ()
                      • +
                    • Cumulative Decoding: + ()
                      • +
                    • Cumulative Decrypting: + ()
                    -
                  • Paused by client:
                    • +
                  • Paused by client:
                  -
                • +
              From c14c4371523d371501e88f0d0103235729f7613e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 13:34:18 -0400 Subject: [PATCH 0107/1054] Make DownloadStatusPage a MultiFormatResource --- src/allmydata/web/status.py | 65 ++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index b20aa679e..85417fcee 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -489,13 +489,25 @@ class _EventJson(Resource, object): return json.dumps(data, indent=1) + "\n" -class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("download-status.xhtml") +class DownloadStatusPage(MultiFormatResource): - def __init__(self, data): - rend.Page.__init__(self, data) - self.download_status = data - self.putChild("event_json", _EventJson(self.download_status)) + def __init__(self, download_status): + super(DownloadStatusPage, self).__init__() + self._download_status = download_status + self.putChild("event_json", _EventJson(self._download_status)) + + def render_HTML(self, req): + elem = DownloadStatusElement(self._download_status) + return renderElement(req, elem) + + +class DownloadStatusElement(Element, DownloadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("download-status.xhtml")) + + def __init__(self, download_status): + super(DownloadStatusElement, self).__init__() + self._download_status = download_status def download_results(self): return defer.maybeDeferred(self.download_status.get_results) @@ -645,33 +657,40 @@ class DownloadStatusPage(DownloadResultsRendererMixin, rend.Page): d.addCallback(_got_results) return d - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + " (%s)" % data.get_started() + @renderer + def started(self, req, tag): + started_s = render_time(self._download_status.get_started()) + return tag(started_s + " (%s)" % self._download_status.get_started()) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._download_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(si_s) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._download_status.using_helper()]) - def render_total_size(self, ctx, data): - size = data.get_size() + @renderer + def total_size(self, req, tag): + size = self._download_status.get_size() if size is None: return "(unknown)" - return size + return tag(str(size)) - def render_progress(self, ctx, data): - progress = data.get_progress() + @renderer + def progress(self, req, tag): + progress = self._download_status.get_progress() # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) + + @renderer + def status(self, req, tag): + return tag(self._download_status.get_status()) - def render_status(self, ctx, data): - return data.get_status() From c4b63ada55d7f1a50ffb2941c677ed2f43213eff Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 14:24:26 -0400 Subject: [PATCH 0108/1054] Update events tables renderer --- src/allmydata/web/status.py | 172 +++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 63 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 85417fcee..bdc5874f5 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -512,14 +512,15 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): def download_results(self): return defer.maybeDeferred(self.download_status.get_results) - def relative_time(self, t): + def _relative_time(self, t): if t is None: return t - if self.download_status.first_timestamp is not None: - return t - self.download_status.first_timestamp + if self._download_status.first_timestamp is not None: + return t - self._download_status.first_timestamp return t - def short_relative_time(self, t): - t = self.relative_time(t) + + def _short_relative_time(self, t): + t = self._relative_time(t) if t is None: return "" return "+%.6fs" % t @@ -532,19 +533,27 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): time_s = self.render_time(None, seconds) if seconds != 0: rate = self.render_rate(None, 1.0 * bytes / seconds) - return T.span(title=rate)[time_s] - return T.span[time_s] + return tags.span(time_s, title=rate) + return tags.span(time_s) - def render_events(self, ctx, data): - if not self.download_status.storage_index: + @renderer + def events(self, req, tag): + if not self._download_status.storage_index: return - srt = self.short_relative_time - l = T.div() - t = T.table(align="left", class_="status-download-events") - t[T.tr[T.th["serverid"], T.th["sent"], T.th["received"], - T.th["shnums"], T.th["RTT"]]] - for d_ev in self.download_status.dyhb_requests: + srt = self._short_relative_time + + l = tags.div() + + t = tags.table(align="left", class_="status-download-events") + + t(tags.tr(tags.th("serverid"), + tags.th("sent"), + tags.th("received"), + tags.th("shnums"), + tags.th("RTT"))) + + for d_ev in self._download_status.dyhb_requests: server = d_ev["server"] sent = d_ev["start_time"] shnums = d_ev["response_shnums"] @@ -554,20 +563,31 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): rtt = received - sent if not shnums: shnums = ["-"] - t[T.tr(style="background: %s" % _color(server))[ - [T.td[server.get_name()], T.td[srt(sent)], T.td[srt(received)], - T.td[",".join([str(shnum) for shnum in shnums])], - T.td[self.render_time(None, rtt)], - ]]] - l[T.h2["DYHB Requests:"], t] - l[T.br(clear="all")] + t(tags.tr(style="background: %s" % _color(server))( + (tags.td(server.get_name()), + tags.td(srt(sent)), + tags.td(srt(received)), + tags.td(",".join([str(shnum) for shnum in shnums])), + tags.td(self.render_time(None, rtt)), + ))) - t = T.table(align="left",class_="status-download-events") - t[T.tr[T.th["range"], T.th["start"], T.th["finish"], T.th["got"], - T.th["time"], T.th["decrypttime"], T.th["pausedtime"], - T.th["speed"]]] - for r_ev in self.download_status.read_events: + l(tags.h2("DYHB Requests:"), t) + l(tags.br(clear="all")) + + t = tags.table(align="left",class_="status-download-events") + + t(tags.tr(( + tags.th("range"), + tags.th("start"), + tags.th("finish"), + tags.th("got"), + tags.th("time"), + tags.th("decrypttime"), + tags.th("pausedtime"), + tags.th("speed")))) + + for r_ev in self._download_status.read_events: start = r_ev["start"] length = r_ev["length"] bytes = r_ev["bytes_returned"] @@ -581,21 +601,33 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): rtt = self.render_time(None, rtt) paused = self.render_time(None, r_ev["paused_time"]) - t[T.tr[T.td["[%d:+%d]" % (start, length)], - T.td[srt(r_ev["start_time"])], T.td[srt(r_ev["finish_time"])], - T.td[bytes], T.td[rtt], - T.td[decrypt_time], T.td[paused], - T.td[speed], - ]] + t(tags.tr( + tags.td("[%d:+%d]" % (start, length)), + tags.td(srt(r_ev["start_time"])), + tags.td(srt(r_ev["finish_time"])), + tags.td(str(bytes)), + tags.td(rtt), + tags.td(decrypt_time), + tags.td(paused), + tags.td(speed), + )) - l[T.h2["Read Events:"], t] - l[T.br(clear="all")] + l(tags.h2("Read Events:"), t) + l(tags.br(clear="all")) - t = T.table(align="left",class_="status-download-events") - t[T.tr[T.th["segnum"], T.th["start"], T.th["active"], T.th["finish"], - T.th["range"], - T.th["decodetime"], T.th["segtime"], T.th["speed"]]] - for s_ev in self.download_status.segment_events: + t = tags.table(align="left",class_="status-download-events") + + t(tags.tr( + tags.th("segnum"), + tags.th("start"), + tags.th("active"), + tags.th("finish"), + tags.th("range"), + tags.th("decodetime"), + tags.th("segtime"), + tags.th("speed"))) + + for s_ev in self._download_status.segment_events: range_s = "-" segtime_s = "-" speed = "-" @@ -615,36 +647,50 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): # not finished yet pass - t[T.tr[T.td["seg%d" % s_ev["segment_number"]], - T.td[srt(s_ev["start_time"])], - T.td[srt(s_ev["active_time"])], - T.td[srt(s_ev["finish_time"])], - T.td[range_s], - T.td[decode_time], - T.td[segtime_s], T.td[speed]]] + t(tags.tr( + tags.td("seg%d" % s_ev["segment_number"]), + tags.td(srt(s_ev["start_time"])), + tags.td(srt(s_ev["active_time"])), + tags.td(srt(s_ev["finish_time"])), + tags.td(range_s), + tags.td(decode_time), + tags.td(segtime_s), + tags.td(speed))) - l[T.h2["Segment Events:"], t] - l[T.br(clear="all")] - t = T.table(align="left",class_="status-download-events") - t[T.tr[T.th["serverid"], T.th["shnum"], T.th["range"], - T.th["txtime"], T.th["rxtime"], - T.th["received"], T.th["RTT"]]] - for r_ev in self.download_status.block_requests: + l(tags.h2("Segment Events:"), t) + + l(tags.br(clear="all")) + + t = tags.table(align="left",class_="status-download-events") + + t(tags.tr( + tags.th("serverid"), + tags.th("shnum"), + tags.th("range"), + tags.th("txtime"), + tags.th("rxtime"), + tags.th("received"), + tags.th("RTT"))) + + for r_ev in self._download_status.block_requests: server = r_ev["server"] rtt = None if r_ev["finish_time"] is not None: rtt = r_ev["finish_time"] - r_ev["start_time"] color = _color(server) - t[T.tr(style="background: %s" % color)[ - T.td[server.get_name()], T.td[r_ev["shnum"]], - T.td["[%d:+%d]" % (r_ev["start"], r_ev["length"])], - T.td[srt(r_ev["start_time"])], T.td[srt(r_ev["finish_time"])], - T.td[r_ev["response_length"] or ""], - T.td[self.render_time(None, rtt)], - ]] + t(tags.tr(style="background: %s" % color) + ( + tags.td(server.get_name()), + tags.td(str(r_ev["shnum"])), + tags.td("[%d:+%d]" % (r_ev["start"], r_ev["length"])), + tags.td(srt(r_ev["start_time"])), + tags.td(srt(r_ev["finish_time"])), + tags.td(str(r_ev["response_length"]) or ""), + tags.td(self.render_time(None, rtt)), + )) - l[T.h2["Requests:"], t] - l[T.br(clear="all")] + l(tags.h2("Requests:"), t) + l(tags.br(clear="all")) return l From c5b2073bf04aad921f48fd49bfde78d58c4b090b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 15:08:44 -0400 Subject: [PATCH 0109/1054] Avoid use of RateAndTimeMixin methods in DownloadStatusElement --- src/allmydata/web/status.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index bdc5874f5..2afe18fef 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -510,7 +510,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): self._download_status = download_status def download_results(self): - return defer.maybeDeferred(self.download_status.get_results) + return defer.maybeDeferred(self._download_status.get_results) def _relative_time(self, t): if t is None: @@ -530,9 +530,9 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): return T.a(href=url.URL.fromContext(ctx).child("timeline"))["timeline"] def _rate_and_time(self, bytes, seconds): - time_s = self.render_time(None, seconds) + time_s = abbreviate_time(seconds) if seconds != 0: - rate = self.render_rate(None, 1.0 * bytes / seconds) + rate = abbreviate_rate(1.0 * bytes / seconds) return tags.span(time_s, title=rate) return tags.span(time_s) @@ -569,7 +569,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): tags.td(srt(sent)), tags.td(srt(received)), tags.td(",".join([str(shnum) for shnum in shnums])), - tags.td(self.render_time(None, rtt)), + tags.td(abbreviate_time(rtt)), ))) l(tags.h2("DYHB Requests:"), t) @@ -597,9 +597,9 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): speed, rtt = "","" if r_ev["finish_time"] is not None: rtt = r_ev["finish_time"] - r_ev["start_time"] - r_ev["paused_time"] - speed = self.render_rate(None, compute_rate(bytes, rtt)) - rtt = self.render_time(None, rtt) - paused = self.render_time(None, r_ev["paused_time"]) + speed = abbreviate_rate(compute_rate(bytes, rtt)) + rtt = abbreviate_time(rtt) + paused = abbreviate_time(r_ev["paused_time"]) t(tags.tr( tags.td("[%d:+%d]" % (start, length)), @@ -635,10 +635,10 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): if s_ev["finish_time"] is not None: if s_ev["success"]: segtime = s_ev["finish_time"] - s_ev["active_time"] - segtime_s = self.render_time(None, segtime) + segtime_s = abbreviate_time(segtime) seglen = s_ev["segment_length"] range_s = "[%d:+%d]" % (s_ev["segment_start"], seglen) - speed = self.render_rate(None, compute_rate(seglen, segtime)) + speed = abbreviate_rate(compute_rate(seglen, segtime)) decode_time = self._rate_and_time(seglen, s_ev["decode_time"]) else: # error @@ -686,7 +686,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): tags.td(srt(r_ev["start_time"])), tags.td(srt(r_ev["finish_time"])), tags.td(str(r_ev["response_length"]) or ""), - tags.td(self.render_time(None, rtt)), + tags.td(abbreviate_time(rtt)), )) l(tags.h2("Requests:"), t) From e0241655e5b24d5db4228b173a1950295dc1e574 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 11 May 2020 15:09:16 -0400 Subject: [PATCH 0110/1054] Update results renderer --- src/allmydata/web/status.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2afe18fef..1e6c05488 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -694,11 +694,12 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): return l - def render_results(self, ctx, data): + @renderer + def results(self, req, tag): d = self.download_results() def _got_results(results): if results: - return ctx.tag + return tag return "" d.addCallback(_got_results) return d From 7e02502268a4083230e59389a041e1cd022ca2a0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 11:41:54 -0400 Subject: [PATCH 0111/1054] Add comments to DownloadStatusPage --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 1e6c05488..9be4ee09a 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -490,8 +490,12 @@ class _EventJson(Resource, object): class DownloadStatusPage(MultiFormatResource): + """Renders /status/down-%d.""" def __init__(self, download_status): + """ + :param IDownloadStatus download_status: stats provider + """ super(DownloadStatusPage, self).__init__() self._download_status = download_status self.putChild("event_json", _EventJson(self._download_status)) From b02cfa5bbe8ff3a07156552464028f0db1a2b863 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 11:46:20 -0400 Subject: [PATCH 0112/1054] Use more descriptive variable names --- src/allmydata/web/status.py | 71 ++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 9be4ee09a..c4ad01dfe 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -547,15 +547,15 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): srt = self._short_relative_time - l = tags.div() + evtag = tags.div() - t = tags.table(align="left", class_="status-download-events") + dyhbtag = tags.table(align="left", class_="status-download-events") - t(tags.tr(tags.th("serverid"), - tags.th("sent"), - tags.th("received"), - tags.th("shnums"), - tags.th("RTT"))) + dyhbtag(tags.tr(tags.th("serverid"), + tags.th("sent"), + tags.th("received"), + tags.th("shnums"), + tags.th("RTT"))) for d_ev in self._download_status.dyhb_requests: server = d_ev["server"] @@ -568,7 +568,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): if not shnums: shnums = ["-"] - t(tags.tr(style="background: %s" % _color(server))( + dyhbtag(tags.tr(style="background: %s" % _color(server))( (tags.td(server.get_name()), tags.td(srt(sent)), tags.td(srt(received)), @@ -576,12 +576,12 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): tags.td(abbreviate_time(rtt)), ))) - l(tags.h2("DYHB Requests:"), t) - l(tags.br(clear="all")) + evtag(tags.h2("DYHB Requests:"), dyhbtag) + evtag(tags.br(clear="all")) - t = tags.table(align="left",class_="status-download-events") + readtag = tags.table(align="left",class_="status-download-events") - t(tags.tr(( + readtag(tags.tr(( tags.th("range"), tags.th("start"), tags.th("finish"), @@ -605,7 +605,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): rtt = abbreviate_time(rtt) paused = abbreviate_time(r_ev["paused_time"]) - t(tags.tr( + readtag(tags.tr( tags.td("[%d:+%d]" % (start, length)), tags.td(srt(r_ev["start_time"])), tags.td(srt(r_ev["finish_time"])), @@ -616,12 +616,12 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): tags.td(speed), )) - l(tags.h2("Read Events:"), t) - l(tags.br(clear="all")) + evtag(tags.h2("Read Events:"), readtag) + evtag(tags.br(clear="all")) - t = tags.table(align="left",class_="status-download-events") + segtag = tags.table(align="left",class_="status-download-events") - t(tags.tr( + segtag(tags.tr( tags.th("segnum"), tags.th("start"), tags.th("active"), @@ -651,7 +651,7 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): # not finished yet pass - t(tags.tr( + segtag(tags.tr( tags.td("seg%d" % s_ev["segment_number"]), tags.td(srt(s_ev["start_time"])), tags.td(srt(s_ev["active_time"])), @@ -661,13 +661,12 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): tags.td(segtime_s), tags.td(speed))) - l(tags.h2("Segment Events:"), t) + evtag(tags.h2("Segment Events:"), segtag) + evtag(tags.br(clear="all")) - l(tags.br(clear="all")) + reqtab = tags.table(align="left",class_="status-download-events") - t = tags.table(align="left",class_="status-download-events") - - t(tags.tr( + reqtab(tags.tr( tags.th("serverid"), tags.th("shnum"), tags.th("range"), @@ -682,21 +681,21 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): if r_ev["finish_time"] is not None: rtt = r_ev["finish_time"] - r_ev["start_time"] color = _color(server) - t(tags.tr(style="background: %s" % color) - ( - tags.td(server.get_name()), - tags.td(str(r_ev["shnum"])), - tags.td("[%d:+%d]" % (r_ev["start"], r_ev["length"])), - tags.td(srt(r_ev["start_time"])), - tags.td(srt(r_ev["finish_time"])), - tags.td(str(r_ev["response_length"]) or ""), - tags.td(abbreviate_time(rtt)), - )) + reqtab(tags.tr(style="background: %s" % color) + ( + tags.td(server.get_name()), + tags.td(str(r_ev["shnum"])), + tags.td("[%d:+%d]" % (r_ev["start"], r_ev["length"])), + tags.td(srt(r_ev["start_time"])), + tags.td(srt(r_ev["finish_time"])), + tags.td(str(r_ev["response_length"]) or ""), + tags.td(abbreviate_time(rtt)), + )) - l(tags.h2("Requests:"), t) - l(tags.br(clear="all")) + evtag(tags.h2("Requests:"), reqtab) + evtag(tags.br(clear="all")) - return l + return evtag @renderer def results(self, req, tag): From f3edc8a48cc2647e05fce7e063a06937b8f96c35 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 12:50:17 -0400 Subject: [PATCH 0113/1054] Add a note about rendering download events tables --- src/allmydata/web/status.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index c4ad01dfe..209a01317 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -540,6 +540,9 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): return tags.span(time_s, title=rate) return tags.span(time_s) + # XXX: This method is a candidate for refactoring. It renders + # four tables from this function. Layout part of those tables + # could be moved to download-status.xhtml. @renderer def events(self, req, tag): if not self._download_status.storage_index: From 1f1f3b5f6100b94122ce01f5bcb669a4b42f7b33 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 12:59:06 -0400 Subject: [PATCH 0114/1054] Remove unused render_timeline_link() As it turns out, nothing is using this method. --- src/allmydata/web/status.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 209a01317..b6d8b18e4 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -529,10 +529,6 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): return "" return "+%.6fs" % t - def render_timeline_link(self, ctx, data): - from nevow import url - return T.a(href=url.URL.fromContext(ctx).child("timeline"))["timeline"] - def _rate_and_time(self, bytes, seconds): time_s = abbreviate_time(seconds) if seconds != 0: From 1fa77d89834ac64035f26a41c9e5561f49cbb89e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 14:07:19 -0400 Subject: [PATCH 0115/1054] Avoid use of DownloadResultsRendererMixin --- src/allmydata/web/status.py | 263 +++++++++++++++++++----------------- 1 file changed, 139 insertions(+), 124 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index b6d8b18e4..0628773da 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -217,129 +217,6 @@ class UploadStatusPage(UploadResultsRendererMixin, rend.Page): def render_status(self, ctx, data): return data.get_status() -class DownloadResultsRendererMixin(RateAndTimeMixin): - # this requires a method named 'download_results' - - def render_servermap(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.servermap) - def _render(servermap): - if servermap is None: - return "None" - l = T.ul() - for peerid in sorted(servermap.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - shares_s = ",".join(["#%d" % shnum - for shnum in servermap[peerid]]) - l[T.li["[%s] has share%s: %s" % (peerid_s, - plural(servermap[peerid]), - shares_s)]] - return l - d.addCallback(_render) - return d - - def render_servers_used(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.servers_used) - def _got(servers_used): - if not servers_used: - return "" - peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) - for peerid in servers_used]) - return T.li["Servers Used: ", peerids_s] - d.addCallback(_got) - return d - - def render_problems(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.server_problems) - def _got(server_problems): - if not server_problems: - return "" - l = T.ul() - for peerid in sorted(server_problems.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - l[T.li["[%s]: %s" % (peerid_s, server_problems[peerid])]] - return T.li["Server Problems:", l] - d.addCallback(_got) - return d - - def data_file_size(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.file_size) - return d - - def _get_time(self, name): - d = self.download_results() - d.addCallback(lambda res: res.timings.get(name)) - return d - - def data_time_total(self, ctx, data): - return self._get_time("total") - - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") - - def data_time_uri_extension(self, ctx, data): - return self._get_time("uri_extension") - - def data_time_hashtrees(self, ctx, data): - return self._get_time("hashtrees") - - def data_time_segments(self, ctx, data): - return self._get_time("segments") - - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") - - def data_time_cumulative_decode(self, ctx, data): - return self._get_time("cumulative_decode") - - def data_time_cumulative_decrypt(self, ctx, data): - return self._get_time("cumulative_decrypt") - - def data_time_paused(self, ctx, data): - return self._get_time("paused") - - def _get_rate(self, name): - d = self.download_results() - def _convert(r): - file_size = r.file_size - duration = r.timings.get(name) - return compute_rate(file_size, duration) - d.addCallback(_convert) - return d - - def data_rate_total(self, ctx, data): - return self._get_rate("total") - - def data_rate_segments(self, ctx, data): - return self._get_rate("segments") - - def data_rate_fetch(self, ctx, data): - return self._get_rate("cumulative_fetch") - - def data_rate_decode(self, ctx, data): - return self._get_rate("cumulative_decode") - - def data_rate_decrypt(self, ctx, data): - return self._get_rate("cumulative_decrypt") - - def render_server_timings(self, ctx, data): - d = self.download_results() - d.addCallback(lambda res: res.timings.get("fetch_per_server")) - def _render(per_server): - if per_server is None: - return "" - l = T.ul() - for peerid in sorted(per_server.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) - for t in per_server[peerid]]) - l[T.li["[%s]: %s" % (peerid_s, times_s)]] - return T.li["Per-Server Segment Fetch Response Times: ", l] - d.addCallback(_render) - return d def _find_overlap(events, start_key, end_key): """ @@ -505,7 +382,7 @@ class DownloadStatusPage(MultiFormatResource): return renderElement(req, elem) -class DownloadStatusElement(Element, DownloadResultsRendererMixin): +class DownloadStatusElement(Element): loader = XMLFile(FilePath(__file__).sibling("download-status.xhtml")) @@ -740,7 +617,145 @@ class DownloadStatusElement(Element, DownloadResultsRendererMixin): def status(self, req, tag): return tag(self._download_status.get_status()) + @renderer + def servers_used(self, req, tag): + d = self.download_results() + d.addCallback(lambda res: res.servers_used) + def _got(servers_used): + if not servers_used: + return "" + peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) + for peerid in servers_used]) + return tags.li("Servers Used: ", peerids_s) + d.addCallback(_got) + return d + @renderer + def servermap(self, req, tag): + d = self.download_results() + d.addCallback(lambda res: res.servermap) + def _render(servermap): + if servermap is None: + return tag("None") + ul = tags.ul() + for peerid in sorted(servermap.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + shares_s = ",".join(["#%d" % shnum + for shnum in servermap[peerid]]) + ul(tags.li("[%s] has share%s: %s" % (peerid_s, + plural(servermap[peerid]), + shares_s))) + return ul + d.addCallback(_render) + return d + + @renderer + def problems(self, req, tag): + d = self.download_results() + d.addCallback(lambda res: res.server_problems) + def _got(server_problems): + if not server_problems: + return tag("") + ul = T.ul() + for peerid in sorted(server_problems.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + ul(tags.li("[%s]: %s" % (peerid_s, server_problems[peerid]))) + return tags.li("Server Problems:", ul) + d.addCallback(_got) + return d + + @renderer + def file_size(self, req, tag): + d = self.download_results() + d.addCallback(lambda res: str(res.file_size)) + return d + + def _get_time(self, name): + d = self.download_results() + d.addCallback(lambda res: res.timings.get(name)) + return d + + @renderer + def time_total(self, req, tag): + return self._get_time("total") + + @renderer + def time_peer_selection(self, req, tag): + return self._get_time("peer_selection") + + @renderer + def time_uri_extension(self, req, tag): + return self._get_time("uri_extension") + + @renderer + def time_hashtrees(self, req, tag): + return self._get_time("hashtrees") + + @renderer + def time_segments(self, req, tag): + return self._get_time("segments") + + @renderer + def time_cumulative_fetch(self, req, tag): + return self._get_time("cumulative_fetch") + + @renderer + def time_cumulative_decode(self, req, tag): + return self._get_time("cumulative_decode") + + @renderer + def time_cumulative_decrypt(self, req, tag): + return self._get_time("cumulative_decrypt") + + @renderer + def time_paused(self, req, tag): + return self._get_time("paused") + + def _get_rate(self, name): + d = self.download_results() + def _convert(r): + file_size = r.file_size + duration = r.timings.get(name) + return compute_rate(file_size, duration) + d.addCallback(_convert) + return d + + @renderer + def rate_total(self, req, tag): + return self._get_rate("total") + + @renderer + def rate_segments(self, req, tag): + return self._get_rate("segments") + + @renderer + def rate_fetch(self, req, tag): + return self._get_rate("cumulative_fetch") + + @renderer + def rate_decode(self, req, tag): + return self._get_rate("cumulative_decode") + + @renderer + def rate_decrypt(self, req, tag): + return self._get_rate("cumulative_decrypt") + + @renderer + def server_timings(self, req, tag): + d = self.download_results() + d.addCallback(lambda res: res.timings.get("fetch_per_server")) + def _render(per_server): + if per_server is None: + return "" + ul = tags.ul() + for peerid in sorted(per_server.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + times_s = ", ".join([self.render_time(None, t) + for t in per_server[peerid]]) + ul(tags.li("[%s]: %s" % (peerid_s, times_s))) + return tags.li("Per-Server Segment Fetch Response Times: ", ul) + d.addCallback(_render) + return d class RetrieveStatusPage(MultiFormatResource): From 68df408229431acaf0b2fdd67d76809368afd820 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 15:20:59 -0400 Subject: [PATCH 0116/1054] Add a note about unimplemented DownloadStatus method --- src/allmydata/web/status.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0628773da..83e588bd9 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -390,6 +390,12 @@ class DownloadStatusElement(Element): super(DownloadStatusElement, self).__init__() self._download_status = download_status + # XXX: fun fact: the `get_results()` method which we wind up + # invoking here (see immutable.downloader.status.DownloadStatus) + # is unimplemented, and simply returns a `None`. As a result, + # `results()` renderer returns an empty tag, and does not invoke + # any of the subsequent renderers. Thus we end up not displaying + # download results on the download status page. def download_results(self): return defer.maybeDeferred(self._download_status.get_results) From 88e8854ad47c13249c346e4c5abfc6b7022a62ca Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 15:28:00 -0400 Subject: [PATCH 0117/1054] Wrap renderer results in tags --- src/allmydata/web/status.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 83e588bd9..3d8ecb975 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -683,39 +683,39 @@ class DownloadStatusElement(Element): @renderer def time_total(self, req, tag): - return self._get_time("total") + return tag(str(self._get_time("total"))) @renderer def time_peer_selection(self, req, tag): - return self._get_time("peer_selection") + return tag(str(self._get_time("peer_selection"))) @renderer def time_uri_extension(self, req, tag): - return self._get_time("uri_extension") + return tag(str(self._get_time("uri_extension"))) @renderer def time_hashtrees(self, req, tag): - return self._get_time("hashtrees") + return tag(str(self._get_time("hashtrees"))) @renderer def time_segments(self, req, tag): - return self._get_time("segments") + return tag(str(self._get_time("segments"))) @renderer def time_cumulative_fetch(self, req, tag): - return self._get_time("cumulative_fetch") + return tag(str(self._get_time("cumulative_fetch"))) @renderer def time_cumulative_decode(self, req, tag): - return self._get_time("cumulative_decode") + return tag(str(self._get_time("cumulative_decode"))) @renderer def time_cumulative_decrypt(self, req, tag): - return self._get_time("cumulative_decrypt") + return tag(str(self._get_time("cumulative_decrypt"))) @renderer def time_paused(self, req, tag): - return self._get_time("paused") + return tag(str(self._get_time("paused"))) def _get_rate(self, name): d = self.download_results() @@ -728,23 +728,23 @@ class DownloadStatusElement(Element): @renderer def rate_total(self, req, tag): - return self._get_rate("total") + return tag(str(self._get_rate("total"))) @renderer def rate_segments(self, req, tag): - return self._get_rate("segments") + return tag(str(self._get_rate("segments"))) @renderer def rate_fetch(self, req, tag): - return self._get_rate("cumulative_fetch") + return tag(str(self._get_rate("cumulative_fetch"))) @renderer def rate_decode(self, req, tag): - return self._get_rate("cumulative_decode") + return tag(str(self._get_rate("cumulative_decode"))) @renderer def rate_decrypt(self, req, tag): - return self._get_rate("cumulative_decrypt") + return tag(str(self._get_rate("cumulative_decrypt"))) @renderer def server_timings(self, req, tag): @@ -756,7 +756,7 @@ class DownloadStatusElement(Element): ul = tags.ul() for peerid in sorted(per_server.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[peerid]]) ul(tags.li("[%s]: %s" % (peerid_s, times_s))) return tags.li("Per-Server Segment Fetch Response Times: ", ul) From ab4811eaa0c5ad48d1920bac90fceeb973d29407 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 15:57:59 -0400 Subject: [PATCH 0118/1054] Move tests for abbreviate time/rate methods to utils tests RateAndTimeMixin.render_time() is really abbreviate_time(), and RateAndTimeMixin.render_rate() is really abbreviate_rate(). This change moves the tests to their rightful place, and exercises them using the right names. We're also trying to avoid use of mixins. RateAndTimeMixin will eventually go. --- src/allmydata/test/web/test_util.py | 6 ++++++ src/allmydata/test/web/test_web.py | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index 2a6f67abc..0e75b0d70 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -22,6 +22,9 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms") self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us") + self.failUnlessReallyEqual(common.abbreviate_time(2.5), "2.50s") + self.failUnlessReallyEqual(common.abbreviate_time(0.25), "250ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.0021), "2.1ms") def test_compute_rate(self): self.failUnlessReallyEqual(common.compute_rate(None, None), None) @@ -44,6 +47,9 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps") self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps") self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps") + self.failUnlessReallyEqual(common.abbreviate_rate(2500000), "2.50MBps") + self.failUnlessReallyEqual(common.abbreviate_rate(30100), "30.1kBps") + self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps") def test_abbreviate_size(self): self.failUnlessReallyEqual(common.abbreviate_size(None), "") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5be0b2f7b..d7b606263 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1035,17 +1035,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_status_numbers(self): - drrm = status.DownloadResultsRendererMixin() - self.failUnlessReallyEqual(drrm.render_time(None, None), "") - self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(drrm.render_rate(None, None), "") - self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps") - urrm = status.UploadResultsRendererMixin() self.failUnlessReallyEqual(urrm.render_time(None, None), "") self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s") From 19e40e2193dd0f0268899a068e12774de7fc41f1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 16:04:26 -0400 Subject: [PATCH 0119/1054] Add newsfragment --- newsfragments/3288.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3288.minor diff --git a/newsfragments/3288.minor b/newsfragments/3288.minor new file mode 100644 index 000000000..e69de29bb From c5342d8ae68acfcdf319ee079f7061ed5528bc20 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 14 May 2020 16:48:43 -0400 Subject: [PATCH 0120/1054] Add comments about events tables --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3d8ecb975..4916030d2 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -431,6 +431,7 @@ class DownloadStatusElement(Element): evtag = tags.div() + # "DYHB Requests" table. dyhbtag = tags.table(align="left", class_="status-download-events") dyhbtag(tags.tr(tags.th("serverid"), @@ -461,6 +462,7 @@ class DownloadStatusElement(Element): evtag(tags.h2("DYHB Requests:"), dyhbtag) evtag(tags.br(clear="all")) + # "Read Events" table. readtag = tags.table(align="left",class_="status-download-events") readtag(tags.tr(( @@ -501,6 +503,7 @@ class DownloadStatusElement(Element): evtag(tags.h2("Read Events:"), readtag) evtag(tags.br(clear="all")) + # "Segment Events" table. segtag = tags.table(align="left",class_="status-download-events") segtag(tags.tr( @@ -546,6 +549,7 @@ class DownloadStatusElement(Element): evtag(tags.h2("Segment Events:"), segtag) evtag(tags.br(clear="all")) + # "Requests" table. reqtab = tags.table(align="left",class_="status-download-events") reqtab(tags.tr( From 993b1e225f6a9038d28d98b601a5ebbb692b2205 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 11:01:14 -0400 Subject: [PATCH 0121/1054] Use transparent tags to render values --- src/allmydata/web/download-status.xhtml | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/download-status.xhtml b/src/allmydata/web/download-status.xhtml index 93bd08408..14fbb4b9e 100644 --- a/src/allmydata/web/download-status.xhtml +++ b/src/allmydata/web/download-status.xhtml @@ -12,12 +12,12 @@

              File Download Status

                -
              • Started:
                • -
              • Storage Index:
                • -
              • Helper?:
                • -
              • Total Size:
                • -
              • Progress:
                • -
              • Status:
                • +
              • Started:
                • +
              • Storage Index:
                • +
              • Helper?:
                • +
              • Total Size:
                • +
              • Progress:
                • +
              • Status:
              @@ -28,28 +28,28 @@
              • -
              • Servermap:
                • +
              • Servermap:
              • Timings:
                  • -
                • File Size: bytes
                  • -
                • Total: - ()
                  • +
                • File Size: bytes
                  • +
                • Total: + ()
                    • -
                  • Peer Selection:
                    • -
                  • UEB Fetch:
                    • -
                  • Hashtree Fetch:
                    • -
                  • Segment Fetch: - ()
                    • +
                  • Peer Selection:
                    • +
                  • UEB Fetch:
                    • +
                  • Hashtree Fetch:
                    • +
                  • Segment Fetch: + ()
                      • -
                    • Cumulative Fetching: - ()
                      • -
                    • Cumulative Decoding: - ()
                      • -
                    • Cumulative Decrypting: - ()
                      • +
                    • Cumulative Fetching: + ()
                      • +
                    • Cumulative Decoding: + ()
                      • +
                    • Cumulative Decrypting: + ()
                    -
                  • Paused by client:
                    • +
                  • Paused by client:
                From 0b3db59742ae727c682bb8a4f42022ef9fc4d6f5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 13:30:39 -0400 Subject: [PATCH 0122/1054] Add related ticket to comment about unimplemented method --- src/allmydata/web/status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4916030d2..c9fa4f1f0 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -396,6 +396,8 @@ class DownloadStatusElement(Element): # `results()` renderer returns an empty tag, and does not invoke # any of the subsequent renderers. Thus we end up not displaying # download results on the download status page. + # + # See #3310: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3310 def download_results(self): return defer.maybeDeferred(self._download_status.get_results) From c23fedbcfeec5d465e2c9719f6b42a6ade4e1fa1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 13:36:30 -0400 Subject: [PATCH 0123/1054] Add related ticket to comment about refactoring --- src/allmydata/web/status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index c9fa4f1f0..e6e422f6f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -424,6 +424,8 @@ class DownloadStatusElement(Element): # XXX: This method is a candidate for refactoring. It renders # four tables from this function. Layout part of those tables # could be moved to download-status.xhtml. + # + # See #3311: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3311 @renderer def events(self, req, tag): if not self._download_status.storage_index: From aec6e5a9d4ca9f1a8bb8416e43642119b2160e45 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 20:03:58 -0400 Subject: [PATCH 0124/1054] Force a CircleCI build --- src/allmydata/web/download-status.xhtml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/download-status.xhtml b/src/allmydata/web/download-status.xhtml index 14fbb4b9e..0c2da523c 100644 --- a/src/allmydata/web/download-status.xhtml +++ b/src/allmydata/web/download-status.xhtml @@ -59,5 +59,4 @@
                Return to the Welcome Page
                - From aac8dee8a0dd0aeab9724800c306b2538bbcae1f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:07:47 -0400 Subject: [PATCH 0125/1054] Use twisted template tags in upload status page template --- src/allmydata/web/upload-status.xhtml | 62 +++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/upload-status.xhtml b/src/allmydata/web/upload-status.xhtml index 8aaecd47a..95a5f7692 100644 --- a/src/allmydata/web/upload-status.xhtml +++ b/src/allmydata/web/upload-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Upload Status @@ -10,46 +10,46 @@

                File Upload Status

                  -
                • Started:
                  • -
                • Storage Index:
                  • -
                • Helper?:
                  • -
                • Total Size:
                  • -
                • Progress (Hash):
                  • -
                • Progress (Ciphertext):
                  • -
                • Progress (Encode+Push):
                  • -
                • Status:
                  • +
                • Started:
                  • +
                • Storage Index:
                  • +
                • Helper?:
                  • +
                • Total Size:
                  • +
                • Progress (Hash):
                  • +
                • Progress (Ciphertext):
                  • +
                • Progress (Encode+Push):
                  • +
                • Status:
                -
                +

                Upload Results

                  -
                • Shares Pushed:
                  • -
                • Shares Already Present:
                  • -
                • Sharemap:
                  • -
                • Servermap:
                  • +
                • Shares Pushed:
                  • +
                • Shares Already Present:
                  • +
                • Sharemap:
                  • +
                • Servermap:
                • Timings:
                    • -
                  • File Size: bytes
                    • -
                  • Total: - ()
                    • +
                  • File Size: bytes
                    • +
                  • Total: + ()
                      • -
                    • Storage Index: - ()
                      • -
                    • [Contacting Helper]:
                      • -
                    • [Upload Ciphertext To Helper]: - ()
                      • +
                    • Storage Index: + ()
                      • +
                    • [Contacting Helper]:
                      • +
                    • [Upload Ciphertext To Helper]: + ()
                      • -
                    • Peer Selection:
                      • -
                    • Encode And Push: - ()
                      • +
                    • Peer Selection:
                      • +
                    • Encode And Push: + ()
                        • -
                      • Cumulative Encoding: - ()
                        • -
                      • Cumulative Pushing: - ()
                        • -
                      • Send Hashes And Close:
                        • +
                      • Cumulative Encoding: + ()
                        • +
                      • Cumulative Pushing: + ()
                        • +
                      • Send Hashes And Close:
                      -
                    • [Helper Total]:
                      • +
                    • [Helper Total]:
                From 41300097c8db4b8ad6c3ed709c1fd78b0eb3d2ec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:08:31 -0400 Subject: [PATCH 0126/1054] Make UploadStatusPage a MultiFormatResource --- src/allmydata/web/status.py | 85 +++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index b20aa679e..2c284caf6 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -160,62 +160,85 @@ class UploadResultsRendererMixin(RateAndTimeMixin): d.addCallback(_convert) return d -class UploadStatusPage(UploadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("upload-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.upload_status = data +class UploadStatusPage(MultiFormatResource): + + def __init__(self, upload_status): + super(UploadStatusPage, self).__init__() + self._upload_status = upload_status + + def render_HTML(self, req): + elem = UploadStatusElement(self._upload_status) + return renderElement(req, elem) + + +class UploadStatusElement(Element, UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) + + def __init__(self, upload_status): + super(UploadStatusElement, self).__init__() + self._upload_status = upload_status def upload_results(self): - return defer.maybeDeferred(self.upload_status.get_results) + return defer.maybeDeferred(self._upload_status.get_results) - def render_results(self, ctx, data): + @renderer + def results(self, req, tag): d = self.upload_results() def _got_results(results): if results: - return ctx.tag + return tag return "" d.addCallback(_got_results) return d - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + @renderer + def started(self, req, tag): + started_s = render_time(self._upload_status.get_started()) + return tag(started_s) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._upload_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(str(si_s)) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._upload_status.using_helper()]) - def render_total_size(self, ctx, data): - size = data.get_size() + @renderer + def total_size(self, req, tag): + size = self._upload_status.get_size() if size is None: return "(unknown)" - return size + return tag(str(size)) - def render_progress_hash(self, ctx, data): - progress = data.get_progress()[0] + @renderer + def progress_hash(self, req, tag): + progress = self._upload_status.get_progress()[0] + # TODO: make an ascii-art bar + return tag("%.1f%%" % (100.0 * progress)) + + @renderer + def progress_ciphertext(self, req, tag): + progress = self._upload_status.get_progress()[1] # TODO: make an ascii-art bar return "%.1f%%" % (100.0 * progress) - def render_progress_ciphertext(self, ctx, data): - progress = data.get_progress()[1] + @renderer + def progress_encode_push(self, req, tag): + progress = self._upload_status.get_progress()[2] # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_progress_encode_push(self, ctx, data): - progress = data.get_progress()[2] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + @renderer + def status(self, req, tag): + return tag(self._upload_status.get_status()) - def render_status(self, ctx, data): - return data.get_status() class DownloadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'download_results' From 647071869173646a2567a9ba844bd8bbdd5884bb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:32 -0400 Subject: [PATCH 0127/1054] Use twisted template tags in upload results page template --- src/allmydata/web/upload-results.xhtml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/upload-results.xhtml b/src/allmydata/web/upload-results.xhtml index fa8894dcd..290c9bd42 100644 --- a/src/allmydata/web/upload-results.xhtml +++ b/src/allmydata/web/upload-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Uploaded @@ -7,37 +7,37 @@ -

                Uploading File...

                +

                Uploading File...

                Upload Results:

                  -
                • URI:
                  • -
                • Download link:
                  • -
                • Sharemap:
                  • -
                • Servermap:
                  • +
                • URI:
                  • +
                • Download link:
                  • +
                • Sharemap:
                  • +
                • Servermap:
                • Timings:
                    • -
                  • File Size: bytes
                    • -
                  • Total: - ()
                    • +
                  • File Size: bytes
                    • +
                  • Total: + ()
                      • -
                    • Storage Index: - ()
                      • -
                    • [Contacting Helper]:
                      • -
                    • [Upload Ciphertext To Helper]: - ()
                      • +
                    • Storage Index: + ()
                      • +
                    • [Contacting Helper]:
                      • +
                    • [Upload Ciphertext To Helper]: + ()
                      • -
                    • Peer Selection:
                      • -
                    • Encode And Push: - ()
                      • +
                    • Peer Selection:
                      • +
                    • Encode And Push: + ()
                        • -
                      • Cumulative Encoding: - ()
                        • -
                      • Cumulative Pushing: - ()
                        • -
                      • Send Hashes And Close:
                        • +
                      • Cumulative Encoding: + ()
                        • +
                      • Cumulative Pushing: + ()
                        • +
                      • Send Hashes And Close:
                      -
                    • [Helper Total]:
                      • +
                    • [Helper Total]:
                From 57b1203d17a713ad58fc5b2e0698ba073f0ce349 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:54 -0400 Subject: [PATCH 0128/1054] Make UploadResultsPage a MultiFormatResource --- src/allmydata/web/unlinked.py | 58 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 8a84a4021..0abf6cc6d 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -2,11 +2,27 @@ import urllib from twisted.web import http from twisted.internet import defer -from nevow import rend, url, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) +from nevow import url, tags as T from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle -from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \ - convert_children_json, WebError, get_format, get_mutable_type +from allmydata.web.common import ( + getxmlfile, + get_arg, + boolean_of_arg, + convert_children_json, + WebError, + get_format, + get_mutable_type, + MultiFormatResource, +) from allmydata.web import status def PUTUnlinkedCHK(req, client): @@ -59,34 +75,50 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(status.UploadResultsRendererMixin, rend.Page): +class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" - docFactory = getxmlfile("upload-results.xhtml") def __init__(self, upload_results): - rend.Page.__init__(self) - self.results = upload_results + super(UploadResultsPage, self).__init__() + self._upload_results = upload_results + + def render_HTML(self, req): + elem = UploadResultsElement(self._upload_results) + return renderElement(req, elem) + + +class UploadResultsElement(Element, status.UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) + + def __init__(self, upload_results): + super(UploadResultsElement, self).__init__() + self._upload_results = upload_results def upload_results(self): - return defer.succeed(self.results) + return defer.succeed(self._upload_results) - def data_done(self, ctx, data): + @renderer + def done(self, req, tag): d = self.upload_results() d.addCallback(lambda res: "done!") return d - def data_uri(self, ctx, data): + @renderer + def uri(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_uri()) return d - def render_download_link(self, ctx, data): + @renderer + def download_link(self, req, tag): d = self.upload_results() d.addCallback(lambda res: - T.a(href="/uri/" + urllib.quote(res.get_uri())) - ["/uri/" + res.get_uri()]) + tags.a("/uri/" + res.get_uri(), + href="/uri/" + urllib.quote(res.get_uri()))) return d + def POSTUnlinkedSSK(req, client, version): # "POST /uri", to create an unlinked file. # SDMF: files are small, and we can only upload data From 3bfa5a5c292c33ff7eb4679938863aa039257d19 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:53:38 -0400 Subject: [PATCH 0129/1054] Use twisted web template in UploadResultsRendererMixin --- src/allmydata/web/status.py | 120 +++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2c284caf6..4edc63234 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -35,128 +35,150 @@ class RateAndTimeMixin(object): def render_rate(self, ctx, data): return abbreviate_rate(data) + class UploadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'upload_results' - def render_pushed_shares(self, ctx, data): + @renderer + def pushed_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_pushed_shares()) + d.addCallback(lambda res: str(res.get_pushed_shares())) return d - def render_preexisting_shares(self, ctx, data): + @renderer + def preexisting_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_preexisting_shares()) + d.addCallback(lambda res: str(res.get_preexisting_shares())) return d - def render_sharemap(self, ctx, data): + @renderer + def sharemap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_sharemap()) def _render(sharemap): if sharemap is None: return "None" - l = T.ul() + ul = tags.ul() for shnum, servers in sorted(sharemap.items()): server_names = ', '.join([s.get_name() for s in servers]) - l[T.li["%d -> placed on [%s]" % (shnum, server_names)]] - return l + ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) + return ul d.addCallback(_render) return d - def render_servermap(self, ctx, data): + @renderer + def servermap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_servermap()) def _render(servermap): if servermap is None: return "None" - l = T.ul() + ul = tags.ul() for server, shnums in sorted(servermap.items()): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) - l[T.li["[%s] got share%s: %s" % (server.get_name(), - plural(shnums), shares_s)]] - return l + ul(tags.li("[%s] got share%s: %s" % (server.get_name(), + plural(shnums), shares_s))) + return ul d.addCallback(_render) return d - def data_file_size(self, ctx, data): + @renderer + def file_size(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_file_size()) + d.addCallback(lambda res: str(res.get_file_size())) return d def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: res.get_timings().get(name)) + d.addCallback(lambda res: str(res.get_timings().get(name))) return d - def data_time_total(self, ctx, data): - return self._get_time("total") + @renderer + def time_total(self, req, tag): + return tag(self._get_time("total")) - def data_time_storage_index(self, ctx, data): - return self._get_time("storage_index") + @renderer + def time_storage_index(self, req, tag): + return tag(self._get_time("storage_index")) - def data_time_contacting_helper(self, ctx, data): - return self._get_time("contacting_helper") + @renderer + def time_contacting_helper(self, req, tag): + return tag(self._get_time("contacting_helper")) - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") + @renderer + def time_cumulative_fetch(self, req, tag): + return tag(self._get_time("cumulative_fetch")) - def data_time_helper_total(self, ctx, data): - return self._get_time("helper_total") + @renderer + def time_helper_total(self, req, tag): + return tag(self._get_time("helper_total")) - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") + @renderer + def time_peer_selection(self, req, tag): + return tag(self._get_time("peer_selection")) - def data_time_total_encode_and_push(self, ctx, data): - return self._get_time("total_encode_and_push") + @renderer + def time_total_encode_and_push(self, req, tag): + return tag(self._get_time("total_encode_and_push")) - def data_time_cumulative_encoding(self, ctx, data): - return self._get_time("cumulative_encoding") + @renderer + def time_cumulative_encoding(self, req, tag): + return tag(self._get_time("cumulative_encoding")) - def data_time_cumulative_sending(self, ctx, data): - return self._get_time("cumulative_sending") + @renderer + def time_cumulative_sending(self, req, tag): + return tag(self._get_time("cumulative_sending")) - def data_time_hashes_and_close(self, ctx, data): - return self._get_time("hashes_and_close") + @renderer + def time_hashes_and_close(self, req, tag): + return tag(self._get_time("hashes_and_close")) def _get_rate(self, name): d = self.upload_results() def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return compute_rate(file_size, duration) + return str(compute_rate(file_size, duration)) d.addCallback(_convert) return d - def data_rate_total(self, ctx, data): - return self._get_rate("total") + @renderer + def rate_total(self, req, tag): + return tag(self._get_rate("total")) - def data_rate_storage_index(self, ctx, data): - return self._get_rate("storage_index") + @renderer + def rate_storage_index(self, req, tag): + return tag(self._get_rate("storage_index")) - def data_rate_encode(self, ctx, data): - return self._get_rate("cumulative_encoding") + @renderer + def rate_encode(self, req, tag): + return tag(self._get_rate("cumulative_encoding")) - def data_rate_push(self, ctx, data): + @renderer + def rate_push(self, req, tag): return self._get_rate("cumulative_sending") - def data_rate_encode_and_push(self, ctx, data): + @renderer + def rate_encode_and_push(self, req, tag): d = self.upload_results() def _convert(r): file_size = r.get_file_size() time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return None + return str(None) else: - return compute_rate(file_size, time1+time2) + return str(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d - def data_rate_ciphertext_fetch(self, ctx, data): + @renderer + def rate_ciphertext_fetch(self, req, tag): d = self.upload_results() def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return compute_rate(fetch_size, duration) + return str(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From 7a82fd671f4377c569957beb234a8c5cc55254b9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 18:00:20 -0400 Subject: [PATCH 0130/1054] Handle "t=upload" in UploadResultsPage POST handling --- src/allmydata/web/unlinked.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 0abf6cc6d..283d8d586 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -86,6 +86,23 @@ class UploadResultsPage(MultiFormatResource): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) + # This is weird but necessary because: + # + # 1. MultiFormatResource.render() uses argument "t" to figure out + # its output format. + # + # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". + # See URIHandler.render_POST(). + # + # MultiFormatResource.render() looks up "t" argument, which in + # this case has the value "upload", and then it would look for a + # render_UPLOAD() method. + # + # We could change upload request to use more descriptive names + # that do not cause name collisions like this. That should be a + # separate change though. + render_UPLOAD = render_HTML + class UploadResultsElement(Element, status.UploadResultsRendererMixin): From 54ac2d224ae449a8da651139a3f60a80de0ed664 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:23:23 -0400 Subject: [PATCH 0131/1054] Remove unused imports --- src/allmydata/web/unlinked.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 283d8d586..33fb7bafa 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -10,11 +10,10 @@ from twisted.web.template import ( renderElement, tags, ) -from nevow import url, tags as T +from nevow import url from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import ( - getxmlfile, get_arg, boolean_of_arg, convert_children_json, From 5fdd61b1464f35b2a5df056d04540eab599917d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:25:09 -0400 Subject: [PATCH 0132/1054] Document UploadStatusPage --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4edc63234..be87cff65 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -184,8 +184,12 @@ class UploadResultsRendererMixin(RateAndTimeMixin): class UploadStatusPage(MultiFormatResource): + """Renders /status/up-%d.""" def __init__(self, upload_status): + """ + :param IUploadStatus upload_status: stats provider. + """ super(UploadStatusPage, self).__init__() self._upload_status = upload_status From 7be18839104b025d78e00f9105714936455a7df4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:28:16 -0400 Subject: [PATCH 0133/1054] Document UploadResultsPage parameter --- src/allmydata/web/unlinked.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 33fb7bafa..6ca219c8b 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -78,6 +78,9 @@ class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): + """ + :param IUploadResults upload_results: stats provider. + """ super(UploadResultsPage, self).__init__() self._upload_results = upload_results From aecd90858a852e242e990729e97f9d9a5af22c51 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 14:53:33 -0400 Subject: [PATCH 0134/1054] Render abbreviated rate and time values --- src/allmydata/web/status.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index be87cff65..61ab0ae26 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -90,7 +90,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: str(res.get_timings().get(name))) + d.addCallback(lambda res: abbreviate_time(res.get_timings().get(name))) return d @renderer @@ -138,7 +138,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return str(compute_rate(file_size, duration)) + return abbreviate_rate(compute_rate(file_size, duration)) d.addCallback(_convert) return d @@ -166,9 +166,9 @@ class UploadResultsRendererMixin(RateAndTimeMixin): time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return str(None) + return abbreviate_rate(None) else: - return str(compute_rate(file_size, time1+time2)) + return abbreviate_rate(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d @@ -178,7 +178,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return str(compute_rate(fetch_size, duration)) + return abbreviate_rate(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From 00f70f6539f3360806feca100802fb8deabc16f3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:08:34 -0400 Subject: [PATCH 0135/1054] Add newsfragment --- newsfragments/3287.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3287.minor diff --git a/newsfragments/3287.minor b/newsfragments/3287.minor new file mode 100644 index 000000000..e69de29bb From 91f8d939f8dd5510c2542178f0326800d66a51f0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:22:59 -0400 Subject: [PATCH 0136/1054] Avoid use of RateAndTimeMixin in UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 61ab0ae26..eb7fedc6f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -36,7 +36,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(RateAndTimeMixin): +class UploadResultsRendererMixin(object): # this requires a method named 'upload_results' @renderer @@ -383,7 +383,7 @@ class DownloadResultsRendererMixin(RateAndTimeMixin): l = T.ul() for peerid in sorted(per_server.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[peerid]]) l[T.li["[%s]: %s" % (peerid_s, times_s)]] return T.li["Per-Server Segment Fetch Response Times: ", l] From b0e3325d03e6af3d2bd1556a91335ef8767b895a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:49:34 -0400 Subject: [PATCH 0137/1054] Move time formatter tests UploadResultsRendererMixin no longer has render_time(); moving tests for render_time() as tests for abbreviate_time() to a more appropriate place. --- src/allmydata/test/web/test_util.py | 10 ++++++++++ src/allmydata/test/web/test_web.py | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index 2a6f67abc..a378c1447 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -23,6 +23,16 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us") + self.failUnlessReallyEqual(common.abbreviate_time(None), "") + self.failUnlessReallyEqual(common.abbreviate_time(2.5), "2.50s") + self.failUnlessReallyEqual(common.abbreviate_time(0.25), "250ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.0021), "2.1ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") + self.failUnlessReallyEqual(common.abbreviate_rate(None), "") + self.failUnlessReallyEqual(common.abbreviate_rate(2500000), "2.50MBps") + self.failUnlessReallyEqual(common.abbreviate_rate(30100), "30.1kBps") + self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps") + def test_compute_rate(self): self.failUnlessReallyEqual(common.compute_rate(None, None), None) self.failUnlessReallyEqual(common.compute_rate(None, 1), None) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5be0b2f7b..d7137ad11 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1046,17 +1046,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps") self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps") - urrm = status.UploadResultsRendererMixin() - self.failUnlessReallyEqual(urrm.render_time(None, None), "") - self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(urrm.render_rate(None, None), "") - self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps") - def test_GET_FILEURL(self): d = self.GET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) From f8562530be3d6c0a07b180a74558b818fadd8ffb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:11:42 -0400 Subject: [PATCH 0138/1054] Make Element the superclass of UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- src/allmydata/web/unlinked.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index eb7fedc6f..8147437fa 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -36,7 +36,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(object): +class UploadResultsRendererMixin(Element): # this requires a method named 'upload_results' @renderer @@ -198,7 +198,7 @@ class UploadStatusPage(MultiFormatResource): return renderElement(req, elem) -class UploadStatusElement(Element, UploadResultsRendererMixin): +class UploadStatusElement(UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 6ca219c8b..d938691a4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -106,7 +106,7 @@ class UploadResultsPage(MultiFormatResource): render_UPLOAD = render_HTML -class UploadResultsElement(Element, status.UploadResultsRendererMixin): +class UploadResultsElement(status.UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) From 9520ad71eb3c6ffa5d42b80d525f26936e79163b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:24:21 -0400 Subject: [PATCH 0139/1054] Rearrange imports --- src/allmydata/web/status.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index b20aa679e..327d610a6 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1,5 +1,7 @@ -import pprint, itertools, hashlib +import pprint +import itertools +import hashlib import json from twisted.internet import defer from twisted.python.filepath import FilePath @@ -24,8 +26,14 @@ from allmydata.web.common import ( MultiFormatPage, MultiFormatResource, ) -from allmydata.interfaces import IUploadStatus, IDownloadStatus, \ - IPublishStatus, IRetrieveStatus, IServermapUpdaterStatus + +from allmydata.interfaces import ( + IUploadStatus, + IDownloadStatus, + IPublishStatus, + IRetrieveStatus, + IServermapUpdaterStatus, +) class RateAndTimeMixin(object): From 03d529c22a91020b893fab8c28678918f3450b6d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:33:30 -0400 Subject: [PATCH 0140/1054] Remove unused import --- src/allmydata/web/unlinked.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index d938691a4..91efff2c4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -4,7 +4,6 @@ from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.web.template import ( - Element, XMLFile, renderer, renderElement, From 81dc63b511704269b2038e7113b3ce753671512a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:32:44 -0400 Subject: [PATCH 0141/1054] Use twisted template tags in status page template --- src/allmydata/web/status.xhtml | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/status.xhtml b/src/allmydata/web/status.xhtml index d827002f4..045d5aa85 100644 --- a/src/allmydata/web/status.xhtml +++ b/src/allmydata/web/status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Recent and Active Operations @@ -11,8 +11,8 @@

                Active Operations:

                - - +
                + @@ -20,21 +20,21 @@ - - - - - - - + + + + + + + - +
                Type Storage Index Helper?Progress Status
                No active operations!
                No active operations!

                Recent Operations:

                - - +
                + @@ -43,16 +43,16 @@ - - - - - - - - + + + + + + + + - +
                Started Type Storage IndexProgress Status
                No recent operations!
                No recent operations!

                From 3357f4c5ceea5a0dcde94d0ff069914821dbde61 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:43:41 -0400 Subject: [PATCH 0142/1054] Make Status a MultiFormatResource --- src/allmydata/web/status.py | 89 ++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 327d610a6..e5165a570 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1125,14 +1125,19 @@ def marshal_json(s): return item -class Status(MultiFormatPage): - docFactory = getxmlfile("status.xhtml") +class Status(MultiFormatResource): + addSlash = True def __init__(self, history): - rend.Page.__init__(self, history) + super(Status, self).__init__() self.history = history + def render_HTML(self, req): + elem = StatusElement(self._get_active_operations(), + self._get_recent_operations()) + return renderElement(req, elem) + def render_JSON(self, req): # modern browsers now render this instead of forcing downloads req.setHeader("content-type", "application/json") @@ -1180,7 +1185,51 @@ class Status(MultiFormatPage): recent.reverse() return recent - def render_row(self, ctx, data): + def childFactory(self, ctx, name): + h = self.history + try: + stype, count_s = name.split("-") + except ValueError: + raise RuntimeError( + "no - in '{}'".format(name) + ) + count = int(count_s) + if stype == "up": + for s in itertools.chain(h.list_all_upload_statuses(), + h.list_all_helper_statuses()): + # immutable-upload helpers use the same status object as a + # regular immutable-upload + if s.get_counter() == count: + return UploadStatusPage(s) + if stype == "down": + for s in h.list_all_download_statuses(): + if s.get_counter() == count: + return DownloadStatusPage(s) + if stype == "mapupdate": + for s in h.list_all_mapupdate_statuses(): + if s.get_counter() == count: + return MapupdateStatusPage(s) + if stype == "publish": + for s in h.list_all_publish_statuses(): + if s.get_counter() == count: + return PublishStatusPage(s) + if stype == "retrieve": + for s in h.list_all_retrieve_statuses(): + if s.get_counter() == count: + return RetrieveStatusPage(s) + + +class StatusElement(Element): + + loader = XMLFile(FilePath(__file__).sibling("status.xhtml")) + + def __init__(self, active, recent): + super(StatusElement, self).__init__() + self._active = active + self._recent = recent + + @renderer + def row(self, req, tag): s = data started_s = render_time(s.get_started()) @@ -1231,38 +1280,6 @@ class Status(MultiFormatPage): ctx.fillSlots("status", T.a(href=link)[s.get_status()]) return ctx.tag - def childFactory(self, ctx, name): - h = self.history - try: - stype, count_s = name.split("-") - except ValueError: - raise RuntimeError( - "no - in '{}'".format(name) - ) - count = int(count_s) - if stype == "up": - for s in itertools.chain(h.list_all_upload_statuses(), - h.list_all_helper_statuses()): - # immutable-upload helpers use the same status object as a - # regular immutable-upload - if s.get_counter() == count: - return UploadStatusPage(s) - if stype == "down": - for s in h.list_all_download_statuses(): - if s.get_counter() == count: - return DownloadStatusPage(s) - if stype == "mapupdate": - for s in h.list_all_mapupdate_statuses(): - if s.get_counter() == count: - return MapupdateStatusPage(s) - if stype == "publish": - for s in h.list_all_publish_statuses(): - if s.get_counter() == count: - return PublishStatusPage(s) - if stype == "retrieve": - for s in h.list_all_retrieve_statuses(): - if s.get_counter() == count: - return RetrieveStatusPage(s) # Render "/helper_status" page. class HelperStatus(MultiFormatResource): From a6e32135e866c8bb3621ae725137cb27953128df Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 22:10:40 -0400 Subject: [PATCH 0143/1054] Update status page child route handler and table renderers Drop nevow-isms and use twisted.web's way of doing things. --- src/allmydata/web/status.py | 162 ++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 72 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index e5165a570..1e0c81b98 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -25,6 +25,7 @@ from allmydata.web.common import ( render_time, MultiFormatPage, MultiFormatResource, + SlotsSequenceElement, ) from allmydata.interfaces import ( @@ -1127,8 +1128,6 @@ def marshal_json(s): class Status(MultiFormatResource): - addSlash = True - def __init__(self, history): super(Status, self).__init__() self.history = history @@ -1153,45 +1152,24 @@ class Status(MultiFormatResource): return json.dumps(data, indent=1) + "\n" - def _get_all_statuses(self): - h = self.history - return itertools.chain(h.list_all_upload_statuses(), - h.list_all_download_statuses(), - h.list_all_mapupdate_statuses(), - h.list_all_publish_statuses(), - h.list_all_retrieve_statuses(), - h.list_all_helper_statuses(), - ) + def getChild(self, path, request): + # The "if (path is empty) return self" line should handle + # trailing slash in request path. + # + # Twisted Web's documentation says this: "If the URL ends in a + # slash, for example ``http://example.com/foo/bar/`` , the + # final URL segment will be an empty string. Resources can + # thus know if they were requested with or without a final + # slash." + if not path: + return self - def data_active_operations(self, ctx, data): - return self._get_active_operations() - - def _get_active_operations(self): - active = [s - for s in self._get_all_statuses() - if s.get_active()] - active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) - active.reverse() - return active - - def data_recent_operations(self, ctx, data): - return self._get_recent_operations() - - def _get_recent_operations(self): - recent = [s - for s in self._get_all_statuses() - if not s.get_active()] - recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) - recent.reverse() - return recent - - def childFactory(self, ctx, name): h = self.history try: - stype, count_s = name.split("-") + stype, count_s = path.split("-") except ValueError: raise RuntimeError( - "no - in '{}'".format(name) + "no - in '{}'".format(path) ) count = int(count_s) if stype == "up": @@ -1218,6 +1196,32 @@ class Status(MultiFormatResource): if s.get_counter() == count: return RetrieveStatusPage(s) + def _get_all_statuses(self): + h = self.history + return itertools.chain(h.list_all_upload_statuses(), + h.list_all_download_statuses(), + h.list_all_mapupdate_statuses(), + h.list_all_publish_statuses(), + h.list_all_retrieve_statuses(), + h.list_all_helper_statuses(), + ) + + def _get_active_operations(self): + active = [s + for s in self._get_all_statuses() + if s.get_active()] + active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + active.reverse() + return active + + def _get_recent_operations(self): + recent = [s + for s in self._get_all_statuses() + if not s.get_active()] + recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + recent.reverse() + return recent + class StatusElement(Element): @@ -1229,56 +1233,70 @@ class StatusElement(Element): self._recent = recent @renderer - def row(self, req, tag): - s = data + def active_operations(self, req, tag): + active = [self.get_op_state(op) for op in self._active] + return SlotsSequenceElement(tag, active) - started_s = render_time(s.get_started()) - ctx.fillSlots("started", started_s) + @renderer + def recent_operations(self, req, tag): + active = [self.get_op_state(op) for op in self._recent] + return SlotsSequenceElement(tag, active) - si_s = base32.b2a_or_none(s.get_storage_index()) + @staticmethod + def get_op_state(op): + result = dict() + + started_s = render_time(op.get_started()) + result.update({"started": started_s}) + + si_s = base32.b2a_or_none(op.get_storage_index()) if si_s is None: si_s = "(None)" - ctx.fillSlots("si", si_s) - ctx.fillSlots("helper", {True: "Yes", - False: "No"}[s.using_helper()]) - size = s.get_size() + result.update({"si": si_s}) + result.update({"helper": + {True: "Yes", False: "No"}[op.using_helper()]}) + + size = op.get_size() if size is None: size = "(unknown)" elif isinstance(size, (int, long, float)): size = abbreviate_size(size) - ctx.fillSlots("total_size", size) - progress = data.get_progress() - if IUploadStatus.providedBy(data): - link = "up-%d" % data.get_counter() - ctx.fillSlots("type", "upload") + result.update({"total_size": size}) + + progress = op.get_progress() + if IUploadStatus.providedBy(op): + link = "up-%d" % op.get_counter() + result.update({"type": "upload"}) # TODO: make an ascii-art bar (chk, ciphertext, encandpush) = progress progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" % - ( (100.0 * chk), - (100.0 * ciphertext), - (100.0 * encandpush) )) - ctx.fillSlots("progress", progress_s) - elif IDownloadStatus.providedBy(data): - link = "down-%d" % data.get_counter() - ctx.fillSlots("type", "download") - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - elif IPublishStatus.providedBy(data): - link = "publish-%d" % data.get_counter() - ctx.fillSlots("type", "publish") - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - elif IRetrieveStatus.providedBy(data): - ctx.fillSlots("type", "retrieve") - link = "retrieve-%d" % data.get_counter() - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) + ((100.0 * chk), + (100.0 * ciphertext), + (100.0 * encandpush))) + result.update({"progress": progress_s}) + elif IDownloadStatus.providedBy(op): + link = "down-%d" % op.get_counter() + result.update({"type": "download"}) + result.update({"progress": "%.1f%%" % (100.0 * progress)}) + elif IPublishStatus.providedBy(op): + link = "publish-%d" % op.get_counter() + result.update({"type": "publish"}) + result.update({"progress": "%.1f%%" % (100.0 * progress)}) + elif IRetrieveStatus.providedBy(op): + result.update({"type": "retrieve"}) + link = "retrieve-%d" % op.get_counter() + result.update({"progress": "%.1f%%" % (100.0 * progress)}) else: - assert IServermapUpdaterStatus.providedBy(data) - ctx.fillSlots("type", "mapupdate %s" % data.get_mode()) - link = "mapupdate-%d" % data.get_counter() - ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) - ctx.fillSlots("status", T.a(href=link)[s.get_status()]) - return ctx.tag + assert IServermapUpdaterStatus.providedBy(op) + result.update({"type": "mapupdate %s" % op.get_mode()}) + link = "mapupdate-%d" % op.get_counter() + result.update({"progress": "%.1f%%" % (100.0 * progress)}) + + result.update({"status": tags.a(op.get_status(), href=link)}) + + return result # Render "/helper_status" page. From bd799d9d726953c08b82ff011c5f84b5d98d987a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 22:18:58 -0400 Subject: [PATCH 0144/1054] Add newsfragment --- newsfragments/3254.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3254.minor diff --git a/newsfragments/3254.minor b/newsfragments/3254.minor new file mode 100644 index 000000000..e69de29bb From 93d015e468dc427743de9dbf2cf35c67ca7ad189 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 22:24:09 -0400 Subject: [PATCH 0145/1054] Document status.Status --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 1e0c81b98..db127db59 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1127,8 +1127,12 @@ def marshal_json(s): class Status(MultiFormatResource): + """Renders /status page.""" def __init__(self, history): + """ + :param allmydata.history.History history: provides operation statuses. + """ super(Status, self).__init__() self.history = history From d3f43d31d998fb97f8ad3e6fac007ed3fe57ab15 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 23 May 2020 06:17:06 -0400 Subject: [PATCH 0146/1054] Remove unused import --- src/allmydata/web/status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index db127db59..ec18426dc 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -23,7 +23,6 @@ from allmydata.web.common import ( plural, compute_rate, render_time, - MultiFormatPage, MultiFormatResource, SlotsSequenceElement, ) From d50ca9af55c2a19ccf35c4ff0b128ecafb75331f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 May 2020 22:43:21 -0600 Subject: [PATCH 0147/1054] proof-of-concept verified-fake for testing --- integration/test_testing.py | 26 +++ src/allmydata/testing/__init__.py | 0 src/allmydata/testing/web.py | 263 ++++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 integration/test_testing.py create mode 100644 src/allmydata/testing/__init__.py create mode 100644 src/allmydata/testing/web.py diff --git a/integration/test_testing.py b/integration/test_testing.py new file mode 100644 index 000000000..47498b1d7 --- /dev/null +++ b/integration/test_testing.py @@ -0,0 +1,26 @@ + + +from allmydata.testing.web import ( + create_fake_tahoe_root, + deterministic_key_generator, +) + +import pytest +import pytest_twisted + + +@pytest_twisted.inlineCallbacks +def test_retrieve_cap(): + """ + WebUI Fake can serve a read-capability back + """ + + keys = deterministic_key_generator() + root = yield create_fake_tahoe_root() + dummy_readcap = yield root.add_data( + next(keys), + "some dummy content\n"*20 + ) + print("readcap: {}".format(dummy_readcap)) + + assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py new file mode 100644 index 000000000..5cf517a54 --- /dev/null +++ b/src/allmydata/testing/web.py @@ -0,0 +1,263 @@ +import io +import os +import string + +import attr + +from twisted.internet import defer +from twisted.python.filepath import ( + FilePath, +) +from twisted.web.resource import ( + Resource, +) +from twisted.web.client import ( + Agent, + FileBodyProducer, +) + +from treq.client import ( + HTTPClient, +) +from treq.testing import ( + RequestTraversalAgent, + RequestSequence, + StubTreq, +) + + +class _FakeTahoeRoot(Resource): + """ + This is a sketch of how an in-memory 'fake' of a Tahoe + WebUI. Ultimately, this will live in Tahoe + """ + + def __init__(self, uri=None): + Resource.__init__(self) # this is an old-style class :( + self._uri = uri + self.putChild(b"uri", self._uri) + + def add_data(self, key, data): + return self._uri.add_data(key, data) + +@attr.s +class _FakeCapability(object): + """ + """ + data=attr.ib() + + +# XXX want to make all kinds of caps, like +# URI:CHK:... URI:DIR2:... etc + +import allmydata.uri +KNOWN_CAPABILITIES = [ + getattr(allmydata.uri, t).BASE_STRING + for t in dir(allmydata.uri) + if hasattr(getattr(allmydata.uri, t), 'BASE_STRING') +] + + +from allmydata.immutable.upload import BaseUploadable +from allmydata.interfaces import IUploadable +from zope.interface import implementer +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) + + +def deterministic_key_generator(): + character = 0 + while character < (26 * 2): + key = string.letters[character] * 16 + character += 1 + yield key + raise RuntimeError("Ran out of keys") + + +@implementer(IUploadable) +class DataUploadable(BaseUploadable): + # Base gives us: + # set_upload_status + # set_default_encoding_parameters + # get_all_encoding_parameters + + def __init__(self, data, key=None): + self._data = data + self._where = 0 + self._key = key if key is not None else urandom(16) + + def get_encryption_key(self): + return succeed(self._key) + + def get_size(self): + return succeed(len(self._data)) + + @inlineCallbacks + def read(self, amount): + data = [self._data[self._where : self._where + amount]] + self._where += amount + yield + returnValue(data) + + def close(self): + pass + +@inlineCallbacks +def create_fake_capability(kind, key, data): + if kind not in KNOWN_CAPABILITIES: + raise ValueError( + "'{}' not a known kind: {}".format( + kind, + ", ".join(list(KNOWN_CAPABILITIES.keys())), + ) + ) + + # XXX to use a allmydata.immutable.upload.CHKUploader directly, + # we'd need to instantiate: + + from allmydata.immutable.upload import ( + CHKUploader, + EncryptAnUploadable, + ) + from allmydata.immutable.encode import ( + Encoder, + ) + + class _FakeSecretHolder(object): + def get_renewal_secret(self): + return "renewal_secret" + + def get_cancel_secret(self): + return "cancel_secret" + + + @attr.s + class _FakeBucket(object): + data = attr.ib(init=False, default="") + + def callRemoteOnly(self, *args): + pass # print("callRemoteOnly({})".format(args)) + + def callRemote(self, verb, *args): + if verb == 'write': + offset, data = args + assert offset >= len(self.data) + while offset > len(self.data): + self.data += 'X' # marker data; we're padding + self.data += data + elif verb == 'close': + pass + else: + print("callRemote({})".format(args)) + + + @attr.s + class _FakeStorageServer(object): + buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) + + def get_buckets(self, storage_index): + return succeed(self.buckets) + + def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): + # returns a 2-tuple .. second one maps share-num to BucketWriter + return succeed(( + {}, + { + i: bucket + for i, bucket in enumerate(self.buckets) + } + )) + + class _FakeServer(object): + def get_serverid(self): + return "fake_server" + + def get_name(self): + return "steven" + + def get_version(self): + return { + "http://allmydata.org/tahoe/protocols/storage/v1": { + "maximum-immutable-share-size": 10*1024*1024*1024, + } + } + + def get_lease_seed(self): + return "decafbaddecafbaddeca" + + def get_storage_server(self): + return _FakeStorageServer() + + + class _FakeStorageBroker(object): + def get_servers_for_psi(self, storage_index): + return [_FakeServer()] + + storage_broker = _FakeStorageBroker() + secret_holder = _FakeSecretHolder() + uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) + uploadable = DataUploadable(data, key=key) + uploadable.set_default_encoding_parameters({ + "n": 1, + "k": 1, + "happy": 1, + }) + encrypted_uploadable = EncryptAnUploadable(uploadable) + + encoder = Encoder() + yield encoder.set_encrypted_uploadable(encrypted_uploadable) + + uploadresults = yield uploader.start(encrypted_uploadable) + + enc_key = yield uploadable.get_encryption_key() + + from allmydata.uri import from_string as uri_from_string + from allmydata.uri import CHKFileURI + + verify_cap = uri_from_string(uploadresults.get_verifycapstr()) + read_cap = CHKFileURI( + enc_key, + verify_cap.uri_extension_hash, + verify_cap.needed_shares, + verify_cap.total_shares, + verify_cap.size, + ) + uploadresults.set_uri(read_cap.to_string()) + + returnValue(read_cap) + + +class _FakeTahoeUriHandler(Resource): + """ + """ + + isLeaf = True + + @inlineCallbacks + def add_data(self, key, data): + """ + adds some data to our grid, returning a capability + """ + cap = yield create_fake_capability("URI:CHK:", key, data) + returnValue(cap) + + def render_GET(self, request): + print(request) + print(request.uri) + return b"URI:DIR2-CHK:some capability" + + +@inlineCallbacks +def create_fake_tahoe_root(): + """ + Probably should take some params to control what this fake does: + return errors, pre-populate capabilities, ... + """ + root = _FakeTahoeRoot( + uri=_FakeTahoeUriHandler(), + ) + yield + returnValue(root) From cf4b3ba008a2ac760c39899570041dfb2bd8e304 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:42:55 -0400 Subject: [PATCH 0148/1054] Correct arguments to super() This was incidentally found when looking at some test failures: super(type, object) is the right invocation, and not the other way around. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index e7a94ff40..a2af857b9 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1269,5 +1269,5 @@ class TrialTestCase(_TrialTestCase): if six.PY2: if isinstance(msg, six.text_type): - return super(self, TrialTestCase).fail(msg.encode("utf8")) - return super(self, TrialTestCase).fail(msg) + return super(TrialTestCase, self).fail(msg.encode("utf8")) + return super(TrialTestCase, self).fail(msg) From b6e00d09f9eee5e6c829b91b3244b6180d239c94 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:49:13 -0400 Subject: [PATCH 0149/1054] Render self in DirectoryNodeHandler.getChild() We need self.render_POST() etc. to be invoked when we have a request such as "POST /uri/URI:DIR:..."; throwing an error here is probably not the right thing to do. --- src/allmydata/web/directory.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 57c222788..2129cec25 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -103,12 +103,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page - name = name.decode('utf8') if not name: - raise EmptyPathnameComponentError( - u"The webapi does not allow empty pathname components", - ) + return self + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From ae7802fc0c5481f51bdc37ec63025191215b30eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 09:24:47 -0400 Subject: [PATCH 0150/1054] Add newsfragment --- newsfragments/3312.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3312.bugfix diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3312.bugfix @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From 0633623eccf113a52f719f56db517b2ff7e0808d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 19:47:12 -0400 Subject: [PATCH 0151/1054] Reject URIs that contain empty segments --- src/allmydata/web/directory.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 2129cec25..fced8c68e 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -106,6 +106,16 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): if not name: return self + # Rejecting URIs that contain empty path pieces (for example: + # "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was + # the old nevow behavior and it is encoded in the test suite; + # we will follow suit. + for segment in req.prepath: + if not segment: + raise EmptyPathnameComponentError( + u"The webapi does not allow empty pathname components", + ) + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) From 31eba00d0fef88d6ba2aad6b28cadda94ff50bd1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:17 -0400 Subject: [PATCH 0152/1054] Reject trailing empty path piece in URIs --- src/allmydata/web/directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index fced8c68e..b1618a3f5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -102,8 +102,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # trying to replicate what I have observed as Nevow behavior # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash - # or no further children) renders "this" page - if not name: + # or no further children) renders "this" page. We also need + # to reject "/uri/URI:DIR2:..//", so we look at postpath. + if not name and req.postpath != ['']: return self # Rejecting URIs that contain empty path pieces (for example: From dce73f7c2fd4c304782342a1bb51645b62a87a53 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:35 -0400 Subject: [PATCH 0153/1054] Make "Refresh" link work This seems to be a subtle difference from nevow: with `href="."`, rendered link target will be `/uri/`, so clicking "Refresh" will result in an error message like so: "GET /uri requires uri=". With `href=""`, the rendered link target will be `/uri/URI:...`, which is what we need. --- src/allmydata/web/directory.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.xhtml b/src/allmydata/web/directory.xhtml index 88058efa4..eeb04692d 100644 --- a/src/allmydata/web/directory.xhtml +++ b/src/allmydata/web/directory.xhtml @@ -31,7 +31,7 @@
                From dd14da4a55e043ab18d067c750ab20eb9050f5b4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:46:23 -0400 Subject: [PATCH 0154/1054] Decode name before null check Make the diff against master a tad bit smaller. --- src/allmydata/web/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index b1618a3f5..a5ba3bed5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -104,6 +104,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page. We also need # to reject "/uri/URI:DIR2:..//", so we look at postpath. + name = name.decode('utf8') if not name and req.postpath != ['']: return self @@ -117,7 +118,6 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): u"The webapi does not allow empty pathname components", ) - name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From 1faa81c7c4b6b7ad9a2dd1e5d1a0851d13eb39f9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 1 Jun 2020 09:06:46 -0600 Subject: [PATCH 0155/1054] actual usable agent --- src/allmydata/testing/web.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5cf517a54..1767cb114 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -261,3 +261,52 @@ def create_fake_tahoe_root(): ) yield returnValue(root) + + +@implementer(IBodyProducer) +class _SynchronousProducer(object): + """ + A partial implementation of an :obj:`IBodyProducer` which produces its + entire payload immediately. There is no way to access to an instance of + this object from :obj:`RequestTraversalAgent` or :obj:`StubTreq`, or even a + :obj:`Resource: passed to :obj:`StubTreq`. + + This does not implement the :func:`IBodyProducer.stopProducing` method, + because that is very difficult to trigger. (The request from + `RequestTraversalAgent` would have to be canceled while it is still in the + transmitting state), and the intent is to use `RequestTraversalAgent` to + make synchronous requests. + """ + + def __init__(self, body): + """ + Create a synchronous producer with some bytes. + """ + if not isinstance(body, bytes): + raise ValueError( + "'body' must be bytes" + ) + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + """ + Immediately produce all data. + """ + consumer.write(self.body) + return succeed(None) + + +@inlineCallbacks +def create_tahoe_treq_client(root=None): + """ + """ + + if root is None: + root = yield create_fake_tahoe_root() + + client = HTTPClient( + agent=RequestTraversalAgent(root), + data_to_body_producer=_SynchronousProducer, + ) + returnValue(client) From 4fa2e6ad97e70e56bcedb6b77d8be292d128e6c6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 11:56:59 -0400 Subject: [PATCH 0156/1054] Use twisted web directives in web.operations.ReloadMixin --- src/allmydata/web/operations.py | 36 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 93de2169e..eb9321344 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,7 +1,11 @@ import time -from nevow import rend, url, tags as T +from nevow import rend, url from nevow.inevow import IRequest +from twisted.web.template import ( + renderer, + tags as T, +) from twisted.python.failure import Failure from twisted.internet import reactor, defer from twisted.web.http import NOT_FOUND @@ -122,31 +126,31 @@ class OphandleTable(rend.Page, service.Service): class ReloadMixin(object): REFRESH_TIME = 1*MINUTE - def render_refresh(self, ctx, data): + @renderer + def refresh(self, req, tag): if self.monitor.is_finished(): return "" # dreid suggests ctx.tag(**dict([("http-equiv", "refresh")])) # but I can't tell if he's joking or not - ctx.tag.attributes["http-equiv"] = "refresh" - ctx.tag.attributes["content"] = str(self.REFRESH_TIME) - return ctx.tag + tag.attributes["http-equiv"] = "refresh" + tag.attributes["content"] = str(self.REFRESH_TIME) + return tag - def render_reload(self, ctx, data): + @renderer + def reload(self, req, tag): if self.monitor.is_finished(): return "" - req = IRequest(ctx) # url.gethere would break a proxy, so the correct thing to do is # req.path[-1] + queryargs ophandle = req.prepath[-1] reload_target = ophandle + "?output=html" cancel_target = ophandle + "?t=cancel" - cancel_button = T.form(action=cancel_target, method="POST", - enctype="multipart/form-data")[ - T.input(type="submit", value="Cancel"), - ] + cancel_button = T.form(T.input(type="submit", value="Cancel"), + action=cancel_target, + method="POST", + enctype="multipart/form-data",) - return [T.h2["Operation still running: ", - T.a(href=reload_target)["Reload"], - ], - cancel_button, - ] + return (T.h2("Operation still running: ", + T.a("Reload", href=reload_target), + ), + cancel_button,) From dd3b567a8635201af8369c46b931ffa09a49ab28 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 12:06:48 -0400 Subject: [PATCH 0157/1054] Add newsfragment --- newsfragments/3315.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3315.minor diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3315.minor @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From 3e973197ac339ee0ff9acc7f0892dc255bbdbae2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 3 Jun 2020 11:32:22 -0600 Subject: [PATCH 0158/1054] synchronous-only up/download tahoe code to keep treq.testing happy --- src/allmydata/testing/web.py | 73 ++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 1767cb114..32fe0c084 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,10 +1,12 @@ import io import os +import json import string import attr -from twisted.internet import defer +from hyperlink import DecodedURL + from twisted.python.filepath import ( FilePath, ) @@ -15,6 +17,9 @@ from twisted.web.client import ( Agent, FileBodyProducer, ) +from twisted.web.iweb import ( + IBodyProducer, +) from treq.client import ( HTTPClient, @@ -40,6 +45,7 @@ class _FakeTahoeRoot(Resource): def add_data(self, key, data): return self._uri.add_data(key, data) + @attr.s class _FakeCapability(object): """ @@ -70,8 +76,8 @@ from twisted.internet.defer import ( def deterministic_key_generator(): character = 0 - while character < (26 * 2): - key = string.letters[character] * 16 + while True:#character < (26 * 2): + key = string.letters[character % 52] * 16 character += 1 yield key raise RuntimeError("Ran out of keys") @@ -159,7 +165,7 @@ def create_fake_capability(kind, key, data): buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) def get_buckets(self, storage_index): - return succeed(self.buckets) + return succeed({0: self.buckets}) def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): # returns a 2-tuple .. second one maps share-num to BucketWriter @@ -230,11 +236,34 @@ def create_fake_capability(kind, key, data): returnValue(read_cap) +def capability_generator(): + number = 0 + while True: + cap = u"URI:CHK:fake capability {}".format(number) + number += 1 + yield cap.encode("ascii") + + class _FakeTahoeUriHandler(Resource): """ """ isLeaf = True + _data = None + _key_generator = None + _capability_generator = None + + def _get_key(self): + if self._key_generator is None: + self._key_generator = deterministic_key_generator() + key = next(self._key_generator) + return key + + def _generate_cap(self): + if self._capability_generator is None: + self._capability_generator = capability_generator() + capability = next(self._capability_generator) + return capability @inlineCallbacks def add_data(self, key, data): @@ -242,12 +271,42 @@ class _FakeTahoeUriHandler(Resource): adds some data to our grid, returning a capability """ cap = yield create_fake_capability("URI:CHK:", key, data) + if self._data is None: + self._data = dict() + self._data[cap] = data returnValue(cap) + def render_PUT(self, request): + data = request.content.read() +# if len(data) == 0: +# raise RuntimeError("No empty data") + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap + + def render_POST(self, request): + kind = request.args[u"t"][0] + data = request.content.read() + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap +# if kind == 'mkdir-immutable': + def render_GET(self, request): - print(request) - print(request.uri) - return b"URI:DIR2-CHK:some capability" + uri = DecodedURL.from_text(request.uri.decode('utf8')) + # XXX FIXME + capability = uri.query[0][1] + + if self._data is None or capability not in self._data: + return u"No data for '{}'".format(capability).decode("ascii") + + return self._data[capability] @inlineCallbacks From d7288ee2381b030b415e86df0461872aa6e70d9e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 5 Jun 2020 09:28:36 -0400 Subject: [PATCH 0159/1054] Update newsfragment --- newsfragments/3315.minor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor index ed00922a6..0536c297a 100644 --- a/newsfragments/3315.minor +++ b/newsfragments/3315.minor @@ -1 +1 @@ -Add your info here \ No newline at end of file +Replace nevow with twisted.web in web.operations.ReloadMixin From fc2203e421b33a50e3ded99eff0583247d6faae8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:26:09 -0600 Subject: [PATCH 0160/1054] simplify tahoe web fakes --- src/allmydata/testing/web.py | 287 ++++++++++------------------------- 1 file changed, 76 insertions(+), 211 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 32fe0c084..78d2b7931 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -2,6 +2,7 @@ import io import os import json import string +import hashlib import attr @@ -20,6 +21,11 @@ from twisted.web.client import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) from treq.client import ( HTTPClient, @@ -29,6 +35,12 @@ from treq.testing import ( RequestSequence, StubTreq, ) +from zope.interface import implementer + +import allmydata.uri +from allmydata.util import ( + base32, +) class _FakeTahoeRoot(Resource): @@ -53,10 +65,6 @@ class _FakeCapability(object): data=attr.ib() -# XXX want to make all kinds of caps, like -# URI:CHK:... URI:DIR2:... etc - -import allmydata.uri KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -64,183 +72,46 @@ KNOWN_CAPABILITIES = [ ] -from allmydata.immutable.upload import BaseUploadable -from allmydata.interfaces import IUploadable -from zope.interface import implementer -from twisted.internet.defer import ( - inlineCallbacks, - succeed, - returnValue, -) +def capability_generator(kind): + """ + :param str kind: the kind of capability, like `URI:CHK` - -def deterministic_key_generator(): - character = 0 - while True:#character < (26 * 2): - key = string.letters[character % 52] * 16 - character += 1 - yield key - raise RuntimeError("Ran out of keys") - - -@implementer(IUploadable) -class DataUploadable(BaseUploadable): - # Base gives us: - # set_upload_status - # set_default_encoding_parameters - # get_all_encoding_parameters - - def __init__(self, data, key=None): - self._data = data - self._where = 0 - self._key = key if key is not None else urandom(16) - - def get_encryption_key(self): - return succeed(self._key) - - def get_size(self): - return succeed(len(self._data)) - - @inlineCallbacks - def read(self, amount): - data = [self._data[self._where : self._where + amount]] - self._where += amount - yield - returnValue(data) - - def close(self): - pass - -@inlineCallbacks -def create_fake_capability(kind, key, data): + :returns: a generator that yields new capablities of a particular + kind. + """ if kind not in KNOWN_CAPABILITIES: raise ValueError( - "'{}' not a known kind: {}".format( + "Unknown capability kind '{} (valid are {})'".format( kind, - ", ".join(list(KNOWN_CAPABILITIES.keys())), + ", ".join(KNOWN_CAPABILITIES), ) ) - - # XXX to use a allmydata.immutable.upload.CHKUploader directly, - # we'd need to instantiate: - - from allmydata.immutable.upload import ( - CHKUploader, - EncryptAnUploadable, - ) - from allmydata.immutable.encode import ( - Encoder, - ) - - class _FakeSecretHolder(object): - def get_renewal_secret(self): - return "renewal_secret" - - def get_cancel_secret(self): - return "cancel_secret" - - - @attr.s - class _FakeBucket(object): - data = attr.ib(init=False, default="") - - def callRemoteOnly(self, *args): - pass # print("callRemoteOnly({})".format(args)) - - def callRemote(self, verb, *args): - if verb == 'write': - offset, data = args - assert offset >= len(self.data) - while offset > len(self.data): - self.data += 'X' # marker data; we're padding - self.data += data - elif verb == 'close': - pass - else: - print("callRemote({})".format(args)) - - - @attr.s - class _FakeStorageServer(object): - buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) - - def get_buckets(self, storage_index): - return succeed({0: self.buckets}) - - def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): - # returns a 2-tuple .. second one maps share-num to BucketWriter - return succeed(( - {}, - { - i: bucket - for i, bucket in enumerate(self.buckets) - } - )) - - class _FakeServer(object): - def get_serverid(self): - return "fake_server" - - def get_name(self): - return "steven" - - def get_version(self): - return { - "http://allmydata.org/tahoe/protocols/storage/v1": { - "maximum-immutable-share-size": 10*1024*1024*1024, - } - } - - def get_lease_seed(self): - return "decafbaddecafbaddeca" - - def get_storage_server(self): - return _FakeStorageServer() - - - class _FakeStorageBroker(object): - def get_servers_for_psi(self, storage_index): - return [_FakeServer()] - - storage_broker = _FakeStorageBroker() - secret_holder = _FakeSecretHolder() - uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) - uploadable = DataUploadable(data, key=key) - uploadable.set_default_encoding_parameters({ - "n": 1, - "k": 1, - "happy": 1, - }) - encrypted_uploadable = EncryptAnUploadable(uploadable) - - encoder = Encoder() - yield encoder.set_encrypted_uploadable(encrypted_uploadable) - - uploadresults = yield uploader.start(encrypted_uploadable) - - enc_key = yield uploadable.get_encryption_key() - - from allmydata.uri import from_string as uri_from_string - from allmydata.uri import CHKFileURI - - verify_cap = uri_from_string(uploadresults.get_verifycapstr()) - read_cap = CHKFileURI( - enc_key, - verify_cap.uri_extension_hash, - verify_cap.needed_shares, - verify_cap.total_shares, - verify_cap.size, - ) - uploadresults.set_uri(read_cap.to_string()) - - returnValue(read_cap) - - -def capability_generator(): + # what we do here is to start with empty hashers for the key and + # ueb_hash and repeatedly feed() them a zero byte on each + # iteration .. so the same sequence of capabilities will always be + # produced. We could add a seed= argument if we wanted to produce + # different sequences. number = 0 + key_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") + + # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: - cap = u"URI:CHK:fake capability {}".format(number) number += 1 + key_hasher.update("\x00") + ueb_hasher.update("\x00") + + key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes + ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes + + cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + kind=kind, + key=key, + ueb_hash=ueb_hash, + n=1, + k=1, + size=number * 1000, + ) yield cap.encode("ascii") @@ -250,53 +121,50 @@ class _FakeTahoeUriHandler(Resource): isLeaf = True _data = None - _key_generator = None - _capability_generator = None + _capability_generators = None - def _get_key(self): - if self._key_generator is None: - self._key_generator = deterministic_key_generator() - key = next(self._key_generator) - return key + def _generate_capability(self, kind): + """ + :param str kind: any valid capability-string type - def _generate_cap(self): - if self._capability_generator is None: - self._capability_generator = capability_generator() - capability = next(self._capability_generator) + :returns: the next capability-string for the given kind + """ + if self._capability_generators is None: + self._capability_generators = dict() + + if kind not in self._capability_generators: + self._capability_generators[kind] = capability_generator(kind) + capability = next(self._capability_generators[kind]) return capability - @inlineCallbacks - def add_data(self, key, data): + def add_data(self, kind, data): """ - adds some data to our grid, returning a capability + adds some data to our grid + + :returns: a capability-string """ - cap = yield create_fake_capability("URI:CHK:", key, data) + assert isinstance(data, bytes) + + cap = self._generate_capability(kind) if self._data is None: self._data = dict() + assert cap not in self._data, "already have '{}'".format(cap) self._data[cap] = data - returnValue(cap) + return cap def render_PUT(self, request): data = request.content.read() -# if len(data) == 0: -# raise RuntimeError("No empty data") - - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap + return self.add_data("URI:CHK:", data) def render_POST(self, request): - kind = request.args[u"t"][0] + t = request.args[u"t"][0] data = request.content.read() - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap -# if kind == 'mkdir-immutable': + type_to_kind = { + "mkdir-immutable": "URI:DIR2-CHK:" + } + kind = type_to_kind[t] + return self.add_data(kind, data) def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -309,17 +177,15 @@ class _FakeTahoeUriHandler(Resource): return self._data[capability] -@inlineCallbacks def create_fake_tahoe_root(): """ - Probably should take some params to control what this fake does: - return errors, pre-populate capabilities, ... + :returns: an IResource instance that will handle certain Tahoe URI + endpoints similar to a real Tahoe server. """ root = _FakeTahoeRoot( uri=_FakeTahoeUriHandler(), ) - yield - returnValue(root) + return root @implementer(IBodyProducer) @@ -356,16 +222,15 @@ class _SynchronousProducer(object): return succeed(None) -@inlineCallbacks def create_tahoe_treq_client(root=None): """ """ if root is None: - root = yield create_fake_tahoe_root() + root = create_fake_tahoe_root() client = HTTPClient( agent=RequestTraversalAgent(root), data_to_body_producer=_SynchronousProducer, ) - returnValue(client) + return client From 213d0348a890fc1ca34ad7c24e8c2478a0f03465 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:36:28 -0600 Subject: [PATCH 0161/1054] news --- newsfragments/3317.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3317.feature diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature new file mode 100644 index 000000000..5ed00acf5 --- /dev/null +++ b/newsfragments/3317.feature @@ -0,0 +1 @@ +Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file From 86a90084bb2082c51b783a207d2a93c4a900393c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 14:14:42 -0600 Subject: [PATCH 0162/1054] simple test of testing/web --- src/allmydata/test/test_testing.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/allmydata/test/test_testing.py diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py new file mode 100644 index 000000000..37e68a0de --- /dev/null +++ b/src/allmydata/test/test_testing.py @@ -0,0 +1,40 @@ +import io + +from twisted.trial.unittest import ( + TestCase, +) +from twisted.internet.defer import ( + inlineCallbacks, +) + +from allmydata.testing.web import ( + create_tahoe_treq_client, +) + + +class FakeWebTest(TestCase): + """ + Test the WebUI verified-fakes infrastucture + """ + + def setUp(self): + super(FakeWebTest, self).setUp() + self.http_client = create_tahoe_treq_client() + + @inlineCallbacks + def test_create_and_download(self): + """ + Upload some content and then download it + """ + content = "fake data\n" * 100 + resp = yield self.http_client.put("http://example.com/uri", content) + cap = yield resp.content() + + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + round_trip_content = yield resp.content() + + self.assertEqual(content, round_trip_content) From 00e7d01a1af8c1dec324305c976e76010cc2c73d Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:40 -0600 Subject: [PATCH 0163/1054] unused imports --- src/allmydata/test/test_testing.py | 2 -- src/allmydata/testing/web.py | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37e68a0de..da37b1de3 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,5 +1,3 @@ -import io - from twisted.trial.unittest import ( TestCase, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 78d2b7931..bc0138326 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,30 +1,17 @@ -import io -import os -import json -import string import hashlib import attr from hyperlink import DecodedURL -from twisted.python.filepath import ( - FilePath, -) from twisted.web.resource import ( Resource, ) -from twisted.web.client import ( - Agent, - FileBodyProducer, -) from twisted.web.iweb import ( IBodyProducer, ) from twisted.internet.defer import ( - inlineCallbacks, succeed, - returnValue, ) from treq.client import ( @@ -32,8 +19,6 @@ from treq.client import ( ) from treq.testing import ( RequestTraversalAgent, - RequestSequence, - StubTreq, ) from zope.interface import implementer From 36d9bf5d0983f66667cf1c9910736d5644e0a1af Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:47 -0600 Subject: [PATCH 0164/1054] new-style classes only --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index bc0138326..de515ebc9 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -28,7 +28,7 @@ from allmydata.util import ( ) -class _FakeTahoeRoot(Resource): +class _FakeTahoeRoot(Resource, object): """ This is a sketch of how an in-memory 'fake' of a Tahoe WebUI. Ultimately, this will live in Tahoe @@ -100,7 +100,7 @@ def capability_generator(kind): yield cap.encode("ascii") -class _FakeTahoeUriHandler(Resource): +class _FakeTahoeUriHandler(Resource, object): """ """ From 0f4201ef193b9af183e06e6141ba659a240de4b0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:55:19 -0600 Subject: [PATCH 0165/1054] the tests are unit-tests --- integration/test_testing.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 integration/test_testing.py diff --git a/integration/test_testing.py b/integration/test_testing.py deleted file mode 100644 index 47498b1d7..000000000 --- a/integration/test_testing.py +++ /dev/null @@ -1,26 +0,0 @@ - - -from allmydata.testing.web import ( - create_fake_tahoe_root, - deterministic_key_generator, -) - -import pytest -import pytest_twisted - - -@pytest_twisted.inlineCallbacks -def test_retrieve_cap(): - """ - WebUI Fake can serve a read-capability back - """ - - keys = deterministic_key_generator() - root = yield create_fake_tahoe_root() - dummy_readcap = yield root.add_data( - next(keys), - "some dummy content\n"*20 - ) - print("readcap: {}".format(dummy_readcap)) - - assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" From 0e5422c24b82f90b7fcb3cf46d2076aec4a31ff9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:11 -0600 Subject: [PATCH 0166/1054] handle more inputs in test body-producer --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index de515ebc9..aa3b2a4a7 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -16,6 +16,7 @@ from twisted.internet.defer import ( from treq.client import ( HTTPClient, + FileBodyProducer, ) from treq.testing import ( RequestTraversalAgent, @@ -192,9 +193,12 @@ class _SynchronousProducer(object): """ Create a synchronous producer with some bytes. """ + if isinstance(body, FileBodyProducer): + body = body._inputFile.read() + if not isinstance(body, bytes): raise ValueError( - "'body' must be bytes" + "'body' must be bytes not '{}'".format(type(body)) ) self.body = body self.length = len(body) From 318dc5092d141e2c2e32f9b18a65aeba78ca65bf Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:21 -0600 Subject: [PATCH 0167/1054] key -> kind --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aa3b2a4a7..fa6a2555a 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -40,8 +40,8 @@ class _FakeTahoeRoot(Resource, object): self._uri = uri self.putChild(b"uri", self._uri) - def add_data(self, key, data): - return self._uri.add_data(key, data) + def add_data(self, kind, data): + return self._uri.add_data(kind, data) @attr.s From 05ea0a2fe9d8525d85c6e6924ce4db35cf479462 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:03 -0600 Subject: [PATCH 0168/1054] copyright, header --- src/allmydata/test/test_testing.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index da37b1de3..4dc9c6bb6 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,6 +1,16 @@ -from twisted.trial.unittest import ( - TestCase, -) +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Tests for the allmydata.testing helpers +""" + from twisted.internet.defer import ( inlineCallbacks, ) From 776ee5f91c12ab3e1577c947f5ed92bf4e414a23 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:12 -0600 Subject: [PATCH 0169/1054] use hypothesis -- which doesn't know about Deferred, so need testtools too --- src/allmydata/test/test_testing.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 4dc9c6bb6..e350546ca 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -19,6 +19,23 @@ from allmydata.testing.web import ( create_tahoe_treq_client, ) +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + binary, +) + +from testtools import ( + TestCase, +) +from testtools.matchers import ( + Always, +) +from testtools.twistedsupport import ( + succeeded, +) + class FakeWebTest(TestCase): """ @@ -29,20 +46,31 @@ class FakeWebTest(TestCase): super(FakeWebTest, self).setUp() self.http_client = create_tahoe_treq_client() - @inlineCallbacks - def test_create_and_download(self): + @given( + content=binary(), + ) + def test_create_and_download(self, content): """ - Upload some content and then download it + Upload some content (via 'PUT /uri') and then download it (via + 'GET /uri?uri=...') """ - content = "fake data\n" * 100 - resp = yield self.http_client.put("http://example.com/uri", content) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + @inlineCallbacks + def do_test(): + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 200) - resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + cap = yield resp.content() + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + self.assertEqual(resp.code, 200) + + round_trip_content = yield resp.content() + self.assertEqual(content, round_trip_content) + self.assertThat( + do_test(), + succeeded(Always()), ) - round_trip_content = yield resp.content() - - self.assertEqual(content, round_trip_content) From 949b09ce278ef65689d853b1bf3b33caa330a02c Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:08:36 -0600 Subject: [PATCH 0170/1054] actually returns CREATED for new data --- src/allmydata/test/test_testing.py | 2 +- src/allmydata/testing/web.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index e350546ca..a5c712d77 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -58,7 +58,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 200) + self.assertEqual(resp.code, 201) cap = yield resp.content() self.assertTrue(cap.startswith("URI:CHK:")) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index fa6a2555a..99dabeddf 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -140,6 +140,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() + request.setResponseCode(201) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From fa597e1735fe64b6df8288ee75f7136b713b58b6 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:57:38 -0600 Subject: [PATCH 0171/1054] use real parser --- src/allmydata/test/test_testing.py | 11 ++++++++--- src/allmydata/testing/web.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a5c712d77..b5bbacad4 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -15,6 +15,10 @@ from twisted.internet.defer import ( inlineCallbacks, ) +from allmydata.uri import ( + from_string, + CHKFileURI, +) from allmydata.testing.web import ( create_tahoe_treq_client, ) @@ -60,11 +64,12 @@ class FakeWebTest(TestCase): resp = yield self.http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 99dabeddf..f6411792d 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -90,7 +90,7 @@ def capability_generator(kind): key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes - cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + cap = u"{kind}{key}:{ueb_hash}:{n}:{k}:{size}".format( kind=kind, key=key, ueb_hash=ueb_hash, From e79534abe8a45e4a5d284e40833310bdda13af71 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:01:02 -0600 Subject: [PATCH 0172/1054] housekeeping --- src/allmydata/testing/web.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index f6411792d..2b0f830f2 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,3 +1,16 @@ +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Test-helpers for clients that use the WebUI. +""" + import hashlib import attr @@ -28,6 +41,11 @@ from allmydata.util import ( base32, ) +__all__ = ( + "create_fake_tahoe_root", + "create_tahoe_treq_client", +) + class _FakeTahoeRoot(Resource, object): """ From 23b17ca02e0bc2e59c3f5982e454bc28f19c77ec Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:05:32 -0600 Subject: [PATCH 0173/1054] cleanup, docstrings --- src/allmydata/testing/web.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 2b0f830f2..7d9b5730b 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -49,8 +49,8 @@ __all__ = ( class _FakeTahoeRoot(Resource, object): """ - This is a sketch of how an in-memory 'fake' of a Tahoe - WebUI. Ultimately, this will live in Tahoe + An in-memory 'fake' of a Tahoe WebUI root. Currently it only + implements (some of) the `/uri` resource. """ def __init__(self, uri=None): @@ -62,13 +62,6 @@ class _FakeTahoeRoot(Resource, object): return self._uri.add_data(kind, data) -@attr.s -class _FakeCapability(object): - """ - """ - data=attr.ib() - - KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -121,6 +114,8 @@ def capability_generator(kind): class _FakeTahoeUriHandler(Resource, object): """ + An in-memory fake of (some of) the `/uri` endpoint of a Tahoe + WebUI """ isLeaf = True @@ -232,6 +227,12 @@ class _SynchronousProducer(object): def create_tahoe_treq_client(root=None): """ + :param root: an instance created via `create_fake_tahoe_root`. The + caller might want a copy of this to call `.add_data` for example. + + :returns: an instance of treq.client.HTTPClient wired up to + in-memory fakes of the Tahoe WebUI. Only a subset of the real + WebUI is available. """ if root is None: From c64e78634d7402df1348cc2ed02ec1559eb851d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:06:33 -0600 Subject: [PATCH 0174/1054] docstring --- src/allmydata/testing/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7d9b5730b..64e615cd6 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -54,6 +54,9 @@ class _FakeTahoeRoot(Resource, object): """ def __init__(self, uri=None): + """ + :param uri: a Resource to handle the `/uri` tree. + """ Resource.__init__(self) # this is an old-style class :( self._uri = uri self.putChild(b"uri", self._uri) From a38188164f7ac7d60b20ff28d029b1c48faea4da Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:08:04 -0600 Subject: [PATCH 0175/1054] comment --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 64e615cd6..c2bced12e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -93,7 +93,7 @@ def capability_generator(kind): # different sequences. number = 0 key_hasher = hashlib.new("sha256") - ueb_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") # ueb means "URI Extension Block" # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: From 9f6fcde2f1f0bdfe59eb234960b63913dfaac867 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:12 -0600 Subject: [PATCH 0176/1054] more docstring --- src/allmydata/testing/web.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index c2bced12e..63f6b9877 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -74,6 +74,10 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ + Deterministically generates a streap of valid capabilities of the + given kind. The N, K and size values aren't related to anything + real. + :param str kind: the kind of capability, like `URI:CHK` :returns: a generator that yields new capablities of a particular From 76040b53f5afd2c7099250ba7e825718f1c1fdf0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:57 -0600 Subject: [PATCH 0177/1054] assert -> raise --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 63f6b9877..90b50d2e0 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -149,7 +149,8 @@ class _FakeTahoeUriHandler(Resource, object): :returns: a capability-string """ - assert isinstance(data, bytes) + if not isinstance(data, bytes): + raise TypeError("'data' must be bytes") cap = self._generate_capability(kind) if self._data is None: From 24899dae8e803d4686c5a534a0784057900ea036 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:41:22 -0600 Subject: [PATCH 0178/1054] assert -> ValueError --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 90b50d2e0..5e6d564fa 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -155,7 +155,8 @@ class _FakeTahoeUriHandler(Resource, object): cap = self._generate_capability(kind) if self._data is None: self._data = dict() - assert cap not in self._data, "already have '{}'".format(cap) + if cap in self._data: + raise ValueError("already have '{}'".format(cap)) self._data[cap] = data return cap From b952ef066c556e5d17d321328abe4222f94523ae Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:46:55 -0600 Subject: [PATCH 0179/1054] match query args --- src/allmydata/testing/web.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5e6d564fa..3231c5820 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -177,8 +177,16 @@ class _FakeTahoeUriHandler(Resource, object): def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) - # XXX FIXME - capability = uri.query[0][1] + capability = None + for arg, value in uri.query: + if arg == u"uri": + capability = value + if capability is None: + raise Exception( + "No ?uri= arguent in GET '{}'".format( + uri.to_string() + ) + ) if self._data is None or capability not in self._data: return u"No data for '{}'".format(capability).decode("ascii") From 0cbdbd9ebef8834d195ea43871d9c2a03e789ece Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:14:45 -0600 Subject: [PATCH 0180/1054] symbols, not magic-numbers --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 3231c5820..5ab870f4f 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -23,6 +23,10 @@ from twisted.web.resource import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.web import ( + http, +) + from twisted.internet.defer import ( succeed, ) @@ -162,7 +166,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() - request.setResponseCode(201) # real code does this for brand-new files + request.setResponseCode(http.CREATED) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From 02a543515f459e2fb8ab92a2b8d96bd21ae216dc Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:55:14 -0600 Subject: [PATCH 0181/1054] replciate Tahoe's behavior for replace=true/false --- src/allmydata/test/test_testing.py | 21 +++++++++++++++++++- src/allmydata/testing/web.py | 31 +++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index b5bbacad4..a995e756d 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -61,7 +61,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri", content) + resp = yield self.http_client.put("http://example.com/uri?replace=true", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() @@ -79,3 +79,22 @@ class FakeWebTest(TestCase): do_test(), succeeded(Always()), ) + + @inlineCallbacks + def test_duplicate_upload(self): + """ + Upload the same content (via 'PUT /uri') twice with no overwrite + """ + + content = "fake content\n" * 200 + + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) + + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) + + # this one fails: same content but no ?replace=true + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 409) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5ab870f4f..ef7cd5bed 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,6 +30,9 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) +from twisted.python.failure import ( + Failure, +) from treq.client import ( HTTPClient, @@ -44,6 +47,13 @@ import allmydata.uri from allmydata.util import ( base32, ) +from allmydata.interfaces import ( + ExistingChildError, +) +from allmydata.web.common import ( + humanize_failure, +) + __all__ = ( "create_fake_tahoe_root", @@ -147,7 +157,7 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self._capability_generators[kind]) return capability - def add_data(self, kind, data): + def add_data(self, kind, data, allow_duplicate=False): """ adds some data to our grid @@ -156,9 +166,18 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - cap = self._generate_capability(kind) if self._data is None: self._data = dict() + + for k in self._data: + if self._data[k] == data: + if allow_duplicate: + return k + raise ValueError( + "Duplicate data" + ) + + cap = self._generate_capability(kind) if cap in self._data: raise ValueError("already have '{}'".format(cap)) self._data[cap] = data @@ -167,7 +186,13 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() request.setResponseCode(http.CREATED) # real code does this for brand-new files - return self.add_data("URI:CHK:", data) + replace = request.args.get("replace", None) + try: + return self.add_data("URI:CHK:", data, allow_duplicate=replace) + except ValueError: + msg, code = humanize_failure(Failure(ExistingChildError())) + request.setResponseCode(code) + return msg def render_POST(self, request): t = request.args[u"t"][0] From d221787541940dbe0e70f24cd8def42188666e8f Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:03:19 -0600 Subject: [PATCH 0182/1054] better docstring --- src/allmydata/testing/web.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ef7cd5bed..5f011f8b1 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -225,6 +225,14 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ + If you wish to pre-populate data into the fake Tahoe grid, retain + a reference to this root and pass it to + `create_tahoe_treq_client`. For example:: + + root = create_fake_tahoe_root() + root.add_data(...) + client = create_tahoe_treq_client(root) + :returns: an IResource instance that will handle certain Tahoe URI endpoints similar to a real Tahoe server. """ From caf77be8ae5b858cf4c7eaf54568baeec29db9fd Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:05:27 -0600 Subject: [PATCH 0183/1054] different words for news --- newsfragments/3317.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature index 5ed00acf5..2a7048397 100644 --- a/newsfragments/3317.feature +++ b/newsfragments/3317.feature @@ -1 +1 @@ -Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file +allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. \ No newline at end of file From 939f60f6207b6bf13e42fa37893a181ed6168288 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:22:17 -0600 Subject: [PATCH 0184/1054] embetter docstring --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5f011f8b1..ca010afa4 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -226,11 +226,11 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ If you wish to pre-populate data into the fake Tahoe grid, retain - a reference to this root and pass it to + a reference to this root by creating it yourself and passing it to `create_tahoe_treq_client`. For example:: root = create_fake_tahoe_root() - root.add_data(...) + cap_string = root.add_data(...) client = create_tahoe_treq_client(root) :returns: an IResource instance that will handle certain Tahoe URI From 6a37f32eeff5020e3a96146433f602c0e532a3be Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:28:28 -0600 Subject: [PATCH 0185/1054] use attr.s --- src/allmydata/testing/web.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ca010afa4..7c6fc1996 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -133,6 +133,7 @@ def capability_generator(kind): yield cap.encode("ascii") +@attr.s class _FakeTahoeUriHandler(Resource, object): """ An in-memory fake of (some of) the `/uri` endpoint of a Tahoe @@ -140,8 +141,9 @@ class _FakeTahoeUriHandler(Resource, object): """ isLeaf = True - _data = None - _capability_generators = None + + data = attr.ib(default=attr.Factory(dict)) + capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): """ @@ -149,12 +151,9 @@ class _FakeTahoeUriHandler(Resource, object): :returns: the next capability-string for the given kind """ - if self._capability_generators is None: - self._capability_generators = dict() - - if kind not in self._capability_generators: - self._capability_generators[kind] = capability_generator(kind) - capability = next(self._capability_generators[kind]) + if kind not in self.capability_generators: + self.capability_generators[kind] = capability_generator(kind) + capability = next(self.capability_generators[kind]) return capability def add_data(self, kind, data, allow_duplicate=False): @@ -166,11 +165,8 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - if self._data is None: - self._data = dict() - - for k in self._data: - if self._data[k] == data: + for k in self.data: + if self.data[k] == data: if allow_duplicate: return k raise ValueError( @@ -178,9 +174,9 @@ class _FakeTahoeUriHandler(Resource, object): ) cap = self._generate_capability(kind) - if cap in self._data: + if cap in self.data: raise ValueError("already have '{}'".format(cap)) - self._data[cap] = data + self.data[cap] = data return cap def render_PUT(self, request): @@ -217,10 +213,10 @@ class _FakeTahoeUriHandler(Resource, object): ) ) - if self._data is None or capability not in self._data: + if capability not in self.data: return u"No data for '{}'".format(capability).decode("ascii") - return self._data[capability] + return self.data[capability] def create_fake_tahoe_root(): From bd20f8d6f4ea89b178ab8389af1a86cb0288b4c7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 16 Jun 2020 12:10:44 -0400 Subject: [PATCH 0186/1054] Update newsfragment text --- newsfragments/3312.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix index ed00922a6..9939fe1f0 100644 --- a/newsfragments/3312.bugfix +++ b/newsfragments/3312.bugfix @@ -1 +1 @@ -Add your info here \ No newline at end of file +Make directory page links work. From 55170dcd2b2191a31b9a5ff101a1e180128ffd1f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 13:49:34 -0400 Subject: [PATCH 0187/1054] Rearrange imports --- src/allmydata/web/check_results.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 7a9badad4..8325a7e8c 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -1,12 +1,30 @@ import time import json -from nevow import rend, inevow, tags as T -from twisted.web import http, html -from allmydata.web.common import getxmlfile, get_arg, get_root, WebError +from nevow import ( + rend, + inevow, + tags as T, +) +from twisted.web import ( + http, + html, +) +from allmydata.web.common import ( + getxmlfile, + get_arg, + get_root, + WebError, +) from allmydata.web.operations import ReloadMixin -from allmydata.interfaces import ICheckAndRepairResults, ICheckResults -from allmydata.util import base32, dictutil +from allmydata.interfaces import ( + ICheckAndRepairResults, + ICheckResults, +) +from allmydata.util import ( + base32, + dictutil, +) def json_check_counts(r): From 950d111793bbc5a48a804d08eb2b145617bfa360 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 13:57:38 -0400 Subject: [PATCH 0188/1054] Use POST target paths without pre-existing query arguments `action="."` on `/uri/URI:DIR2:$DIRCAP` will render as `/uri/`, which is not the result we want. --- src/allmydata/web/info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index fa62ea23d..7eccfea9e 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -261,7 +261,7 @@ class MoreInfoElement(Element): @renderer def deep_check_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_check = T.form(action=".", method="post", + deep_check = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-check"), @@ -287,7 +287,7 @@ class MoreInfoElement(Element): @renderer def deep_size_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_size = T.form(action=".", method="post", + deep_size = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-size"), @@ -300,7 +300,7 @@ class MoreInfoElement(Element): @renderer def deep_stats_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - deep_stats = T.form(action=".", method="post", + deep_stats = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-deep-stats"), @@ -313,7 +313,7 @@ class MoreInfoElement(Element): @renderer def manifest_form(self, req, tag): ophandle = base32.b2a(os.urandom(16)) - manifest = T.form(action=".", method="post", + manifest = T.form(action=req.path, method="post", enctype="multipart/form-data")( T.fieldset( T.input(type="hidden", name="t", value="start-manifest"), From ebf23a4f0e76c0d61276686a32fe31b838857738 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 16:44:29 -0400 Subject: [PATCH 0189/1054] Use twisted web tags in literal check results template --- src/allmydata/web/literal-check-results.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/literal-check-results.xhtml b/src/allmydata/web/literal-check-results.xhtml index 95b4b0cf7..35723417a 100644 --- a/src/allmydata/web/literal-check-results.xhtml +++ b/src/allmydata/web/literal-check-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -11,7 +11,7 @@
                Literal files are always healthy: their data is contained in the URI
                -
                +
                From 3901559e346ca42f98078355de27d48b05c09de5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 16:51:37 -0400 Subject: [PATCH 0190/1054] Make LiteralCheckResultsRenderer a MultiFormatResource --- src/allmydata/web/check_results.py | 40 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 8325a7e8c..2a250efe1 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -10,11 +10,20 @@ from twisted.web import ( http, html, ) +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) from allmydata.web.common import ( getxmlfile, get_arg, get_root, WebError, + MultiFormatResource, ) from allmydata.web.operations import ReloadMixin from allmydata.interfaces import ( @@ -199,30 +208,37 @@ class ResultsBase(object): target = target + "?output=%s" % output return T.a(href=target)[si_s] -class LiteralCheckResultsRenderer(rend.Page, ResultsBase): - docFactory = getxmlfile("literal-check-results.xhtml") + +class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): def __init__(self, client): + super(LiteralCheckResultsRenderer, self).__init__() self.client = client - rend.Page.__init__(self, client) - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + return renderElement(req, LiteralCheckResultsElement()) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") data = json_check_results(None) return json.dumps(data, indent=1) + "\n" - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + +class LiteralCheckResultsElement(Element): + + loader = XMLFile(FilePath(__file__).sibling("literal-check-results.xhtml")) + + def __init__(self): + super(LiteralCheckResultsElement, self).__init__() + + @renderer + def return(self, req, tag): return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file."]] + return tags.div(tags.a("Return to file.", href=return_to)) return "" + class CheckerBase(object): def renderHTTP(self, ctx): From c15001269ffad7a3e12e1893ebbc5ad06985fa1a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 16:52:09 -0400 Subject: [PATCH 0191/1054] Avoid using a Python keyword as a renderer name --- src/allmydata/web/check_results.py | 2 +- src/allmydata/web/literal-check-results.xhtml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 2a250efe1..d3c868298 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -232,7 +232,7 @@ class LiteralCheckResultsElement(Element): super(LiteralCheckResultsElement, self).__init__() @renderer - def return(self, req, tag): + def return_to(self, req, tag): return_to = get_arg(req, "return_to", None) if return_to: return tags.div(tags.a("Return to file.", href=return_to)) diff --git a/src/allmydata/web/literal-check-results.xhtml b/src/allmydata/web/literal-check-results.xhtml index 35723417a..f82ddf8d3 100644 --- a/src/allmydata/web/literal-check-results.xhtml +++ b/src/allmydata/web/literal-check-results.xhtml @@ -11,7 +11,7 @@
                Literal files are always healthy: their data is contained in the URI
                -
                +
                From cbc414725c169e40eda4612ebf08d1fdb9d20472 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 3 Jun 2020 14:06:55 -0400 Subject: [PATCH 0192/1054] Use "output" as format argument Nearly everywhere else the query argument for output format is "t"; but here "t" is used for operations, and "output" is used for output format. The test suite will fail without this. --- src/allmydata/web/check_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d3c868298..bfe785d41 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -211,6 +211,8 @@ class ResultsBase(object): class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): + formatArgument = "output" + def __init__(self, client): super(LiteralCheckResultsRenderer, self).__init__() self.client = client From 9d104f5893343e208f1c0cb57969161f3842bf06 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 3 Jun 2020 17:57:27 -0400 Subject: [PATCH 0193/1054] Use twisted tags in check results template file --- src/allmydata/web/check-results.xhtml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/check-results.xhtml b/src/allmydata/web/check-results.xhtml index 5367552fc..2c9f5c283 100644 --- a/src/allmydata/web/check-results.xhtml +++ b/src/allmydata/web/check-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -7,17 +7,17 @@ -

                File Check Results for SI=

                +

                File Check Results for SI=

                - +
                -
                +
                -
                +
                -
                +
                From fea4e516f3485465024e16b8a552b7d32e0e6e3b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 3 Jun 2020 17:56:22 -0400 Subject: [PATCH 0194/1054] Make CheckResultsRenderer a MultiFormatResource --- src/allmydata/web/check_results.py | 77 +++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index bfe785d41..94b55d52d 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -243,49 +243,77 @@ class LiteralCheckResultsElement(Element): class CheckerBase(object): - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + # def renderHTTP(self, ctx): + # if self.want_json(ctx): + # return self.json(ctx) + # return rend.Page.renderHTTP(self, ctx) - def render_storage_index(self, ctx, data): + @renderer + def storage_index(self, req, tag): return self.r.get_storage_index_string() - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def return_to(self, req, tag): + # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" -class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase): - docFactory = getxmlfile("check-results.xhtml") + +class CheckResultsRenderer(MultiFormatResource, ResultsBase): + + formatArgument = "output" def __init__(self, client, results): + """ + TODO: update these with the correct values. + :param allmydata.test.no_network._NoNetworkClient client: + :param allmydata.check_results.CheckResults results: + """ + super(CheckResultsRenderer, self).__init__() self.client = client self.r = ICheckResults(results) - rend.Page.__init__(self, results) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_HTML(self, req): + return renderElement(req, CheckResultsRendererElement(self.r)) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") data = json_check_results(self.r) return json.dumps(data, indent=1) + "\n" - def render_summary(self, ctx, data): + +class CheckResultsRendererElement(Element, CheckerBase): + + loader = XMLFile(FilePath(__file__).sibling("check-results.xhtml")) + + def __init__(self, r): + super(CheckResultsRendererElement, self).__init__() + # TODO: use a better name + self.r = r + + @renderer + def summary(self, req, tag): results = [] - if data.is_healthy(): + # XXX: how did `data` get populated? + if self.r.is_healthy(): results.append("Healthy") - elif data.is_recoverable(): + elif self.r.is_recoverable(): results.append("Not Healthy!") else: results.append("Not Recoverable!") results.append(" : ") - results.append(self._html(data.get_summary())) - return ctx.tag[results] + # TODO: what is self._html? + # results.append(self._html(self.r.get_summary())) + results.append(self.r.get_summary()) + return tag(results) - def render_repair(self, ctx, data): - if data.is_healthy(): + @renderer + def repair(self, req, tag): + if self.r.is_healthy(): return "" + #repair = T.form(action=".", method="post", # enctype="multipart/form-data")[ # T.fieldset[ @@ -294,12 +322,15 @@ class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase): # T.input(type="submit", value="Repair"), # ]] #return ctx.tag[repair] + return "" # repair button disabled until we make it work correctly, # see #622 for details - def render_results(self, ctx, data): - cr = self._render_results(ctx, data) - return ctx.tag[cr] + @renderer + def results(self, req, tag): + cr = self._render_results(req, self.r) + return tag(cr) + class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): docFactory = getxmlfile("check-and-repair-results.xhtml") From 8e37bb1ebbd0048791f72f5d04275bc289a6b3d7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 3 Jun 2020 21:42:20 -0400 Subject: [PATCH 0195/1054] Make ResultsBase a base class of CheckResultsRendererElement --- src/allmydata/web/check_results.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 94b55d52d..c6a3d25ea 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -261,7 +261,7 @@ class CheckerBase(object): return "" -class CheckResultsRenderer(MultiFormatResource, ResultsBase): +class CheckResultsRenderer(MultiFormatResource): formatArgument = "output" @@ -276,7 +276,7 @@ class CheckResultsRenderer(MultiFormatResource, ResultsBase): self.r = ICheckResults(results) def render_HTML(self, req): - return renderElement(req, CheckResultsRendererElement(self.r)) + return renderElement(req, CheckResultsRendererElement(self.client, self.r)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") @@ -284,12 +284,13 @@ class CheckResultsRenderer(MultiFormatResource, ResultsBase): return json.dumps(data, indent=1) + "\n" -class CheckResultsRendererElement(Element, CheckerBase): +class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): loader = XMLFile(FilePath(__file__).sibling("check-results.xhtml")) - def __init__(self, r): + def __init__(self, client, r): super(CheckResultsRendererElement, self).__init__() + self.client = client # TODO: use a better name self.r = r From d2540b89d370f1144e088734da593967fc3fb970 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 3 Jun 2020 21:43:15 -0400 Subject: [PATCH 0196/1054] Use twisted web tags in ResultsBase --- src/allmydata/web/check_results.py | 71 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index c6a3d25ea..0a56a55e8 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -107,32 +107,35 @@ class ResultsBase(object): sb = c.get_storage_broker() r = [] def add(name, value): - r.append(T.li[name + ": ", value]) + r.append(tags.li(name + ": ", value)) + + add("Report", tags.pre("\n".join(self._html(cr.get_report())))) - add("Report", T.pre["\n".join(self._html(cr.get_report()))]) add("Share Counts", "need %d-of-%d, have %d" % (cr.get_encoding_needed(), cr.get_encoding_expected(), cr.get_share_counter_good())) - add("Happiness Level", cr.get_happiness()) - add("Hosts with good shares", cr.get_host_counter_good_shares()) + add("Happiness Level", str(cr.get_happiness())) + add("Hosts with good shares", str(cr.get_host_counter_good_shares())) + + rrr = cr.get_corrupt_shares() if cr.get_corrupt_shares(): badsharemap = [] for (s, si, shnum) in cr.get_corrupt_shares(): - d = T.tr[T.td["sh#%d" % shnum], - T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]], - ] + d = tags.tr(tags.td("sh#%d" % shnum), + tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid")),) badsharemap.append(d) - add("Corrupt shares", T.table()[ - T.tr[T.th["Share ID"], - T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], - badsharemap]) + add("Corrupt shares", + tags.table( + tags.tr(tags.th("Share ID"), + tags.th((tags.div("Nickname"), tags.div("Node ID", class_="nodeid")), class_="nickname-and-peerid")), + badsharemap)) else: add("Corrupt shares", "none") - add("Wrong Shares", cr.get_share_counter_wrong()) + add("Wrong Shares", str(cr.get_share_counter_wrong())) sharemap_data = [] shares_on_server = dictutil.DictOfSets() @@ -147,18 +150,19 @@ class ResultsBase(object): shareid_s = "" if i == 0: shareid_s = shareid - d = T.tr[T.td[shareid_s], - T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]] - ] + d = tags.tr(tags.td(shareid_s), + tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid"))) sharemap_data.append(d) + add("Good Shares (sorted in share order)", - T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], - sharemap_data]) + tags.table(tags.tr(tags.th("Share ID"), + tags.th(tags.div("Nickname"), + tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid")), + sharemap_data)) - - add("Recoverable Versions", cr.get_version_counter_recoverable()) - add("Unrecoverable Versions", cr.get_version_counter_unrecoverable()) + add("Recoverable Versions", str(cr.get_version_counter_recoverable())) + add("Unrecoverable Versions", str(cr.get_version_counter_unrecoverable())) # this table is sorted by permuted order permuted_servers = [s @@ -171,20 +175,23 @@ class ResultsBase(object): for s in permuted_servers: shareids = list(shares_on_server.get(s, [])) shareids.reverse() - shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ] - d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()], - T.div(class_="nodeid")[T.tt[s.get_name()]]], - T.td[shareids_s], - ] + shareids_s = [tags.tt(shareid, " ") for shareid in sorted(shareids)] + + d = tags.tr(tags.td(tags.div(s.get_nickname(), class_="nickname"), + tags.div(tags.tt(s.get_name()), class_="nodeid")), + tags.td(shareids_s), ) servermap.append(d) num_shares_left -= len(shareids) if not num_shares_left: break - add("Share Balancing (servers in permuted order)", - T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]], - servermap]) - return T.ul[r] + add("Share Balancing (servers in permuted order)", + tags.table(tags.tr(tags.th(tags.div("Nickname"), + tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid"), + tags.th("Share IDs")), + servermap)) + + return tags.ul(r) def _html(self, s): if isinstance(s, (str, unicode)): @@ -206,7 +213,7 @@ class ResultsBase(object): output = get_arg(ctx, "output") if output: target = target + "?output=%s" % output - return T.a(href=target)[si_s] + return tags.a(si_s, href=target) class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): From 6f1601aeebcd247f643ff26e39de3ea696129b4f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 16:18:04 -0400 Subject: [PATCH 0197/1054] Stringify numbers before wrapping them up in tags --- src/allmydata/web/check_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 0a56a55e8..d798fac4f 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -149,7 +149,7 @@ class ResultsBase(object): shares_on_server.add(s, shareid) shareid_s = "" if i == 0: - shareid_s = shareid + shareid_s = str(shareid) d = tags.tr(tags.td(shareid_s), tags.td(tags.div(s.get_nickname(), class_="nickname"), tags.div(tags.tt(s.get_name()), class_="nodeid"))) @@ -175,7 +175,7 @@ class ResultsBase(object): for s in permuted_servers: shareids = list(shares_on_server.get(s, [])) shareids.reverse() - shareids_s = [tags.tt(shareid, " ") for shareid in sorted(shareids)] + shareids_s = [tags.tt(str(shareid), " ") for shareid in sorted(shareids)] d = tags.tr(tags.td(tags.div(s.get_nickname(), class_="nickname"), tags.div(tags.tt(s.get_name()), class_="nodeid")), From 80e9f48551df94f7e69f81a7e175181c6fbf5f2d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 16:18:27 -0400 Subject: [PATCH 0198/1054] Add line breaks in a comment --- src/allmydata/web/check_results.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d798fac4f..a6f65d52b 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -140,7 +140,10 @@ class ResultsBase(object): sharemap_data = [] shares_on_server = dictutil.DictOfSets() - # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined. + # FIXME: The two tables below contain nickname-and-nodeid + # table column markup which is duplicated with each other, + # introducer.xhtml, and deep-check-results.xhtml. All of these + # (and any other presentations of nickname-and-nodeid) should be combined. for shareid in sorted(cr.get_sharemap().keys()): servers = sorted(cr.get_sharemap()[shareid], From d9005fcccc4cb4ab290d241bb8ef3d1481258a13 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 16:39:13 -0400 Subject: [PATCH 0199/1054] Use twisted tags in check and repair results template --- src/allmydata/web/check-and-repair-results.xhtml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/check-and-repair-results.xhtml b/src/allmydata/web/check-and-repair-results.xhtml index 34eb31787..d2c9c3a47 100644 --- a/src/allmydata/web/check-and-repair-results.xhtml +++ b/src/allmydata/web/check-and-repair-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Check Results @@ -7,17 +7,17 @@ -

                File Check-And-Repair Results for SI=

                +

                File Check-And-Repair Results for SI=

                -
                +
                -
                +
                -
                +
                -
                +
                -
                +
                From 0eea108123a33acc92ac95d92f2a25b93795cb01 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 16:41:30 -0400 Subject: [PATCH 0200/1054] Make CheckAndRepairResultsRenderer a MultiFormatResource --- src/allmydata/web/check_results.py | 62 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index a6f65d52b..3f272f991 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -343,23 +343,40 @@ class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): return tag(cr) -class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): - docFactory = getxmlfile("check-and-repair-results.xhtml") +class CheckAndRepairResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, results): + # TODO: document params + super(CheckAndRepairResultsRenderer, self).__init__() self.client = client + # TODO: use a better name self.r = None if results: self.r = ICheckAndRepairResults(results) - rend.Page.__init__(self, results) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + def render_HTML(self, req): + elem = CheckAndRepairResultsRendererElement(self.r) + return renderElement(req, elem) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") data = json_check_and_repair_results(self.r) return json.dumps(data, indent=1) + "\n" - def render_summary(self, ctx, data): - cr = data.get_post_repair_results() + +class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): + + loader = XMLFile(FilePath(__file__).sibling("check-and-repair-results.xhtml")) + + def __init__(self, r): + super(CheckAndRepairResultsRendererElement, self).__init__() + self.r = r + + @renderer + def summary(self, req, tag): + cr = self.r.get_post_repair_results() results = [] if cr.is_healthy(): results.append("Healthy") @@ -369,24 +386,27 @@ class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): results.append("Not Recoverable!") results.append(" : ") results.append(self._html(cr.get_summary())) - return ctx.tag[results] + return tag(results) - def render_repair_results(self, ctx, data): - if data.get_repair_attempted(): - if data.get_repair_successful(): - return ctx.tag["Repair successful"] + @renderer + def repair_results(self, req, tag): + if self.r.get_repair_attempted(): + if self.r.get_repair_successful(): + return tag("Repair successful") else: - return ctx.tag["Repair unsuccessful"] - return ctx.tag["No repair necessary"] + return tag("Repair unsuccessful") + return tag("No repair necessary") - def render_post_repair_results(self, ctx, data): - cr = self._render_results(ctx, data.get_post_repair_results()) - return ctx.tag[T.div["Post-Repair Checker Results:"], cr] + @renderer + def post_repair_results(self, req, tag): + cr = self._render_results(req, self.r.get_post_repair_results()) + return tag(tags.div("Post-Repair Checker Results:"), cr) - def render_maybe_pre_repair_results(self, ctx, data): - if data.get_repair_attempted(): - cr = self._render_results(ctx, data.get_pre_repair_results()) - return ctx.tag[T.div["Pre-Repair Checker Results:"], cr] + @renderer + def maybe_pre_repair_results(self, req, tag): + if self.r.get_repair_attempted(): + cr = self._render_results(req, self.r.get_pre_repair_results()) + return tag(tags.div("Pre-Repair Checker Results:"), cr) return "" From 9dcaa104c5ea121859585da17ef6ffe76e1478fd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 17:03:51 -0400 Subject: [PATCH 0201/1054] Use twisted tags in deep check results template --- src/allmydata/web/deep-check-results.xhtml | 71 +++++++++++----------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index cb2330b7d..f9770eb17 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -1,87 +1,86 @@ - + Tahoe-LAFS - Deep Check Results - + -

                Deep-Check Results for root SI=

                +

                Deep-Check Results for root SI=

                -

                +

                Counters:

                  -
                • Objects Checked:
                  • -
                • Objects Healthy:
                  • -
                • Objects Unhealthy:
                  • -
                • Objects Unrecoverable:
                  • -
                • Corrupt Shares:
                  • - +
                • Objects Checked:
                  • +
                • Objects Healthy:
                  • +
                • Objects Unhealthy:
                  • +
                • Objects Unrecoverable:
                  • +
                • Corrupt Shares:
                -
                +

                Files/Directories That Had Problems:

                -
                  -
                • -
                • None
                  • +
                    +
                  • +
                  • None
                -
                +

                Servers on which corrupt shares were found

                -
                  -
                • -
                • None
                  • +
                    +
                  • +
                  • None
                -
                +

                Corrupt Shares

                If repair fails, these shares need to be manually inspected and removed.

                - - +
                + - - - - - + + + + +
                Server Server Nickname Storage Index Share Number
                -
                +

                All Results

                - - +
                + - - - - - - + + + + + +
                Relative Path Healthy Recoverable Storage Index Summary
                -
                +
                From c33c0a884de10b61ffbf78e0fec0034c0933b9c6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 17:46:06 -0400 Subject: [PATCH 0202/1054] Make DeepCheckResultsRenderer a MultiFormatResource --- src/allmydata/web/check_results.py | 148 +++++++++++++++++++---------- 1 file changed, 99 insertions(+), 49 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 3f272f991..4199b2c47 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -410,14 +410,17 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): return "" -class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): - docFactory = getxmlfile("deep-check-results.xhtml") +class DeepCheckResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, monitor): + # TODO: document params + super(DeepCheckResultsRenderer, self).__init__() self.client = client self.monitor = monitor - def childFactory(self, ctx, name): + def getChild(self, name, req): if not name: return self # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information @@ -431,13 +434,18 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): raise WebError("No detailed results for SI %s" % html.escape(name), http.NOT_FOUND) - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + elem = DeepCheckResultsRendererElement(self.monitor) + return renderElement(req, elem) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + # def renderHTTP(self, ctx): + # if self.want_json(ctx): + # return self.json(ctx) + # return rend.Page.renderHTTP(self, ctx) + + def render_JSON(self, req): + # inevow.IRequest(ctx).setHeader("content-type", "text/plain") + req.setHeader("content-type", "text/plain") data = {} data["finished"] = self.monitor.is_finished() res = self.monitor.get_status() @@ -459,28 +467,49 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): data["stats"] = res.get_stats() return json.dumps(data, indent=1) + "\n" - def render_root_storage_index(self, ctx, data): + +class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): + + loader = XMLFile(FilePath(__file__).sibling("deep-check-results.xhtml")) + + def __init__(self, monitor): + super(DeepCheckResultsRendererElement, self).__init__() + self.monitor = monitor + + @renderer + def root_storage_index(self, req, tag): return self.monitor.get_status().get_root_storage_index_string() - def data_objects_checked(self, ctx, data): + @renderer + def objects_checked(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-checked"] - def data_objects_healthy(self, ctx, data): + + @renderer + def objects_healthy(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-healthy"] - def data_objects_unhealthy(self, ctx, data): + + @renderer + def objects_unhealthy(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-unhealthy"] - def data_objects_unrecoverable(self, ctx, data): + + @renderer + def objects_unrecoverable(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-unrecoverable"] - def data_count_corrupt_shares(self, ctx, data): + @renderer + def count_corrupt_shares(self, req, tag): return self.monitor.get_status().get_counters()["count-corrupt-shares"] - def render_problems_p(self, ctx, data): + @renderer + def problems_p(self, req, tag): c = self.monitor.get_status().get_counters() if c["count-objects-unhealthy"]: - return ctx.tag + return tag return "" - def data_problems(self, ctx, data): + # TODO: use SlotsSequenceElement to render this. + @renderer + def problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() for path in sorted(all_objects.keys()): cr = all_objects[path] @@ -488,78 +517,99 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): if not cr.is_healthy(): yield path, cr - def render_problem(self, ctx, data): + @renderer + def render_problem(self, req, tag): + # TODO: deal with this. path, cr = data summary_text = "" summary = cr.get_summary() if summary: summary_text = ": " + summary summary_text += " [SI: %s]" % cr.get_storage_index_string() - return ctx.tag[self._join_pathstring(path), self._html(summary_text)] + return tag(self._join_pathstring(path), self._html(summary_text)) - - def render_servers_with_corrupt_shares_p(self, ctx, data): + @renderer + def servers_with_corrupt_shares_p(self, req, tag): if self.monitor.get_status().get_counters()["count-corrupt-shares"]: - return ctx.tag + return tag return "" - def data_servers_with_corrupt_shares(self, ctx, data): + # TODO: use SlotsSequenceElement to render this. + @renderer + def servers_with_corrupt_shares(self, ctx, data): servers = [s for (s, storage_index, sharenum) in self.monitor.get_status().get_corrupt_shares()] servers.sort(key=lambda s: s.get_longname()) return servers - def render_server_problem(self, ctx, server): + @renderer + def server_problem(self, req, tag): + # def server_problem(self, ctx, server): + # TODO: where do `server` come from now? data = [server.get_name()] nickname = server.get_nickname() if nickname: data.append(" (%s)" % self._html(nickname)) - return ctx.tag[data] + return tag(data) - - def render_corrupt_shares_p(self, ctx, data): + @renderer + def corrupt_shares_p(self, req, tag): if self.monitor.get_status().get_counters()["count-corrupt-shares"]: - return ctx.tag + return tag return "" - def data_corrupt_shares(self, ctx, data): + + # TODO: Probably should use SlotsSequenceElement to render this. + @renderer + def corrupt_shares(self, req, tag): return self.monitor.get_status().get_corrupt_shares() - def render_share_problem(self, ctx, data): + + @renderer + def share_problem(self, req, tag): + # def render_share_problem(self, req, tag): server, storage_index, sharenum = data nickname = server.get_nickname() - ctx.fillSlots("serverid", server.get_name()) + tag.fillSlots("serverid", server.get_name()) if nickname: - ctx.fillSlots("nickname", self._html(nickname)) - ctx.fillSlots("si", self._render_si_link(ctx, storage_index)) - ctx.fillSlots("shnum", str(sharenum)) - return ctx.tag + tag.fillSlots("nickname", self._html(nickname)) + tag.fillSlots("si", self._render_si_link(ctx, storage_index)) + tag.fillSlots("shnum", str(sharenum)) + return tag - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def return_to(self, req, tag): + # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" + # TODO: use SlotsSequenceElement to render this. + @renderer def data_all_objects(self, ctx, data): r = self.monitor.get_status().get_all_results() for path in sorted(r.keys()): yield (path, r[path]) - def render_object(self, ctx, data): + @renderer + def object(self, ctx, data): + # def render_object(self, ctx, data): + # TODO: figure `data` out path, r = data - ctx.fillSlots("path", self._join_pathstring(path)) - ctx.fillSlots("healthy", str(r.is_healthy())) - ctx.fillSlots("recoverable", str(r.is_recoverable())) + tag.fillSlots("path", self._join_pathstring(path)) + tag.fillSlots("healthy", str(r.is_healthy())) + tag.fillSlots("recoverable", str(r.is_recoverable())) storage_index = r.get_storage_index() - ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index)) - ctx.fillSlots("summary", self._html(r.get_summary())) - return ctx.tag + tag.fillSlots("storage_index", self._render_si_link(req, storage_index)) + tag.fillSlots("summary", self._html(r.get_summary())) + return tag - def render_runtime(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def runtime(self, req, tag): + # req = inevow.IRequest(ctx) runtime = time.time() - req.processing_started_timestamp - return ctx.tag["runtime: %s seconds" % runtime] + return tag("runtime: %s seconds" % runtime) + class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): docFactory = getxmlfile("deep-check-and-repair-results.xhtml") From 27e5ce090aa7bb29aa05a2da212eba299022bb0c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 17:46:34 -0400 Subject: [PATCH 0203/1054] Update deep check results template --- src/allmydata/web/deep-check-results.xhtml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index f9770eb17..11a2e7f4b 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -24,7 +24,7 @@

                Files/Directories That Had Problems:

                -
                  +
                  • None
                  @@ -33,7 +33,7 @@

                  Servers on which corrupt shares were found

                  -
                    +
                    • None
                    @@ -42,7 +42,7 @@

                    Corrupt Shares

                    If repair fails, these shares need to be manually inspected and removed.

                    - +
                    @@ -58,11 +58,11 @@
                    Server Server Nickname
                    -
                    +

                    All Results

                    - +
                    From 842b2b6be511afd5aff6a6b8204e787a68c7162e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 18:03:33 -0400 Subject: [PATCH 0204/1054] Use twisted tags in "deep check and repair results" template --- .../web/deep-check-and-repair-results.xhtml | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index a7db34837..366dcf346 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -1,76 +1,76 @@ - + Tahoe-LAFS - Deep Check Results - +

                    Deep-Check-And-Repair Results for root - SI=

                    + SI= -

                    +

                    Counters:

                      -
                    • Objects Checked:
                      • +
                    • Objects Checked:
                      • -
                    • Objects Healthy (before repair):
                      • -
                    • Objects Unhealthy (before repair):
                      • -
                    • Corrupt Shares (before repair):
                      • +
                    • Objects Healthy (before repair):
                      • +
                    • Objects Unhealthy (before repair):
                      • +
                    • Corrupt Shares (before repair):
                      • -
                    • Repairs Attempted:
                      • -
                    • Repairs Successful:
                      • -
                    • Repairs Unsuccessful:
                      • +
                    • Repairs Attempted:
                      • +
                    • Repairs Successful:
                      • +
                    • Repairs Unsuccessful:
                      • -
                    • Objects Healthy (after repair):
                      • -
                    • Objects Unhealthy (after repair):
                      • -
                    • Corrupt Shares (after repair):
                      • +
                    • Objects Healthy (after repair):
                      • +
                    • Objects Unhealthy (after repair):
                      • +
                    • Corrupt Shares (after repair):
                    -
                    +

                    Files/Directories That Had Problems:

                    -
                      -
                    • -
                    • None
                      • +
                        +
                      • +
                      • None
                    -
                    +

                    Files/Directories That Still Have Problems:

                    -
                      -
                    • -
                    • None
                      • +
                        +
                      • +
                      • None
                    -
                    +

                    Servers on which corrupt shares were found

                    -
                      -
                    • -
                    • None
                      • +
                        +
                      • +
                      • None
                    -
                    +

                    Remaining Corrupt Shares

                    These shares need to be manually inspected and removed.

                    -
                      -
                    • -
                    • None
                      • +
                        +
                      • +
                      • None
                    -
                    +
                    -

                    Relative Path Healthy
                    - +
                    + @@ -78,18 +78,18 @@ - - - - - - - + + + + + + +
                    Relative Path Healthy Pre-Repair Recoverable Pre-RepairStorage Index Summary
                    -
                    +
                    From e4efcdd925682486039bd23bfd2e83d7214f747e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 18:17:33 -0400 Subject: [PATCH 0205/1054] Make DeepCheckAndRepairResultsRenderer a MultiFormatResource --- src/allmydata/web/check_results.py | 168 +++++++++++++++++++---------- 1 file changed, 111 insertions(+), 57 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 4199b2c47..19cfefe87 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -586,7 +586,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): # TODO: use SlotsSequenceElement to render this. @renderer - def data_all_objects(self, ctx, data): + def all_objects(self, ctx, data): r = self.monitor.get_status().get_all_results() for path in sorted(r.keys()): yield (path, r[path]) @@ -611,14 +611,15 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return tag("runtime: %s seconds" % runtime) -class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): - docFactory = getxmlfile("deep-check-and-repair-results.xhtml") +class DeepCheckAndRepairResultsRenderer(MultiFormatResource): + + formatArgument = "output" def __init__(self, client, monitor): self.client = client self.monitor = monitor - def childFactory(self, ctx, name): + def getChild(self, name, req): if not name: return self # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information @@ -632,13 +633,18 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): raise WebError("No detailed results for SI %s" % html.escape(name), http.NOT_FOUND) - def renderHTTP(self, ctx): - if self.want_json(ctx): - return self.json(ctx) - return rend.Page.renderHTTP(self, ctx) + def render_HTML(self, req): + elem = DeepCheckAndRepairResultsRendererElement(self.monitor) + return renderElement(req, elem) - def json(self, ctx): - inevow.IRequest(ctx).setHeader("content-type", "text/plain") + # def renderHTTP(self, ctx): + # if self.want_json(ctx): + # return self.json(ctx) + # return rend.Page.renderHTTP(self, ctx) + + def render_JSON(self, req): + # inevow.IRequest(ctx).setHeader("content-type", "text/plain") + req.setHeader("content-type", "text/plain") res = self.monitor.get_status() data = {} data["finished"] = self.monitor.is_finished() @@ -679,40 +685,70 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): data["stats"] = res.get_stats() return json.dumps(data, indent=1) + "\n" - def render_root_storage_index(self, ctx, data): + +class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin): + + loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml")) + + def __init__(self, monitor): + # TODO: document params + super(DeepCheckAndRepairResultsRendererElement, self).__init__() + self.monitor = monitor + + @renderer + def root_storage_index(self, req, tag): return self.monitor.get_status().get_root_storage_index_string() - def data_objects_checked(self, ctx, data): + @renderer + def objects_checked(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-checked"] - def data_objects_healthy(self, ctx, data): + @renderer + def objects_healthy(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"] - def data_objects_unhealthy(self, ctx, data): + + @renderer + def objects_unhealthy(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"] - def data_corrupt_shares(self, ctx, data): + + @renderer + def corrupt_shares(self, req, tag): return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"] - def data_repairs_attempted(self, ctx, data): + @renderer + def repairs_attempted(self, req, tag): return self.monitor.get_status().get_counters()["count-repairs-attempted"] - def data_repairs_successful(self, ctx, data): + + @renderer + def repairs_successful(self, req, tag): return self.monitor.get_status().get_counters()["count-repairs-successful"] - def data_repairs_unsuccessful(self, ctx, data): + + @renderer + def repairs_unsuccessful(self, req, tag): return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"] - def data_objects_healthy_post(self, ctx, data): + @renderer + def objects_healthy_post(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"] - def data_objects_unhealthy_post(self, ctx, data): + + @renderer + def objects_unhealthy_post(self, req, tag): return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"] - def data_corrupt_shares_post(self, ctx, data): + + @renderer + def corrupt_shares_post(self, req, tag): return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"] - def render_pre_repair_problems_p(self, ctx, data): + @renderer + def pre_repair_problems_p(self, req, tag): c = self.monitor.get_status().get_counters() if c["count-objects-unhealthy-pre-repair"]: - return ctx.tag + return tag return "" - def data_pre_repair_problems(self, ctx, data): + # TODO: use SlotsSequenceElement + @renderer + def pre_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() for path in sorted(all_objects.keys()): r = all_objects[path] @@ -721,19 +757,24 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): if not cr.is_healthy(): yield path, cr - def render_problem(self, ctx, data): - path, cr = data - return ctx.tag[self._join_pathstring(path), ": ", - self._html(cr.get_summary())] + @renderer + def problem(self, req, tag): + # TODO: figure this out + # path, cr = data + return tag(self._join_pathstring(path), ": ", + self._html(cr.get_summary())) - def render_post_repair_problems_p(self, ctx, data): + @renderer + def post_repair_problems_p(self, req, tag): c = self.monitor.get_status().get_counters() if (c["count-objects-unhealthy-post-repair"] or c["count-corrupt-shares-post-repair"]): - return ctx.tag + return tag return "" - def data_post_repair_problems(self, ctx, data): + # TODO: use SlotsSequenceElement + @renderer + def post_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() for path in sorted(all_objects.keys()): r = all_objects[path] @@ -742,56 +783,69 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): if not cr.is_healthy(): yield path, cr - def render_servers_with_corrupt_shares_p(self, ctx, data): + @renderer + def servers_with_corrupt_shares_p(self, req, tag): if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]: - return ctx.tag + return tag return "" - def data_servers_with_corrupt_shares(self, ctx, data): + + @renderer + def servers_with_corrupt_shares(self, req, tag): return [] # TODO - def render_server_problem(self, ctx, data): + + @renderer + def server_problem(self, req, tag): pass - - def render_remaining_corrupt_shares_p(self, ctx, data): + @renderer + def remaining_corrupt_shares_p(self, req, tag): if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]: - return ctx.tag + return tag return "" - def data_post_repair_corrupt_shares(self, ctx, data): + + @renderer + def post_repair_corrupt_shares(self, req, tag): return [] # TODO - def render_share_problem(self, ctx, data): + @renderer + def render_share_problem(self, req, tag): pass - - def render_return(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def return_to(self, req, tag): + # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: - return T.div[T.a(href=return_to)["Return to file/directory."]] + return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" - def data_all_objects(self, ctx, data): + # TODO: use SlotsSequenceElement + @renderer + def all_objects(self, req, tag): r = self.monitor.get_status().get_all_results() for path in sorted(r.keys()): yield (path, r[path]) - def render_object(self, ctx, data): - path, r = data - ctx.fillSlots("path", self._join_pathstring(path)) - ctx.fillSlots("healthy_pre_repair", + @renderer + def object(self, req, tag): + # TODO: figure this out + # path, r = data + tag.fillSlots("path", self._join_pathstring(path)) + tag.fillSlots("healthy_pre_repair", str(r.get_pre_repair_results().is_healthy())) - ctx.fillSlots("recoverable_pre_repair", + tag.fillSlots("recoverable_pre_repair", str(r.get_pre_repair_results().is_recoverable())) - ctx.fillSlots("healthy_post_repair", + tag.fillSlots("healthy_post_repair", str(r.get_post_repair_results().is_healthy())) storage_index = r.get_storage_index() - ctx.fillSlots("storage_index", + tag.fillSlots("storage_index", self._render_si_link(ctx, storage_index)) - ctx.fillSlots("summary", + tag.fillSlots("summary", self._html(r.get_pre_repair_results().get_summary())) - return ctx.tag + return tag - def render_runtime(self, ctx, data): - req = inevow.IRequest(ctx) + @renderer + def runtime(self, req, tag): + # req = inevow.IRequest(ctx) runtime = time.time() - req.processing_started_timestamp - return ctx.tag["runtime: %s seconds" % runtime] + return tag("runtime: %s seconds" % runtime) From 19db5fb3128732ae7880dad351e42924fc1f02d0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 18:17:44 -0400 Subject: [PATCH 0206/1054] Update "deep check and repair results" template --- src/allmydata/web/deep-check-and-repair-results.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 366dcf346..60cb10fba 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -66,7 +66,7 @@
                  -
                  +
                  From 7237fef6307675b8ec0cf5d900e475e2a34b7822 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 19:04:42 -0400 Subject: [PATCH 0207/1054] Add property client to CheckAndRepairResultsRendererElement --- src/allmydata/web/check_results.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 19cfefe87..4339e5e30 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -357,7 +357,7 @@ class CheckAndRepairResultsRenderer(MultiFormatResource): self.r = ICheckAndRepairResults(results) def render_HTML(self, req): - elem = CheckAndRepairResultsRendererElement(self.r) + elem = CheckAndRepairResultsRendererElement(self.client, self.r) return renderElement(req, elem) def render_JSON(self, req): @@ -370,8 +370,9 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): loader = XMLFile(FilePath(__file__).sibling("check-and-repair-results.xhtml")) - def __init__(self, r): + def __init__(self, client, r): super(CheckAndRepairResultsRendererElement, self).__init__() + self.client = client self.r = r @renderer From 0d1269264a6044afe2ece2d5ec80ce5b12ce28ac Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 4 Jun 2020 19:14:34 -0400 Subject: [PATCH 0208/1054] Stringify numbers in DeepCheckResultsRenderer --- src/allmydata/web/check_results.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 4339e5e30..757326bf7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -477,29 +477,31 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): super(DeepCheckResultsRendererElement, self).__init__() self.monitor = monitor + # TODO: Add a helper to query monitor.get_status.get_counters() + @renderer def root_storage_index(self, req, tag): return self.monitor.get_status().get_root_storage_index_string() @renderer def objects_checked(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-checked"] + return str(self.monitor.get_status().get_counters()["count-objects-checked"]) @renderer def objects_healthy(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-healthy"] + return str(self.monitor.get_status().get_counters()["count-objects-healthy"]) @renderer def objects_unhealthy(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-unhealthy"] + return str(self.monitor.get_status().get_counters()["count-objects-unhealthy"]) @renderer def objects_unrecoverable(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-unrecoverable"] + return str(self.monitor.get_status().get_counters()["count-objects-unrecoverable"]) @renderer def count_corrupt_shares(self, req, tag): - return self.monitor.get_status().get_counters()["count-corrupt-shares"] + return str(self.monitor.get_status().get_counters()["count-corrupt-shares"]) @renderer def problems_p(self, req, tag): From 77120ba70c9c54a51a71ca41a76602f3f1cf9790 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 11 Jun 2020 12:44:48 -0400 Subject: [PATCH 0209/1054] Update tests for ResultsRenderer classes --- src/allmydata/test/test_checker.py | 98 +++++++++++++++++++----------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 5eed6f21f..bc4215ab3 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -3,6 +3,13 @@ import json import os.path, shutil from twisted.trial import unittest from twisted.internet import defer + +from nevow.inevow import IRequest +from zope.interface import implementer +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel +from twisted.web.template import flattenString + from allmydata import check_results, uri from allmydata import uri as tahoe_uri from allmydata.util import base32 @@ -23,6 +30,17 @@ class FakeClient(object): def get_storage_broker(self): return self.storage_broker +# XXX: We have to have this class because `common.get_arg()` expects a +# `nevow.inevow.IRequest`, which `twisted.web.server.Request` isn't. +# Also, the request needs to have `args` and `fields` properties so +# that `allmydata.web.common.get_arg()` won't complain. +@implementer(IRequest) +class TestRequest(Request): + def __init__(self, args=None, fields=None): + Request.__init__(self, DummyChannel()) + self.args = args or {} + self.fields = fields or {} + class WebResultsRendering(unittest.TestCase, WebRenderingMixin): def create_fake_client(self): @@ -51,34 +69,39 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): c.storage_broker = sb return c - def render_json(self, page): - d = self.render1(page, args={"output": ["json"]}) - return d + def render_json(self, resource): + return resource.render(TestRequest(args={"output": ["json"]})) + + def render_element(self, element, args=None): + d = flattenString(TestRequest(args), element) + return unittest.TestCase().successResultOf(d) def test_literal(self): - c = self.create_fake_client() - lcr = web_check_results.LiteralCheckResultsRenderer(c) + lcr = web_check_results.LiteralCheckResultsElement() - d = self.render1(lcr) + d = self.render_element(lcr) def _check(html): s = self.remove_tags(html) self.failUnlessIn("Literal files are always healthy", s) - d.addCallback(_check) - d.addCallback(lambda ignored: - self.render1(lcr, args={"return_to": ["FOOURL"]})) + _check(d) + + d = self.render_element(lcr, args={"return_to": ["FOOURL"]}) def _check_return_to(html): s = self.remove_tags(html) self.failUnlessIn("Literal files are always healthy", s) self.failUnlessIn('Return to file.', html) - d.addCallback(_check_return_to) - d.addCallback(lambda ignored: self.render_json(lcr)) + _check_return_to(d) + + c = self.create_fake_client() + lcr = web_check_results.LiteralCheckResultsRenderer(c) + + d = self.render_json(lcr) def _check_json(js): j = json.loads(js) self.failUnlessEqual(j["storage-index"], "") self.failUnlessEqual(j["results"]["healthy"], True) - d.addCallback(_check_json) - return d + _check_json(d) def test_check(self): c = self.create_fake_client() @@ -108,8 +131,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): healthy=True, recoverable=True, summary="groovy", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Healthy : groovy", s) @@ -126,8 +149,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): healthy=False, recoverable=True, summary="ungroovy", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Healthy! : ungroovy", s) @@ -138,22 +161,23 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): healthy=False, recoverable=False, summary="rather dead", **data) - w = web_check_results.CheckResultsRenderer(c, cr) - html = self.render2(w) + w = web_check_results.CheckResultsRendererElement(c, cr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) - html = self.render2(w, args={"return_to": ["FOOURL"]}) + html = self.render_element(w, args={"return_to": ["FOOURL"]}) self.failUnlessIn('Return to file/directory.', html) + w = web_check_results.CheckResultsRenderer(c, cr) d = self.render_json(w) def _check_json(jdata): j = json.loads(jdata) @@ -178,15 +202,15 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): 'recoverable': False, } self.failUnlessEqual(j["results"], expected) - d.addCallback(_check_json) - d.addCallback(lambda ignored: self.render1(w)) + _check_json(d) + + w = web_check_results.CheckResultsRendererElement(c, cr) + d = self.render_element(w) def _check(html): s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) self.failUnlessIn("Not Recoverable! : rather dead", s) - d.addCallback(_check) - return d - + _check(html) def test_check_and_repair(self): c = self.create_fake_client() @@ -244,8 +268,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): crr.post_repair_results = post_cr crr.repair_attempted = False - w = web_check_results.CheckAndRepairResultsRenderer(c, crr) - html = self.render2(w) + w = web_check_results.CheckAndRepairResultsRendererElement(c, crr) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -256,7 +280,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): crr.repair_attempted = True crr.repair_successful = True - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -271,7 +295,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): summary="better", **data) crr.post_repair_results = post_cr - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -286,7 +310,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): summary="worse", **data) crr.post_repair_results = post_cr - html = self.render2(w) + html = self.render_element(w) s = self.remove_tags(html) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) @@ -294,7 +318,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): self.failUnlessIn("Repair unsuccessful", s) self.failUnlessIn("Post-Repair Checker Results:", s) - d = self.render_json(w) + w2 = web_check_results.CheckAndRepairResultsRenderer(c, crr) + d = self.render_json(w2) def _got_json(data): j = json.loads(data) self.failUnlessEqual(j["repair-attempted"], True) @@ -302,16 +327,15 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "2k6avpjga3dho3zsjo6nnkt7n4") self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") self.failUnlessEqual(j["post-repair-results"]["summary"], "worse") - d.addCallback(_got_json) + _got_json(d) - w2 = web_check_results.CheckAndRepairResultsRenderer(c, None) - d.addCallback(lambda ignored: self.render_json(w2)) + w3 = web_check_results.CheckAndRepairResultsRenderer(c, None) + d = self.render_json(w3) def _got_lit_results(data): j = json.loads(data) self.failUnlessEqual(j["repair-attempted"], False) self.failUnlessEqual(j["storage-index"], "") - d.addCallback(_got_lit_results) - return d + _got_lit_results(d) class BalancingAct(GridTestMixin, unittest.TestCase): # test for #1115 regarding the 'count-good-share-hosts' metric From 02cd42c130e9ff0c9d1dc7a799ffa078a29983a1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 11 Jun 2020 15:48:44 -0400 Subject: [PATCH 0210/1054] Simplify WebResultsRendering.test_literal --- src/allmydata/test/test_checker.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index bc4215ab3..62725da90 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -79,29 +79,23 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): def test_literal(self): lcr = web_check_results.LiteralCheckResultsElement() - d = self.render_element(lcr) - def _check(html): - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) - _check(d) + html = self.render_element(lcr) + s = self.remove_tags(html) + self.failUnlessIn("Literal files are always healthy", s) - d = self.render_element(lcr, args={"return_to": ["FOOURL"]}) - def _check_return_to(html): - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) - self.failUnlessIn('Return to file.', - html) - _check_return_to(d) + html = self.render_element(lcr, args={"return_to": ["FOOURL"]}) + s = self.remove_tags(html) + self.failUnlessIn("Literal files are always healthy", s) + self.failUnlessIn('Return to file.', html) c = self.create_fake_client() lcr = web_check_results.LiteralCheckResultsRenderer(c) - d = self.render_json(lcr) - def _check_json(js): - j = json.loads(js) - self.failUnlessEqual(j["storage-index"], "") - self.failUnlessEqual(j["results"]["healthy"], True) - _check_json(d) + js = self.render_json(lcr) + j = json.loads(js) + self.failUnlessEqual(j["storage-index"], "") + self.failUnlessEqual(j["results"]["healthy"], True) + def test_check(self): c = self.create_fake_client() From c44a167a660763465721432e925261cef0835d2a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 11 Jun 2020 15:51:39 -0400 Subject: [PATCH 0211/1054] Remove WebRenderingMixin from WebResultsRendering --- src/allmydata/test/test_checker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 62725da90..2bea98280 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -1,4 +1,5 @@ +import re import json import os.path, shutil from twisted.trial import unittest @@ -41,7 +42,12 @@ class TestRequest(Request): self.args = args or {} self.fields = fields or {} -class WebResultsRendering(unittest.TestCase, WebRenderingMixin): +class WebResultsRendering(unittest.TestCase): + + def remove_tags(self, s): + s = re.sub(r'<[^>]*>', ' ', s) + s = re.sub(r'\s+', ' ', s) + return s def create_fake_client(self): sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) From 048840ac99251bff328d7b963de90e6ff401e68c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 11 Jun 2020 16:00:21 -0400 Subject: [PATCH 0212/1054] Stringify numbers --- src/allmydata/web/check_results.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 757326bf7..c1edb799c 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -702,45 +702,48 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin def root_storage_index(self, req, tag): return self.monitor.get_status().get_root_storage_index_string() + def _get_counter(self, name): + return str(self.monitor.get_status().get_counters().get(name)) + @renderer def objects_checked(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-checked"] + return self._get_counter("count-objects-checked") @renderer def objects_healthy(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"] + return self._get_counter("count-objects-healthy-pre-repair") @renderer def objects_unhealthy(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"] + return self._get_counter("count-objects-unhealthy-pre-repair") @renderer def corrupt_shares(self, req, tag): - return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"] + return self._get_counter("count-corrupt-shares-pre-repair") @renderer def repairs_attempted(self, req, tag): - return self.monitor.get_status().get_counters()["count-repairs-attempted"] + return self._get_counter("count-repairs-attempted") @renderer def repairs_successful(self, req, tag): - return self.monitor.get_status().get_counters()["count-repairs-successful"] + return self._get_counter("count-repairs-successful") @renderer def repairs_unsuccessful(self, req, tag): - return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"] + return self._get_counter("count-repairs-unsuccessful") @renderer def objects_healthy_post(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"] + return self._get_counter("count-objects-healthy-post-repair") @renderer def objects_unhealthy_post(self, req, tag): - return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"] + return self._get_counter("count-objects-unhealthy-post-repair") @renderer def corrupt_shares_post(self, req, tag): - return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"] + return self._get_counter("count-corrupt-shares-post-repair") @renderer def pre_repair_problems_p(self, req, tag): From c4be4b566cab402c2c6af6158de67be783525988 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 11 Jun 2020 20:50:03 -0400 Subject: [PATCH 0213/1054] Make TestRequest an object Wouldn't pass PythonTwoRegressions.test_new_style_classes without this. --- src/allmydata/test/test_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 2bea98280..75619beaa 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -36,7 +36,7 @@ class FakeClient(object): # Also, the request needs to have `args` and `fields` properties so # that `allmydata.web.common.get_arg()` won't complain. @implementer(IRequest) -class TestRequest(Request): +class TestRequest(object, Request): def __init__(self, args=None, fields=None): Request.__init__(self, DummyChannel()) self.args = args or {} From 3783349ea7bf280b42205c6744bb9f5c2544af84 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 09:38:18 -0400 Subject: [PATCH 0214/1054] Add span tags to "deep check and repair results" template Tests expect tags around these numbers, let us add those. --- .../web/deep-check-and-repair-results.xhtml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 60cb10fba..72ee726db 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -15,19 +15,19 @@

                  Counters:

                    -
                  • Objects Checked:
                    • +
                  • Objects Checked:
                    • -
                  • Objects Healthy (before repair):
                    • -
                  • Objects Unhealthy (before repair):
                    • -
                  • Corrupt Shares (before repair):
                    • +
                  • Objects Healthy (before repair):
                    • +
                  • Objects Unhealthy (before repair):
                    • +
                  • Corrupt Shares (before repair):
                    • -
                  • Repairs Attempted:
                    • -
                  • Repairs Successful:
                    • -
                  • Repairs Unsuccessful:
                    • +
                  • Repairs Attempted:
                    • +
                  • Repairs Successful:
                    • +
                  • Repairs Unsuccessful:
                    • -
                  • Objects Healthy (after repair):
                    • -
                  • Objects Unhealthy (after repair):
                    • -
                  • Corrupt Shares (after repair):
                    • +
                  • Objects Healthy (after repair):
                    • +
                  • Objects Unhealthy (after repair):
                    • +
                  • Corrupt Shares (after repair):
                  From cfb1560d154f460abda8b77d38b1e06ce863a23c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 14:01:41 -0400 Subject: [PATCH 0215/1054] Render "all objects" table using SlotsSequenceElement --- src/allmydata/web/check_results.py | 38 +++++++++---------- .../web/deep-check-and-repair-results.xhtml | 2 +- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index c1edb799c..14cf8b5c5 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -24,6 +24,7 @@ from allmydata.web.common import ( get_root, WebError, MultiFormatResource, + SlotsSequenceElement, ) from allmydata.web.operations import ReloadMixin from allmydata.interfaces import ( @@ -825,30 +826,25 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" - # TODO: use SlotsSequenceElement @renderer def all_objects(self, req, tag): - r = self.monitor.get_status().get_all_results() - for path in sorted(r.keys()): - yield (path, r[path]) + results = self.monitor.get_status().get_all_results() + objects = [] - @renderer - def object(self, req, tag): - # TODO: figure this out - # path, r = data - tag.fillSlots("path", self._join_pathstring(path)) - tag.fillSlots("healthy_pre_repair", - str(r.get_pre_repair_results().is_healthy())) - tag.fillSlots("recoverable_pre_repair", - str(r.get_pre_repair_results().is_recoverable())) - tag.fillSlots("healthy_post_repair", - str(r.get_post_repair_results().is_healthy())) - storage_index = r.get_storage_index() - tag.fillSlots("storage_index", - self._render_si_link(ctx, storage_index)) - tag.fillSlots("summary", - self._html(r.get_pre_repair_results().get_summary())) - return tag + for path in sorted(results.keys()): + result = results[path] + storage_index = result.get_storage_index() + obj = { + "path": self._join_pathstring(path), + "healthy_pre_repair": str(result.get_pre_repair_results().is_healthy()), + "recoverable_pre_repair": str(result.get_pre_repair_results().is_recoverable()), + "healthy_post_repair": str(result.get_post_repair_results().is_healthy()), + "storage_index": self._render_si_link(req, storage_index), + "summary": self._html(result.get_pre_repair_results().get_summary()), + } + objects.append(obj) + + return SlotsSequenceElement(tag, objects) @renderer def runtime(self, req, tag): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 72ee726db..9d16e27b9 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -78,7 +78,7 @@                   - + From c7aa789a22024fc053f8de0e578b81d516beef2a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:28:54 -0400 Subject: [PATCH 0216/1054] Fix "class has no children attribute" error --- src/allmydata/web/check_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 14cf8b5c5..5cd2bdbdc 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -622,6 +622,7 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): def __init__(self, client, monitor): self.client = client self.monitor = monitor + self.children = {} def getChild(self, name, req): if not name: From f168dcd19c78639c1a3ff94bc67fb58b6bb74842 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:30:39 -0400 Subject: [PATCH 0217/1054] Use SlotsSequenceElement to render pre-repair problems --- src/allmydata/web/check_results.py | 13 +++++-------- .../web/deep-check-and-repair-results.xhtml | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 5cd2bdbdc..ad02385f1 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -754,23 +754,20 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin return tag return "" - # TODO: use SlotsSequenceElement @renderer def pre_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): r = all_objects[path] assert ICheckAndRepairResults.providedBy(r) cr = r.get_pre_repair_results() if not cr.is_healthy(): - yield path, cr + problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) + problems.append(problem) - @renderer - def problem(self, req, tag): - # TODO: figure this out - # path, cr = data - return tag(self._join_pathstring(path), ": ", - self._html(cr.get_summary())) + return SlotsSequenceElement(tag, problems) @renderer def post_repair_problems_p(self, req, tag): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 9d16e27b9..50131ac60 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -35,8 +35,8 @@

                  Files/Directories That Had Problems:

                    -
                  • -
                  • None
                    • +
                  • +
                  • None
                  From e0c686bb923301cb725875f7fc8fbff3557dddfb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:36:34 -0400 Subject: [PATCH 0218/1054] Use SlotsSequenceElement to render post-repair problems --- src/allmydata/web/check_results.py | 8 ++++++-- src/allmydata/web/deep-check-and-repair-results.xhtml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index ad02385f1..f8ad4241a 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -777,16 +777,20 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin return tag return "" - # TODO: use SlotsSequenceElement @renderer def post_repair_problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): r = all_objects[path] assert ICheckAndRepairResults.providedBy(r) cr = r.get_post_repair_results() if not cr.is_healthy(): - yield path, cr + problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) + problems.append(problem) + + return SlotsSequenceElement(tag, problems) @renderer def servers_with_corrupt_shares_p(self, req, tag): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 50131ac60..b0700a71f 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -44,8 +44,8 @@

                  Files/Directories That Still Have Problems:

                    -
                  • -
                  • None
                    • +
                  • +
                  • None
                  From 19f6ec71026fa3ed70e3ec978a634208c14f7b75 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:43:47 -0400 Subject: [PATCH 0219/1054] Use SlotsSequenceElement to render servers with corrupt shares This was previously not implemented; leaving it as such. --- src/allmydata/web/check_results.py | 9 ++++----- src/allmydata/web/deep-check-and-repair-results.xhtml | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index f8ad4241a..8b281045b 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -800,11 +800,10 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def servers_with_corrupt_shares(self, req, tag): - return [] # TODO - - @renderer - def server_problem(self, req, tag): - pass + # TODO: this was originally unimplemented before porting to + # twisted.web.template; leaving it as such. + corrupt = [] + return SlotsSequenceElement(tag, corrupt) @renderer def remaining_corrupt_shares_p(self, req, tag): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index b0700a71f..71fa92878 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -52,8 +52,8 @@

                  Servers on which corrupt shares were found

                    -
                  • -
                  • None
                    • +
                  • +
                  • None
                  From bbd6bde478daa14d789f826caa2ab4934f9b4f9f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:47:51 -0400 Subject: [PATCH 0220/1054] Use SlotsSequenceElement to render remaining corrupt shares This was previously not implemented; leaving it as such. --- src/allmydata/web/check_results.py | 9 ++++----- src/allmydata/web/deep-check-and-repair-results.xhtml | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 8b281045b..370f1c308 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -813,11 +813,10 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def post_repair_corrupt_shares(self, req, tag): - return [] # TODO - - @renderer - def render_share_problem(self, req, tag): - pass + # TODO: this was not implemented before porting to + # twisted.web.template; leaving it as such. + corrupt = [] + return SlotsSequenceElement(tag, corrupt) @renderer def return_to(self, req, tag): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 71fa92878..c9bd998e5 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -61,8 +61,8 @@

                  Remaining Corrupt Shares

                  These shares need to be manually inspected and removed.

                    -
                  • -
                  • None
                    • +
                  • +
                  • None
                  From c41bca24f5f8e5feebcf037fe3c1ce4535da5405 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sun, 14 Jun 2020 22:15:01 -0400 Subject: [PATCH 0221/1054] Use SlotsSequenceElement to render deep check results problems --- src/allmydata/web/check_results.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 370f1c308..748fd0aa3 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -511,26 +511,24 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return tag return "" - # TODO: use SlotsSequenceElement to render this. @renderer def problems(self, req, tag): all_objects = self.monitor.get_status().get_all_results() + problems = [] + for path in sorted(all_objects.keys()): cr = all_objects[path] assert ICheckResults.providedBy(cr) if not cr.is_healthy(): - yield path, cr + summary_text = "" + summary = cr.get_summary() + if summary: + summary_text = ": " + summary + summary_text += " [SI: %s]" % cr.get_storage_index_string() + problem = self._join_pathstring(path), ":", self._html(summary_text) + problems.append(problem) - @renderer - def render_problem(self, req, tag): - # TODO: deal with this. - path, cr = data - summary_text = "" - summary = cr.get_summary() - if summary: - summary_text = ": " + summary - summary_text += " [SI: %s]" % cr.get_storage_index_string() - return tag(self._join_pathstring(path), self._html(summary_text)) + return SlotsSequenceElement(tag, problems) @renderer def servers_with_corrupt_shares_p(self, req, tag): From 46cb3b6bc3affdefc98672b54ac23418e486da7f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 13:34:11 -0400 Subject: [PATCH 0222/1054] Use SlotsSequenceElement to render servers with corrupt shares --- src/allmydata/web/check_results.py | 21 ++++++++++----------- src/allmydata/web/deep-check-results.xhtml | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 748fd0aa3..94c42ded7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -536,24 +536,23 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return tag return "" - # TODO: use SlotsSequenceElement to render this. @renderer def servers_with_corrupt_shares(self, ctx, data): servers = [s for (s, storage_index, sharenum) in self.monitor.get_status().get_corrupt_shares()] servers.sort(key=lambda s: s.get_longname()) - return servers - @renderer - def server_problem(self, req, tag): - # def server_problem(self, ctx, server): - # TODO: where do `server` come from now? - data = [server.get_name()] - nickname = server.get_nickname() - if nickname: - data.append(" (%s)" % self._html(nickname)) - return tag(data) + problems = [] + + for server in servers: + name = [server.get_name()] + nick = server.get_nickname() + if nick: + name.append(" (%s)" % self._html(nickname)) + problems.append(name) + + return SlotsSequenceElement(tag, problems) @renderer def corrupt_shares_p(self, req, tag): diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 11a2e7f4b..5a2cf3542 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -34,8 +34,8 @@

                  Servers on which corrupt shares were found

                    -
                  • -
                  • None
                    • +
                  • +
                  • None
                  From 4e594b09bf719b6f76d6cb52d9b44faecdc1f644 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 13:59:22 -0400 Subject: [PATCH 0223/1054] Use SlotsSequenceElement to render corrupt shares --- src/allmydata/web/check_results.py | 26 +++++++++++----------- src/allmydata/web/deep-check-results.xhtml | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 94c42ded7..fde245dbe 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -560,22 +560,22 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return tag return "" - # TODO: Probably should use SlotsSequenceElement to render this. @renderer def corrupt_shares(self, req, tag): - return self.monitor.get_status().get_corrupt_shares() + shares = self.monitor.get_status().get_corrupt_shares() + problems = [] - @renderer - def share_problem(self, req, tag): - # def render_share_problem(self, req, tag): - server, storage_index, sharenum = data - nickname = server.get_nickname() - tag.fillSlots("serverid", server.get_name()) - if nickname: - tag.fillSlots("nickname", self._html(nickname)) - tag.fillSlots("si", self._render_si_link(ctx, storage_index)) - tag.fillSlots("shnum", str(sharenum)) - return tag + for share in shares: + server, storage_index, sharenum = share + problem = { + "serverid": server.get_name(), + "nickname": self._html(nickname), + "si": self._render_si_link(req, storage_index), + "shnum": str(sharenum), + } + problems.append(problem) + + return SlotsSequenceElement(tag, problems) @renderer def return_to(self, req, tag): diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 5a2cf3542..2c5c18544 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -43,13 +43,13 @@

                  Corrupt Shares

                  If repair fails, these shares need to be manually inspected and removed.

                  Storage Index Summary
                  - + - + From 8094b3144d570da7df6ff8644a3056f523bc1ca6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 14:23:23 -0400 Subject: [PATCH 0224/1054] Use SlotsSequenceElement to render all objects --- src/allmydata/web/check_results.py | 33 +++++++++++----------- src/allmydata/web/deep-check-results.xhtml | 4 +-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index fde245dbe..9189b8da0 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -585,25 +585,24 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return tags.div(tags.a("Return to file/directory.", href=return_to)) return "" - # TODO: use SlotsSequenceElement to render this. @renderer - def all_objects(self, ctx, data): - r = self.monitor.get_status().get_all_results() - for path in sorted(r.keys()): - yield (path, r[path]) + def all_objects(self, req, tag): + results = self.monitor.get_status().get_all_results() + objects = [] - @renderer - def object(self, ctx, data): - # def render_object(self, ctx, data): - # TODO: figure `data` out - path, r = data - tag.fillSlots("path", self._join_pathstring(path)) - tag.fillSlots("healthy", str(r.is_healthy())) - tag.fillSlots("recoverable", str(r.is_recoverable())) - storage_index = r.get_storage_index() - tag.fillSlots("storage_index", self._render_si_link(req, storage_index)) - tag.fillSlots("summary", self._html(r.get_summary())) - return tag + for path in sorted(results.keys()): + result = results.get(path) + storage_index = result.get_storage_index() + object = { + "path": self._join_pathstring(path), + "healthy": str(result.is_healthy()), + "recoverable": str(result.is_recoverable()), + "storage_index": self._render_si_link(req, storage_index), + "summary": self._html(result.get_summary()), + } + objects.append(object) + + return SlotsSequenceElement(tag, objects) @renderer def runtime(self, req, tag): diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 2c5c18544..e868c6e4f 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -63,14 +63,14 @@

                  All Results

                  Server Server Nickname Storage Index Share Number
                  - + - + From 33ddac879cfe65e506c735555cd72d2f8f812161 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Jun 2020 17:48:56 -0400 Subject: [PATCH 0225/1054] Remove unused code --- src/allmydata/web/check_results.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 9189b8da0..248c771a8 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -254,18 +254,12 @@ class LiteralCheckResultsElement(Element): class CheckerBase(object): - # def renderHTTP(self, ctx): - # if self.want_json(ctx): - # return self.json(ctx) - # return rend.Page.renderHTTP(self, ctx) - @renderer def storage_index(self, req, tag): return self.r.get_storage_index_string() @renderer def return_to(self, req, tag): - # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: return tags.div(tags.a("Return to file/directory.", href=return_to)) @@ -440,13 +434,7 @@ class DeepCheckResultsRenderer(MultiFormatResource): elem = DeepCheckResultsRendererElement(self.monitor) return renderElement(req, elem) - # def renderHTTP(self, ctx): - # if self.want_json(ctx): - # return self.json(ctx) - # return rend.Page.renderHTTP(self, ctx) - def render_JSON(self, req): - # inevow.IRequest(ctx).setHeader("content-type", "text/plain") req.setHeader("content-type", "text/plain") data = {} data["finished"] = self.monitor.is_finished() @@ -579,7 +567,6 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def return_to(self, req, tag): - # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: return tags.div(tags.a("Return to file/directory.", href=return_to)) @@ -606,7 +593,6 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def runtime(self, req, tag): - # req = inevow.IRequest(ctx) runtime = time.time() - req.processing_started_timestamp return tag("runtime: %s seconds" % runtime) @@ -638,13 +624,7 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): elem = DeepCheckAndRepairResultsRendererElement(self.monitor) return renderElement(req, elem) - # def renderHTTP(self, ctx): - # if self.want_json(ctx): - # return self.json(ctx) - # return rend.Page.renderHTTP(self, ctx) - def render_JSON(self, req): - # inevow.IRequest(ctx).setHeader("content-type", "text/plain") req.setHeader("content-type", "text/plain") res = self.monitor.get_status() data = {} @@ -816,7 +796,6 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def return_to(self, req, tag): - # req = inevow.IRequest(ctx) return_to = get_arg(req, "return_to", None) if return_to: return tags.div(tags.a("Return to file/directory.", href=return_to)) @@ -844,6 +823,5 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def runtime(self, req, tag): - # req = inevow.IRequest(ctx) runtime = time.time() - req.processing_started_timestamp return tag("runtime: %s seconds" % runtime) From 2e60408bf62dc510cf6871929a5e102995423df2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 14:29:33 -0400 Subject: [PATCH 0226/1054] Wrap counters in tags Unless the rendered page contains strings of the form "Objects Checked: n" etc, test_POST_DIRURL_deepcheck will not pass. --- src/allmydata/web/deep-check-results.xhtml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index e868c6e4f..572e44ff1 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -14,11 +14,11 @@

                  Counters:

                    -
                  • Objects Checked:
                    • -
                  • Objects Healthy:
                    • -
                  • Objects Unhealthy:
                    • -
                  • Objects Unrecoverable:
                    • -
                  • Corrupt Shares:
                    • +
                  • Objects Checked:
                    • +
                  • Objects Healthy:
                    • +
                  • Objects Unhealthy:
                    • +
                  • Objects Unrecoverable:
                    • +
                  • Corrupt Shares:
                  From 1ee756c8df8e61a332fb70de2e4752b1842fa076 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 14:36:44 -0400 Subject: [PATCH 0227/1054] Use Beautiful Soup to check for favicon --- src/allmydata/test/web/test_grid.py | 15 ++++++++++++--- src/allmydata/test/web/test_web.py | 9 ++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 2b953b82c..73c354567 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -21,7 +21,12 @@ from allmydata.mutable import publish from .. import common_util as testutil from ..common import WebErrorMixin, ShouldFailMixin from ..no_network import GridTestMixin -from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP +from .common import ( + assert_soup_has_favicon, + unknown_immcap, + unknown_rocap, + unknown_rwcap, +) DIR_HTML_TAG = '' @@ -92,7 +97,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _got_html_good(res): self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) + d.addCallback(_got_html_good) d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") def _got_html_good_return_to(res): @@ -235,7 +242,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) self.failUnlessIn("No repair necessary", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) + d.addCallback(_got_html_good) d.addCallback(self.CHECK, "sick", "t=check&repair=true") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5be0b2f7b..94e628ef9 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -3080,13 +3080,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi res = yield self.get_operation_results(None, "123", "html") self.failUnlessIn("Objects Checked: 11", res) self.failUnlessIn("Objects Healthy: 11", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) res = yield self.GET("/operations/123/") # should be the same as without the slash self.failUnlessIn("Objects Checked: 11", res) self.failUnlessIn("Objects Healthy: 11", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) yield self.shouldFail2(error.Error, "one", "404 Not Found", "No detailed results for SI bogus", @@ -3136,7 +3138,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Objects Unhealthy (after repair): 0", res) self.failUnlessIn("Corrupt Shares (after repair): 0", res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_favicon(self, soup) d.addCallback(_check_html) return d From de999126413b4e782d521c3e09373d20f01ef13a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 15:17:43 -0400 Subject: [PATCH 0228/1054] Replace `ctx` references with `req` references --- src/allmydata/web/check_results.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 248c771a8..af12fa370 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -203,18 +203,17 @@ class ResultsBase(object): assert isinstance(s, (list, tuple)) return [html.escape(w) for w in s] - def want_json(self, ctx): - output = get_arg(inevow.IRequest(ctx), "output", "").lower() + def want_json(self, req): + output = get_arg(req, "output", "").lower() if output.lower() == "json": return True return False - def _render_si_link(self, ctx, storage_index): + def _render_si_link(self, req, storage_index): si_s = base32.b2a(storage_index) - req = inevow.IRequest(ctx) ophandle = req.prepath[-1] - target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s) - output = get_arg(ctx, "output") + target = "%s/operations/%s/%s" % (get_root(req), ophandle, si_s) + output = get_arg(req, "output") if output: target = target + "?output=%s" % output return tags.a(si_s, href=target) From cdb3747470afd707349dd52723fb87dca5097728 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 15:18:11 -0400 Subject: [PATCH 0229/1054] Remove nevow imports --- src/allmydata/web/check_results.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index af12fa370..c109966e5 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -1,11 +1,7 @@ import time import json -from nevow import ( - rend, - inevow, - tags as T, -) + from twisted.web import ( http, html, From fae4f44404ff81f182ee4f848b963fdceacd837c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 22:27:22 -0400 Subject: [PATCH 0230/1054] Prefer `req` to `ctx` --- src/allmydata/web/check_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index c109966e5..4051e1379 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -98,7 +98,7 @@ class ResultsBase(object): pathstring = "" return pathstring - def _render_results(self, ctx, cr): + def _render_results(self, req, cr): assert ICheckResults(cr) c = self.client sb = c.get_storage_broker() From 23a51117d2fea008beeaef0ca7d96922d904b834 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 15 Jun 2020 22:28:28 -0400 Subject: [PATCH 0231/1054] Rename `r` to `check_results` --- src/allmydata/web/check_results.py | 51 ++++++++++++++---------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 4051e1379..ffd366385 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -251,7 +251,7 @@ class CheckerBase(object): @renderer def storage_index(self, req, tag): - return self.r.get_storage_index_string() + return self.check_results.get_storage_index_string() @renderer def return_to(self, req, tag): @@ -273,14 +273,14 @@ class CheckResultsRenderer(MultiFormatResource): """ super(CheckResultsRenderer, self).__init__() self.client = client - self.r = ICheckResults(results) + self.check_results = ICheckResults(results) def render_HTML(self, req): - return renderElement(req, CheckResultsRendererElement(self.client, self.r)) + return renderElement(req, CheckResultsRendererElement(self.client, self.check_results)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - data = json_check_results(self.r) + data = json_check_results(self.check_results) return json.dumps(data, indent=1) + "\n" @@ -288,31 +288,27 @@ class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): loader = XMLFile(FilePath(__file__).sibling("check-results.xhtml")) - def __init__(self, client, r): + def __init__(self, client, results): super(CheckResultsRendererElement, self).__init__() self.client = client - # TODO: use a better name - self.r = r + self.check_results = results @renderer def summary(self, req, tag): results = [] - # XXX: how did `data` get populated? - if self.r.is_healthy(): + if self.check_results.is_healthy(): results.append("Healthy") - elif self.r.is_recoverable(): + elif self.check_results.is_recoverable(): results.append("Not Healthy!") else: results.append("Not Recoverable!") results.append(" : ") - # TODO: what is self._html? - # results.append(self._html(self.r.get_summary())) - results.append(self.r.get_summary()) + results.append(self._html(self.check_results.get_summary())) return tag(results) @renderer def repair(self, req, tag): - if self.r.is_healthy(): + if self.check_results.is_healthy(): return "" #repair = T.form(action=".", method="post", @@ -329,10 +325,9 @@ class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def results(self, req, tag): - cr = self._render_results(req, self.r) + cr = self._render_results(req, self.check_results) return tag(cr) - class CheckAndRepairResultsRenderer(MultiFormatResource): formatArgument = "output" @@ -342,17 +337,17 @@ class CheckAndRepairResultsRenderer(MultiFormatResource): super(CheckAndRepairResultsRenderer, self).__init__() self.client = client # TODO: use a better name - self.r = None + self.check_results = None if results: - self.r = ICheckAndRepairResults(results) + self.check_results = ICheckAndRepairResults(results) def render_HTML(self, req): - elem = CheckAndRepairResultsRendererElement(self.client, self.r) + elem = CheckAndRepairResultsRendererElement(self.client, self.check_results) return renderElement(req, elem) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - data = json_check_and_repair_results(self.r) + data = json_check_and_repair_results(self.check_results) return json.dumps(data, indent=1) + "\n" @@ -360,14 +355,14 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): loader = XMLFile(FilePath(__file__).sibling("check-and-repair-results.xhtml")) - def __init__(self, client, r): + def __init__(self, client, results): super(CheckAndRepairResultsRendererElement, self).__init__() self.client = client - self.r = r + self.check_results = results @renderer def summary(self, req, tag): - cr = self.r.get_post_repair_results() + cr = self.check_results.get_post_repair_results() results = [] if cr.is_healthy(): results.append("Healthy") @@ -381,8 +376,8 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def repair_results(self, req, tag): - if self.r.get_repair_attempted(): - if self.r.get_repair_successful(): + if self.check_results.get_repair_attempted(): + if self.check_results.get_repair_successful(): return tag("Repair successful") else: return tag("Repair unsuccessful") @@ -390,13 +385,13 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def post_repair_results(self, req, tag): - cr = self._render_results(req, self.r.get_post_repair_results()) + cr = self._render_results(req, self.check_results.get_post_repair_results()) return tag(tags.div("Post-Repair Checker Results:"), cr) @renderer def maybe_pre_repair_results(self, req, tag): - if self.r.get_repair_attempted(): - cr = self._render_results(req, self.r.get_pre_repair_results()) + if self.check_results.get_repair_attempted(): + cr = self._render_results(req, self.check_results.get_pre_repair_results()) return tag(tags.div("Pre-Repair Checker Results:"), cr) return "" From 79be7955660fe3e7283c1063d48571f17f78c745 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 16 Jun 2020 07:12:08 -0400 Subject: [PATCH 0232/1054] Remove unused want_json() --- src/allmydata/web/check_results.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index ffd366385..f675c53bb 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -199,12 +199,6 @@ class ResultsBase(object): assert isinstance(s, (list, tuple)) return [html.escape(w) for w in s] - def want_json(self, req): - output = get_arg(req, "output", "").lower() - if output.lower() == "json": - return True - return False - def _render_si_link(self, req, storage_index): si_s = base32.b2a(storage_index) ophandle = req.prepath[-1] From 22977352bb6669355d20f72ac7ed269231596075 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 16 Jun 2020 09:40:37 -0400 Subject: [PATCH 0233/1054] Document parameters of ResultsRenderer classes --- src/allmydata/web/check_results.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index f675c53bb..8a3ecc61b 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -214,6 +214,9 @@ class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): formatArgument = "output" def __init__(self, client): + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + """ super(LiteralCheckResultsRenderer, self).__init__() self.client = client @@ -261,9 +264,8 @@ class CheckResultsRenderer(MultiFormatResource): def __init__(self, client, results): """ - TODO: update these with the correct values. - :param allmydata.test.no_network._NoNetworkClient client: - :param allmydata.check_results.CheckResults results: + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.interfaces.ICheckResults results: results of check/vefify operation. """ super(CheckResultsRenderer, self).__init__() self.client = client @@ -327,10 +329,12 @@ class CheckAndRepairResultsRenderer(MultiFormatResource): formatArgument = "output" def __init__(self, client, results): - # TODO: document params + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.interfaces.ICheckResults results: check/verify results. + """ super(CheckAndRepairResultsRenderer, self).__init__() self.client = client - # TODO: use a better name self.check_results = None if results: self.check_results = ICheckAndRepairResults(results) @@ -395,7 +399,10 @@ class DeepCheckResultsRenderer(MultiFormatResource): formatArgument = "output" def __init__(self, client, monitor): - # TODO: document params + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. + """ super(DeepCheckResultsRenderer, self).__init__() self.client = client self.monitor = monitor @@ -586,6 +593,10 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): formatArgument = "output" def __init__(self, client, monitor): + """ + :param allmydata.interfaces.IStatsProducer client: stats provider. + :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. + """ self.client = client self.monitor = monitor self.children = {} @@ -656,7 +667,6 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml")) def __init__(self, monitor): - # TODO: document params super(DeepCheckAndRepairResultsRendererElement, self).__init__() self.monitor = monitor From fe1b0cb7ed3801b709e5491ccd77ef6627ce0f53 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 17:51:04 -0400 Subject: [PATCH 0234/1054] Rename class variables --- src/allmydata/web/check_results.py | 64 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 8a3ecc61b..db6b28796 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -88,7 +88,7 @@ def json_check_and_repair_results(r): return data class ResultsBase(object): - # self.client must point to the Client, so we can get nicknames and + # self._client must point to the Client, so we can get nicknames and # determine the permuted peer order def _join_pathstring(self, path): @@ -100,7 +100,7 @@ class ResultsBase(object): def _render_results(self, req, cr): assert ICheckResults(cr) - c = self.client + c = self._client sb = c.get_storage_broker() r = [] def add(name, value): @@ -218,7 +218,7 @@ class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): :param allmydata.interfaces.IStatsProducer client: stats provider. """ super(LiteralCheckResultsRenderer, self).__init__() - self.client = client + self._client = client def render_HTML(self, req): return renderElement(req, LiteralCheckResultsElement()) @@ -248,7 +248,7 @@ class CheckerBase(object): @renderer def storage_index(self, req, tag): - return self.check_results.get_storage_index_string() + return self._results.get_storage_index_string() @renderer def return_to(self, req, tag): @@ -268,15 +268,15 @@ class CheckResultsRenderer(MultiFormatResource): :param allmydata.interfaces.ICheckResults results: results of check/vefify operation. """ super(CheckResultsRenderer, self).__init__() - self.client = client - self.check_results = ICheckResults(results) + self._client = client + self._results = ICheckResults(results) def render_HTML(self, req): - return renderElement(req, CheckResultsRendererElement(self.client, self.check_results)) + return renderElement(req, CheckResultsRendererElement(self._client, self._results)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - data = json_check_results(self.check_results) + data = json_check_results(self._results) return json.dumps(data, indent=1) + "\n" @@ -286,25 +286,25 @@ class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): def __init__(self, client, results): super(CheckResultsRendererElement, self).__init__() - self.client = client - self.check_results = results + self._client = client + self._results = results @renderer def summary(self, req, tag): results = [] - if self.check_results.is_healthy(): + if self._results.is_healthy(): results.append("Healthy") - elif self.check_results.is_recoverable(): + elif self._results.is_recoverable(): results.append("Not Healthy!") else: results.append("Not Recoverable!") results.append(" : ") - results.append(self._html(self.check_results.get_summary())) + results.append(self._html(self._results.get_summary())) return tag(results) @renderer def repair(self, req, tag): - if self.check_results.is_healthy(): + if self._results.is_healthy(): return "" #repair = T.form(action=".", method="post", @@ -321,7 +321,7 @@ class CheckResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def results(self, req, tag): - cr = self._render_results(req, self.check_results) + cr = self._render_results(req, self._results) return tag(cr) class CheckAndRepairResultsRenderer(MultiFormatResource): @@ -334,18 +334,18 @@ class CheckAndRepairResultsRenderer(MultiFormatResource): :param allmydata.interfaces.ICheckResults results: check/verify results. """ super(CheckAndRepairResultsRenderer, self).__init__() - self.client = client - self.check_results = None + self._client = client + self._results = None if results: - self.check_results = ICheckAndRepairResults(results) + self._results = ICheckAndRepairResults(results) def render_HTML(self, req): - elem = CheckAndRepairResultsRendererElement(self.client, self.check_results) + elem = CheckAndRepairResultsRendererElement(self._client, self._results) return renderElement(req, elem) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - data = json_check_and_repair_results(self.check_results) + data = json_check_and_repair_results(self._results) return json.dumps(data, indent=1) + "\n" @@ -355,12 +355,12 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): def __init__(self, client, results): super(CheckAndRepairResultsRendererElement, self).__init__() - self.client = client - self.check_results = results + self._client = client + self._results = results @renderer def summary(self, req, tag): - cr = self.check_results.get_post_repair_results() + cr = self._results.get_post_repair_results() results = [] if cr.is_healthy(): results.append("Healthy") @@ -374,8 +374,8 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def repair_results(self, req, tag): - if self.check_results.get_repair_attempted(): - if self.check_results.get_repair_successful(): + if self._results.get_repair_attempted(): + if self._results.get_repair_successful(): return tag("Repair successful") else: return tag("Repair unsuccessful") @@ -383,13 +383,13 @@ class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase): @renderer def post_repair_results(self, req, tag): - cr = self._render_results(req, self.check_results.get_post_repair_results()) + cr = self._render_results(req, self._results.get_post_repair_results()) return tag(tags.div("Post-Repair Checker Results:"), cr) @renderer def maybe_pre_repair_results(self, req, tag): - if self.check_results.get_repair_attempted(): - cr = self._render_results(req, self.check_results.get_pre_repair_results()) + if self._results.get_repair_attempted(): + cr = self._render_results(req, self._results.get_pre_repair_results()) return tag(tags.div("Pre-Repair Checker Results:"), cr) return "" @@ -404,7 +404,7 @@ class DeepCheckResultsRenderer(MultiFormatResource): :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. """ super(DeepCheckResultsRenderer, self).__init__() - self.client = client + self._client = client self.monitor = monitor def getChild(self, name, req): @@ -415,7 +415,7 @@ class DeepCheckResultsRenderer(MultiFormatResource): si = base32.a2b(name) r = self.monitor.get_status() try: - return CheckResultsRenderer(self.client, + return CheckResultsRenderer(self._client, r.get_results_for_storage_index(si)) except KeyError: raise WebError("No detailed results for SI %s" % html.escape(name), @@ -597,7 +597,7 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): :param allmydata.interfaces.IStatsProducer client: stats provider. :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. """ - self.client = client + self._client = client self.monitor = monitor self.children = {} @@ -610,7 +610,7 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): s = self.monitor.get_status() try: results = s.get_results_for_storage_index(si) - return CheckAndRepairResultsRenderer(self.client, results) + return CheckAndRepairResultsRenderer(self._client, results) except KeyError: raise WebError("No detailed results for SI %s" % html.escape(name), http.NOT_FOUND) From c8d218fb9f10831faceff6019c3143b7f047eb79 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 18:09:21 -0400 Subject: [PATCH 0235/1054] Add newsfragment --- newsfragments/3316.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3316.minor diff --git a/newsfragments/3316.minor b/newsfragments/3316.minor new file mode 100644 index 000000000..9457b486e --- /dev/null +++ b/newsfragments/3316.minor @@ -0,0 +1 @@ +Port checker result pages' rendering from nevow to twisted web templates. From 47f9147e165f860ed0b6b8897f5992faf6e4e01c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 21:32:49 -0400 Subject: [PATCH 0236/1054] Rename a class for consistency's sake --- src/allmydata/test/test_checker.py | 2 +- src/allmydata/web/check_results.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 75619beaa..ba212c65e 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -83,7 +83,7 @@ class WebResultsRendering(unittest.TestCase): return unittest.TestCase().successResultOf(d) def test_literal(self): - lcr = web_check_results.LiteralCheckResultsElement() + lcr = web_check_results.LiteralCheckResultsRendererElement() html = self.render_element(lcr) s = self.remove_tags(html) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index db6b28796..89bc41b79 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -221,7 +221,7 @@ class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): self._client = client def render_HTML(self, req): - return renderElement(req, LiteralCheckResultsElement()) + return renderElement(req, LiteralCheckResultsRendererElement()) def render_JSON(self, req): req.setHeader("content-type", "text/plain") @@ -229,12 +229,12 @@ class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase): return json.dumps(data, indent=1) + "\n" -class LiteralCheckResultsElement(Element): +class LiteralCheckResultsRendererElement(Element): loader = XMLFile(FilePath(__file__).sibling("literal-check-results.xhtml")) def __init__(self): - super(LiteralCheckResultsElement, self).__init__() + super(LiteralCheckResultsRendererElement, self).__init__() @renderer def return_to(self, req, tag): From c3440ce21312da568e51fd1d529c3dd5f185bcef Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 21:53:11 -0400 Subject: [PATCH 0237/1054] Use a helper to query monitor counters --- src/allmydata/web/check_results.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 89bc41b79..290d9a1c4 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -457,31 +457,32 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): super(DeepCheckResultsRendererElement, self).__init__() self.monitor = monitor - # TODO: Add a helper to query monitor.get_status.get_counters() - @renderer def root_storage_index(self, req, tag): return self.monitor.get_status().get_root_storage_index_string() + def _get_monitor_counter(self, name): + return self.monitor.get_status().get_counters().get(name) + @renderer def objects_checked(self, req, tag): - return str(self.monitor.get_status().get_counters()["count-objects-checked"]) + return str(self._get_monitor_counter("count-objects-checked")) @renderer def objects_healthy(self, req, tag): - return str(self.monitor.get_status().get_counters()["count-objects-healthy"]) + return str(self._get_monitor_counter("count-objects-healthy")) @renderer def objects_unhealthy(self, req, tag): - return str(self.monitor.get_status().get_counters()["count-objects-unhealthy"]) + return str(self._get_monitor_counter("count-objects-unhealthy")) @renderer def objects_unrecoverable(self, req, tag): - return str(self.monitor.get_status().get_counters()["count-objects-unrecoverable"]) + return str(self._get_monitor_counter("count-objects-unrecoverable")) @renderer def count_corrupt_shares(self, req, tag): - return str(self.monitor.get_status().get_counters()["count-corrupt-shares"]) + return str(self._get_monitor_counter("count-corrupt-shares")) @renderer def problems_p(self, req, tag): @@ -511,7 +512,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def servers_with_corrupt_shares_p(self, req, tag): - if self.monitor.get_status().get_counters()["count-corrupt-shares"]: + if self._get_monitor_counter("count-corrupt-shares"): return tag return "" @@ -535,7 +536,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def corrupt_shares_p(self, req, tag): - if self.monitor.get_status().get_counters()["count-corrupt-shares"]: + if self._get_monitor_counter("count-corrupt-shares"): return tag return "" From 0483fc91c1dd5045f6f6797a0bf27b3aa22d2b8e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 21:56:42 -0400 Subject: [PATCH 0238/1054] Remove unused import --- src/allmydata/web/check_results.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 290d9a1c4..8fdfece7c 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -15,7 +15,6 @@ from twisted.web.template import ( tags, ) from allmydata.web.common import ( - getxmlfile, get_arg, get_root, WebError, From 4f213e93bed80a5cdfdc04232dd06f0886c90f2b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 21:57:14 -0400 Subject: [PATCH 0239/1054] Remove an unused variable --- src/allmydata/web/check_results.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 8fdfece7c..fdb6a39d5 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -114,8 +114,6 @@ class ResultsBase(object): add("Happiness Level", str(cr.get_happiness())) add("Hosts with good shares", str(cr.get_host_counter_good_shares())) - rrr = cr.get_corrupt_shares() - if cr.get_corrupt_shares(): badsharemap = [] for (s, si, shnum) in cr.get_corrupt_shares(): From e121cc0a84d126d38416b3f6a7e8fb1f8c761bb1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 22:01:21 -0400 Subject: [PATCH 0240/1054] Use the right names Missed this in testing because this code was not reached in testing. --- src/allmydata/web/check_results.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index fdb6a39d5..811885ec3 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -514,7 +514,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): return "" @renderer - def servers_with_corrupt_shares(self, ctx, data): + def servers_with_corrupt_shares(self, req, tag): servers = [s for (s, storage_index, sharenum) in self.monitor.get_status().get_corrupt_shares()] @@ -524,8 +524,8 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): for server in servers: name = [server.get_name()] - nick = server.get_nickname() - if nick: + nickname = server.get_nickname() + if nickname: name.append(" (%s)" % self._html(nickname)) problems.append(name) @@ -544,6 +544,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): for share in shares: server, storage_index, sharenum = share + nickname = server.get_nickname() problem = { "serverid": server.get_name(), "nickname": self._html(nickname), From dfb477a2b14f0683063d228d0e1859e445e5e4a5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 17 Jun 2020 22:02:15 -0400 Subject: [PATCH 0241/1054] Remove unused import --- src/allmydata/test/test_checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index ba212c65e..bd1bd87c6 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -20,7 +20,6 @@ from allmydata.storage.server import storage_index_to_dir from allmydata.monitor import Monitor from allmydata.test.no_network import GridTestMixin from allmydata.immutable.upload import Data -from allmydata.test.common_web import WebRenderingMixin from allmydata.mutable.publish import MutableData from .common import ( From 6f8af688dddf865e607a313a234699415578d46e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 1 Jun 2020 16:16:04 -0400 Subject: [PATCH 0242/1054] Rearrange imports --- src/allmydata/web/operations.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index eb9321344..044a9fa64 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -12,8 +12,12 @@ from twisted.web.http import NOT_FOUND from twisted.web.html import escape from twisted.application import service -from allmydata.web.common import WebError, \ - get_root, get_arg, boolean_of_arg +from allmydata.web.common import ( + WebError, + get_root, + get_arg, + boolean_of_arg, +) MINUTE = 60 HOUR = 60*MINUTE From 5ec78641dbdd17a6717e7f682ee24e8a3b458d85 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 18 Jun 2020 18:29:27 -0400 Subject: [PATCH 0243/1054] Make web.operations.OphandleTable a twisted web Resource --- src/allmydata/web/operations.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 044a9fa64..206542b06 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,13 +1,13 @@ import time -from nevow import rend, url -from nevow.inevow import IRequest +from nevow import url from twisted.web.template import ( renderer, tags as T, ) from twisted.python.failure import Failure from twisted.internet import reactor, defer +from twisted.web import resource from twisted.web.http import NOT_FOUND from twisted.web.html import escape from twisted.application import service @@ -25,13 +25,14 @@ DAY = 24*HOUR (MONITOR, RENDERER, WHEN_ADDED) = range(3) -class OphandleTable(rend.Page, service.Service): +class OphandleTable(resource.Resource, service.Service): name = "operations" UNCOLLECTED_HANDLE_LIFETIME = 4*DAY COLLECTED_HANDLE_LIFETIME = 1*DAY def __init__(self, clock=None): + super(OphandleTable, self).__init__() # both of these are indexed by ophandle self.handles = {} # tuple of (monitor, renderer, when_added) self.timers = {} @@ -80,27 +81,26 @@ class OphandleTable(rend.Page, service.Service): target = target + "?output=%s" % output return url.URL.fromString(target) - def childFactory(self, ctx, name): + def getChild(self, name, req): ophandle = name if ophandle not in self.handles: raise WebError("unknown/expired handle '%s'" % escape(ophandle), NOT_FOUND) (monitor, renderer, when_added) = self.handles[ophandle] - request = IRequest(ctx) - t = get_arg(ctx, "t", "status") - if t == "cancel" and request.method == "POST": + t = get_arg(req, "t", "status") + if t == "cancel" and req.method == "POST": monitor.cancel() # return the status anyways, but release the handle self._release_ophandle(ophandle) else: - retain_for = get_arg(ctx, "retain-for", None) + retain_for = get_arg(req, "retain-for", None) if retain_for is not None: self._set_timer(ophandle, int(retain_for)) if monitor.is_finished(): - if boolean_of_arg(get_arg(ctx, "release-after-complete", "false")): + if boolean_of_arg(get_arg(req, "release-after-complete", "false")): self._release_ophandle(ophandle) if retain_for is None: # this GET is collecting the ophandle, so change its timer @@ -127,6 +127,7 @@ class OphandleTable(rend.Page, service.Service): self.timers.pop(ophandle, None) self.handles.pop(ophandle, None) + class ReloadMixin(object): REFRESH_TIME = 1*MINUTE From bbee23b4937de49e6d331637c1d454425612256c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 1 Jun 2020 18:19:08 -0400 Subject: [PATCH 0244/1054] Rename a parameter for clarity "context" is nevow terminology, whereas these are really requests. --- src/allmydata/web/operations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 206542b06..75be2f975 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -50,12 +50,12 @@ class OphandleTable(resource.Resource, service.Service): del self.timers return service.Service.stopService(self) - def add_monitor(self, ctx, monitor, renderer): - ophandle = get_arg(ctx, "ophandle") + def add_monitor(self, req, monitor, renderer): + ophandle = get_arg(req, "ophandle") assert ophandle now = time.time() self.handles[ophandle] = (monitor, renderer, now) - retain_for = get_arg(ctx, "retain-for", None) + retain_for = get_arg(req, "retain-for", None) if retain_for is not None: self._set_timer(ophandle, int(retain_for)) monitor.when_done().addBoth(self._operation_complete, ophandle) @@ -72,11 +72,11 @@ class OphandleTable(resource.Resource, service.Service): # if we already have a timer, the client must have provided the # retain-for= value, so don't touch it. - def redirect_to(self, ctx): - ophandle = get_arg(ctx, "ophandle") + def redirect_to(self, req): + ophandle = get_arg(req, "ophandle") assert ophandle - target = get_root(ctx) + "/operations/" + ophandle - output = get_arg(ctx, "output") + target = get_root(req) + "/operations/" + ophandle + output = get_arg(req, "output") if output: target = target + "?output=%s" % output return url.URL.fromString(target) From 2d2e8a58724f88d63f329b08a934d407320f3277 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 08:49:41 -0400 Subject: [PATCH 0245/1054] Add comments to web.operations.OphandleTable --- src/allmydata/web/operations.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 75be2f975..857fdf2c9 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -26,6 +26,8 @@ DAY = 24*HOUR (MONITOR, RENDERER, WHEN_ADDED) = range(3) class OphandleTable(resource.Resource, service.Service): + """Renders /operations/%d.""" + name = "operations" UNCOLLECTED_HANDLE_LIFETIME = 4*DAY @@ -51,6 +53,11 @@ class OphandleTable(resource.Resource, service.Service): return service.Service.stopService(self) def add_monitor(self, req, monitor, renderer): + """ + :param allmydata.webish.MyRequest req: + :param allmydata.monitor.Monitor monitor: + :param allmydata.web.directory.ManifestResults renderer: + """ ophandle = get_arg(req, "ophandle") assert ophandle now = time.time() @@ -73,6 +80,9 @@ class OphandleTable(resource.Resource, service.Service): # retain-for= value, so don't touch it. def redirect_to(self, req): + """ + :param allmydata.webish.MyRequest req: + """ ophandle = get_arg(req, "ophandle") assert ophandle target = get_root(req) + "/operations/" + ophandle From 982ad8942e2417816ff91743001f3b5418d4b255 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 18 Jun 2020 18:30:18 -0400 Subject: [PATCH 0246/1054] Comment about nevow.url usage in web.operations.OphandleTable --- src/allmydata/web/operations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 857fdf2c9..4dcad0028 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -89,6 +89,10 @@ class OphandleTable(resource.Resource, service.Service): output = get_arg(req, "output") if output: target = target + "?output=%s" % output + + # XXX: We have to use nevow.url here because nevow.appserver + # is unhappy with anything else; so this gets its own ticket. + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3314 return url.URL.fromString(target) def getChild(self, name, req): From df01c65540eb98168abd8c68bbb4a832f7509259 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 10:13:53 -0400 Subject: [PATCH 0247/1054] Add newsfragment --- newsfragments/3313.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3313.minor diff --git a/newsfragments/3313.minor b/newsfragments/3313.minor new file mode 100644 index 000000000..c4eecd956 --- /dev/null +++ b/newsfragments/3313.minor @@ -0,0 +1 @@ +Replace nevow with twisted.web in web.operations.OphandleTable From ef71fc2bfc743548e47a76578a88a18d9bd7ed26 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:15:01 -0600 Subject: [PATCH 0248/1054] Get rid of ?replace= handling entirely and more-correctly support /uri variants --- src/allmydata/test/test_testing.py | 98 ++++++++++++++++++++++++------ src/allmydata/testing/web.py | 52 ++++++++-------- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a995e756d..67c38ed57 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -21,6 +21,11 @@ from allmydata.uri import ( ) from allmydata.testing.web import ( create_tahoe_treq_client, + capability_generator, +) + +from hyperlink import ( + DecodedURL, ) from hypothesis import ( @@ -35,9 +40,12 @@ from testtools import ( ) from testtools.matchers import ( Always, + Equals, + MatchesStructure, ) from testtools.twistedsupport import ( succeeded, + failed, ) @@ -46,9 +54,9 @@ class FakeWebTest(TestCase): Test the WebUI verified-fakes infrastucture """ - def setUp(self): - super(FakeWebTest, self).setUp() - self.http_client = create_tahoe_treq_client() + # Note: do NOT use setUp() because Hypothesis doesn't work + # properly with it. You must instead do all fixture-type work + # yourself in each test. @given( content=binary(), @@ -58,21 +66,31 @@ class FakeWebTest(TestCase): Upload some content (via 'PUT /uri') and then download it (via 'GET /uri?uri=...') """ + http_client = create_tahoe_treq_client() @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri?replace=true", content) + resp = yield http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() cap = from_string(cap_raw) self.assertIsInstance(cap, CHKFileURI) - resp = yield self.http_client.get( + resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() + + # using the form "/uri/" is also valid + + resp = yield http_client.get( + "http://example.com/uri/{}".format(cap.to_string()) + ) + self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() self.assertEqual(content, round_trip_content) self.assertThat( @@ -80,21 +98,67 @@ class FakeWebTest(TestCase): succeeded(Always()), ) - @inlineCallbacks - def test_duplicate_upload(self): + @given( + content=binary(), + ) + def test_duplicate_upload(self, content): """ - Upload the same content (via 'PUT /uri') twice with no overwrite + Upload the same content (via 'PUT /uri') twice """ - content = "fake content\n" * 200 + http_client = create_tahoe_treq_client() - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + @inlineCallbacks + def do_test(): + resp = yield http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) - cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) - # this one fails: same content but no ?replace=true - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 409) + resp = yield http_client.put("http://example.com/uri", content) + self.assertThat(resp.code, Equals(200)) + self.assertThat( + do_test(), + succeeded(Always()), + ) + + def test_download_missing(self): + """ + Error if we download a capability that doesn't exist + """ + + http_client = create_tahoe_treq_client() + cap_gen = capability_generator("URI:CHK:") + + uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(next(cap_gen))) + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(500) + ) + ) + ) + + def test_download_no_arg(self): + """ + Error if we GET from "/uri" with no ?uri= query-arg + """ + + http_client = create_tahoe_treq_client() + + uri = DecodedURL.from_text(u"http://example.com/uri/") + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(400) + ) + ) + ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7c6fc1996..aaa795d06 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -76,7 +76,8 @@ class _FakeTahoeRoot(Resource, object): self.putChild(b"uri", self._uri) def add_data(self, kind, data): - return self._uri.add_data(kind, data) + fresh, cap = self._uri.add_data(kind, data) + return cap KNOWN_CAPABILITIES = [ @@ -156,39 +157,32 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self.capability_generators[kind]) return capability - def add_data(self, kind, data, allow_duplicate=False): + def add_data(self, kind, data): """ adds some data to our grid - :returns: a capability-string + :returns: a two-tuple: a bool (True if the data is freshly added) and a capability-string """ if not isinstance(data, bytes): raise TypeError("'data' must be bytes") for k in self.data: if self.data[k] == data: - if allow_duplicate: - return k - raise ValueError( - "Duplicate data" - ) + return (False, k) cap = self._generate_capability(kind) - if cap in self.data: - raise ValueError("already have '{}'".format(cap)) + assert cap not in self.data, "logic error" # shouldn't happen self.data[cap] = data - return cap + return (True, cap) def render_PUT(self, request): data = request.content.read() - request.setResponseCode(http.CREATED) # real code does this for brand-new files - replace = request.args.get("replace", None) - try: - return self.add_data("URI:CHK:", data, allow_duplicate=replace) - except ValueError: - msg, code = humanize_failure(Failure(ExistingChildError())) - request.setResponseCode(code) - return msg + fresh, cap = self.add_data("URI:CHK:", data) + if fresh: + request.setResponseCode(http.CREATED) # real code does this for brand-new files + else: + request.setResponseCode(http.OK) # replaced/modified files + return cap def render_POST(self, request): t = request.args[u"t"][0] @@ -198,7 +192,8 @@ class _FakeTahoeUriHandler(Resource, object): "mkdir-immutable": "URI:DIR2-CHK:" } kind = type_to_kind[t] - return self.add_data(kind, data) + fresh, cap = self.add_data(kind, data) + return cap def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -206,14 +201,19 @@ class _FakeTahoeUriHandler(Resource, object): for arg, value in uri.query: if arg == u"uri": capability = value - if capability is None: - raise Exception( - "No ?uri= arguent in GET '{}'".format( - uri.to_string() - ) - ) + # it's legal to use the form "/uri/" + if capability is None and request.postpath and request.postpath[0]: + capability = request.postpath[0] + # if we don't yet have a capability, that's an error + if capability is None: + request.setResponseCode(http.BAD_REQUEST) + return b"GET /uri requires uri=" + + # the user gave us a capability; if our Grid doesn't have any + # data for it, that's an error. if capability not in self.data: + request.setResponseCode(http.BAD_REQUEST) return u"No data for '{}'".format(capability).decode("ascii") return self.data[capability] From 57eb875354f42b6bd037bb8f3ec0fda25a6ce333 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:16:39 -0600 Subject: [PATCH 0249/1054] testtool's asserts --- src/allmydata/test/test_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 67c38ed57..6ba622f3c 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -71,7 +71,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + self.assertThat(resp.code, Equals(201)) cap_raw = yield resp.content() cap = from_string(cap_raw) @@ -80,7 +80,7 @@ class FakeWebTest(TestCase): resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) - self.assertEqual(resp.code, 200) + self.assertThat(resp.code, Equals(200)) round_trip_content = yield resp.content() From f0ac5a2cef89080c7fe77f73fd63e27f6bc6d232 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:19:30 -0600 Subject: [PATCH 0250/1054] more-testtools-y asserts --- src/allmydata/test/test_testing.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 6ba622f3c..37a5f3c7b 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -41,7 +41,9 @@ from testtools import ( from testtools.matchers import ( Always, Equals, + IsInstance, MatchesStructure, + AfterPreprocessing, ) from testtools.twistedsupport import ( succeeded, @@ -75,7 +77,7 @@ class FakeWebTest(TestCase): cap_raw = yield resp.content() cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat(cap, IsInstance(CHKFileURI)) resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) @@ -114,8 +116,13 @@ class FakeWebTest(TestCase): self.assertEqual(resp.code, 201) cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat( + cap_raw, + AfterPreprocessing( + from_string, + IsInstance(CHKFileURI) + ) + ) resp = yield http_client.put("http://example.com/uri", content) self.assertThat(resp.code, Equals(200)) From aedc58ef5158f43b6809b1afeab207ad7c9ba2af Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:22:42 -0600 Subject: [PATCH 0251/1054] unused imports --- src/allmydata/test/test_testing.py | 1 - src/allmydata/testing/web.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37a5f3c7b..ec6935914 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -47,7 +47,6 @@ from testtools.matchers import ( ) from testtools.twistedsupport import ( succeeded, - failed, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aaa795d06..6656236ca 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,9 +30,6 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) -from twisted.python.failure import ( - Failure, -) from treq.client import ( HTTPClient, @@ -47,12 +44,6 @@ import allmydata.uri from allmydata.util import ( base32, ) -from allmydata.interfaces import ( - ExistingChildError, -) -from allmydata.web.common import ( - humanize_failure, -) __all__ = ( From 9b68b484bdb7a92944990a6bdd34d6ddbd48916e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 22 Jun 2020 16:27:18 -0400 Subject: [PATCH 0252/1054] Run integration tests on Windows on GitHub Actions Integration tests are currently not run on Windows, because they turned out to be a little unreliable: sometimes they fail, and when that happens restarting the test would make things pass. We will re-enable them and see what happens. --- .github/workflows/ci.yml | 1 + newsfragments/3320.minor | 0 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3320.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ded8f418..7cd97dcca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,7 @@ jobs: matrix: os: - macos-latest + - windows-latest python-version: - 2.7 diff --git a/newsfragments/3320.minor b/newsfragments/3320.minor new file mode 100644 index 000000000..e69de29bb From 7d93ae9213aae1e4957bf580152a91ead586a511 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 22 Jun 2020 20:16:19 -0400 Subject: [PATCH 0253/1054] Skip Tor tests on Windows --- integration/conftest.py | 8 ++++++++ integration/test_tor.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index 5395d7c5f..35e550416 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -359,6 +359,10 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques @pytest.fixture(scope='session') def chutney(reactor, temp_dir): + + if sys.platform.startswith('win'): + pytest.skip('Tor tests are unstable on Windows') + chutney_dir = join(temp_dir, 'chutney') mkdir(chutney_dir) @@ -389,6 +393,10 @@ def chutney(reactor, temp_dir): @pytest.fixture(scope='session') def tor_network(reactor, temp_dir, chutney, request): + + if sys.platform.startswith('win'): + pytest.skip('Tor tests are unstable on Windows') + # this is the actual "chutney" script at the root of a chutney checkout chutney_dir = chutney chut = join(chutney_dir, 'chutney') diff --git a/integration/test_tor.py b/integration/test_tor.py index 633def8de..cec88a483 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -10,12 +10,17 @@ from six.moves import StringIO from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone from twisted.internet.defer import inlineCallbacks, Deferred + +import pytest import pytest_twisted import util # see "conftest.py" for the fixtures (e.g. "tor_network") +if sys.platform.startswith('win'): + pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) + @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) From b14c0a0495020b0bf39bdb6a1804f2f0acf63184 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 18:06:54 -0400 Subject: [PATCH 0254/1054] Make DownloadStatusPage a Resource No need of this being a MultiFormatResource, because it renders just HTML. --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index e6e422f6f..4f715fd5d 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -366,7 +366,7 @@ class _EventJson(Resource, object): return json.dumps(data, indent=1) + "\n" -class DownloadStatusPage(MultiFormatResource): +class DownloadStatusPage(Resource): """Renders /status/down-%d.""" def __init__(self, download_status): From 36ecd8f38455166ea2b275b2149361fdc4df6996 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 18:07:27 -0400 Subject: [PATCH 0255/1054] Use render_GET() Implementing `render_METHOD()` is preferred over overriding render() --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4f715fd5d..2efd2a940 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -377,7 +377,7 @@ class DownloadStatusPage(Resource): self._download_status = download_status self.putChild("event_json", _EventJson(self._download_status)) - def render_HTML(self, req): + def render_GET(self, req): elem = DownloadStatusElement(self._download_status) return renderElement(req, elem) From 00e852c465ab211f449674932c0002da6afef6d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 20:51:26 -0400 Subject: [PATCH 0256/1054] Make DownloadStatus an object PythonTwoRegressions.test_new_style_classes would be unhappy without this. --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2efd2a940..5100a9a3d 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -366,7 +366,7 @@ class _EventJson(Resource, object): return json.dumps(data, indent=1) + "\n" -class DownloadStatusPage(Resource): +class DownloadStatusPage(Resource, object): """Renders /status/down-%d.""" def __init__(self, download_status): From 6ab7f26faf4899caee412279c3812fd4ba6de486 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:28:31 -0400 Subject: [PATCH 0257/1054] Make UploadStatusPage a Resource --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 8147437fa..ee0e924e0 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -183,7 +183,7 @@ class UploadResultsRendererMixin(Element): return d -class UploadStatusPage(MultiFormatResource): +class UploadStatusPage(Resource, object): """Renders /status/up-%d.""" def __init__(self, upload_status): @@ -193,7 +193,7 @@ class UploadStatusPage(MultiFormatResource): super(UploadStatusPage, self).__init__() self._upload_status = upload_status - def render_HTML(self, req): + def render_GET(self, req): elem = UploadStatusElement(self._upload_status) return renderElement(req, elem) From dfbb8e7743f07cb5369f64c8949c1ac99bab5036 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:53:39 -0400 Subject: [PATCH 0258/1054] Make UploadResultsPage a Resource --- src/allmydata/web/unlinked.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 91efff2c4..09795101a 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -3,6 +3,7 @@ import urllib from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath +from twisted.web.resource import Resource from twisted.web.template import ( XMLFile, renderer, @@ -19,7 +20,6 @@ from allmydata.web.common import ( WebError, get_format, get_mutable_type, - MultiFormatResource, ) from allmydata.web import status @@ -73,7 +73,7 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(MultiFormatResource): +class UploadResultsPage(Resource, object): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): @@ -83,7 +83,7 @@ class UploadResultsPage(MultiFormatResource): super(UploadResultsPage, self).__init__() self._upload_results = upload_results - def render_HTML(self, req): + def render_POST(self, req): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) From 88d3fa7415505c3c7a305d1613a9faf7e3fc111d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:54:23 -0400 Subject: [PATCH 0259/1054] Remove now-unnecessary UploadResultsPage.render_UPLOAD --- src/allmydata/web/unlinked.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 09795101a..df342d7dd 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -87,23 +87,6 @@ class UploadResultsPage(Resource, object): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) - # This is weird but necessary because: - # - # 1. MultiFormatResource.render() uses argument "t" to figure out - # its output format. - # - # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". - # See URIHandler.render_POST(). - # - # MultiFormatResource.render() looks up "t" argument, which in - # this case has the value "upload", and then it would look for a - # render_UPLOAD() method. - # - # We could change upload request to use more descriptive names - # that do not cause name collisions like this. That should be a - # separate change though. - render_UPLOAD = render_HTML - class UploadResultsElement(status.UploadResultsRendererMixin): From f9e864c51cf079a61cff449074f35ec3d7bc2a53 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 24 Jun 2020 08:15:18 -0400 Subject: [PATCH 0260/1054] Use simpler syntax to update map --- src/allmydata/web/status.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index ec18426dc..8214913de 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1250,15 +1250,14 @@ class StatusElement(Element): result = dict() started_s = render_time(op.get_started()) - result.update({"started": started_s}) + result["started"] = started_s si_s = base32.b2a_or_none(op.get_storage_index()) if si_s is None: si_s = "(None)" - result.update({"si": si_s}) - result.update({"helper": - {True: "Yes", False: "No"}[op.using_helper()]}) + result["si"] = si_s + result["helper"] = {True: "Yes", False: "No"}[op.using_helper()] size = op.get_size() if size is None: @@ -1266,38 +1265,38 @@ class StatusElement(Element): elif isinstance(size, (int, long, float)): size = abbreviate_size(size) - result.update({"total_size": size}) + result["total_size"] = size progress = op.get_progress() if IUploadStatus.providedBy(op): link = "up-%d" % op.get_counter() - result.update({"type": "upload"}) + result["type"] = "upload" # TODO: make an ascii-art bar (chk, ciphertext, encandpush) = progress progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" % ((100.0 * chk), (100.0 * ciphertext), (100.0 * encandpush))) - result.update({"progress": progress_s}) + result["progress"] = progress_s elif IDownloadStatus.providedBy(op): link = "down-%d" % op.get_counter() - result.update({"type": "download"}) - result.update({"progress": "%.1f%%" % (100.0 * progress)}) + result["type"] = "download" + result["progress"] = "%.1f%%" % (100.0 * progress) elif IPublishStatus.providedBy(op): link = "publish-%d" % op.get_counter() - result.update({"type": "publish"}) - result.update({"progress": "%.1f%%" % (100.0 * progress)}) + result["type"] = "publish" + result["progress"] = "%.1f%%" % (100.0 * progress) elif IRetrieveStatus.providedBy(op): - result.update({"type": "retrieve"}) + result["type"] = "retrieve" link = "retrieve-%d" % op.get_counter() - result.update({"progress": "%.1f%%" % (100.0 * progress)}) + result["progress"] = "%.1f%%" % (100.0 * progress) else: assert IServermapUpdaterStatus.providedBy(op) - result.update({"type": "mapupdate %s" % op.get_mode()}) + result["type"] = "mapupdate %s" % op.get_mode() link = "mapupdate-%d" % op.get_counter() - result.update({"progress": "%.1f%%" % (100.0 * progress)}) + result["progress"] = "%.1f%%" % (100.0 * progress) - result.update({"status": tags.a(op.get_status(), href=link)}) + result["status"] = tags.a(op.get_status(), href=link) return result From a9f4210014df81b1a7d5b914f758aecd64e34b54 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:15:58 -0600 Subject: [PATCH 0261/1054] streap -> stream --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 6656236ca..0d6e7cb6e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -80,7 +80,7 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ - Deterministically generates a streap of valid capabilities of the + Deterministically generates a stream of valid capabilities of the given kind. The N, K and size values aren't related to anything real. From 80359e02e6e255d48ade2060e413447b2728a63e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:17:34 -0600 Subject: [PATCH 0262/1054] assert -> Exception --- src/allmydata/testing/web.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 0d6e7cb6e..12f5ef40c 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -162,7 +162,10 @@ class _FakeTahoeUriHandler(Resource, object): return (False, k) cap = self._generate_capability(kind) - assert cap not in self.data, "logic error" # shouldn't happen + # it should be impossible for this to already be in our data, + # but check anyway to be sure + if cap in self.data: + raise Exception("Internal error; key already exists somehow") self.data[cap] = data return (True, cap) From 9a2da37241305472484f3ae1a6c6698fbe45cd49 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:41:17 -0400 Subject: [PATCH 0263/1054] Here's a start --- docs/CODE_OF_CONDUCT.md | 53 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 54 insertions(+) create mode 100644 docs/CODE_OF_CONDUCT.md diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..868f59c6a --- /dev/null +++ b/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,53 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer (see below). All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + +The following community members have made themselves available for conduct issues: + +- Jean-Paul Calderone (jean-paul at leastauthority dot com) + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/docs/index.rst b/docs/index.rst index 98b4f241b..b125cdd4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING + ../.github/CODE_OF_CONDUCT servers helper From f9cab479fd55dca3d8e1e4239f5503d96898ad7e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:48:39 -0400 Subject: [PATCH 0264/1054] as far as I can tell there is nothing magic about putting it in .github --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index b125cdd4e..581e74bbb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING - ../.github/CODE_OF_CONDUCT + CODE_OF_CONDUCT servers helper From f9ac0a7a9eb178dc998a2bbb76f3e8efb3fcf212 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:50:47 -0400 Subject: [PATCH 0265/1054] news fragment --- newsfragments/2755.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2755.other diff --git a/newsfragments/2755.other b/newsfragments/2755.other new file mode 100644 index 000000000..e3e31465d --- /dev/null +++ b/newsfragments/2755.other @@ -0,0 +1 @@ +The Tahoe-LAFS project has adopted a formal code of conduct. From b7f88b278f3c157213577faf23f397a7c3709b83 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 13:40:40 -0400 Subject: [PATCH 0266/1054] Split out tests for humanreadable into their own module, so we can port just humanreadable.py in isolation. --- src/allmydata/test/test_humanreadable.py | 46 ++++++++++++++++++++++++ src/allmydata/test/test_util.py | 40 ++------------------- 2 files changed, 49 insertions(+), 37 deletions(-) create mode 100644 src/allmydata/test/test_humanreadable.py diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py new file mode 100644 index 000000000..ea1d9ce9f --- /dev/null +++ b/src/allmydata/test/test_humanreadable.py @@ -0,0 +1,46 @@ +"""Tests for allmydata.util.humanreadable.""" + +from twisted.trial import unittest + +from allmydata.util import humanreadable + + + + +def foo(): pass # keep the line number constant + + +class NoArgumentException(Exception): + def __init__(self): + pass + +class HumanReadable(unittest.TestCase): + def test_repr(self): + hr = humanreadable.hr + self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(self.test_repr), + ">") + self.failUnlessEqual(hr(long(1)), "1") + self.failUnlessEqual(hr(10**40), + "100000000000000000...000000000000000000") + self.failUnlessEqual(hr(self), "") + self.failUnlessEqual(hr([1,2]), "[1, 2]") + self.failUnlessEqual(hr({1:2}), "{1:2}") + try: + raise ValueError + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "ValueError()") # python-2.5 + try: + raise ValueError("oops") + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "ValueError('oops',)") # python-2.5 + try: + raise NoArgumentException + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "NoArgumentException()") # python-2.5 diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 3bddc8bcc..670304a8b 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,8 +1,5 @@ from __future__ import print_function - -def foo(): pass # keep the line number constant - import binascii import six import hashlib @@ -17,7 +14,7 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil +from allmydata.util import base32, idlib, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil @@ -57,44 +54,13 @@ class Base32(unittest.TestCase): self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34") self.failUnlessRaises(AssertionError, base32.a2b, "b0gus") + class IDLib(unittest.TestCase): def test_nodeid_b2a(self): self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) -class NoArgumentException(Exception): - def __init__(self): - pass -class HumanReadable(unittest.TestCase): - def test_repr(self): - hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") - self.failUnlessEqual(hr(self.test_repr), - ">") - self.failUnlessEqual(hr(long(1)), "1") - self.failUnlessEqual(hr(10**40), - "100000000000000000...000000000000000000") - self.failUnlessEqual(hr(self), "") - self.failUnlessEqual(hr([1,2]), "[1, 2]") - self.failUnlessEqual(hr({1:2}), "{1:2}") - try: - raise ValueError - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "ValueError()") # python-2.5 - try: - raise ValueError("oops") - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "ValueError('oops',)") # python-2.5 - try: - raise NoArgumentException - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "NoArgumentException()") # python-2.5 +class Abbreviate(unittest.TestCase): def test_abbrev_time_1s(self): diff = timedelta(seconds=1) From 99b62775e901cd705538df598f1d27b7aa524414 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 13:42:30 -0400 Subject: [PATCH 0267/1054] Initial list of ported modules. --- src/allmydata/util/_python3.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/allmydata/util/_python3.py diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py new file mode 100644 index 000000000..c91e8c400 --- /dev/null +++ b/src/allmydata/util/_python3.py @@ -0,0 +1,3 @@ +"""Track the port to Python 3.""" + +PORTED_MODULES = [] From c207a0c9329faef523e1c76eca1c7b27e2198c19 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:10:38 -0400 Subject: [PATCH 0268/1054] Port test_humanreadable.py to Python 3. --- setup.py | 3 +++ src/allmydata/test/test_humanreadable.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index caa37f16e..f773652af 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,9 @@ install_requires = [ # WebSocket library for twisted and asyncio "autobahn >= 19.5.2", + + # Support for Python 3 transition + "future >= 0.18.2", ] setup_requires = [ diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index ea1d9ce9f..7df6e2cf3 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -1,12 +1,17 @@ """Tests for allmydata.util.humanreadable.""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from future import standard_library +standard_library.install_aliases() +from builtins import * from twisted.trial import unittest from allmydata.util import humanreadable - - def foo(): pass # keep the line number constant @@ -17,7 +22,7 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(foo), "") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") @@ -37,7 +42,9 @@ class HumanReadable(unittest.TestCase): except Exception as e: self.failUnless( hr(e) == "" # python-2.4 - or hr(e) == "ValueError('oops',)") # python-2.5 + or hr(e) == "ValueError('oops',)" # python-2.5 + or hr(e) == "ValueError(u'oops',)" # python 2 during py3 transition + ) try: raise NoArgumentException except Exception as e: From 70a029fb81aa79a2082d30fe81147ab5fd04c6fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:26:35 -0400 Subject: [PATCH 0269/1054] Tests pass on both Python 2 and Python 3. --- setup.py | 4 +++- src/allmydata/test/test_humanreadable.py | 10 ++++---- src/allmydata/util/humanreadable.py | 29 +++++++++++++++--------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index f773652af..9b0436035 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,9 @@ install_requires = [ # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely - "foolscap == 0.13.1", + # * foolscap >= 20.4 is necessary for Python 3 + "foolscap == 0.13.1; ; python_version < '3.0'", + "foolscap >= 20.4.0; ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 7df6e2cf3..2cb66cdaf 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -6,7 +6,7 @@ from __future__ import print_function from future import standard_library standard_library.install_aliases() -from builtins import * +from past.builtins import long from twisted.trial import unittest from allmydata.util import humanreadable @@ -26,8 +26,9 @@ class HumanReadable(unittest.TestCase): self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") - self.failUnlessEqual(hr(10**40), - "100000000000000000...000000000000000000") + self.assertIn(hr(10**40), + ["100000000000000000...000000000000000000", + "100000000000000000...0000000000000000000"]) self.failUnlessEqual(hr(self), "") self.failUnlessEqual(hr([1,2]), "[1, 2]") self.failUnlessEqual(hr({1:2}), "{1:2}") @@ -50,4 +51,5 @@ class HumanReadable(unittest.TestCase): except Exception as e: self.failUnless( hr(e) == "" # python-2.4 - or hr(e) == "NoArgumentException()") # python-2.5 + or hr(e) == "NoArgumentException()" # python-2.5 + or hr(e) == "", hr(e)) # python-3 diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 4d97effe9..8ef15b346 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -1,5 +1,12 @@ -import exceptions, os -from repr import Repr +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals +from future import standard_library +standard_library.install_aliases() + +import os +from reprlib import Repr class BetterRepr(Repr, object): def __init__(self): @@ -14,21 +21,21 @@ class BetterRepr(Repr, object): self.maxother = 300 def repr_function(self, obj, level): - if hasattr(obj, 'func_code'): - return '<' + obj.func_name + '() at ' + os.path.basename(obj.func_code.co_filename) + ':' + str(obj.func_code.co_firstlineno) + '>' + if hasattr(obj, '__code__'): + return '<' + obj.__name__ + '() at ' + os.path.basename(obj.__code__.co_filename) + ':' + str(obj.__code__.co_firstlineno) + '>' else: - return '<' + obj.func_name + '() at (builtin)' + return '<' + obj.__name__ + '() at (builtin)' def repr_instance_method(self, obj, level): - if hasattr(obj, 'func_code'): - return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at ' + os.path.basename(obj.im_func.func_code.co_filename) + ':' + str(obj.im_func.func_code.co_firstlineno) + '>' + if hasattr(obj, '__code__'): + return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at ' + os.path.basename(obj.__func__.__code__.co_filename) + ':' + str(obj.__func__.__code__.co_firstlineno) + '>' else: - return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at (builtin)' + return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at (builtin)' def repr_long(self, obj, level): s = repr(obj) # XXX Hope this isn't too slow... if len(s) > self.maxlong: - i = max(0, (self.maxlong-3)/2) + i = max(0, (self.maxlong-3) // 2) j = max(0, self.maxlong-3-i) s = s[:i] + '...' + s[len(s)-j:] if s[-1] == 'L': @@ -43,7 +50,7 @@ class BetterRepr(Repr, object): on it. If it is an instance of list call self.repr_list() on it. Else call Repr.repr_instance(). """ - if isinstance(obj, exceptions.Exception): + if isinstance(obj, Exception): # Don't cut down exception strings so much. tms = self.maxstring self.maxstring = max(512, tms * 4) @@ -91,7 +98,7 @@ class BetterRepr(Repr, object): if level <= 0: return '{...}' s = '' n = len(obj) - items = obj.items()[:min(n, self.maxdict)] + items = list(obj.items())[:min(n, self.maxdict)] items.sort() for key, val in items: entry = self.repr1(key, level-1) + ':' + self.repr1(val, level-1) From ad7f44db99c856f502ae1f25f0545572111fdcf9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:27:43 -0400 Subject: [PATCH 0270/1054] Listed ported modules. --- src/allmydata/util/_python3.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c91e8c400..1dbbb4a1a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,3 +1,10 @@ """Track the port to Python 3.""" -PORTED_MODULES = [] +# Keep these sorted alphabetically, to reduce merge conflicts: +PORTED_MODULES = [ + "allmydata.util.humanreadable", +] + +PORTED_TEST_MODULES = [ + "allmydata.test.test_humanreadable", +] From b77807c558b455a5a7077a1e968b54d60a3bf0c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:29:15 -0400 Subject: [PATCH 0271/1054] News entry. --- newsfragments/3324.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3324.other diff --git a/newsfragments/3324.other b/newsfragments/3324.other new file mode 100644 index 000000000..e69de29bb From 12691eb597d25eed39ae7edc727a25bf789c3a0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:31:33 -0400 Subject: [PATCH 0272/1054] Only install module mappings once. --- src/allmydata/__init__.py | 5 +++++ src/allmydata/test/test_humanreadable.py | 4 ++-- src/allmydata/util/humanreadable.py | 2 -- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 915b4c013..bbfcb89d4 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -37,3 +37,8 @@ __appname__ = "tahoe-lafs" # in the "application" part of the Tahoe versioning scheme: # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning __full_version__ = __appname__ + '/' + str(__version__) + + +# Install Python 3 module locations in Python 2: +from future import standard_library +standard_library.install_aliases() diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 2cb66cdaf..94fe07504 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,14 +4,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future import standard_library -standard_library.install_aliases() from past.builtins import long + from twisted.trial import unittest from allmydata.util import humanreadable + def foo(): pass # keep the line number constant diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 8ef15b346..696fc72bb 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -2,8 +2,6 @@ from __future__ import division from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import os from reprlib import Repr From 83182e1d381510e3000ffd0ece8ed2379dfe0144 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:15:13 -0400 Subject: [PATCH 0273/1054] Remove extraneous semicolon. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9b0436035..fd8d7bcff 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,8 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 20.4 is necessary for Python 3 - "foolscap == 0.13.1; ; python_version < '3.0'", - "foolscap >= 20.4.0; ; python_version > '3.0'", + "foolscap == 0.13.1 ; python_version < '3.0'", + "foolscap >= 20.4.0 ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] From 140cb80ff7be474dcbf5f3db4521ca0ee131ec4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:15:57 -0400 Subject: [PATCH 0274/1054] Clarify comment. --- src/allmydata/test/test_humanreadable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 94fe07504..6d90b562e 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -12,7 +12,7 @@ from allmydata.util import humanreadable -def foo(): pass # keep the line number constant +def foo(): pass # FYI foo()'s line number is used in the test below class NoArgumentException(Exception): From 83b9bbf7433825f2f2e4390f51d3dedfd6ec5752 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:28:36 -0400 Subject: [PATCH 0275/1054] Switch definitely to _python3.py for listing modules, but also note if a module has been ported in its docstring. --- src/allmydata/ported-modules.txt | 1 - src/allmydata/test/test_humanreadable.py | 7 ++++++- src/allmydata/util/_python3.py | 1 + src/allmydata/util/humanreadable.py | 6 ++++++ src/allmydata/util/namespace.py | 3 +++ 5 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 src/allmydata/ported-modules.txt diff --git a/src/allmydata/ported-modules.txt b/src/allmydata/ported-modules.txt deleted file mode 100644 index cd150cbfb..000000000 --- a/src/allmydata/ported-modules.txt +++ /dev/null @@ -1 +0,0 @@ -allmydata.util.namespace diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 6d90b562e..cbe478daf 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -1,4 +1,9 @@ -"""Tests for allmydata.util.humanreadable.""" +""" +Tests for allmydata.util.humanreadable. + +This module has been ported to Python 3. +""" + from __future__ import unicode_literals from __future__ import absolute_import from __future__ import division diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1dbbb4a1a..4f023ade7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -3,6 +3,7 @@ # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.humanreadable", + "allmydata.util.namespace", ] PORTED_TEST_MODULES = [ diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 696fc72bb..edddcdf4b 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -1,3 +1,9 @@ +""" +Utilities for turning objects into human-readable strings. + +This module has been ported to Python 3. +""" + from __future__ import division from __future__ import absolute_import from __future__ import print_function diff --git a/src/allmydata/util/namespace.py b/src/allmydata/util/namespace.py index c386b7376..247827c47 100644 --- a/src/allmydata/util/namespace.py +++ b/src/allmydata/util/namespace.py @@ -1,3 +1,6 @@ +""" +This module has been ported to Python 3. +""" class Namespace(object): pass From 6d4ce3b7ad5ddc48c1a883f7855bc4d773a0bf62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:44:47 -0400 Subject: [PATCH 0276/1054] Additional hidden imports due to use of Future. --- pyinstaller.spec | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 6d1bd21f2..875629c13 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -27,15 +27,29 @@ added_files = [ ('src/allmydata/web/static/img/*.png', 'allmydata/web/static/img')] hidden_imports = [ + '__builtin__', 'allmydata.client', 'allmydata.introducer', 'allmydata.stats', + 'base64', 'cffi', + 'collections', + 'commands', 'Crypto', + 'functools', + 'future.backports.misc', + 'itertools', + 'math', 'packaging.specifiers', + 're', + 'reprlib', 'six.moves.html_parser', + 'subprocess', + 'UserDict', + 'UserList', + 'UserString', 'yaml', - 'zfec' + 'zfec', ] a = Analysis( From 8a85bf783645e7510087001b67a3ce06d0c7fc7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 6 Jul 2020 10:41:38 -0400 Subject: [PATCH 0277/1054] add a contributor covenant badge --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d53fa0104..274cee41d 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,8 @@ function correctly, preserving your privacy and security. For full documentation, please see http://tahoe-lafs.readthedocs.io/en/latest/ . -|readthedocs| |travis| |circleci| |codecov| +|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov| + INSTALLING ========== @@ -105,3 +106,5 @@ slides. .. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md From 906d6ac29151c4e5f69c3f70b7e6aa03af7bc4c9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 13:50:06 -0400 Subject: [PATCH 0278/1054] Switch to flake8 as a better way of running pyflakes. --- Makefile | 2 +- newsfragments/3329.other | 0 setup.cfg | 5 +++++ setup.py | 3 ++- tox.ini | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3329.other diff --git a/Makefile b/Makefile index 6e0009ddf..6dd87409e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ default: PYTHON=python export PYTHON -PYFLAKES=pyflakes +PYFLAKES=flake8 export PYFLAKES SOURCES=src/allmydata static misc setup.py diff --git a/newsfragments/3329.other b/newsfragments/3329.other new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index de40c9e23..c2d224225 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,8 @@ install = update_version install develop = update_version develop bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel + +[flake8] +# For now, only use pyflakes errors; flake8 is still helpful because it allows +# ignoring specific errors/warnings when needed. +select = F \ No newline at end of file diff --git a/setup.py b/setup.py index caa37f16e..0d43294cc 100644 --- a/setup.py +++ b/setup.py @@ -353,11 +353,12 @@ setup(name="tahoe-lafs", # also set in __init__.py # discussion. ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ + "flake8", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. - "pyflakes == 2.1.0", + "pyflakes == 2.2.0", # coverage 5.0 breaks the integration tests in some opaque way. # This probably needs to be addressed in a more permanent way # eventually... diff --git a/tox.ini b/tox.ini index 3134a0d9f..44b583baa 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = whitelist_externals = /bin/mv commands = - pyflakes src static misc setup.py + flake8 src static misc setup.py python misc/coding_tools/check-umids.py src python misc/coding_tools/check-debugging.py python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py From b99033410431b3434d0e6f4614c127b118d36584 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 14:00:02 -0400 Subject: [PATCH 0279/1054] Add the builtins import to make Python 2 builtins match Python 3's. --- src/allmydata/test/test_humanreadable.py | 6 +++++- src/allmydata/util/humanreadable.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index cbe478daf..ec6d3f666 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -9,6 +9,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from past.builtins import long from twisted.trial import unittest @@ -27,7 +31,7 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(foo), "") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index edddcdf4b..8c7079c3a 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -9,6 +9,10 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import os from reprlib import Repr From f548593f374bcf499c358faa1f56c6d85a4acf9e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 14:18:18 -0400 Subject: [PATCH 0280/1054] Fix test_python3, and mark these two modules as being ported as well. --- src/allmydata/test/test_python3.py | 40 +++++++++++++++++++++++------- src/allmydata/util/_python3.py | 17 ++++++++++++- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 8a25af889..294323b68 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -1,6 +1,18 @@ """ Tests related to the Python 3 porting effort itself. + +This module has been ported to Python 3. """ +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if False:#PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from unittest import expectedFailure from pkg_resources import ( resource_stream, @@ -13,11 +25,18 @@ from twisted.trial.unittest import ( SynchronousTestCase, ) +from allmydata.util._python3 import PORTED_MODULES, PORTED_TEST_MODULES + class Python3PortingEffortTests(SynchronousTestCase): + def test_finished_porting(self): """ Tahoe-LAFS has been ported to Python 3. + + Once + https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 + is completed this test should pass (and can be deleted!). """ tahoe_lafs_module_names = set(all_module_names("allmydata")) ported_names = set(ported_module_names()) @@ -31,7 +50,10 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed" + if PY2: + test_finished_porting.skip = "For some reason todo isn't working on Python 2 now" + else: + test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed" def test_ported_modules_exist(self): """ @@ -70,18 +92,18 @@ def all_module_names(toplevel): """ allmydata = getModule(toplevel) for module in allmydata.walkModules(): - yield module.name.decode("utf-8") + name = module.name + if PY2: + name = name.decode("utf-8") + yield name def ported_module_names(): """ - :return list[unicode]: A ``set`` of ``unicode`` giving the names of + :return list[unicode]: A ``list`` of ``unicode`` giving the names of Tahoe-LAFS modules which have been ported to Python 3. """ - return resource_stream( - "allmydata", - u"ported-modules.txt", - ).read().splitlines() + return PORTED_MODULES + PORTED_TEST_MODULES def unported_report(tahoe_lafs_module_names, ported_names): @@ -100,8 +122,8 @@ def count_lines(module_name): try: source = module.filePath.getContent() except Exception as e: - print(module_name, e) + print((module_name, e)) return 0 lines = source.splitlines() - nonblank = filter(None, lines) + nonblank = [_f for _f in lines if _f] return len(nonblank) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4f023ade7..fd679e014 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,11 +1,26 @@ -"""Track the port to Python 3.""" +""" +Track the port to Python 3. + +This module has been ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.humanreadable", "allmydata.util.namespace", + "allmydata.util._python3", ] PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", + "allmydata.test.test_python3", ] From b9737febd0ee1f497bc5e6fc92acb342fdd52013 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 6 Jul 2020 13:49:07 -0600 Subject: [PATCH 0281/1054] add meejah to code-of-conduct --- docs/CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 868f59c6a..3b14f2d70 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -44,6 +44,7 @@ incident. The following community members have made themselves available for conduct issues: - Jean-Paul Calderone (jean-paul at leastauthority dot com) +- meejah (meejah at meejah dot ca) This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.3.0, available at From e6eb3877073ec97a37970b8632b46868abea99fc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Jul 2020 17:05:52 -0400 Subject: [PATCH 0282/1054] Remove status.RateAndTimeMixin Both UploadResultsRenderer and DownloadResultsRenderer do not use RateAndTimeMixin anymore: safe to remove it now. Tests for methods formerly in RateAndTimeMixin have been moved to test.web.test_util: specifically test_abbreviate_rate() and test_abbreviate_time(). --- src/allmydata/test/web/test_web.py | 23 ----------------------- src/allmydata/web/status.py | 8 -------- 2 files changed, 31 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 01cb6b1ca..eb7b0b2c1 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1035,29 +1035,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d - def test_status_numbers(self): - urrm = status.UploadResultsRendererMixin() - self.failUnlessReallyEqual(urrm.render_time(None, None), "") - self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(urrm.render_rate(None, None), "") - self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps") - - drrm = status.DownloadResultsRendererMixin() - self.failUnlessReallyEqual(drrm.render_time(None, None), "") - self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(drrm.render_rate(None, None), "") - self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps") - def test_GET_FILEURL(self): d = self.GET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 38c9f8440..fe9043563 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -27,14 +27,6 @@ from allmydata.web.common import ( from allmydata.interfaces import IUploadStatus, IDownloadStatus, \ IPublishStatus, IRetrieveStatus, IServermapUpdaterStatus -class RateAndTimeMixin(object): - - def render_time(self, ctx, data): - return abbreviate_time(data) - - def render_rate(self, ctx, data): - return abbreviate_rate(data) - class UploadResultsRendererMixin(Element): # this requires a method named 'upload_results' From a3bf4e0269301eeb074ff8b8ee8efc367b111e74 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Jul 2020 20:18:22 -0400 Subject: [PATCH 0283/1054] Remove an unused import --- src/allmydata/test/web/test_web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index eb7b0b2c1..f68da2dae 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -33,7 +33,6 @@ from allmydata.immutable import upload from allmydata.immutable.downloader.status import DownloadStatus from allmydata.dirnode import DirectoryNode from allmydata.nodemaker import NodeMaker -from allmydata.web import status from allmydata.web.common import WebError, MultiFormatPage from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data From 0ed8a7709e4ff1d78d0ba19db4c537c824f1aa41 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:18:11 -0400 Subject: [PATCH 0284/1054] Perhaps this is the preferred formatting? --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 274cee41d..082925670 100644 --- a/README.rst +++ b/README.rst @@ -107,4 +107,5 @@ slides. :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master -.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :target: docs/CODE_OF_CONDUCT.md From f08e2ba832f829c5469b87752235b0161c54902b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:19:30 -0400 Subject: [PATCH 0285/1054] add some alt text --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 082925670..d3b089186 100644 --- a/README.rst +++ b/README.rst @@ -108,4 +108,5 @@ slides. :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :alt: code of conduct :target: docs/CODE_OF_CONDUCT.md From e6268f3dd5a762d3e6b6abf1327f6046d321875c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:13:45 -0400 Subject: [PATCH 0286/1054] Add additional dependencies. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index cc0eea479..3b54519f1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib +, html5lib, future }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; From 238b17098e1fff93cdcd2f61abdaf1b7bafc646a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:28:11 -0400 Subject: [PATCH 0287/1054] Add future in another place. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 3b54519f1..d82c97db7 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,6 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools + future ]; checkInputs = with python.pkgs; [ From 3d0e4df5cec43570d0b5cb672db14b945efd381c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:30:01 -0400 Subject: [PATCH 0288/1054] Lint fixes. --- src/allmydata/test/test_python3.py | 8 +--- src/allmydata/test/test_util.py | 74 ++++++++++++++---------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 294323b68..1326c8282 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -9,15 +9,9 @@ from __future__ import division from __future__ import print_function from future.utils import PY2 -if False:#PY2: +if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -from unittest import expectedFailure - -from pkg_resources import ( - resource_stream, -) - from twisted.python.modules import ( getModule, ) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 670304a8b..65ba308d9 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -60,45 +60,6 @@ class IDLib(unittest.TestCase): self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) -class Abbreviate(unittest.TestCase): - - def test_abbrev_time_1s(self): - diff = timedelta(seconds=1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('1 second ago', s) - - def test_abbrev_time_25s(self): - diff = timedelta(seconds=25) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('25 seconds ago', s) - - def test_abbrev_time_future_5_minutes(self): - diff = timedelta(minutes=-5) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes in the future', s) - - def test_abbrev_time_hours(self): - diff = timedelta(hours=4) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('4 hours ago', s) - - def test_abbrev_time_day(self): - diff = timedelta(hours=49) # must be more than 2 days - s = abbreviate.abbreviate_time(diff) - self.assertEqual('2 days ago', s) - - def test_abbrev_time_month(self): - diff = timedelta(days=91) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('3 months ago', s) - - def test_abbrev_time_year(self): - diff = timedelta(weeks=(5 * 52) + 1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 years ago', s) - - - class MyList(list): pass @@ -950,6 +911,41 @@ class HashUtilTests(unittest.TestCase): ) class Abbreviate(unittest.TestCase): + def test_abbrev_time_1s(self): + diff = timedelta(seconds=1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('1 second ago', s) + + def test_abbrev_time_25s(self): + diff = timedelta(seconds=25) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('25 seconds ago', s) + + def test_abbrev_time_future_5_minutes(self): + diff = timedelta(minutes=-5) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 minutes in the future', s) + + def test_abbrev_time_hours(self): + diff = timedelta(hours=4) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('4 hours ago', s) + + def test_abbrev_time_day(self): + diff = timedelta(hours=49) # must be more than 2 days + s = abbreviate.abbreviate_time(diff) + self.assertEqual('2 days ago', s) + + def test_abbrev_time_month(self): + diff = timedelta(days=91) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('3 months ago', s) + + def test_abbrev_time_year(self): + diff = timedelta(weeks=(5 * 52) + 1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 years ago', s) + def test_time(self): a = abbreviate.abbreviate_time self.failUnlessEqual(a(None), "unknown") From 7877d8a5cc9cfebddb48c76cd971632f9e9dac24 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 7 Jul 2020 19:55:14 -0400 Subject: [PATCH 0289/1054] Use an accessor --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index fe9043563..db11cc6e7 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -468,8 +468,8 @@ class DownloadStatusElement(Element): # See #3311: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3311 @renderer def events(self, req, tag): - if not self._download_status.storage_index: - return + if not self._download_status.get_storage_index(): + return tag srt = self._short_relative_time From 0c4d24a2ac8eba0d0995d207b214e0871c2307cf Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 7 Jul 2020 20:24:56 -0400 Subject: [PATCH 0290/1054] Avoid use of deferred within an Element This causes the final HTML to be rendered funny, with a bunch of "", which is clearly not what we want. --- src/allmydata/web/status.py | 121 ++++++++++++++---------------------- 1 file changed, 48 insertions(+), 73 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index db11cc6e7..4a9594a1c 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -439,7 +439,7 @@ class DownloadStatusElement(Element): # # See #3310: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3310 def download_results(self): - return defer.maybeDeferred(self._download_status.get_results) + return self._download_status.get_results() def _relative_time(self, t): if t is None: @@ -629,13 +629,9 @@ class DownloadStatusElement(Element): @renderer def results(self, req, tag): - d = self.download_results() - def _got_results(results): - if results: - return tag - return "" - d.addCallback(_got_results) - return d + if self.download_results(): + return tag + return "" @renderer def started(self, req, tag): @@ -673,61 +669,47 @@ class DownloadStatusElement(Element): @renderer def servers_used(self, req, tag): - d = self.download_results() - d.addCallback(lambda res: res.servers_used) - def _got(servers_used): - if not servers_used: - return "" - peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) - for peerid in servers_used]) - return tags.li("Servers Used: ", peerids_s) - d.addCallback(_got) - return d + servers_used = self.download_results().servers_used + + if not servers_used: + return "" + + peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) + for peerid in servers_used]) + return tags.li("Servers Used: ", peerids_s) @renderer def servermap(self, req, tag): - d = self.download_results() - d.addCallback(lambda res: res.servermap) - def _render(servermap): - if servermap is None: - return tag("None") - ul = tags.ul() - for peerid in sorted(servermap.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - shares_s = ",".join(["#%d" % shnum - for shnum in servermap[peerid]]) - ul(tags.li("[%s] has share%s: %s" % (peerid_s, - plural(servermap[peerid]), - shares_s))) - return ul - d.addCallback(_render) - return d + servermap = self.download_results().servermap + if not servermap: + return tag("None") + ul = tags.ul() + for peerid in sorted(servermap.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + shares_s = ",".join(["#%d" % shnum + for shnum in servermap[peerid]]) + ul(tags.li("[%s] has share%s: %s" % (peerid_s, + plural(servermap[peerid]), + shares_s))) + return ul @renderer def problems(self, req, tag): - d = self.download_results() - d.addCallback(lambda res: res.server_problems) - def _got(server_problems): - if not server_problems: - return tag("") - ul = T.ul() - for peerid in sorted(server_problems.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - ul(tags.li("[%s]: %s" % (peerid_s, server_problems[peerid]))) - return tags.li("Server Problems:", ul) - d.addCallback(_got) - return d + server_problems = self.download_results().server_problems + if not server_problems: + return "" + ul = T.ul() + for peerid in sorted(server_problems.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + ul(tags.li("[%s]: %s" % (peerid_s, server_problems[peerid]))) + return tags.li("Server Problems:", ul) @renderer def file_size(self, req, tag): - d = self.download_results() - d.addCallback(lambda res: str(res.file_size)) - return d + return tag(str(self.download_results().file_size)) def _get_time(self, name): - d = self.download_results() - d.addCallback(lambda res: res.timings.get(name)) - return d + return self.download_results().timings.get(name) @renderer def time_total(self, req, tag): @@ -766,13 +748,10 @@ class DownloadStatusElement(Element): return tag(str(self._get_time("paused"))) def _get_rate(self, name): - d = self.download_results() - def _convert(r): - file_size = r.file_size - duration = r.timings.get(name) - return compute_rate(file_size, duration) - d.addCallback(_convert) - return d + r = self.download_results() + file_size = r.file_size + duration = r.timings.get(name) + return compute_rate(file_size, duration) @renderer def rate_total(self, req, tag): @@ -796,20 +775,16 @@ class DownloadStatusElement(Element): @renderer def server_timings(self, req, tag): - d = self.download_results() - d.addCallback(lambda res: res.timings.get("fetch_per_server")) - def _render(per_server): - if per_server is None: - return "" - ul = tags.ul() - for peerid in sorted(per_server.keys()): - peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([abbreviate_time(t) - for t in per_server[peerid]]) - ul(tags.li("[%s]: %s" % (peerid_s, times_s))) - return tags.li("Per-Server Segment Fetch Response Times: ", ul) - d.addCallback(_render) - return d + per_server = self._get_time("fetch_per_server") + if per_server is None: + return "" + ul = tags.ul() + for peerid in sorted(per_server.keys()): + peerid_s = idlib.shortnodeid_b2a(peerid) + times_s = ", ".join([abbreviate_time(t) + for t in per_server[peerid]]) + ul(tags.li("[%s]: %s" % (peerid_s, times_s))) + return tags.li("Per-Server Segment Fetch Response Times: ", ul) class RetrieveStatusPage(MultiFormatResource): From 7757756a418809ce69749ac48ccc266f9d3080cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 7 Jul 2020 20:45:30 -0400 Subject: [PATCH 0291/1054] Use correct tag type --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4a9594a1c..94d35285b 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -698,7 +698,7 @@ class DownloadStatusElement(Element): server_problems = self.download_results().server_problems if not server_problems: return "" - ul = T.ul() + ul = tags.ul() for peerid in sorted(server_problems.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) ul(tags.li("[%s]: %s" % (peerid_s, server_problems[peerid]))) From 0db56f99cf28f73aa616447c595d633f74177735 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 7 Jul 2020 20:49:58 -0400 Subject: [PATCH 0292/1054] Add some tests for DownloadStatusElement --- src/allmydata/test/web/test_status.py | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/allmydata/test/web/test_status.py diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py new file mode 100644 index 000000000..bc7d46fc4 --- /dev/null +++ b/src/allmydata/test/web/test_status.py @@ -0,0 +1,82 @@ + +from bs4 import BeautifulSoup +from twisted.web.template import flattenString +from zope.interface import implementer + +from allmydata.interfaces import ( + IDownloadResults, + IDownloadStatus, +) +from allmydata.web.status import DownloadStatusElement + +from .common import ( + assert_soup_has_favicon, + assert_soup_has_text, + assert_soup_has_tag_with_content, +) +from ..common import TrialTestCase + + +@implementer(IDownloadResults) +class FakeDownloadResults(object): + file_size = 0 + servers_used = 0 + server_problems = {"s-1": "unknown problem"} + servermap = dict() + timings = dict() + + +@implementer(IDownloadStatus) +class FakeDownloadStatus(object): + + def __init__(self, storage_index, size): + self.storage_index = storage_index + self.size = size + self.dyhb_requests = [] + self.read_events = [] + self.segment_events = [] + self.block_requests = [] + + def get_started(self): + return None + + def get_storage_index(self): + return self.storage_index + + def get_size(self): + return self.size + + def using_helper(self): + return False + + def get_status(self): + return "FakeDownloadStatus" + + def get_progress(self): + return 0 + + def get_active(): + return False + + def get_counter(): + return 0 + + def get_results(self): + return FakeDownloadResults() + + +class StatusTests(TrialTestCase): + + def _render_download_status_element(self): + elem = DownloadStatusElement(FakeDownloadStatus("si-1", 123)) + d = flattenString(None, elem) + return self.successResultOf(d) + + def test_download_status_element(self): + result = self._render_download_status_element() + soup = BeautifulSoup(result, 'html5lib') + + assert_soup_has_text(self, soup, u'Tahoe-LAFS - File Download Status') + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content(self, soup, u'li', u'[omwtc]: unknown problem') From d6ea30b9356c6dd792603551fb17398224601dec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 7 Jul 2020 21:56:35 -0400 Subject: [PATCH 0293/1054] Test more markup rendered by DownloadStatusElement --- src/allmydata/test/web/test_status.py | 39 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index bc7d46fc4..a04b7cdd7 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -1,3 +1,4 @@ +# Tests for code in allmydata.web.status from bs4 import BeautifulSoup from twisted.web.template import flattenString @@ -19,11 +20,13 @@ from ..common import TrialTestCase @implementer(IDownloadResults) class FakeDownloadResults(object): - file_size = 0 - servers_used = 0 - server_problems = {"s-1": "unknown problem"} - servermap = dict() - timings = dict() + + def __init__(self, file_size): + self.file_size = file_size + self.servers_used = ["s-1", "s-2", "s-3"] + self.server_problems = {"s-1": "unknown problem"} + self.servermap = {"s-1": [1,2,3], "s-2": [2,3,4], "s-3": [0,1,3]} + self.timings = { "fetch_per_server": {"s-1": [1,2,3], "s-2": [2], "s-3": [3]}} @implementer(IDownloadStatus) @@ -62,10 +65,10 @@ class FakeDownloadStatus(object): return 0 def get_results(self): - return FakeDownloadResults() + return FakeDownloadResults(self.size) - -class StatusTests(TrialTestCase): +# Tests for code in allmydata.web.status.DownloadStatusElement +class DownloadStatusElementTests(TrialTestCase): def _render_download_status_element(self): elem = DownloadStatusElement(FakeDownloadStatus("si-1", 123)) @@ -76,7 +79,23 @@ class StatusTests(TrialTestCase): result = self._render_download_status_element() soup = BeautifulSoup(result, 'html5lib') - assert_soup_has_text(self, soup, u'Tahoe-LAFS - File Download Status') + assert_soup_has_text(self, soup, u"Tahoe-LAFS - File Download Status") assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content(self, soup, u'li', u'[omwtc]: unknown problem') + assert_soup_has_tag_with_content(self, soup, u"li", u"File Size: 123 bytes") + assert_soup_has_tag_with_content(self, soup, u"li", u"Progress: 0.0%") + + assert_soup_has_tag_with_content(self, soup, u"li", u"Servers Used: [omwtc], [omwte], [omwtg]") + + assert_soup_has_tag_with_content(self, soup, u"li", u"Server Problems:") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: unknown problem") + + assert_soup_has_tag_with_content(self, soup, u"li", u"Servermap:") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc] has shares: #1,#2,#3") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte] has shares: #2,#3,#4") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg] has shares: #0,#1,#3") + + assert_soup_has_tag_with_content(self, soup, u"li", u"Per-Server Segment Fetch Response Times:") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: 1.00s, 2.00s, 3.00s") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte]: 2.00s") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg]: 3.00s") From 3d80a6c17e13f3eb51564423b58df2a25c2adc6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 10:11:28 -0400 Subject: [PATCH 0294/1054] New packages so we can get future 0.18.2. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..ffb0eb4d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,7 +329,7 @@ jobs: - image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03-small.tar.gz" steps: - "checkout" From 81216f18bf393f08216ed0b986ebfe5fbd581225 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 10:15:33 -0400 Subject: [PATCH 0295/1054] Remove some whitespace --- src/allmydata/test/web/test_status.py | 2 +- src/allmydata/web/status.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index a04b7cdd7..15b641a8a 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -26,7 +26,7 @@ class FakeDownloadResults(object): self.servers_used = ["s-1", "s-2", "s-3"] self.server_problems = {"s-1": "unknown problem"} self.servermap = {"s-1": [1,2,3], "s-2": [2,3,4], "s-3": [0,1,3]} - self.timings = { "fetch_per_server": {"s-1": [1,2,3], "s-2": [2], "s-3": [3]}} + self.timings = {"fetch_per_server": {"s-1": [1,2,3], "s-2": [2], "s-3": [3]}} @implementer(IDownloadStatus) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 94d35285b..5d0b2a861 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -670,10 +670,8 @@ class DownloadStatusElement(Element): @renderer def servers_used(self, req, tag): servers_used = self.download_results().servers_used - if not servers_used: return "" - peerids_s = ", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid) for peerid in servers_used]) return tags.li("Servers Used: ", peerids_s) From 0bc8a441a9af800d618dac9cbfac8f93a24af8a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 11:56:53 -0400 Subject: [PATCH 0296/1054] Add pyutil as direct dependency; it was previously indirect dependency via zfec. --- nix/tahoe-lafs.nix | 4 ++-- setup.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index d82c97db7..c26572ec4 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, future +, html5lib, future, pyutil }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future + future pyutil ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index 919c84818..047d08030 100644 --- a/setup.py +++ b/setup.py @@ -124,6 +124,9 @@ install_requires = [ # Support for Python 3 transition "future >= 0.18.2", + + # Utility code: + "pyutil >= 3.3.0", ] setup_requires = [ From 016c7754a6ab509a9a0e6286f2bed8a2062e8a7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 11:57:10 -0400 Subject: [PATCH 0297/1054] Switch to use pyutil's assertutil implementation. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/assertutil.py | 64 +++++++------------------------- 2 files changed, 15 insertions(+), 50 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index fd679e014..6532a23d2 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.util.assertutil", "allmydata.util.humanreadable", "allmydata.util.namespace", "allmydata.util._python3", diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index 5c48500c3..67083c2d4 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -1,57 +1,21 @@ """ Tests useful in assertion checking, prints out nicely formated messages too. + +Backwards compatibility layer, the versions in pyutil are better maintained and +have tests. + +Ported to Python 3. """ -from allmydata.util.humanreadable import hr +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals -def _assert(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=[] - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 - raise AssertionError("".join(msgbuf)) -def precondition(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=["precondition", ] - if ___args or ___kwargs: - msgbuf.append(": ") - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) - - raise AssertionError("".join(msgbuf)) - -def postcondition(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=["postcondition", ] - if ___args or ___kwargs: - msgbuf.append(": ") - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) - - raise AssertionError("".join(msgbuf)) +# The API importers expect: +from pyutil.asserutil import _assert, precondition, postcondition From 229583c59621c3d464c2b937152690c4f14bc351 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 12:02:58 -0400 Subject: [PATCH 0298/1054] Switch to mathutil from pyutil (it has tests, and has already been ported to Python 3). --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/mathutil.py | 77 ++++++---------------------------- 2 files changed, 14 insertions(+), 64 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6532a23d2..c8fe8fddd 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,6 +17,7 @@ if PY2: PORTED_MODULES = [ "allmydata.util.assertutil", "allmydata.util.humanreadable", + "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util._python3", ] diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index bbdb40b08..3d65d883b 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -1,71 +1,20 @@ """ A few commonly needed functions. + +Backwards compatibility for direct imports. + +Ported to Python 3. """ -import math +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals -def div_ceil(n, d): - """ - The smallest integer k such that k*d >= n. - """ - return (n/d) + (n%d != 0) +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -def next_multiple(n, k): - """ - The smallest multiple of k which is >= n. - """ - return div_ceil(n, k) * k -def pad_size(n, k): - """ - The smallest number that has to be added to n so that n is a multiple of k. - """ - if n%k: - return k - n%k - else: - return 0 - -def is_power_of_k(n, k): - return k**int(math.log(n, k) + 0.5) == n - -def next_power_of_k(n, k): - if n == 0: - x = 0 - else: - x = int(math.log(n, k) + 0.5) - if k**x < n: - return k**(x+1) - else: - return k**x - -def ave(l): - return sum(l) / len(l) - -def log_ceil(n, b): - """ - The smallest integer k such that b^k >= n. - - log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. - the number of bits needed to store any of 128 possible values is 7. - """ - p = 1 - k = 0 - while p < n: - p *= b - k += 1 - return k - -def log_floor(n, b): - """ - The largest integer k such that b^k <= n. - """ - p = 1 - k = 0 - while p <= n: - p *= b - k += 1 - return k - 1 - -def round_sigfigs(f, n): - fmt = "%." + str(n-1) + "e" - return float(fmt % f) +# The API importers expect: +from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor, round_sigfigs From 939648c04fbcde890a90edde10d853f00e0fdbc0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 12:13:19 -0400 Subject: [PATCH 0299/1054] Add back missing function. --- src/allmydata/util/mathutil.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index 3d65d883b..b518c3591 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -17,4 +17,11 @@ if PY2: # The API importers expect: -from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor, round_sigfigs +from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor + + +# This function is not present in pyutil.mathutil: +def round_sigfigs(f, n): + fmt = "%." + str(n-1) + "e" + return float(fmt % f) + From bd6ff36b2fee31ca7d65d347737a416f78f78771 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 12:13:24 -0400 Subject: [PATCH 0300/1054] Fix typo. --- src/allmydata/util/assertutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index 67083c2d4..21fae0411 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -18,4 +18,4 @@ if PY2: # The API importers expect: -from pyutil.asserutil import _assert, precondition, postcondition +from pyutil.assertutil import _assert, precondition, postcondition From 9e3bc3e8889da3947d0f0402ee54d840775532fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 12:16:26 -0400 Subject: [PATCH 0301/1054] These tests are no longer necessary, since we are no longer implementing this code ourselves. Before deleting the tests I ensured they either passed, or the failure were due to slightly different string formatting in pyutil's assertutil. As such I believe pyutil has the same behavior. --- src/allmydata/test/test_util.py | 150 -------------------------------- 1 file changed, 150 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 65ba308d9..f2e89af28 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -64,97 +64,6 @@ class MyList(list): pass class Math(unittest.TestCase): - def test_div_ceil(self): - f = mathutil.div_ceil - self.failUnlessEqual(f(0, 1), 0) - self.failUnlessEqual(f(0, 2), 0) - self.failUnlessEqual(f(0, 3), 0) - self.failUnlessEqual(f(1, 3), 1) - self.failUnlessEqual(f(2, 3), 1) - self.failUnlessEqual(f(3, 3), 1) - self.failUnlessEqual(f(4, 3), 2) - self.failUnlessEqual(f(5, 3), 2) - self.failUnlessEqual(f(6, 3), 2) - self.failUnlessEqual(f(7, 3), 3) - - def test_next_multiple(self): - f = mathutil.next_multiple - self.failUnlessEqual(f(5, 1), 5) - self.failUnlessEqual(f(5, 2), 6) - self.failUnlessEqual(f(5, 3), 6) - self.failUnlessEqual(f(5, 4), 8) - self.failUnlessEqual(f(5, 5), 5) - self.failUnlessEqual(f(5, 6), 6) - self.failUnlessEqual(f(32, 1), 32) - self.failUnlessEqual(f(32, 2), 32) - self.failUnlessEqual(f(32, 3), 33) - self.failUnlessEqual(f(32, 4), 32) - self.failUnlessEqual(f(32, 5), 35) - self.failUnlessEqual(f(32, 6), 36) - self.failUnlessEqual(f(32, 7), 35) - self.failUnlessEqual(f(32, 8), 32) - self.failUnlessEqual(f(32, 9), 36) - self.failUnlessEqual(f(32, 10), 40) - self.failUnlessEqual(f(32, 11), 33) - self.failUnlessEqual(f(32, 12), 36) - self.failUnlessEqual(f(32, 13), 39) - self.failUnlessEqual(f(32, 14), 42) - self.failUnlessEqual(f(32, 15), 45) - self.failUnlessEqual(f(32, 16), 32) - self.failUnlessEqual(f(32, 17), 34) - self.failUnlessEqual(f(32, 18), 36) - self.failUnlessEqual(f(32, 589), 589) - - def test_pad_size(self): - f = mathutil.pad_size - self.failUnlessEqual(f(0, 4), 0) - self.failUnlessEqual(f(1, 4), 3) - self.failUnlessEqual(f(2, 4), 2) - self.failUnlessEqual(f(3, 4), 1) - self.failUnlessEqual(f(4, 4), 0) - self.failUnlessEqual(f(5, 4), 3) - - def test_is_power_of_k(self): - f = mathutil.is_power_of_k - for i in range(1, 100): - if i in (1, 2, 4, 8, 16, 32, 64): - self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) - else: - self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) - for i in range(1, 100): - if i in (1, 3, 9, 27, 81): - self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) - else: - self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) - - def test_next_power_of_k(self): - f = mathutil.next_power_of_k - self.failUnlessEqual(f(0,2), 1) - self.failUnlessEqual(f(1,2), 1) - self.failUnlessEqual(f(2,2), 2) - self.failUnlessEqual(f(3,2), 4) - self.failUnlessEqual(f(4,2), 4) - for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) - for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) - for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) - for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) - for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) - - self.failUnlessEqual(f(0,3), 1) - self.failUnlessEqual(f(1,3), 1) - self.failUnlessEqual(f(2,3), 3) - self.failUnlessEqual(f(3,3), 3) - for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) - for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) - for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) - for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) - - def test_ave(self): - f = mathutil.ave - self.failUnlessEqual(f([1,2,3]), 2) - self.failUnlessEqual(f([0,0,0,4]), 1) - self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) - def test_round_sigfigs(self): f = mathutil.round_sigfigs self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002) @@ -297,65 +206,6 @@ class Statistics(unittest.TestCase): self.failUnlessEqual(f(plist, .5, 3), .02734375) -class Asserts(unittest.TestCase): - def should_assert(self, func, *args, **kwargs): - try: - func(*args, **kwargs) - except AssertionError as e: - return str(e) - except Exception as e: - self.fail("assert failed with non-AssertionError: %s" % e) - self.fail("assert was not caught") - - def should_not_assert(self, func, *args, **kwargs): - try: - func(*args, **kwargs) - except AssertionError as e: - self.fail("assertion fired when it should not have: %s" % e) - except Exception as e: - self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e) - return # we're happy - - - def test_assert(self): - f = assertutil._assert - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual(m, "'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("othermsg: 'message2' ", m) - - def test_precondition(self): - f = assertutil.precondition - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual("precondition: 'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("precondition: 'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("precondition: othermsg: 'message2' ", m) - - def test_postcondition(self): - f = assertutil.postcondition - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual("postcondition: 'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("postcondition: 'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("postcondition: othermsg: 'message2' ", m) - class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0o777): fn = os.path.join(basedir, path) From f75ff70924a0c7363cb129078737136c9aec3410 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 12:21:33 -0400 Subject: [PATCH 0302/1054] News fragment. --- newsfragments/3339.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3339.other diff --git a/newsfragments/3339.other b/newsfragments/3339.other new file mode 100644 index 000000000..e69de29bb From c187e2752b1fc455e440524d376a4b51edf43038 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 12:44:09 -0400 Subject: [PATCH 0303/1054] Parameterize FakeDownloadResults and FakeDownloadStatus --- src/allmydata/test/web/test_status.py | 75 ++++++++++++++++++++------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 15b641a8a..4366d88ea 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -21,25 +21,47 @@ from ..common import TrialTestCase @implementer(IDownloadResults) class FakeDownloadResults(object): - def __init__(self, file_size): + def __init__(self, + file_size=0, + servers_used=None, + server_problems=None, + servermap=None, + timings=None): + """ + See IDownloadResults for parameters. + """ self.file_size = file_size - self.servers_used = ["s-1", "s-2", "s-3"] - self.server_problems = {"s-1": "unknown problem"} - self.servermap = {"s-1": [1,2,3], "s-2": [2,3,4], "s-3": [0,1,3]} - self.timings = {"fetch_per_server": {"s-1": [1,2,3], "s-2": [2], "s-3": [3]}} + self.servers_used = servers_used + self.server_problems = server_problems + self.servermap = servermap + self.timings = timings @implementer(IDownloadStatus) class FakeDownloadStatus(object): - def __init__(self, storage_index, size): + def __init__(self, + storage_index = None, + file_size = 0, + servers_used = None, + server_problems = None, + servermap = None, + timings = None): + """ + See IDownloadStatus and IDownloadResults for parameters. + """ self.storage_index = storage_index - self.size = size + self.file_size = file_size self.dyhb_requests = [] self.read_events = [] self.segment_events = [] self.block_requests = [] + self.servers_used = servers_used + self.server_problems = server_problems + self.servermap = servermap + self.timings = timings + def get_started(self): return None @@ -47,7 +69,7 @@ class FakeDownloadStatus(object): return self.storage_index def get_size(self): - return self.size + return self.file_size def using_helper(self): return False @@ -65,18 +87,35 @@ class FakeDownloadStatus(object): return 0 def get_results(self): - return FakeDownloadResults(self.size) + return FakeDownloadResults(self.file_size, + self.servers_used, + self.server_problems, + self.servermap, + self.timings) # Tests for code in allmydata.web.status.DownloadStatusElement class DownloadStatusElementTests(TrialTestCase): - def _render_download_status_element(self): - elem = DownloadStatusElement(FakeDownloadStatus("si-1", 123)) + def _render_download_status_element(self, status): + """ + :param IDownloadStatus status: + :return: HTML string rendered by DownloadStatusElement + """ + elem = DownloadStatusElement(status) d = flattenString(None, elem) return self.successResultOf(d) def test_download_status_element(self): - result = self._render_download_status_element() + """ + See if we can render the page almost fully. + """ + status = FakeDownloadStatus("si-1", 123, + ["s-1", "s-2", "s-3"], + {"s-1": "unknown problem"}, + {"s-1": [1], "s-2": [1,2], "s-3": [2,3]}, + {"fetch_per_server": {"s-1": [1], "s-2": [2,3], "s-3": [3,2]}}) + + result = self._render_download_status_element(status) soup = BeautifulSoup(result, 'html5lib') assert_soup_has_text(self, soup, u"Tahoe-LAFS - File Download Status") @@ -91,11 +130,11 @@ class DownloadStatusElementTests(TrialTestCase): assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: unknown problem") assert_soup_has_tag_with_content(self, soup, u"li", u"Servermap:") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc] has shares: #1,#2,#3") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte] has shares: #2,#3,#4") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg] has shares: #0,#1,#3") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc] has share: #1") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte] has shares: #1,#2") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg] has shares: #2,#3") assert_soup_has_tag_with_content(self, soup, u"li", u"Per-Server Segment Fetch Response Times:") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: 1.00s, 2.00s, 3.00s") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte]: 2.00s") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg]: 3.00s") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: 1.00s") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte]: 2.00s, 3.00s") + assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg]: 3.00s, 2.00s") From 112cfc1da5278bd6896dbd58772178f0dd464a6b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 13:07:19 -0400 Subject: [PATCH 0304/1054] Test if download status page can be rendered partially --- src/allmydata/test/web/test_status.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 4366d88ea..e555c2229 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -138,3 +138,15 @@ class DownloadStatusElementTests(TrialTestCase): assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: 1.00s") assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte]: 2.00s, 3.00s") assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg]: 3.00s, 2.00s") + + def test_download_status_element_partial(self): + """ + See if we can render the page with incomplete download status. + """ + status = FakeDownloadStatus() + result = self._render_download_status_element(status) + soup = BeautifulSoup(result, 'html5lib') + + assert_soup_has_tag_with_content(self, soup, u"li", u"Servermap: None") + assert_soup_has_tag_with_content(self, soup, u"li", u"File Size: 0 bytes") + assert_soup_has_tag_with_content(self, soup, u"li", u"Total: None (None)") From 51d6307b45ae62cf6040cb1faa590f8a0ccdfcf4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 13:08:19 -0400 Subject: [PATCH 0305/1054] Fail better when timings data isn't available --- src/allmydata/web/status.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 5d0b2a861..b2e54f8cf 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -707,7 +707,9 @@ class DownloadStatusElement(Element): return tag(str(self.download_results().file_size)) def _get_time(self, name): - return self.download_results().timings.get(name) + if self.download_results().timings: + return self.download_results().timings.get(name) + return None @renderer def time_total(self, req, tag): @@ -748,7 +750,9 @@ class DownloadStatusElement(Element): def _get_rate(self, name): r = self.download_results() file_size = r.file_size - duration = r.timings.get(name) + duration = None + if r.timings: + duration = r.timings.get(name) return compute_rate(file_size, duration) @renderer From 1cbf7981276a49b5034eb664141b6c7a472341c5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:07 -0400 Subject: [PATCH 0306/1054] Package Future 0.18.2 for us --- nix/future.nix | 35 +++++++++++++++++++++++++++++++++++ nix/overlays.nix | 5 +++++ nix/tahoe-lafs.nix | 1 + 3 files changed, 41 insertions(+) create mode 100644 nix/future.nix diff --git a/nix/future.nix b/nix/future.nix new file mode 100644 index 000000000..814b7c1b5 --- /dev/null +++ b/nix/future.nix @@ -0,0 +1,35 @@ +{ lib +, buildPythonPackage +, fetchPypi +}: + +buildPythonPackage rec { + pname = "future"; + version = "0.18.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; + }; + + doCheck = false; + + meta = { + description = "Clean single-source support for Python 3 and 2"; + longDescription = '' + python-future is the missing compatibility layer between Python 2 and + Python 3. It allows you to use a single, clean Python 3.x-compatible + codebase to support both Python 2 and Python 3 with minimal overhead. + + It provides future and past packages with backports and forward ports + of features from Python 3 and 2. It also comes with futurize and + pasteurize, customized 2to3-based scripts that helps you to convert + either Py2 or Py3 code easily to support both Python 2 and 3 in a + single clean Py3-style codebase, module by module. + ''; + homepage = https://python-future.org; + downloadPage = https://github.com/PythonCharmers/python-future/releases; + license = with lib.licenses; [ mit ]; + maintainers = with lib.maintainers; [ prikhi ]; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 2a9fe8e5e..7f19a8f1a 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -10,6 +10,11 @@ self: super: { # NixOS autobahn package has trollius as a dependency, although # it is optional. Trollius is unmaintained and fails on CI. autobahn = python-super.callPackage ./autobahn.nix { }; + + # Porting to Python 3 is greatly aided by the future package. A + # slightly newer version than appears in nixos 19.09 is helpful. + future = python-super.callPackage ./future.nix { }; + }; }; } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index cc0eea479..ae5c05f70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,6 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools + future ]; checkInputs = with python.pkgs; [ From 7e16148f83cabb0507560ef45c60b9d84b38de8a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:42 -0400 Subject: [PATCH 0307/1054] news fragment --- newsfragments/3338.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3338.minor diff --git a/newsfragments/3338.minor b/newsfragments/3338.minor new file mode 100644 index 000000000..e69de29bb From 4d22390b59e503169953f9ece4ac4e168b420a68 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 13:29:01 -0400 Subject: [PATCH 0308/1054] Untabify --- src/allmydata/test/web/test_status.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index e555c2229..2c0fe032c 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -31,10 +31,10 @@ class FakeDownloadResults(object): See IDownloadResults for parameters. """ self.file_size = file_size - self.servers_used = servers_used - self.server_problems = server_problems - self.servermap = servermap - self.timings = timings + self.servers_used = servers_used + self.server_problems = server_problems + self.servermap = servermap + self.timings = timings @implementer(IDownloadStatus) From 7bee9ff54007b3a71a7905febf741e69e026165f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 8 Jul 2020 13:56:52 -0400 Subject: [PATCH 0309/1054] Untabify again --- src/allmydata/test/web/test_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 2c0fe032c..42f2b9794 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -30,7 +30,7 @@ class FakeDownloadResults(object): """ See IDownloadResults for parameters. """ - self.file_size = file_size + self.file_size = file_size self.servers_used = servers_used self.server_problems = server_problems self.servermap = servermap From 98c8b2fd00b3823988d4d253ad5600ed56bd80f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 14:53:38 -0400 Subject: [PATCH 0310/1054] Revert changes obsoleted by #3338. --- .circleci/config.yml | 2 +- nix/tahoe-lafs.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffb0eb4d8..d0f596fc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,7 +329,7 @@ jobs: - image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03-small.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" steps: - "checkout" diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index d82c97db7..ae5c05f70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, future +, html5lib }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; From 28f219ad3e036a8db35a81a587d443671f50f16c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 16:46:54 -0400 Subject: [PATCH 0311/1054] Update to version that supports _python3.py variant of port list. --- misc/python3/tahoe-depgraph.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/misc/python3/tahoe-depgraph.py b/misc/python3/tahoe-depgraph.py index 1bfe89c73..0f624fa70 100644 --- a/misc/python3/tahoe-depgraph.py +++ b/misc/python3/tahoe-depgraph.py @@ -60,7 +60,8 @@ class mymf(modulefinder.ModuleFinder): self._depgraph[last_caller.__name__].add(fqname) return r - def load_module(self, fqname, fp, pathname, (suffix, mode, type)): + def load_module(self, fqname, fp, pathname, additional_info): + (suffix, mode, type) = additional_info r = modulefinder.ModuleFinder.load_module( self, fqname, fp, pathname, (suffix, mode, type)) if r is not None: @@ -71,7 +72,7 @@ class mymf(modulefinder.ModuleFinder): return { 'depgraph': { name: dict.fromkeys(deps, 1) - for name, deps in self._depgraph.iteritems()}, + for name, deps in self._depgraph.items()}, 'types': self._types, } @@ -101,20 +102,25 @@ def main(target): filepath = path moduleNames.append(reflect.filenameToModuleName(filepath)) - with tempfile.NamedTemporaryFile() as tmpfile: + with tempfile.NamedTemporaryFile("w") as tmpfile: for moduleName in moduleNames: tmpfile.write('import %s\n' % moduleName) tmpfile.flush() mf.run_script(tmpfile.name) - with open('tahoe-deps.json', 'wb') as outfile: + with open('tahoe-deps.json', 'w') as outfile: json_dump(mf.as_json(), outfile) outfile.write('\n') - ported_modules_path = os.path.join(target, "src", "allmydata", "ported-modules.txt") - with open(ported_modules_path) as ported_modules: - port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported") - with open('tahoe-ported.json', 'wb') as outfile: + ported_modules_path = os.path.join(target, "src", "allmydata", "util", "_python3.py") + with open(ported_modules_path) as f: + ported_modules = {} + exec(f.read(), ported_modules, ported_modules) + port_status = dict.fromkeys( + ported_modules["PORTED_MODULES"] + ported_modules["PORTED_TEST_MODULES"], + "ported" + ) + with open('tahoe-ported.json', 'w') as outfile: json_dump(port_status, outfile) outfile.write('\n') From 12d5131568fece49852b1792bbbea4a9a9faca17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 13:12:30 -0400 Subject: [PATCH 0312/1054] Fix pyflakes issues. --- src/allmydata/test/test_util.py | 2 +- src/allmydata/util/assertutil.py | 2 ++ src/allmydata/util/mathutil.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index f2e89af28..14d9f5dd7 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,7 +15,7 @@ from twisted.python.failure import Failure from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil -from allmydata.util import assertutil, fileutil, deferredutil, abbreviate +from allmydata.util import fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index 21fae0411..af8817702 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -19,3 +19,5 @@ if PY2: # The API importers expect: from pyutil.assertutil import _assert, precondition, postcondition + +__all__ = ["_assert", "precondition", "postcondition"] diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index b518c3591..be88f60b4 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -25,3 +25,4 @@ def round_sigfigs(f, n): fmt = "%." + str(n-1) + "e" return float(fmt % f) +__all__ = ["div_ceil", "next_multiple", "pad_size", "is_power_of_k", "next_power_of_k", "ave", "log_ceil", "log_floor", "round_sigfigs"] From 8efc12a3ef4089435b40c161ae879f29b07c5bba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 13:21:02 -0400 Subject: [PATCH 0313/1054] Use newer version of pyutil. --- nix/overlays.nix | 3 +++ nix/pyutil.nix | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 nix/pyutil.nix diff --git a/nix/overlays.nix b/nix/overlays.nix index 7f19a8f1a..ba3c9c885 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -15,6 +15,9 @@ self: super: { # slightly newer version than appears in nixos 19.09 is helpful. future = python-super.callPackage ./future.nix { }; + # Need version of pyutil that supports Python 3. The version in 19.09 + # is too old. + pyutil = python-super.callPackage ./pyutil.nix { }; }; }; } diff --git a/nix/pyutil.nix b/nix/pyutil.nix new file mode 100644 index 000000000..6852c2acc --- /dev/null +++ b/nix/pyutil.nix @@ -0,0 +1,48 @@ +{ stdenv +, buildPythonPackage +, fetchPypi +, setuptoolsDarcs +, setuptoolsTrial +, simplejson +, twisted +, isPyPy +}: + +buildPythonPackage rec { + pname = "pyutil"; + version = "3.3.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848"; + }; + + buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []); + propagatedBuildInputs = [ twisted ]; + + # Tests fail because they try to write new code into the twisted + # package, apparently some kind of plugin. + doCheck = false; + + prePatch = stdenv.lib.optionalString isPyPy '' + grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g" + ''; + + meta = with stdenv.lib; { + description = "Pyutil, a collection of mature utilities for Python programmers"; + + longDescription = '' + These are a few data structures, classes and functions which + we've needed over many years of Python programming and which + seem to be of general use to other Python programmers. Many of + the modules that have existed in pyutil over the years have + subsequently been obsoleted by new features added to the + Python language or its standard library, thus showing that + we're not alone in wanting tools like these. + ''; + + homepage = "http://allmydata.org/trac/pyutil"; + license = licenses.gpl2Plus; + }; + +} \ No newline at end of file From af86066eab671b3f6582d20cea3dfd4a71ac358f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 13:48:33 -0400 Subject: [PATCH 0314/1054] Delete some dead code. --- newsfragments/3340.other | 0 src/allmydata/test/common_util.py | 33 ---------------- src/allmydata/test/test_client.py | 2 +- src/allmydata/util/eliotutil.py | 66 ------------------------------- src/allmydata/util/fileutil.py | 54 ------------------------- src/allmydata/util/rrefutil.py | 10 +---- 6 files changed, 3 insertions(+), 162 deletions(-) create mode 100644 newsfragments/3340.other diff --git a/newsfragments/3340.other b/newsfragments/3340.other new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index f9e0f2b01..025459961 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -89,39 +89,6 @@ class ReallyEqualMixin(object): self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) -class NonASCIIPathMixin(object): - def mkdir_nonascii(self, dirpath): - # Kludge to work around the fact that buildbot can't remove a directory tree that has - # any non-ASCII directory names on Windows. (#1472) - if sys.platform == "win32": - def _cleanup(): - try: - fileutil.rm_dir(dirpath) - finally: - if os.path.exists(dirpath): - msg = ("We were unable to delete a non-ASCII directory %r created by the test. " - "This is liable to cause failures on future builds." % (dirpath,)) - print(msg) - log.err(msg) - self.addCleanup(_cleanup) - os.mkdir(dirpath) - - def unicode_or_fallback(self, unicode_name, fallback_name, io_as_well=False): - if not unicode_platform(): - try: - unicode_name.encode(get_filesystem_encoding()) - except UnicodeEncodeError: - return fallback_name - - if io_as_well: - try: - unicode_name.encode(get_io_encoding()) - except UnicodeEncodeError: - return fallback_name - - return unicode_name - - class SignalMixin(object): # This class is necessary for any code which wants to use Processes # outside the usual reactor.run() environment. It is copied from diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 41a44d5a6..b188008fd 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -83,7 +83,7 @@ BASECONFIG_I = ("[client]\n" "introducer.furl = %s\n" ) -class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.TestCase): +class Basic(testutil.ReallyEqualMixin, unittest.TestCase): def test_loadable(self): basedir = "test_client.Basic.test_loadable" os.mkdir(basedir) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index f6f40945d..c4805ff3c 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -14,8 +14,6 @@ __all__ = [ "eliot_logging_service", "opt_eliot_destination", "opt_help_eliot_destinations", - "validateInstanceOf", - "validateSetMembership", ] from sys import ( @@ -75,23 +73,6 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -def validateInstanceOf(t): - """ - Return an Eliot validator that requires values to be instances of ``t``. - """ - def validator(v): - if not isinstance(v, t): - raise ValidationError("{} not an instance of {}".format(v, t)) - return validator - -def validateSetMembership(s): - """ - Return an Eliot validator that requires values to be elements of ``s``. - """ - def validator(v): - if v not in s: - raise ValidationError("{} not in {}".format(v, s)) - return validator def eliot_logging_service(reactor, destinations): """ @@ -246,53 +227,6 @@ class _DestinationParser(object): else: return parser(kind, args) - def _get_arg(self, arg_name, default, arg_list): - return dict( - arg.split(u"=", 1) - for arg - in arg_list - ).get( - arg_name, - default, - ) - - def _parse_file(self, kind, arg_text): - # Reserve the possibility of an escape character in the future. \ is - # the standard choice but it's the path separator on Windows which - # pretty much ruins it in this context. Most other symbols already - # have some shell-assigned meaning which makes them treacherous to use - # in a CLI interface. Eliminating all such dangerous symbols leaves - # approximately @. - if u"@" in arg_text: - raise ValueError( - u"Unsupported escape character (@) in destination text ({!r}).".format(arg_text), - ) - arg_list = arg_text.split(u",") - path_name = arg_list.pop(0) - if path_name == "-": - get_file = lambda: stdout - else: - path = FilePath(path_name) - rotate_length = int(self._get_arg( - u"rotate_length", - 1024 * 1024 * 1024, - arg_list, - )) - max_rotated_files = int(self._get_arg( - u"max_rotated_files", - 10, - arg_list, - )) - def get_file(): - path.parent().makedirs(ignoreExistingDirectory=True) - return LogFile( - path.basename(), - path.dirname(), - rotateLength=rotate_length, - maxRotatedFiles=max_rotated_files, - ) - return lambda reactor: FileDestination(get_file()) - _parse_destination_description = _DestinationParser().parse diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 162852c38..16adf2f43 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -538,60 +538,6 @@ def get_available_space(whichdir, reserved_space): return 0 -if sys.platform == "win32": - # - CreateFileW = WINFUNCTYPE( - HANDLE, LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE, - use_last_error=True - )(("CreateFileW", windll.kernel32)) - - GENERIC_WRITE = 0x40000000 - FILE_SHARE_READ = 0x00000001 - FILE_SHARE_WRITE = 0x00000002 - OPEN_EXISTING = 3 - INVALID_HANDLE_VALUE = 0xFFFFFFFF - - # - FlushFileBuffers = WINFUNCTYPE( - BOOL, HANDLE, - use_last_error=True - )(("FlushFileBuffers", windll.kernel32)) - - # - CloseHandle = WINFUNCTYPE( - BOOL, HANDLE, - use_last_error=True - )(("CloseHandle", windll.kernel32)) - - # - def flush_volume(path): - abspath = os.path.realpath(path) - if abspath.startswith("\\\\?\\"): - abspath = abspath[4 :] - drive = os.path.splitdrive(abspath)[0] - - print("flushing %r" % (drive,)) - hVolume = CreateFileW(u"\\\\.\\" + drive, - GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, - None, - OPEN_EXISTING, - 0, - None - ) - if hVolume == INVALID_HANDLE_VALUE: - raise WinError(get_last_error()) - - if FlushFileBuffers(hVolume) == 0: - raise WinError(get_last_error()) - - CloseHandle(hVolume) -else: - def flush_volume(path): - # use sync()? - pass - - class ConflictError(Exception): pass diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 0dde651d3..40e921507 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,7 +1,7 @@ from twisted.internet import address -from foolscap.api import Violation, RemoteException, DeadReferenceError, \ - SturdyRef +from foolscap.api import Violation, RemoteException, SturdyRef + def add_version_to_remote_reference(rref, default): """I try to add a .version attribute to the given RemoteReference. I call @@ -19,12 +19,6 @@ def add_version_to_remote_reference(rref, default): d.addCallbacks(_got_version, _no_get_version) return d -def trap_and_discard(f, *errorTypes): - f.trap(*errorTypes) - -def trap_deadref(f): - return trap_and_discard(f, DeadReferenceError) - def connection_hints_for_furl(furl): hints = [] From e108ecb4b0fef5a6f5e2a1ef6cfea53e5f951234 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 13:56:25 -0400 Subject: [PATCH 0315/1054] Fix lint. --- src/allmydata/test/common_util.py | 3 +-- src/allmydata/util/eliotutil.py | 13 ------------- src/allmydata/util/fileutil.py | 2 +- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 025459961..bf44785a5 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -import os, signal, sys, time +import os, signal, time from random import randrange from six.moves import StringIO @@ -8,7 +8,6 @@ from twisted.internet import reactor, defer from twisted.python import failure from twisted.trial import unittest -from allmydata.util import fileutil, log from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index c4805ff3c..14c867ed2 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -16,9 +16,6 @@ __all__ = [ "opt_help_eliot_destinations", ] -from sys import ( - stdout, -) from functools import wraps from logging import ( INFO, @@ -40,15 +37,11 @@ from attr.validators import ( from eliot import ( ILogger, Message, - FileDestination, add_destinations, remove_destination, write_traceback, start_action, ) -from eliot._validation import ( - ValidationError, -) from eliot.twisted import ( DeferredContext, inline_callbacks, @@ -57,12 +50,6 @@ from eliot.twisted import ( from twisted.python.usage import ( UsageError, ) -from twisted.python.filepath import ( - FilePath, -) -from twisted.python.logfile import ( - LogFile, -) from twisted.logger import ( ILogObserver, eventAsJSON, diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 16adf2f43..269a8a356 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -12,7 +12,7 @@ from errno import ENOENT if sys.platform == "win32": from ctypes import WINFUNCTYPE, WinError, windll, POINTER, byref, c_ulonglong, \ create_unicode_buffer, get_last_error - from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR, LPVOID, HANDLE + from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR, LPVOID from twisted.python import log From 0cf1117165bc022c2b519a49a592189afa1da45c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:20:12 -0400 Subject: [PATCH 0316/1054] Move the tests for deferredutil into their own module. --- src/allmydata/test/test_deferredutil.py | 76 +++++++++++++++++++++++++ src/allmydata/test/test_util.py | 56 +----------------- src/allmydata/util/_python3.py | 1 + 3 files changed, 78 insertions(+), 55 deletions(-) create mode 100644 src/allmydata/test/test_deferredutil.py diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py new file mode 100644 index 000000000..fcff77b54 --- /dev/null +++ b/src/allmydata/test/test_deferredutil.py @@ -0,0 +1,76 @@ +""" +Tests for allmydata.util.deferredutil. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.python.failure import Failure + +from allmydata.util import deferredutil + + +class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin): + def test_gather_results(self): + d1 = defer.Deferred() + d2 = defer.Deferred() + res = deferredutil.gatherResults([d1, d2]) + d1.errback(ValueError("BAD")) + def _callb(res): + self.fail("Should have errbacked, not resulted in %s" % (res,)) + def _errb(thef): + thef.trap(ValueError) + res.addCallbacks(_callb, _errb) + return res + + def test_success(self): + d1, d2 = defer.Deferred(), defer.Deferred() + good = [] + bad = [] + dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) + dlss.addCallbacks(good.append, bad.append) + d1.callback(1) + d2.callback(2) + self.failUnlessEqual(good, [[1,2]]) + self.failUnlessEqual(bad, []) + + def test_failure(self): + d1, d2 = defer.Deferred(), defer.Deferred() + good = [] + bad = [] + dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) + dlss.addCallbacks(good.append, bad.append) + d1.addErrback(lambda _ignore: None) + d2.addErrback(lambda _ignore: None) + d1.callback(1) + d2.errback(ValueError()) + self.failUnlessEqual(good, []) + self.failUnlessEqual(len(bad), 1) + f = bad[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(ValueError)) + + def test_wait_for_delayed_calls(self): + """ + This tests that 'wait_for_delayed_calls' does in fact wait for a + delayed call that is active when the test returns. If it didn't, + Trial would report an unclean reactor error for this test. + """ + def _trigger(): + #print "trigger" + pass + reactor.callLater(0.1, _trigger) + + d = defer.succeed(None) + d.addBoth(self.wait_for_delayed_calls) + return d diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 65ba308d9..b6f2074aa 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,7 +15,7 @@ from twisted.python.failure import Failure from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil -from allmydata.util import assertutil, fileutil, deferredutil, abbreviate +from allmydata.util import assertutil, fileutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log @@ -750,60 +750,6 @@ class PollMixinTests(unittest.TestCase): d.addCallbacks(_suc, _err) return d -class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin): - def test_gather_results(self): - d1 = defer.Deferred() - d2 = defer.Deferred() - res = deferredutil.gatherResults([d1, d2]) - d1.errback(ValueError("BAD")) - def _callb(res): - self.fail("Should have errbacked, not resulted in %s" % (res,)) - def _errb(thef): - thef.trap(ValueError) - res.addCallbacks(_callb, _errb) - return res - - def test_success(self): - d1, d2 = defer.Deferred(), defer.Deferred() - good = [] - bad = [] - dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) - dlss.addCallbacks(good.append, bad.append) - d1.callback(1) - d2.callback(2) - self.failUnlessEqual(good, [[1,2]]) - self.failUnlessEqual(bad, []) - - def test_failure(self): - d1, d2 = defer.Deferred(), defer.Deferred() - good = [] - bad = [] - dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) - dlss.addCallbacks(good.append, bad.append) - d1.addErrback(lambda _ignore: None) - d2.addErrback(lambda _ignore: None) - d1.callback(1) - d2.errback(ValueError()) - self.failUnlessEqual(good, []) - self.failUnlessEqual(len(bad), 1) - f = bad[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(ValueError)) - - def test_wait_for_delayed_calls(self): - """ - This tests that 'wait_for_delayed_calls' does in fact wait for a - delayed call that is active when the test returns. If it didn't, - Trial would report an unclean reactor error for this test. - """ - def _trigger(): - #print "trigger" - pass - reactor.callLater(0.1, _trigger) - - d = defer.succeed(None) - d.addBoth(self.wait_for_delayed_calls) - return d class HashUtilTests(unittest.TestCase): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index fd679e014..3196a8f90 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_python3", ] From 7d1fdd8284c6901e85985927ef4f6424834cb46c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:32:16 -0400 Subject: [PATCH 0317/1054] Port modules to Python 3. --- newsfragments/3341.other | 0 src/allmydata/util/deferredutil.py | 59 ++++++++++-------------------- src/allmydata/util/pollmixin.py | 13 +++++++ 3 files changed, 33 insertions(+), 39 deletions(-) create mode 100644 newsfragments/3341.other diff --git a/newsfragments/3341.other b/newsfragments/3341.other new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 671d8bfd9..e36892f96 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -1,7 +1,21 @@ +""" +Utilities for working with Twisted Deferreds. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import time -from foolscap.api import eventually, fireEventually +from foolscap.api import eventually from twisted.internet import defer, reactor, error from twisted.python.failure import Failure @@ -130,7 +144,7 @@ class HookMixin(object): self._hooks[name] = (d, ignore_count) return d - def _call_hook(self, res, name, async=False): + def _call_hook(self, res, name, **kwargs): """ Called to trigger the hook, with argument 'res'. This is a no-op if the hook is unset. If the hook's ignore_count is positive, it will be @@ -142,7 +156,10 @@ class HookMixin(object): which will typically cause the test to also fail. 'res' is returned so that the current result or failure will be passed through. + + Accepts a single keyword argument, async, defaulting to False. """ + async_ = kwargs.get("async", False) hook = self._hooks[name] if hook is None: return res # pass on error/result @@ -153,7 +170,7 @@ class HookMixin(object): self._hooks[name] = (d, ignore_count - 1) else: self._hooks[name] = None - if async: + if async_: _with_log(eventually_callback(d), res) else: _with_log(d.callback, res) @@ -163,42 +180,6 @@ class HookMixin(object): log.msg(msg, level=log.NOISY) -def async_iterate(process, iterable, *extra_args, **kwargs): - """ - I iterate over the elements of 'iterable' (which may be deferred), eventually - applying 'process' to each one, optionally with 'extra_args' and 'kwargs'. - 'process' should return a (possibly deferred) boolean: True to continue the - iteration, False to stop. - - I return a Deferred that fires with True if all elements of the iterable - were processed (i.e. 'process' only returned True values); with False if - the iteration was stopped by 'process' returning False; or that fails with - the first failure of either 'process' or the iterator. - """ - iterator = iter(iterable) - - d = defer.succeed(None) - def _iterate(ign): - d2 = defer.maybeDeferred(iterator.next) - def _cb(item): - d3 = defer.maybeDeferred(process, item, *extra_args, **kwargs) - def _maybe_iterate(res): - if res: - d4 = fireEventually() - d4.addCallback(_iterate) - return d4 - return False - d3.addCallback(_maybe_iterate) - return d3 - def _eb(f): - f.trap(StopIteration) - return True - d2.addCallbacks(_cb, _eb) - return d2 - d.addCallback(_iterate) - return d - - def for_items(cb, mapping): """ For each (key, value) pair in a mapping, I add a callback to cb(None, key, value) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 14f5cd390..4eb19b8e3 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -1,4 +1,17 @@ +""" +Polling utility that returns Deferred. + +Ported to Python 3. +""" + from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import time from twisted.internet import task From 54248f334ed8a7eaab0df7ffa6e5ce981e4187d5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 9 Jul 2020 10:01:13 -0400 Subject: [PATCH 0318/1054] Use WebError, not RuntimeError, to indicate error Several problems with using RuntimeError to signal error here: - It dumps a rather unhelpful webpage at the user. - The exception backtrace on Tahoe console is not quite necessary here. - It really is not a runtime error: it is just an expected failure. - But mainly, testing for RuntimeError is harder. --- src/allmydata/web/status.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 8214913de..0c9e1ebb1 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -25,6 +25,7 @@ from allmydata.web.common import ( render_time, MultiFormatResource, SlotsSequenceElement, + WebError, ) from allmydata.interfaces import ( @@ -1171,9 +1172,7 @@ class Status(MultiFormatResource): try: stype, count_s = path.split("-") except ValueError: - raise RuntimeError( - "no - in '{}'".format(path) - ) + raise WebError("no '-' in '{}'".format(path)) count = int(count_s) if stype == "up": for s in itertools.chain(h.list_all_upload_statuses(), From 2e160c15266284d84e1e94b9025814b0206fe0a1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 9 Jul 2020 14:23:26 -0400 Subject: [PATCH 0319/1054] Test for failure caused by invalid path A hyphen is expected when rendering /status page child nodes: "/status/up" is wrong; "/status/up-0" is right. --- src/allmydata/test/web/test_web.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5be0b2f7b..c3779c0be 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1034,6 +1034,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d + def test_status_path_error(self): + # Expect an error, because path is expected to be of the form + # "/status/{up,down,..}-%number", with a hyphen. + return self.assertFailure(self.GET("/storage/nodash"), error.Error) + def test_status_numbers(self): drrm = status.DownloadResultsRendererMixin() self.failUnlessReallyEqual(drrm.render_time(None, None), "") From 1e41df9f48337b7f4411f21618d120972174128e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:38:07 -0400 Subject: [PATCH 0320/1054] Add to list of ported modules. --- src/allmydata/util/_python3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 3196a8f90..74f5d0385 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,8 +15,10 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.util.deferredutil", "allmydata.util.humanreadable", "allmydata.util.namespace", + "allmydata.util.pollmixin", "allmydata.util._python3", ] From 4dd6c8695076ce579f96f3af01116bdd6c0694e0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 11:56:59 -0400 Subject: [PATCH 0321/1054] Use twisted web directives in web.operations.ReloadMixin --- src/allmydata/web/operations.py | 36 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 93de2169e..eb9321344 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,7 +1,11 @@ import time -from nevow import rend, url, tags as T +from nevow import rend, url from nevow.inevow import IRequest +from twisted.web.template import ( + renderer, + tags as T, +) from twisted.python.failure import Failure from twisted.internet import reactor, defer from twisted.web.http import NOT_FOUND @@ -122,31 +126,31 @@ class OphandleTable(rend.Page, service.Service): class ReloadMixin(object): REFRESH_TIME = 1*MINUTE - def render_refresh(self, ctx, data): + @renderer + def refresh(self, req, tag): if self.monitor.is_finished(): return "" # dreid suggests ctx.tag(**dict([("http-equiv", "refresh")])) # but I can't tell if he's joking or not - ctx.tag.attributes["http-equiv"] = "refresh" - ctx.tag.attributes["content"] = str(self.REFRESH_TIME) - return ctx.tag + tag.attributes["http-equiv"] = "refresh" + tag.attributes["content"] = str(self.REFRESH_TIME) + return tag - def render_reload(self, ctx, data): + @renderer + def reload(self, req, tag): if self.monitor.is_finished(): return "" - req = IRequest(ctx) # url.gethere would break a proxy, so the correct thing to do is # req.path[-1] + queryargs ophandle = req.prepath[-1] reload_target = ophandle + "?output=html" cancel_target = ophandle + "?t=cancel" - cancel_button = T.form(action=cancel_target, method="POST", - enctype="multipart/form-data")[ - T.input(type="submit", value="Cancel"), - ] + cancel_button = T.form(T.input(type="submit", value="Cancel"), + action=cancel_target, + method="POST", + enctype="multipart/form-data",) - return [T.h2["Operation still running: ", - T.a(href=reload_target)["Reload"], - ], - cancel_button, - ] + return (T.h2("Operation still running: ", + T.a("Reload", href=reload_target), + ), + cancel_button,) From c043d1beadde39dc79385729c8603ba2117b2926 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 12:06:48 -0400 Subject: [PATCH 0322/1054] Add newsfragment --- newsfragments/3315.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3315.minor diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3315.minor @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From 88accc23c3c0614a2f1d09ee10467a9cc7d56b0e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 5 Jun 2020 09:28:36 -0400 Subject: [PATCH 0323/1054] Update newsfragment --- newsfragments/3315.minor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor index ed00922a6..0536c297a 100644 --- a/newsfragments/3315.minor +++ b/newsfragments/3315.minor @@ -1 +1 @@ -Add your info here \ No newline at end of file +Replace nevow with twisted.web in web.operations.ReloadMixin From 7444d6b7a9a1c6ca71e6b499e304c1bb0f807f14 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:42:55 -0400 Subject: [PATCH 0324/1054] Correct arguments to super() This was incidentally found when looking at some test failures: super(type, object) is the right invocation, and not the other way around. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index e7a94ff40..a2af857b9 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1269,5 +1269,5 @@ class TrialTestCase(_TrialTestCase): if six.PY2: if isinstance(msg, six.text_type): - return super(self, TrialTestCase).fail(msg.encode("utf8")) - return super(self, TrialTestCase).fail(msg) + return super(TrialTestCase, self).fail(msg.encode("utf8")) + return super(TrialTestCase, self).fail(msg) From 5892eae580a71d2b99760ec72685532df9a8cb45 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:49:13 -0400 Subject: [PATCH 0325/1054] Render self in DirectoryNodeHandler.getChild() We need self.render_POST() etc. to be invoked when we have a request such as "POST /uri/URI:DIR:..."; throwing an error here is probably not the right thing to do. --- src/allmydata/web/directory.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 57c222788..2129cec25 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -103,12 +103,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page - name = name.decode('utf8') if not name: - raise EmptyPathnameComponentError( - u"The webapi does not allow empty pathname components", - ) + return self + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From 83b5807ca119a4641ee1a5e928e6a10d3ef5d147 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 09:24:47 -0400 Subject: [PATCH 0326/1054] Add newsfragment --- newsfragments/3312.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3312.bugfix diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3312.bugfix @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From d3d0d0285596ac75c055be5662fb7b9099630821 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 19:47:12 -0400 Subject: [PATCH 0327/1054] Reject URIs that contain empty segments --- src/allmydata/web/directory.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 2129cec25..fced8c68e 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -106,6 +106,16 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): if not name: return self + # Rejecting URIs that contain empty path pieces (for example: + # "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was + # the old nevow behavior and it is encoded in the test suite; + # we will follow suit. + for segment in req.prepath: + if not segment: + raise EmptyPathnameComponentError( + u"The webapi does not allow empty pathname components", + ) + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) From 378b20c51a489a22deb80567f4a18807c7ed55d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:17 -0400 Subject: [PATCH 0328/1054] Reject trailing empty path piece in URIs --- src/allmydata/web/directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index fced8c68e..b1618a3f5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -102,8 +102,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # trying to replicate what I have observed as Nevow behavior # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash - # or no further children) renders "this" page - if not name: + # or no further children) renders "this" page. We also need + # to reject "/uri/URI:DIR2:..//", so we look at postpath. + if not name and req.postpath != ['']: return self # Rejecting URIs that contain empty path pieces (for example: From fccc3316711523b021f46f0b2b932da9080f5ff4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:35 -0400 Subject: [PATCH 0329/1054] Make "Refresh" link work This seems to be a subtle difference from nevow: with `href="."`, rendered link target will be `/uri/`, so clicking "Refresh" will result in an error message like so: "GET /uri requires uri=". With `href=""`, the rendered link target will be `/uri/URI:...`, which is what we need. --- src/allmydata/web/directory.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.xhtml b/src/allmydata/web/directory.xhtml index 88058efa4..eeb04692d 100644 --- a/src/allmydata/web/directory.xhtml +++ b/src/allmydata/web/directory.xhtml @@ -31,7 +31,7 @@
                  From b956d9a58c2f4e92b5871071a861318a669c8428 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:46:23 -0400 Subject: [PATCH 0330/1054] Decode name before null check Make the diff against master a tad bit smaller. --- src/allmydata/web/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index b1618a3f5..a5ba3bed5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -104,6 +104,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page. We also need # to reject "/uri/URI:DIR2:..//", so we look at postpath. + name = name.decode('utf8') if not name and req.postpath != ['']: return self @@ -117,7 +118,6 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): u"The webapi does not allow empty pathname components", ) - name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From 1fd220dba0796d0b7abf1aeb45c4e02b9e331f7a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 16 Jun 2020 12:10:44 -0400 Subject: [PATCH 0331/1054] Update newsfragment text --- newsfragments/3312.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix index ed00922a6..9939fe1f0 100644 --- a/newsfragments/3312.bugfix +++ b/newsfragments/3312.bugfix @@ -1 +1 @@ -Add your info here \ No newline at end of file +Make directory page links work. From 6e2285ed2c024f271a8b656bbeb5a2563c64bbfa Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 May 2020 22:43:21 -0600 Subject: [PATCH 0332/1054] proof-of-concept verified-fake for testing --- integration/test_testing.py | 26 +++ src/allmydata/testing/__init__.py | 0 src/allmydata/testing/web.py | 263 ++++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 integration/test_testing.py create mode 100644 src/allmydata/testing/__init__.py create mode 100644 src/allmydata/testing/web.py diff --git a/integration/test_testing.py b/integration/test_testing.py new file mode 100644 index 000000000..47498b1d7 --- /dev/null +++ b/integration/test_testing.py @@ -0,0 +1,26 @@ + + +from allmydata.testing.web import ( + create_fake_tahoe_root, + deterministic_key_generator, +) + +import pytest +import pytest_twisted + + +@pytest_twisted.inlineCallbacks +def test_retrieve_cap(): + """ + WebUI Fake can serve a read-capability back + """ + + keys = deterministic_key_generator() + root = yield create_fake_tahoe_root() + dummy_readcap = yield root.add_data( + next(keys), + "some dummy content\n"*20 + ) + print("readcap: {}".format(dummy_readcap)) + + assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py new file mode 100644 index 000000000..5cf517a54 --- /dev/null +++ b/src/allmydata/testing/web.py @@ -0,0 +1,263 @@ +import io +import os +import string + +import attr + +from twisted.internet import defer +from twisted.python.filepath import ( + FilePath, +) +from twisted.web.resource import ( + Resource, +) +from twisted.web.client import ( + Agent, + FileBodyProducer, +) + +from treq.client import ( + HTTPClient, +) +from treq.testing import ( + RequestTraversalAgent, + RequestSequence, + StubTreq, +) + + +class _FakeTahoeRoot(Resource): + """ + This is a sketch of how an in-memory 'fake' of a Tahoe + WebUI. Ultimately, this will live in Tahoe + """ + + def __init__(self, uri=None): + Resource.__init__(self) # this is an old-style class :( + self._uri = uri + self.putChild(b"uri", self._uri) + + def add_data(self, key, data): + return self._uri.add_data(key, data) + +@attr.s +class _FakeCapability(object): + """ + """ + data=attr.ib() + + +# XXX want to make all kinds of caps, like +# URI:CHK:... URI:DIR2:... etc + +import allmydata.uri +KNOWN_CAPABILITIES = [ + getattr(allmydata.uri, t).BASE_STRING + for t in dir(allmydata.uri) + if hasattr(getattr(allmydata.uri, t), 'BASE_STRING') +] + + +from allmydata.immutable.upload import BaseUploadable +from allmydata.interfaces import IUploadable +from zope.interface import implementer +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) + + +def deterministic_key_generator(): + character = 0 + while character < (26 * 2): + key = string.letters[character] * 16 + character += 1 + yield key + raise RuntimeError("Ran out of keys") + + +@implementer(IUploadable) +class DataUploadable(BaseUploadable): + # Base gives us: + # set_upload_status + # set_default_encoding_parameters + # get_all_encoding_parameters + + def __init__(self, data, key=None): + self._data = data + self._where = 0 + self._key = key if key is not None else urandom(16) + + def get_encryption_key(self): + return succeed(self._key) + + def get_size(self): + return succeed(len(self._data)) + + @inlineCallbacks + def read(self, amount): + data = [self._data[self._where : self._where + amount]] + self._where += amount + yield + returnValue(data) + + def close(self): + pass + +@inlineCallbacks +def create_fake_capability(kind, key, data): + if kind not in KNOWN_CAPABILITIES: + raise ValueError( + "'{}' not a known kind: {}".format( + kind, + ", ".join(list(KNOWN_CAPABILITIES.keys())), + ) + ) + + # XXX to use a allmydata.immutable.upload.CHKUploader directly, + # we'd need to instantiate: + + from allmydata.immutable.upload import ( + CHKUploader, + EncryptAnUploadable, + ) + from allmydata.immutable.encode import ( + Encoder, + ) + + class _FakeSecretHolder(object): + def get_renewal_secret(self): + return "renewal_secret" + + def get_cancel_secret(self): + return "cancel_secret" + + + @attr.s + class _FakeBucket(object): + data = attr.ib(init=False, default="") + + def callRemoteOnly(self, *args): + pass # print("callRemoteOnly({})".format(args)) + + def callRemote(self, verb, *args): + if verb == 'write': + offset, data = args + assert offset >= len(self.data) + while offset > len(self.data): + self.data += 'X' # marker data; we're padding + self.data += data + elif verb == 'close': + pass + else: + print("callRemote({})".format(args)) + + + @attr.s + class _FakeStorageServer(object): + buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) + + def get_buckets(self, storage_index): + return succeed(self.buckets) + + def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): + # returns a 2-tuple .. second one maps share-num to BucketWriter + return succeed(( + {}, + { + i: bucket + for i, bucket in enumerate(self.buckets) + } + )) + + class _FakeServer(object): + def get_serverid(self): + return "fake_server" + + def get_name(self): + return "steven" + + def get_version(self): + return { + "http://allmydata.org/tahoe/protocols/storage/v1": { + "maximum-immutable-share-size": 10*1024*1024*1024, + } + } + + def get_lease_seed(self): + return "decafbaddecafbaddeca" + + def get_storage_server(self): + return _FakeStorageServer() + + + class _FakeStorageBroker(object): + def get_servers_for_psi(self, storage_index): + return [_FakeServer()] + + storage_broker = _FakeStorageBroker() + secret_holder = _FakeSecretHolder() + uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) + uploadable = DataUploadable(data, key=key) + uploadable.set_default_encoding_parameters({ + "n": 1, + "k": 1, + "happy": 1, + }) + encrypted_uploadable = EncryptAnUploadable(uploadable) + + encoder = Encoder() + yield encoder.set_encrypted_uploadable(encrypted_uploadable) + + uploadresults = yield uploader.start(encrypted_uploadable) + + enc_key = yield uploadable.get_encryption_key() + + from allmydata.uri import from_string as uri_from_string + from allmydata.uri import CHKFileURI + + verify_cap = uri_from_string(uploadresults.get_verifycapstr()) + read_cap = CHKFileURI( + enc_key, + verify_cap.uri_extension_hash, + verify_cap.needed_shares, + verify_cap.total_shares, + verify_cap.size, + ) + uploadresults.set_uri(read_cap.to_string()) + + returnValue(read_cap) + + +class _FakeTahoeUriHandler(Resource): + """ + """ + + isLeaf = True + + @inlineCallbacks + def add_data(self, key, data): + """ + adds some data to our grid, returning a capability + """ + cap = yield create_fake_capability("URI:CHK:", key, data) + returnValue(cap) + + def render_GET(self, request): + print(request) + print(request.uri) + return b"URI:DIR2-CHK:some capability" + + +@inlineCallbacks +def create_fake_tahoe_root(): + """ + Probably should take some params to control what this fake does: + return errors, pre-populate capabilities, ... + """ + root = _FakeTahoeRoot( + uri=_FakeTahoeUriHandler(), + ) + yield + returnValue(root) From ab7936db56a2326c43980a69b5ec38ba5dac4c77 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 1 Jun 2020 09:06:46 -0600 Subject: [PATCH 0333/1054] actual usable agent --- src/allmydata/testing/web.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5cf517a54..1767cb114 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -261,3 +261,52 @@ def create_fake_tahoe_root(): ) yield returnValue(root) + + +@implementer(IBodyProducer) +class _SynchronousProducer(object): + """ + A partial implementation of an :obj:`IBodyProducer` which produces its + entire payload immediately. There is no way to access to an instance of + this object from :obj:`RequestTraversalAgent` or :obj:`StubTreq`, or even a + :obj:`Resource: passed to :obj:`StubTreq`. + + This does not implement the :func:`IBodyProducer.stopProducing` method, + because that is very difficult to trigger. (The request from + `RequestTraversalAgent` would have to be canceled while it is still in the + transmitting state), and the intent is to use `RequestTraversalAgent` to + make synchronous requests. + """ + + def __init__(self, body): + """ + Create a synchronous producer with some bytes. + """ + if not isinstance(body, bytes): + raise ValueError( + "'body' must be bytes" + ) + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + """ + Immediately produce all data. + """ + consumer.write(self.body) + return succeed(None) + + +@inlineCallbacks +def create_tahoe_treq_client(root=None): + """ + """ + + if root is None: + root = yield create_fake_tahoe_root() + + client = HTTPClient( + agent=RequestTraversalAgent(root), + data_to_body_producer=_SynchronousProducer, + ) + returnValue(client) From 95531fa7295c24837d5b53054dc61d36777e8155 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 3 Jun 2020 11:32:22 -0600 Subject: [PATCH 0334/1054] synchronous-only up/download tahoe code to keep treq.testing happy --- src/allmydata/testing/web.py | 73 ++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 1767cb114..32fe0c084 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,10 +1,12 @@ import io import os +import json import string import attr -from twisted.internet import defer +from hyperlink import DecodedURL + from twisted.python.filepath import ( FilePath, ) @@ -15,6 +17,9 @@ from twisted.web.client import ( Agent, FileBodyProducer, ) +from twisted.web.iweb import ( + IBodyProducer, +) from treq.client import ( HTTPClient, @@ -40,6 +45,7 @@ class _FakeTahoeRoot(Resource): def add_data(self, key, data): return self._uri.add_data(key, data) + @attr.s class _FakeCapability(object): """ @@ -70,8 +76,8 @@ from twisted.internet.defer import ( def deterministic_key_generator(): character = 0 - while character < (26 * 2): - key = string.letters[character] * 16 + while True:#character < (26 * 2): + key = string.letters[character % 52] * 16 character += 1 yield key raise RuntimeError("Ran out of keys") @@ -159,7 +165,7 @@ def create_fake_capability(kind, key, data): buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) def get_buckets(self, storage_index): - return succeed(self.buckets) + return succeed({0: self.buckets}) def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): # returns a 2-tuple .. second one maps share-num to BucketWriter @@ -230,11 +236,34 @@ def create_fake_capability(kind, key, data): returnValue(read_cap) +def capability_generator(): + number = 0 + while True: + cap = u"URI:CHK:fake capability {}".format(number) + number += 1 + yield cap.encode("ascii") + + class _FakeTahoeUriHandler(Resource): """ """ isLeaf = True + _data = None + _key_generator = None + _capability_generator = None + + def _get_key(self): + if self._key_generator is None: + self._key_generator = deterministic_key_generator() + key = next(self._key_generator) + return key + + def _generate_cap(self): + if self._capability_generator is None: + self._capability_generator = capability_generator() + capability = next(self._capability_generator) + return capability @inlineCallbacks def add_data(self, key, data): @@ -242,12 +271,42 @@ class _FakeTahoeUriHandler(Resource): adds some data to our grid, returning a capability """ cap = yield create_fake_capability("URI:CHK:", key, data) + if self._data is None: + self._data = dict() + self._data[cap] = data returnValue(cap) + def render_PUT(self, request): + data = request.content.read() +# if len(data) == 0: +# raise RuntimeError("No empty data") + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap + + def render_POST(self, request): + kind = request.args[u"t"][0] + data = request.content.read() + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap +# if kind == 'mkdir-immutable': + def render_GET(self, request): - print(request) - print(request.uri) - return b"URI:DIR2-CHK:some capability" + uri = DecodedURL.from_text(request.uri.decode('utf8')) + # XXX FIXME + capability = uri.query[0][1] + + if self._data is None or capability not in self._data: + return u"No data for '{}'".format(capability).decode("ascii") + + return self._data[capability] @inlineCallbacks From 66116020fb78da074f1d7752fe2540dd686744c3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:26:09 -0600 Subject: [PATCH 0335/1054] simplify tahoe web fakes --- src/allmydata/testing/web.py | 287 ++++++++++------------------------- 1 file changed, 76 insertions(+), 211 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 32fe0c084..78d2b7931 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -2,6 +2,7 @@ import io import os import json import string +import hashlib import attr @@ -20,6 +21,11 @@ from twisted.web.client import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) from treq.client import ( HTTPClient, @@ -29,6 +35,12 @@ from treq.testing import ( RequestSequence, StubTreq, ) +from zope.interface import implementer + +import allmydata.uri +from allmydata.util import ( + base32, +) class _FakeTahoeRoot(Resource): @@ -53,10 +65,6 @@ class _FakeCapability(object): data=attr.ib() -# XXX want to make all kinds of caps, like -# URI:CHK:... URI:DIR2:... etc - -import allmydata.uri KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -64,183 +72,46 @@ KNOWN_CAPABILITIES = [ ] -from allmydata.immutable.upload import BaseUploadable -from allmydata.interfaces import IUploadable -from zope.interface import implementer -from twisted.internet.defer import ( - inlineCallbacks, - succeed, - returnValue, -) +def capability_generator(kind): + """ + :param str kind: the kind of capability, like `URI:CHK` - -def deterministic_key_generator(): - character = 0 - while True:#character < (26 * 2): - key = string.letters[character % 52] * 16 - character += 1 - yield key - raise RuntimeError("Ran out of keys") - - -@implementer(IUploadable) -class DataUploadable(BaseUploadable): - # Base gives us: - # set_upload_status - # set_default_encoding_parameters - # get_all_encoding_parameters - - def __init__(self, data, key=None): - self._data = data - self._where = 0 - self._key = key if key is not None else urandom(16) - - def get_encryption_key(self): - return succeed(self._key) - - def get_size(self): - return succeed(len(self._data)) - - @inlineCallbacks - def read(self, amount): - data = [self._data[self._where : self._where + amount]] - self._where += amount - yield - returnValue(data) - - def close(self): - pass - -@inlineCallbacks -def create_fake_capability(kind, key, data): + :returns: a generator that yields new capablities of a particular + kind. + """ if kind not in KNOWN_CAPABILITIES: raise ValueError( - "'{}' not a known kind: {}".format( + "Unknown capability kind '{} (valid are {})'".format( kind, - ", ".join(list(KNOWN_CAPABILITIES.keys())), + ", ".join(KNOWN_CAPABILITIES), ) ) - - # XXX to use a allmydata.immutable.upload.CHKUploader directly, - # we'd need to instantiate: - - from allmydata.immutable.upload import ( - CHKUploader, - EncryptAnUploadable, - ) - from allmydata.immutable.encode import ( - Encoder, - ) - - class _FakeSecretHolder(object): - def get_renewal_secret(self): - return "renewal_secret" - - def get_cancel_secret(self): - return "cancel_secret" - - - @attr.s - class _FakeBucket(object): - data = attr.ib(init=False, default="") - - def callRemoteOnly(self, *args): - pass # print("callRemoteOnly({})".format(args)) - - def callRemote(self, verb, *args): - if verb == 'write': - offset, data = args - assert offset >= len(self.data) - while offset > len(self.data): - self.data += 'X' # marker data; we're padding - self.data += data - elif verb == 'close': - pass - else: - print("callRemote({})".format(args)) - - - @attr.s - class _FakeStorageServer(object): - buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) - - def get_buckets(self, storage_index): - return succeed({0: self.buckets}) - - def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): - # returns a 2-tuple .. second one maps share-num to BucketWriter - return succeed(( - {}, - { - i: bucket - for i, bucket in enumerate(self.buckets) - } - )) - - class _FakeServer(object): - def get_serverid(self): - return "fake_server" - - def get_name(self): - return "steven" - - def get_version(self): - return { - "http://allmydata.org/tahoe/protocols/storage/v1": { - "maximum-immutable-share-size": 10*1024*1024*1024, - } - } - - def get_lease_seed(self): - return "decafbaddecafbaddeca" - - def get_storage_server(self): - return _FakeStorageServer() - - - class _FakeStorageBroker(object): - def get_servers_for_psi(self, storage_index): - return [_FakeServer()] - - storage_broker = _FakeStorageBroker() - secret_holder = _FakeSecretHolder() - uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) - uploadable = DataUploadable(data, key=key) - uploadable.set_default_encoding_parameters({ - "n": 1, - "k": 1, - "happy": 1, - }) - encrypted_uploadable = EncryptAnUploadable(uploadable) - - encoder = Encoder() - yield encoder.set_encrypted_uploadable(encrypted_uploadable) - - uploadresults = yield uploader.start(encrypted_uploadable) - - enc_key = yield uploadable.get_encryption_key() - - from allmydata.uri import from_string as uri_from_string - from allmydata.uri import CHKFileURI - - verify_cap = uri_from_string(uploadresults.get_verifycapstr()) - read_cap = CHKFileURI( - enc_key, - verify_cap.uri_extension_hash, - verify_cap.needed_shares, - verify_cap.total_shares, - verify_cap.size, - ) - uploadresults.set_uri(read_cap.to_string()) - - returnValue(read_cap) - - -def capability_generator(): + # what we do here is to start with empty hashers for the key and + # ueb_hash and repeatedly feed() them a zero byte on each + # iteration .. so the same sequence of capabilities will always be + # produced. We could add a seed= argument if we wanted to produce + # different sequences. number = 0 + key_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") + + # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: - cap = u"URI:CHK:fake capability {}".format(number) number += 1 + key_hasher.update("\x00") + ueb_hasher.update("\x00") + + key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes + ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes + + cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + kind=kind, + key=key, + ueb_hash=ueb_hash, + n=1, + k=1, + size=number * 1000, + ) yield cap.encode("ascii") @@ -250,53 +121,50 @@ class _FakeTahoeUriHandler(Resource): isLeaf = True _data = None - _key_generator = None - _capability_generator = None + _capability_generators = None - def _get_key(self): - if self._key_generator is None: - self._key_generator = deterministic_key_generator() - key = next(self._key_generator) - return key + def _generate_capability(self, kind): + """ + :param str kind: any valid capability-string type - def _generate_cap(self): - if self._capability_generator is None: - self._capability_generator = capability_generator() - capability = next(self._capability_generator) + :returns: the next capability-string for the given kind + """ + if self._capability_generators is None: + self._capability_generators = dict() + + if kind not in self._capability_generators: + self._capability_generators[kind] = capability_generator(kind) + capability = next(self._capability_generators[kind]) return capability - @inlineCallbacks - def add_data(self, key, data): + def add_data(self, kind, data): """ - adds some data to our grid, returning a capability + adds some data to our grid + + :returns: a capability-string """ - cap = yield create_fake_capability("URI:CHK:", key, data) + assert isinstance(data, bytes) + + cap = self._generate_capability(kind) if self._data is None: self._data = dict() + assert cap not in self._data, "already have '{}'".format(cap) self._data[cap] = data - returnValue(cap) + return cap def render_PUT(self, request): data = request.content.read() -# if len(data) == 0: -# raise RuntimeError("No empty data") - - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap + return self.add_data("URI:CHK:", data) def render_POST(self, request): - kind = request.args[u"t"][0] + t = request.args[u"t"][0] data = request.content.read() - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap -# if kind == 'mkdir-immutable': + type_to_kind = { + "mkdir-immutable": "URI:DIR2-CHK:" + } + kind = type_to_kind[t] + return self.add_data(kind, data) def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -309,17 +177,15 @@ class _FakeTahoeUriHandler(Resource): return self._data[capability] -@inlineCallbacks def create_fake_tahoe_root(): """ - Probably should take some params to control what this fake does: - return errors, pre-populate capabilities, ... + :returns: an IResource instance that will handle certain Tahoe URI + endpoints similar to a real Tahoe server. """ root = _FakeTahoeRoot( uri=_FakeTahoeUriHandler(), ) - yield - returnValue(root) + return root @implementer(IBodyProducer) @@ -356,16 +222,15 @@ class _SynchronousProducer(object): return succeed(None) -@inlineCallbacks def create_tahoe_treq_client(root=None): """ """ if root is None: - root = yield create_fake_tahoe_root() + root = create_fake_tahoe_root() client = HTTPClient( agent=RequestTraversalAgent(root), data_to_body_producer=_SynchronousProducer, ) - returnValue(client) + return client From f216f977f0a9efc62b20639d2f5bebeef55d404c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:36:28 -0600 Subject: [PATCH 0336/1054] news --- newsfragments/3317.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3317.feature diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature new file mode 100644 index 000000000..5ed00acf5 --- /dev/null +++ b/newsfragments/3317.feature @@ -0,0 +1 @@ +Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file From 05140a58813573160d76a244850192a2a89ec15e Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 14:14:42 -0600 Subject: [PATCH 0337/1054] simple test of testing/web --- src/allmydata/test/test_testing.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/allmydata/test/test_testing.py diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py new file mode 100644 index 000000000..37e68a0de --- /dev/null +++ b/src/allmydata/test/test_testing.py @@ -0,0 +1,40 @@ +import io + +from twisted.trial.unittest import ( + TestCase, +) +from twisted.internet.defer import ( + inlineCallbacks, +) + +from allmydata.testing.web import ( + create_tahoe_treq_client, +) + + +class FakeWebTest(TestCase): + """ + Test the WebUI verified-fakes infrastucture + """ + + def setUp(self): + super(FakeWebTest, self).setUp() + self.http_client = create_tahoe_treq_client() + + @inlineCallbacks + def test_create_and_download(self): + """ + Upload some content and then download it + """ + content = "fake data\n" * 100 + resp = yield self.http_client.put("http://example.com/uri", content) + cap = yield resp.content() + + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + round_trip_content = yield resp.content() + + self.assertEqual(content, round_trip_content) From 18b62bf7465c1bc00bf5a8391b6e019e35604165 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:40 -0600 Subject: [PATCH 0338/1054] unused imports --- src/allmydata/test/test_testing.py | 2 -- src/allmydata/testing/web.py | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37e68a0de..da37b1de3 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,5 +1,3 @@ -import io - from twisted.trial.unittest import ( TestCase, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 78d2b7931..bc0138326 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,30 +1,17 @@ -import io -import os -import json -import string import hashlib import attr from hyperlink import DecodedURL -from twisted.python.filepath import ( - FilePath, -) from twisted.web.resource import ( Resource, ) -from twisted.web.client import ( - Agent, - FileBodyProducer, -) from twisted.web.iweb import ( IBodyProducer, ) from twisted.internet.defer import ( - inlineCallbacks, succeed, - returnValue, ) from treq.client import ( @@ -32,8 +19,6 @@ from treq.client import ( ) from treq.testing import ( RequestTraversalAgent, - RequestSequence, - StubTreq, ) from zope.interface import implementer From 02a663c244189c916d0c27ba473dc5b3d4ea7419 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:47 -0600 Subject: [PATCH 0339/1054] new-style classes only --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index bc0138326..de515ebc9 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -28,7 +28,7 @@ from allmydata.util import ( ) -class _FakeTahoeRoot(Resource): +class _FakeTahoeRoot(Resource, object): """ This is a sketch of how an in-memory 'fake' of a Tahoe WebUI. Ultimately, this will live in Tahoe @@ -100,7 +100,7 @@ def capability_generator(kind): yield cap.encode("ascii") -class _FakeTahoeUriHandler(Resource): +class _FakeTahoeUriHandler(Resource, object): """ """ From 1f2ad6e634ded72a47fd1494ffc597dba88620dd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:55:19 -0600 Subject: [PATCH 0340/1054] the tests are unit-tests --- integration/test_testing.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 integration/test_testing.py diff --git a/integration/test_testing.py b/integration/test_testing.py deleted file mode 100644 index 47498b1d7..000000000 --- a/integration/test_testing.py +++ /dev/null @@ -1,26 +0,0 @@ - - -from allmydata.testing.web import ( - create_fake_tahoe_root, - deterministic_key_generator, -) - -import pytest -import pytest_twisted - - -@pytest_twisted.inlineCallbacks -def test_retrieve_cap(): - """ - WebUI Fake can serve a read-capability back - """ - - keys = deterministic_key_generator() - root = yield create_fake_tahoe_root() - dummy_readcap = yield root.add_data( - next(keys), - "some dummy content\n"*20 - ) - print("readcap: {}".format(dummy_readcap)) - - assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" From c0ce7164309b3aea9f7d983eb9e2d315cca0eb2f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:11 -0600 Subject: [PATCH 0341/1054] handle more inputs in test body-producer --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index de515ebc9..aa3b2a4a7 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -16,6 +16,7 @@ from twisted.internet.defer import ( from treq.client import ( HTTPClient, + FileBodyProducer, ) from treq.testing import ( RequestTraversalAgent, @@ -192,9 +193,12 @@ class _SynchronousProducer(object): """ Create a synchronous producer with some bytes. """ + if isinstance(body, FileBodyProducer): + body = body._inputFile.read() + if not isinstance(body, bytes): raise ValueError( - "'body' must be bytes" + "'body' must be bytes not '{}'".format(type(body)) ) self.body = body self.length = len(body) From 1710530b6bc7c884a705b0bf4f578ee01f8e3ffd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:21 -0600 Subject: [PATCH 0342/1054] key -> kind --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aa3b2a4a7..fa6a2555a 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -40,8 +40,8 @@ class _FakeTahoeRoot(Resource, object): self._uri = uri self.putChild(b"uri", self._uri) - def add_data(self, key, data): - return self._uri.add_data(key, data) + def add_data(self, kind, data): + return self._uri.add_data(kind, data) @attr.s From 0cc020b5f224f967cb01e05bc53f45b85b78dff7 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:03 -0600 Subject: [PATCH 0343/1054] copyright, header --- src/allmydata/test/test_testing.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index da37b1de3..4dc9c6bb6 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,6 +1,16 @@ -from twisted.trial.unittest import ( - TestCase, -) +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Tests for the allmydata.testing helpers +""" + from twisted.internet.defer import ( inlineCallbacks, ) From b01995f158870a45ab8d848f8fae9e74191c6d6c Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:12 -0600 Subject: [PATCH 0344/1054] use hypothesis -- which doesn't know about Deferred, so need testtools too --- src/allmydata/test/test_testing.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 4dc9c6bb6..e350546ca 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -19,6 +19,23 @@ from allmydata.testing.web import ( create_tahoe_treq_client, ) +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + binary, +) + +from testtools import ( + TestCase, +) +from testtools.matchers import ( + Always, +) +from testtools.twistedsupport import ( + succeeded, +) + class FakeWebTest(TestCase): """ @@ -29,20 +46,31 @@ class FakeWebTest(TestCase): super(FakeWebTest, self).setUp() self.http_client = create_tahoe_treq_client() - @inlineCallbacks - def test_create_and_download(self): + @given( + content=binary(), + ) + def test_create_and_download(self, content): """ - Upload some content and then download it + Upload some content (via 'PUT /uri') and then download it (via + 'GET /uri?uri=...') """ - content = "fake data\n" * 100 - resp = yield self.http_client.put("http://example.com/uri", content) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + @inlineCallbacks + def do_test(): + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 200) - resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + cap = yield resp.content() + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + self.assertEqual(resp.code, 200) + + round_trip_content = yield resp.content() + self.assertEqual(content, round_trip_content) + self.assertThat( + do_test(), + succeeded(Always()), ) - round_trip_content = yield resp.content() - - self.assertEqual(content, round_trip_content) From 683c8d3e4f0ddac997cdc16ede90778bd597db75 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:08:36 -0600 Subject: [PATCH 0345/1054] actually returns CREATED for new data --- src/allmydata/test/test_testing.py | 2 +- src/allmydata/testing/web.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index e350546ca..a5c712d77 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -58,7 +58,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 200) + self.assertEqual(resp.code, 201) cap = yield resp.content() self.assertTrue(cap.startswith("URI:CHK:")) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index fa6a2555a..99dabeddf 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -140,6 +140,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() + request.setResponseCode(201) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From 9d5e438c2d36d336da24d081d1bd2acc7f7501c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:57:38 -0600 Subject: [PATCH 0346/1054] use real parser --- src/allmydata/test/test_testing.py | 11 ++++++++--- src/allmydata/testing/web.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a5c712d77..b5bbacad4 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -15,6 +15,10 @@ from twisted.internet.defer import ( inlineCallbacks, ) +from allmydata.uri import ( + from_string, + CHKFileURI, +) from allmydata.testing.web import ( create_tahoe_treq_client, ) @@ -60,11 +64,12 @@ class FakeWebTest(TestCase): resp = yield self.http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 99dabeddf..f6411792d 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -90,7 +90,7 @@ def capability_generator(kind): key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes - cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + cap = u"{kind}{key}:{ueb_hash}:{n}:{k}:{size}".format( kind=kind, key=key, ueb_hash=ueb_hash, From e481e37beba39ab5a1de9b1630bd35a578b193ea Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:01:02 -0600 Subject: [PATCH 0347/1054] housekeeping --- src/allmydata/testing/web.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index f6411792d..2b0f830f2 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,3 +1,16 @@ +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Test-helpers for clients that use the WebUI. +""" + import hashlib import attr @@ -28,6 +41,11 @@ from allmydata.util import ( base32, ) +__all__ = ( + "create_fake_tahoe_root", + "create_tahoe_treq_client", +) + class _FakeTahoeRoot(Resource, object): """ From 94207778109cbdf60fc2df6cfa455e53474ca08a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:05:32 -0600 Subject: [PATCH 0348/1054] cleanup, docstrings --- src/allmydata/testing/web.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 2b0f830f2..7d9b5730b 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -49,8 +49,8 @@ __all__ = ( class _FakeTahoeRoot(Resource, object): """ - This is a sketch of how an in-memory 'fake' of a Tahoe - WebUI. Ultimately, this will live in Tahoe + An in-memory 'fake' of a Tahoe WebUI root. Currently it only + implements (some of) the `/uri` resource. """ def __init__(self, uri=None): @@ -62,13 +62,6 @@ class _FakeTahoeRoot(Resource, object): return self._uri.add_data(kind, data) -@attr.s -class _FakeCapability(object): - """ - """ - data=attr.ib() - - KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -121,6 +114,8 @@ def capability_generator(kind): class _FakeTahoeUriHandler(Resource, object): """ + An in-memory fake of (some of) the `/uri` endpoint of a Tahoe + WebUI """ isLeaf = True @@ -232,6 +227,12 @@ class _SynchronousProducer(object): def create_tahoe_treq_client(root=None): """ + :param root: an instance created via `create_fake_tahoe_root`. The + caller might want a copy of this to call `.add_data` for example. + + :returns: an instance of treq.client.HTTPClient wired up to + in-memory fakes of the Tahoe WebUI. Only a subset of the real + WebUI is available. """ if root is None: From 10715fa26b88e3999d081a3c1b9623b6aef781d5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:06:33 -0600 Subject: [PATCH 0349/1054] docstring --- src/allmydata/testing/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7d9b5730b..64e615cd6 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -54,6 +54,9 @@ class _FakeTahoeRoot(Resource, object): """ def __init__(self, uri=None): + """ + :param uri: a Resource to handle the `/uri` tree. + """ Resource.__init__(self) # this is an old-style class :( self._uri = uri self.putChild(b"uri", self._uri) From 6fd0d61d5e2083b2141658c542897824b7682295 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:08:04 -0600 Subject: [PATCH 0350/1054] comment --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 64e615cd6..c2bced12e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -93,7 +93,7 @@ def capability_generator(kind): # different sequences. number = 0 key_hasher = hashlib.new("sha256") - ueb_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") # ueb means "URI Extension Block" # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: From 835341f2c24ac7f47b9ac315b695efa292d71124 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:12 -0600 Subject: [PATCH 0351/1054] more docstring --- src/allmydata/testing/web.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index c2bced12e..63f6b9877 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -74,6 +74,10 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ + Deterministically generates a streap of valid capabilities of the + given kind. The N, K and size values aren't related to anything + real. + :param str kind: the kind of capability, like `URI:CHK` :returns: a generator that yields new capablities of a particular From bd858f8fef6848abe0bc9d34f11073fb5c44762f Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:57 -0600 Subject: [PATCH 0352/1054] assert -> raise --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 63f6b9877..90b50d2e0 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -149,7 +149,8 @@ class _FakeTahoeUriHandler(Resource, object): :returns: a capability-string """ - assert isinstance(data, bytes) + if not isinstance(data, bytes): + raise TypeError("'data' must be bytes") cap = self._generate_capability(kind) if self._data is None: From 66f61c514b987c42b152ff2204aa86ed834b8f7d Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:41:22 -0600 Subject: [PATCH 0353/1054] assert -> ValueError --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 90b50d2e0..5e6d564fa 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -155,7 +155,8 @@ class _FakeTahoeUriHandler(Resource, object): cap = self._generate_capability(kind) if self._data is None: self._data = dict() - assert cap not in self._data, "already have '{}'".format(cap) + if cap in self._data: + raise ValueError("already have '{}'".format(cap)) self._data[cap] = data return cap From 0f6b4c33ac206d384e2d6adba0ad562e40ad3c59 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:46:55 -0600 Subject: [PATCH 0354/1054] match query args --- src/allmydata/testing/web.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5e6d564fa..3231c5820 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -177,8 +177,16 @@ class _FakeTahoeUriHandler(Resource, object): def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) - # XXX FIXME - capability = uri.query[0][1] + capability = None + for arg, value in uri.query: + if arg == u"uri": + capability = value + if capability is None: + raise Exception( + "No ?uri= arguent in GET '{}'".format( + uri.to_string() + ) + ) if self._data is None or capability not in self._data: return u"No data for '{}'".format(capability).decode("ascii") From a3f4cc698327ee7e791cef853fe14528ff88e558 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:14:45 -0600 Subject: [PATCH 0355/1054] symbols, not magic-numbers --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 3231c5820..5ab870f4f 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -23,6 +23,10 @@ from twisted.web.resource import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.web import ( + http, +) + from twisted.internet.defer import ( succeed, ) @@ -162,7 +166,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() - request.setResponseCode(201) # real code does this for brand-new files + request.setResponseCode(http.CREATED) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From eaf96b22fef6a128c944e9b649aa32f55b9d8dce Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:55:14 -0600 Subject: [PATCH 0356/1054] replciate Tahoe's behavior for replace=true/false --- src/allmydata/test/test_testing.py | 21 +++++++++++++++++++- src/allmydata/testing/web.py | 31 +++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index b5bbacad4..a995e756d 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -61,7 +61,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri", content) + resp = yield self.http_client.put("http://example.com/uri?replace=true", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() @@ -79,3 +79,22 @@ class FakeWebTest(TestCase): do_test(), succeeded(Always()), ) + + @inlineCallbacks + def test_duplicate_upload(self): + """ + Upload the same content (via 'PUT /uri') twice with no overwrite + """ + + content = "fake content\n" * 200 + + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) + + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) + + # this one fails: same content but no ?replace=true + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 409) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5ab870f4f..ef7cd5bed 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,6 +30,9 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) +from twisted.python.failure import ( + Failure, +) from treq.client import ( HTTPClient, @@ -44,6 +47,13 @@ import allmydata.uri from allmydata.util import ( base32, ) +from allmydata.interfaces import ( + ExistingChildError, +) +from allmydata.web.common import ( + humanize_failure, +) + __all__ = ( "create_fake_tahoe_root", @@ -147,7 +157,7 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self._capability_generators[kind]) return capability - def add_data(self, kind, data): + def add_data(self, kind, data, allow_duplicate=False): """ adds some data to our grid @@ -156,9 +166,18 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - cap = self._generate_capability(kind) if self._data is None: self._data = dict() + + for k in self._data: + if self._data[k] == data: + if allow_duplicate: + return k + raise ValueError( + "Duplicate data" + ) + + cap = self._generate_capability(kind) if cap in self._data: raise ValueError("already have '{}'".format(cap)) self._data[cap] = data @@ -167,7 +186,13 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() request.setResponseCode(http.CREATED) # real code does this for brand-new files - return self.add_data("URI:CHK:", data) + replace = request.args.get("replace", None) + try: + return self.add_data("URI:CHK:", data, allow_duplicate=replace) + except ValueError: + msg, code = humanize_failure(Failure(ExistingChildError())) + request.setResponseCode(code) + return msg def render_POST(self, request): t = request.args[u"t"][0] From d91996aa827b15e4090b042c562377b52920bf6f Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:03:19 -0600 Subject: [PATCH 0357/1054] better docstring --- src/allmydata/testing/web.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ef7cd5bed..5f011f8b1 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -225,6 +225,14 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ + If you wish to pre-populate data into the fake Tahoe grid, retain + a reference to this root and pass it to + `create_tahoe_treq_client`. For example:: + + root = create_fake_tahoe_root() + root.add_data(...) + client = create_tahoe_treq_client(root) + :returns: an IResource instance that will handle certain Tahoe URI endpoints similar to a real Tahoe server. """ From ced9bdeb2590ec0121036ff35d4d81bff2462202 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:05:27 -0600 Subject: [PATCH 0358/1054] different words for news --- newsfragments/3317.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature index 5ed00acf5..2a7048397 100644 --- a/newsfragments/3317.feature +++ b/newsfragments/3317.feature @@ -1 +1 @@ -Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file +allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. \ No newline at end of file From af70911e2f38c1443a6e5e1e9144f39514c249f6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:22:17 -0600 Subject: [PATCH 0359/1054] embetter docstring --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5f011f8b1..ca010afa4 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -226,11 +226,11 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ If you wish to pre-populate data into the fake Tahoe grid, retain - a reference to this root and pass it to + a reference to this root by creating it yourself and passing it to `create_tahoe_treq_client`. For example:: root = create_fake_tahoe_root() - root.add_data(...) + cap_string = root.add_data(...) client = create_tahoe_treq_client(root) :returns: an IResource instance that will handle certain Tahoe URI From e0287a7bfe654c40ce34b98d3af02d9cd8194d75 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:28:28 -0600 Subject: [PATCH 0360/1054] use attr.s --- src/allmydata/testing/web.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ca010afa4..7c6fc1996 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -133,6 +133,7 @@ def capability_generator(kind): yield cap.encode("ascii") +@attr.s class _FakeTahoeUriHandler(Resource, object): """ An in-memory fake of (some of) the `/uri` endpoint of a Tahoe @@ -140,8 +141,9 @@ class _FakeTahoeUriHandler(Resource, object): """ isLeaf = True - _data = None - _capability_generators = None + + data = attr.ib(default=attr.Factory(dict)) + capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): """ @@ -149,12 +151,9 @@ class _FakeTahoeUriHandler(Resource, object): :returns: the next capability-string for the given kind """ - if self._capability_generators is None: - self._capability_generators = dict() - - if kind not in self._capability_generators: - self._capability_generators[kind] = capability_generator(kind) - capability = next(self._capability_generators[kind]) + if kind not in self.capability_generators: + self.capability_generators[kind] = capability_generator(kind) + capability = next(self.capability_generators[kind]) return capability def add_data(self, kind, data, allow_duplicate=False): @@ -166,11 +165,8 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - if self._data is None: - self._data = dict() - - for k in self._data: - if self._data[k] == data: + for k in self.data: + if self.data[k] == data: if allow_duplicate: return k raise ValueError( @@ -178,9 +174,9 @@ class _FakeTahoeUriHandler(Resource, object): ) cap = self._generate_capability(kind) - if cap in self._data: + if cap in self.data: raise ValueError("already have '{}'".format(cap)) - self._data[cap] = data + self.data[cap] = data return cap def render_PUT(self, request): @@ -217,10 +213,10 @@ class _FakeTahoeUriHandler(Resource, object): ) ) - if self._data is None or capability not in self._data: + if capability not in self.data: return u"No data for '{}'".format(capability).decode("ascii") - return self._data[capability] + return self.data[capability] def create_fake_tahoe_root(): From 26d7a3a957faf35520ce349d235da840fa8b2c15 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:15:01 -0600 Subject: [PATCH 0361/1054] Get rid of ?replace= handling entirely and more-correctly support /uri variants --- src/allmydata/test/test_testing.py | 98 ++++++++++++++++++++++++------ src/allmydata/testing/web.py | 52 ++++++++-------- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a995e756d..67c38ed57 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -21,6 +21,11 @@ from allmydata.uri import ( ) from allmydata.testing.web import ( create_tahoe_treq_client, + capability_generator, +) + +from hyperlink import ( + DecodedURL, ) from hypothesis import ( @@ -35,9 +40,12 @@ from testtools import ( ) from testtools.matchers import ( Always, + Equals, + MatchesStructure, ) from testtools.twistedsupport import ( succeeded, + failed, ) @@ -46,9 +54,9 @@ class FakeWebTest(TestCase): Test the WebUI verified-fakes infrastucture """ - def setUp(self): - super(FakeWebTest, self).setUp() - self.http_client = create_tahoe_treq_client() + # Note: do NOT use setUp() because Hypothesis doesn't work + # properly with it. You must instead do all fixture-type work + # yourself in each test. @given( content=binary(), @@ -58,21 +66,31 @@ class FakeWebTest(TestCase): Upload some content (via 'PUT /uri') and then download it (via 'GET /uri?uri=...') """ + http_client = create_tahoe_treq_client() @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri?replace=true", content) + resp = yield http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() cap = from_string(cap_raw) self.assertIsInstance(cap, CHKFileURI) - resp = yield self.http_client.get( + resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() + + # using the form "/uri/" is also valid + + resp = yield http_client.get( + "http://example.com/uri/{}".format(cap.to_string()) + ) + self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() self.assertEqual(content, round_trip_content) self.assertThat( @@ -80,21 +98,67 @@ class FakeWebTest(TestCase): succeeded(Always()), ) - @inlineCallbacks - def test_duplicate_upload(self): + @given( + content=binary(), + ) + def test_duplicate_upload(self, content): """ - Upload the same content (via 'PUT /uri') twice with no overwrite + Upload the same content (via 'PUT /uri') twice """ - content = "fake content\n" * 200 + http_client = create_tahoe_treq_client() - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + @inlineCallbacks + def do_test(): + resp = yield http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) - cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) - # this one fails: same content but no ?replace=true - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 409) + resp = yield http_client.put("http://example.com/uri", content) + self.assertThat(resp.code, Equals(200)) + self.assertThat( + do_test(), + succeeded(Always()), + ) + + def test_download_missing(self): + """ + Error if we download a capability that doesn't exist + """ + + http_client = create_tahoe_treq_client() + cap_gen = capability_generator("URI:CHK:") + + uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(next(cap_gen))) + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(500) + ) + ) + ) + + def test_download_no_arg(self): + """ + Error if we GET from "/uri" with no ?uri= query-arg + """ + + http_client = create_tahoe_treq_client() + + uri = DecodedURL.from_text(u"http://example.com/uri/") + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(400) + ) + ) + ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7c6fc1996..aaa795d06 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -76,7 +76,8 @@ class _FakeTahoeRoot(Resource, object): self.putChild(b"uri", self._uri) def add_data(self, kind, data): - return self._uri.add_data(kind, data) + fresh, cap = self._uri.add_data(kind, data) + return cap KNOWN_CAPABILITIES = [ @@ -156,39 +157,32 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self.capability_generators[kind]) return capability - def add_data(self, kind, data, allow_duplicate=False): + def add_data(self, kind, data): """ adds some data to our grid - :returns: a capability-string + :returns: a two-tuple: a bool (True if the data is freshly added) and a capability-string """ if not isinstance(data, bytes): raise TypeError("'data' must be bytes") for k in self.data: if self.data[k] == data: - if allow_duplicate: - return k - raise ValueError( - "Duplicate data" - ) + return (False, k) cap = self._generate_capability(kind) - if cap in self.data: - raise ValueError("already have '{}'".format(cap)) + assert cap not in self.data, "logic error" # shouldn't happen self.data[cap] = data - return cap + return (True, cap) def render_PUT(self, request): data = request.content.read() - request.setResponseCode(http.CREATED) # real code does this for brand-new files - replace = request.args.get("replace", None) - try: - return self.add_data("URI:CHK:", data, allow_duplicate=replace) - except ValueError: - msg, code = humanize_failure(Failure(ExistingChildError())) - request.setResponseCode(code) - return msg + fresh, cap = self.add_data("URI:CHK:", data) + if fresh: + request.setResponseCode(http.CREATED) # real code does this for brand-new files + else: + request.setResponseCode(http.OK) # replaced/modified files + return cap def render_POST(self, request): t = request.args[u"t"][0] @@ -198,7 +192,8 @@ class _FakeTahoeUriHandler(Resource, object): "mkdir-immutable": "URI:DIR2-CHK:" } kind = type_to_kind[t] - return self.add_data(kind, data) + fresh, cap = self.add_data(kind, data) + return cap def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -206,14 +201,19 @@ class _FakeTahoeUriHandler(Resource, object): for arg, value in uri.query: if arg == u"uri": capability = value - if capability is None: - raise Exception( - "No ?uri= arguent in GET '{}'".format( - uri.to_string() - ) - ) + # it's legal to use the form "/uri/" + if capability is None and request.postpath and request.postpath[0]: + capability = request.postpath[0] + # if we don't yet have a capability, that's an error + if capability is None: + request.setResponseCode(http.BAD_REQUEST) + return b"GET /uri requires uri=" + + # the user gave us a capability; if our Grid doesn't have any + # data for it, that's an error. if capability not in self.data: + request.setResponseCode(http.BAD_REQUEST) return u"No data for '{}'".format(capability).decode("ascii") return self.data[capability] From 1ce760b2370ea156d680b0291f6fdf3d9b89a081 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:16:39 -0600 Subject: [PATCH 0362/1054] testtool's asserts --- src/allmydata/test/test_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 67c38ed57..6ba622f3c 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -71,7 +71,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + self.assertThat(resp.code, Equals(201)) cap_raw = yield resp.content() cap = from_string(cap_raw) @@ -80,7 +80,7 @@ class FakeWebTest(TestCase): resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) - self.assertEqual(resp.code, 200) + self.assertThat(resp.code, Equals(200)) round_trip_content = yield resp.content() From 77a2f11694c68cc0b07ee10f97550f4aadb01951 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:19:30 -0600 Subject: [PATCH 0363/1054] more-testtools-y asserts --- src/allmydata/test/test_testing.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 6ba622f3c..37a5f3c7b 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -41,7 +41,9 @@ from testtools import ( from testtools.matchers import ( Always, Equals, + IsInstance, MatchesStructure, + AfterPreprocessing, ) from testtools.twistedsupport import ( succeeded, @@ -75,7 +77,7 @@ class FakeWebTest(TestCase): cap_raw = yield resp.content() cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat(cap, IsInstance(CHKFileURI)) resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) @@ -114,8 +116,13 @@ class FakeWebTest(TestCase): self.assertEqual(resp.code, 201) cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat( + cap_raw, + AfterPreprocessing( + from_string, + IsInstance(CHKFileURI) + ) + ) resp = yield http_client.put("http://example.com/uri", content) self.assertThat(resp.code, Equals(200)) From 1f4b619fda37cab02762c1f50dff74cba0f33c18 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:22:42 -0600 Subject: [PATCH 0364/1054] unused imports --- src/allmydata/test/test_testing.py | 1 - src/allmydata/testing/web.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37a5f3c7b..ec6935914 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -47,7 +47,6 @@ from testtools.matchers import ( ) from testtools.twistedsupport import ( succeeded, - failed, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aaa795d06..6656236ca 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,9 +30,6 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) -from twisted.python.failure import ( - Failure, -) from treq.client import ( HTTPClient, @@ -47,12 +44,6 @@ import allmydata.uri from allmydata.util import ( base32, ) -from allmydata.interfaces import ( - ExistingChildError, -) -from allmydata.web.common import ( - humanize_failure, -) __all__ = ( From 5cfb35077185d2d87ab3b5b13d8ef61bf89d8933 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:15:58 -0600 Subject: [PATCH 0365/1054] streap -> stream --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 6656236ca..0d6e7cb6e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -80,7 +80,7 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ - Deterministically generates a streap of valid capabilities of the + Deterministically generates a stream of valid capabilities of the given kind. The N, K and size values aren't related to anything real. From 7a37eeee678d5765d85cbd52e6c6515c90b6a538 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:17:34 -0600 Subject: [PATCH 0366/1054] assert -> Exception --- src/allmydata/testing/web.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 0d6e7cb6e..12f5ef40c 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -162,7 +162,10 @@ class _FakeTahoeUriHandler(Resource, object): return (False, k) cap = self._generate_capability(kind) - assert cap not in self.data, "logic error" # shouldn't happen + # it should be impossible for this to already be in our data, + # but check anyway to be sure + if cap in self.data: + raise Exception("Internal error; key already exists somehow") self.data[cap] = data return (True, cap) From aad3557d2b6452dc1c0245c080c3df2062c9c06b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 06:46:08 -0500 Subject: [PATCH 0367/1054] Use twisted.web.template in web/storage.py Related to ticket:3247. Nevow usage has been removed, and generated page looks the same as its former self, but tests are failing because test_storage.py assumes that we're using nevow. --- src/allmydata/web/storage.py | 258 ++++++++++++++----------- src/allmydata/web/storage_status.xhtml | 60 +++--- 2 files changed, 182 insertions(+), 136 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 79c0a8c38..3000fb70d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,10 +1,10 @@ import time, json -from nevow import rend, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile from allmydata.web.common import ( - getxmlfile, abbreviate_time, - MultiFormatPage, + MultiFormatResource ) from allmydata.util.abbreviate import abbreviate_space from allmydata.util import time_format, idlib @@ -16,91 +16,100 @@ def remove_prefix(s, prefix): return s[len(prefix):] -class StorageStatus(MultiFormatPage): - docFactory = getxmlfile("storage_status.xhtml") - # the default 'data' argument is the StorageServer instance +class StorageStatusElement(Element): + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname=""): - rend.Page.__init__(self, storage) + def __init__(self, storage, nickname): + super(StorageStatusElement, self).__init__() self.storage = storage - self.nickname = nickname + self.nick = nickname - def render_JSON(self, req): - req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), - } - return json.dumps(d, indent=1) + "\n" + @renderer + def nickname(self, req, tag): + return self.nick - def data_nickname(self, ctx, storage): - return self.nickname - def data_nodeid(self, ctx, storage): + @renderer + def nodeid(self, req, tag): return idlib.nodeid_b2a(self.storage.my_nodeid) - def render_storage_running(self, ctx, storage): - if storage: - return ctx.tag - else: - return T.h1["No Storage Server Running"] + def get_stat(self, key): + return self.storage.get_stats().get(key) - def render_bool(self, ctx, data): - return {True: "Yes", False: "No"}[bool(data)] + def str(self, tag, val): + if val is None: + return tag("?") + return tag(str(val)) - def render_abbrev_space(self, ctx, size): - if size is None: - return "?" - return abbreviate_space(size) + def abbr(self, tag, val): + if val is None: + return tag("?") + return tag(abbreviate_space(val)) - def render_space(self, ctx, size): - if size is None: - return "?" - return "%d" % size + @renderer + def disk_total(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_total")) - def data_stats(self, ctx, data): - # FYI: 'data' appears to be self, rather than the StorageServer - # object in self.original that gets passed to render_* methods. I - # still don't understand Nevow. + @renderer + def disk_total_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_total")) - # Nevow has nevow.accessors.DictionaryContainer: Any data= directive - # that appears in a context in which the current data is a dictionary - # will be looked up as keys in that dictionary. So if data_stats() - # returns a dictionary, then we can use something like this: - # - #
                    - #
                  • disk_total:
                    • - #
                  + @renderer + def disk_used(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_used")) - # to use get_stats()["storage_server.disk_total"] . However, - # DictionaryContainer does a raw d[] instead of d.get(), so any - # missing keys will cause an error, even if the renderer can tolerate - # None values. To overcome this, we either need a dict-like object - # that always returns None for unknown keys, or we must pre-populate - # our dict with those missing keys, or we should get rid of data_ - # methods that return dicts (or find some way to override Nevow's - # handling of dictionaries). + @renderer + def disk_used_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_used")) - d = dict([ (remove_prefix(k, "storage_server."), v) - for k,v in self.storage.get_stats().items() ]) - d.setdefault("disk_total", None) - d.setdefault("disk_used", None) - d.setdefault("disk_free_for_root", None) - d.setdefault("disk_free_for_nonroot", None) - d.setdefault("reserved_space", None) - d.setdefault("disk_avail", None) - return d + @renderer + def disk_free_for_root(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) - def data_last_complete_bucket_count(self, ctx, data): + @renderer + def disk_free_for_root_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + + @renderer + def disk_free_for_nonroot(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def disk_free_for_nonroot_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def reserved_space(self, req, tag): + return self.str(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def reserved_space_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def disk_avail(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def disk_avail_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def accepting_immutable_shares(self, req, tag): + accepting = self.get_stat("storage_server.accepting_immutable_shares") + return {True: "Yes", False: "No"}[bool(accepting)] + + @renderer + def last_complete_bucket_count(self, req, tag): s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" - return count + return str(count) - def render_count_crawler_status(self, ctx, storage): + @renderer + def count_crawler_status(self, req, tag): p = self.storage.bucket_counter.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return self.format_crawler_progress(p) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] @@ -127,55 +136,51 @@ class StorageStatus(MultiFormatPage): return ["Next crawl in %s" % abbreviate_time(soon), cycletime_s] - def render_lease_expiration_enabled(self, ctx, data): + @renderer + def storage_running(self, req, tag): + if self.storage: + return tag + return tag("No Storage Server Running") + + @renderer + def lease_expiration_enabled(self, req, tag): lc = self.storage.lease_checker if lc.expiration_enabled: - return ctx.tag["Enabled: expired leases will be removed"] + return tag("Enabled: expired leases will be removed") else: - return ctx.tag["Disabled: scan-only mode, no leases will be removed"] + return tag("Disabled: scan-only mode, no leases will be removed") - def render_lease_expiration_mode(self, ctx, data): + @renderer + def lease_expiration_mode(self, req, tag): lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: - ctx.tag["Leases will expire naturally, probably 31 days after " - "creation or renewal."] + tag("Leases will expire naturally, probably 31 days after " + "creation or renewal.") else: - ctx.tag["Leases created or last renewed more than %s ago " - "will be considered expired." - % abbreviate_time(lc.override_lease_duration)] + tag("Leases created or last renewed more than %s ago " + "will be considered expired." + % abbreviate_time(lc.override_lease_duration)) else: assert lc.mode == "cutoff-date" localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date)) isoutcdate = time_format.iso_utc_date(lc.cutoff_date) - ctx.tag["Leases created or last renewed before %s (%s) UTC " - "will be considered expired." % (isoutcdate, localizedutcdate, )] + tag("Leases created or last renewed before %s (%s) UTC " + "will be considered expired." + % (isoutcdate, localizedutcdate, )) if len(lc.mode) > 2: - ctx.tag[" The following sharetypes will be expired: ", - " ".join(sorted(lc.sharetypes_to_expire)), "."] - return ctx.tag + tag(" The following sharetypes will be expired: ", + " ".join(sorted(lc.sharetypes_to_expire)), ".") + return tag - def format_recovered(self, sr, a): - def maybe(d): - if d is None: - return "?" - return "%d" % d - return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ - (maybe(sr["%s-shares" % a]), - maybe(sr["%s-buckets" % a]), - maybe(sr["%s-buckets-mutable" % a]), - maybe(sr["%s-buckets-immutable" % a]), - abbreviate_space(sr["%s-diskbytes" % a]), - abbreviate_space(sr["%s-diskbytes-mutable" % a]), - abbreviate_space(sr["%s-diskbytes-immutable" % a]), - ) - - def render_lease_current_cycle_progress(self, ctx, data): + @renderer + def lease_current_cycle_progress(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return tag(self.format_crawler_progress(p)) - def render_lease_current_cycle_results(self, ctx, data): + @renderer + def lease_current_cycle_results(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -229,10 +234,10 @@ class StorageStatus(MultiFormatPage): T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] ]]]) + return tag("Current cycle:", p) - return ctx.tag["Current cycle:", p] - - def render_lease_last_cycle_results(self, ctx, data): + @renderer + def lease_last_cycle_results(self, req, tag): lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: @@ -240,15 +245,15 @@ class StorageStatus(MultiFormatPage): last = h[max(h.keys())] start, end = last["cycle-start-finish-times"] - ctx.tag["Last complete cycle (which took %s and finished %s ago)" - " recovered: " % (abbreviate_time(end-start), - abbreviate_time(time.time() - end)), - self.format_recovered(last["space-recovered"], "actual") - ] + tag("Last complete cycle (which took %s and finished %s ago)" + " recovered: " % (abbreviate_time(end-start), + abbreviate_time(time.time() - end)), + self.format_recovered(last["space-recovered"], "actual")) p = T.ul() + def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) saw = self.format_recovered(last["space-recovered"], "examined") add("and saw a total of ", saw) @@ -264,4 +269,37 @@ class StorageStatus(MultiFormatPage): for corrupt_share in last["corrupt-shares"] ] ]]]) - return ctx.tag[p] + return tag(p) + + def format_recovered(self, sr, a): + def maybe(d): + if d is None: + return "?" + return "%d" % d + return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ + (maybe(sr["%s-shares" % a]), + maybe(sr["%s-buckets" % a]), + maybe(sr["%s-buckets-mutable" % a]), + maybe(sr["%s-buckets-immutable" % a]), + abbreviate_space(sr["%s-diskbytes" % a]), + abbreviate_space(sr["%s-diskbytes-mutable" % a]), + abbreviate_space(sr["%s-diskbytes-immutable" % a]), + ) + +class StorageStatus(MultiFormatResource): + def __init__(self, storage, nickname=""): + super(StorageStatus, self).__init__() + self.storage = storage + self.nickname = nickname + + def render_HTML(self, req): + return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") + d = {"stats": self.storage.get_stats(), + "bucket-counter": self.storage.bucket_counter.get_state(), + "lease-checker": self.storage.lease_checker.get_state(), + "lease-checker-progress": self.storage.lease_checker.get_progress(), + } + return json.dumps(d, indent=1) + "\n" diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d97daf9af..cfd7a860c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Storage Server Status @@ -7,19 +7,19 @@ -
                  +

                  Storage Server Status

                  -
                  Relative Path Healthy Recoverable Storage Index Summary
                  +
                  - - + + - - + + - - + + - - + + - - + + - - + +
                  Total disk space:()()
                  Disk space used:- ()- ()
                  @@ -28,18 +28,18 @@
                  Disk space free (root):()() [see 1]
                  Disk space free (non-root):()() [see 2]
                  Reserved space:- ()- ()
                  @@ -48,23 +48,31 @@
                  Space Available to Tahoe:()()
                    -
                  • Server Nickname:
                    • -
                  • Server Nodeid:
                    • -
                  • Accepting new shares: -
                    • +
                  • Server Nickname: + + + +
                    • +
                  • Server Nodeid: + + + +
                    • +
                  • Accepting new shares: +
                  • Total buckets: - + (the number of files and directories for which this server is holding a share)
                      -
                    • +
                  @@ -72,11 +80,11 @@

                  Lease Expiration Crawler

                    -
                  • Expiration
                    • -
                  • -
                  • -
                  • -
                  • +
                  • Expiration
                    • +
                  • +
                  • +
                  • +
                  From 75488c4aff704f84ad061230ab18ca2f48dca0a1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 22:59:45 -0500 Subject: [PATCH 0368/1054] Add `StorageStatus::renderSynchronously` Related to ticket:3247 test_storage.py wants a `StorageStatus::renderSynchronously()` method and a `StorageStatus::renderHTTP()` method. Let us begin with the goofy first-cut. Both these methods are not only wrong, but they will also not please the test suite. However error messages produced in CI can be shared, and that way I can hopefully get unstuck. --- src/allmydata/web/storage.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 3000fb70d..8f36cf44d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,7 +1,8 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile +from twisted.web.template import tags as T, \ + renderer, Element, renderElement, XMLFile from allmydata.web.common import ( abbreviate_time, MultiFormatResource @@ -303,3 +304,14 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self.storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" + + def renderSynchronously(self): + # to appease the test suite. + elem = StorageStatusElement(self.storage, self.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result + + def renderHTTP(self, ctx=None): + # to appease the test suite. + self.renderSynchronously() From 805b880c2eee1c46abf98728991e12213a5f35c1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 23:16:48 -0500 Subject: [PATCH 0369/1054] Add missing `flattenString` import --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 8f36cf44d..a912d79f6 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -2,7 +2,7 @@ import time, json from twisted.python.filepath import FilePath from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile + renderer, Element, renderElement, XMLFile, flattenString from allmydata.web.common import ( abbreviate_time, MultiFormatResource From d1f17e1019f71dfa5a514fdf71dcd169e0c4b97b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:05:33 -0500 Subject: [PATCH 0370/1054] Second version of renderSynchronously --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a912d79f6..4ad5a2baf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -310,7 +310,7 @@ class StorageStatus(MultiFormatResource): elem = StorageStatusElement(self.storage, self.nickname) result = [] flattenString(None, elem).addCallback(result.append) - return result + return result[0] def renderHTTP(self, ctx=None): # to appease the test suite. From 42c3e276a351f6cb62b6f8617cbb2ba11f77af7f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:09:16 -0500 Subject: [PATCH 0371/1054] Check storage server status before using it --- src/allmydata/web/storage.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 4ad5a2baf..9a99286bf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -31,9 +31,13 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): + if not self.storage: + return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def get_stat(self, key): + if not self.storage: + return None return self.storage.get_stats().get(key) def str(self, tag, val): @@ -101,6 +105,8 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): + if not self.storage: + return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -109,6 +115,8 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): + if not self.storage: + return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -145,6 +153,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -153,6 +163,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -176,12 +188,16 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -239,6 +255,8 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From 923219e54a850e678c0e512ddac8b7614e4f08e4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 23:09:39 -0500 Subject: [PATCH 0372/1054] Use right syntax for twisted.web.template tags --- src/allmydata/web/storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9a99286bf..1e206d8cf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -212,7 +212,7 @@ class StorageStatusElement(Element): p = T.ul() def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) def maybe(d): if d is None: @@ -248,9 +248,9 @@ class StorageStatusElement(Element): if so_far["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] - ]]]) + )))) return tag("Current cycle:", p) @renderer @@ -284,9 +284,9 @@ class StorageStatusElement(Element): if last["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in last["corrupt-shares"] ] - ]]]) + )))) return tag(p) From d5af7c0d2e9492302db57a41de23c869ea5a1a52 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:17:35 -0500 Subject: [PATCH 0373/1054] Refactor so that test_util pass --- src/allmydata/test/test_storage.py | 5 +++-- src/allmydata/web/storage.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 73c183a38..56dba406e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -38,7 +38,8 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, remove_prefix +from allmydata.web.storage import StorageStatus, StorageStatusElement, \ + remove_prefix from allmydata.storage_client import ( _StorageServer, ) @@ -4221,7 +4222,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) def test_util(self): - w = StorageStatus(None) + w = StorageStatusElement(None, None) self.failUnlessEqual(w.render_space(None, None), "?") self.failUnlessEqual(w.render_space(None, 10e6), "10000000") self.failUnlessEqual(w.render_abbrev_space(None, None), "?") diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1e206d8cf..6b033bc5a 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,6 +50,16 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) + def render_abbrev_space(self, ctx, size): + if size is None: + return "?" + return abbreviate_space(size) + + def render_space(self, ctx, size): + if size is None: + return "?" + return "%d" % size + @renderer def disk_total(self, req, tag): return self.str(tag, self.get_stat("storage_server.disk_total")) From dbc6b2afa00fb2b0e8be59b745f8b75366a02f01 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:20:45 -0500 Subject: [PATCH 0374/1054] Drop unused `ctx` argument from render_space methods --- src/allmydata/test/test_storage.py | 8 ++++---- src/allmydata/web/storage.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 56dba406e..b7d1b6e9f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -4223,9 +4223,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_util(self): w = StorageStatusElement(None, None) - self.failUnlessEqual(w.render_space(None, None), "?") - self.failUnlessEqual(w.render_space(None, 10e6), "10000000") - self.failUnlessEqual(w.render_abbrev_space(None, None), "?") - self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB") + self.failUnlessEqual(w.render_space(None), "?") + self.failUnlessEqual(w.render_space(10e6), "10000000") + self.failUnlessEqual(w.render_abbrev_space(None), "?") + self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB") self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 6b033bc5a..88f099b24 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,12 +50,12 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) - def render_abbrev_space(self, ctx, size): + def render_abbrev_space(self, size): if size is None: return "?" return abbreviate_space(size) - def render_space(self, ctx, size): + def render_space(self, size): if size is None: return "?" return "%d" % size From 6ccb9d29cbfc49efa1fd594fed802df2ba2f6a01 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:49:11 -0500 Subject: [PATCH 0375/1054] Use render_space methods to render space --- src/allmydata/web/storage.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 88f099b24..496766b92 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -40,16 +40,6 @@ class StorageStatusElement(Element): return None return self.storage.get_stats().get(key) - def str(self, tag, val): - if val is None: - return tag("?") - return tag(str(val)) - - def abbr(self, tag, val): - if val is None: - return tag("?") - return tag(abbreviate_space(val)) - def render_abbrev_space(self, size): if size is None: return "?" @@ -62,51 +52,63 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - return self.str(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): From 8dd4029428d352662452d4521b0ab1f38b391181 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 06:22:25 -0500 Subject: [PATCH 0376/1054] Use within table cells --- src/allmydata/web/storage_status.xhtml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index cfd7a860c..6e0e26008 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -13,13 +13,13 @@ - - + + - - + + - - + + - - + + - - + + - - + +
                  Total disk space:()()
                  Disk space used:- ()- ()
                  @@ -28,18 +28,18 @@
                  Disk space free (root):()() [see 1]
                  Disk space free (non-root):()() [see 2]
                  Reserved space:- ()- ()
                  @@ -48,8 +48,8 @@
                  Space Available to Tahoe:()()
                  From 1be736e4d1c04a04028253630e1b4f8db9437a57 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 07:34:55 -0500 Subject: [PATCH 0377/1054] Fix test_storage.WebStatus.test_no_server failure --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 496766b92..c2e517dce 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -161,7 +161,7 @@ class StorageStatusElement(Element): def storage_running(self, req, tag): if self.storage: return tag - return tag("No Storage Server Running") + return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): From 62a997ce205e08468efb622f152d4168b5eceab6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 08:19:29 -0500 Subject: [PATCH 0378/1054] Use parentheses in import statement --- src/allmydata/web/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index c2e517dce..d45541457 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,8 +1,14 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile, flattenString +from twisted.web.template import ( + Element, + XMLFile, + tags as T, + renderer, + renderElement, + flattenString +) from allmydata.web.common import ( abbreviate_time, MultiFormatResource From ac9a73eddb39a81c2548ba32c45433eaae9d482c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 09:55:33 -0500 Subject: [PATCH 0379/1054] Use parentheses in test suite's import statement --- src/allmydata/test/test_storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index b7d1b6e9f..a507596e8 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -38,8 +38,11 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, StorageStatusElement, \ +from allmydata.web.storage import ( + StorageStatus, + StorageStatusElement, remove_prefix +) from allmydata.storage_client import ( _StorageServer, ) From e67fa7cd95e590f88cda1d69df2d7154155932ef Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 17:02:14 -0500 Subject: [PATCH 0380/1054] Add docstrings to StorageStatusElement --- src/allmydata/web/storage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d45541457..df8354a38 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -24,9 +24,15 @@ def remove_prefix(s, prefix): class StorageStatusElement(Element): + """Class to render a storage status page.""" + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname): + def __init__(self, storage, nickname=""): + """ + :param _StorageServer storage: data about storage. + :param string nickname: friendly name for storage. + """ super(StorageStatusElement, self).__init__() self.storage = storage self.nick = nickname From abb8834be1930f85cffe254ba9ad422f8668153c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:14:24 -0500 Subject: [PATCH 0381/1054] Rename function for clarity --- src/allmydata/web/storage.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index df8354a38..a5d65fa85 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -47,7 +47,7 @@ class StorageStatusElement(Element): return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) - def get_stat(self, key): + def _get_storage_stat(self, key): if not self.storage: return None return self.storage.get_stats().get(key) @@ -64,67 +64,67 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): - accepting = self.get_stat("storage_server.accepting_immutable_shares") + accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") return {True: "Yes", False: "No"}[bool(accepting)] @renderer From 6b5e530cb381b74d23fd1ee611e5be2bd9feb1b4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:15:26 -0500 Subject: [PATCH 0382/1054] Add a docstring --- src/allmydata/web/storage.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a5d65fa85..263fa9344 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -48,6 +48,30 @@ class StorageStatusElement(Element): return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): + """Get storage server statistics. + + Storage Server keeps a dict that contains various usage and + latency statistics. The dict looks like this: + + { + 'storage_server.accepting_immutable_shares': 1, + 'storage_server.allocated': 0, + 'storage_server.disk_avail': 106539192320, + 'storage_server.disk_free_for_nonroot': 106539192320, + 'storage_server.disk_free_for_root': 154415284224, + 'storage_server.disk_total': 941088460800, + 'storage_server.disk_used': 786673176576, + 'storage_server.latencies.add-lease.01_0_percentile': None, + 'storage_server.latencies.add-lease.10_0_percentile': None, + ... + } + + ``StorageServer.get_stats()`` returns the above dict. Storage + status page uses a subset of the items in the dict, concerning + disk usage. + + :param str key: storage server statistic we want to know. + """ if not self.storage: return None return self.storage.get_stats().get(key) From affe10b3e9982edbd4def2f9b67d777df6a08f06 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 12:25:20 -0500 Subject: [PATCH 0383/1054] Return result from renderHTTP Bogus renderHTTP, but test failures are now down to three. We just need to handle requests for JSON now. --- src/allmydata/web/storage.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 263fa9344..1656df70b 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -378,6 +378,18 @@ class StorageStatus(MultiFormatResource): flattenString(None, elem).addCallback(result.append) return result[0] + # to appease the test suite def renderHTTP(self, ctx=None): - # to appease the test suite. - self.renderSynchronously() + """Send HTML or JSON formatted data, based on request. + + This function contains a bit of nevow-ism, but since this is + only called from the test suite, the nevow-ism should go away + as we update things. + + :param _nevow.context.WovenContext ctx: context is passed on + from the test suite. We get a request out of this + context, and use the request to render a result. + + """ + from nevow.inevow import IRequest + return self.render(IRequest(ctx)) From 92f6e4bfe5b7dd98c794b946a1bb8796e5516ab3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 15:29:42 -0500 Subject: [PATCH 0384/1054] Use explicit `None`-check on self.storage --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1656df70b..76a09fda3 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -43,7 +43,7 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if not self.storage: + if self.storage is None: return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) From ee8e7307275f2285d1474cefcb8bfba2fa430bf9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 16:49:15 -0500 Subject: [PATCH 0385/1054] Move `renderSynchronously` to test suite --- src/allmydata/test/test_storage.py | 36 +++++++++++++++++++----------- src/allmydata/web/storage.py | 10 +-------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index a507596e8..88b4ab57c 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -12,6 +12,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service +from twisted.web.template import flattenString from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -2976,6 +2977,15 @@ def remove_tags(s): s = re.sub(r'\s+', ' ', s) return s +def renderSynchronously(ss): + """ + :param _StorageStatus ss: a StorageStatus instance. + """ + elem = StorageStatusElement(ss.storage, ss.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result[0] + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -3012,7 +3022,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): w = StorageStatus(ss) # this sample is before the crawler has started doing anything - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -3035,7 +3045,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(state["last-complete-prefix"], ss.bucket_counter.prefixes[0]) ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn(" Current crawl ", s) self.failUnlessIn(" (next work in ", s) @@ -3047,7 +3057,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): d.addCallback(lambda ignored: self.poll(_watch)) def _check2(ignored): ss.bucket_counter.cpu_slice = orig_cpu_slice - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("Total buckets: 0 (the number of", s) self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) @@ -3109,20 +3119,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): def _check_1(ignored): # no ETA is available yet - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (next work", s) def _check_2(ignored): # one prefix has finished, so an ETA based upon that elapsed time # should be available. - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) def _check_3(ignored): # two prefixes have finished - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) d.callback("done") @@ -4077,7 +4087,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_no_server(self): w = StorageStatus(None) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  No Storage Server Running

                  ", html) def test_status(self): @@ -4123,7 +4133,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -4143,7 +4153,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4179,7 +4189,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) @@ -4197,7 +4207,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4208,7 +4218,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) @@ -4219,7 +4229,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

                  Storage Server Status

                  ", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 76a09fda3..e9c2f6bb5 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -6,8 +6,7 @@ from twisted.web.template import ( XMLFile, tags as T, renderer, - renderElement, - flattenString + renderElement ) from allmydata.web.common import ( abbreviate_time, @@ -371,13 +370,6 @@ class StorageStatus(MultiFormatResource): } return json.dumps(d, indent=1) + "\n" - def renderSynchronously(self): - # to appease the test suite. - elem = StorageStatusElement(self.storage, self.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] - # to appease the test suite def renderHTTP(self, ctx=None): """Send HTML or JSON formatted data, based on request. From db0e6407dc0309ee4825d936dd6dfd706f0620fd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:00:54 -0500 Subject: [PATCH 0386/1054] Use `successResultOf` in `renderSynchronously` Get rid of [].append trick when dealing with the deferred. --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 88b4ab57c..2bedffac2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2982,9 +2982,8 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss.storage, ss.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] + deferred = flattenString(None, elem) + return unittest.TestCase().successResultOf(deferred) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): From 0b4c235fbd3197626bf051899ef16f1d6b49519e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:18:37 -0500 Subject: [PATCH 0387/1054] Remove redundant `None`-checks on `self.storage` When no storage is up, `storage_running()` renderer will return a big honking `no storage server running` message, and no further renderers will be invoked. Therefore the extra defense is probably not required. (I tested this hypothesis. The extra defense is not required, unless there's something I have not seen.) --- src/allmydata/web/storage.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index e9c2f6bb5..cbee2780f 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -42,8 +42,6 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if self.storage is None: - return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): @@ -71,8 +69,6 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - if not self.storage: - return None return self.storage.get_stats().get(key) def render_abbrev_space(self, size): @@ -152,8 +148,6 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - if not self.storage: - return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -162,8 +156,6 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - if not self.storage: - return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -200,8 +192,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -210,8 +200,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -235,16 +223,12 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -302,8 +286,6 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From bc8a86e1065c3c37286cd3e7a0378376dddbb6cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:23:35 -0500 Subject: [PATCH 0388/1054] Use Unicode strings to render space --- src/allmydata/web/storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index cbee2780f..607eeff79 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -73,13 +73,13 @@ class StorageStatusElement(Element): def render_abbrev_space(self, size): if size is None: - return "?" + return u"?" return abbreviate_space(size) def render_space(self, size): if size is None: - return "?" - return "%d" % size + return u"?" + return u"%d" % size @renderer def disk_total(self, req, tag): From 0bd386384e3a83d29c4aca9e00d02bfe829431c1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 22:02:30 -0500 Subject: [PATCH 0389/1054] Mark `format_recovered` as static method --- src/allmydata/web/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 607eeff79..d670c4231 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -319,7 +319,8 @@ class StorageStatusElement(Element): return tag(p) - def format_recovered(self, sr, a): + @staticmethod + def format_recovered(sr, a): def maybe(d): if d is None: return "?" From 0423e49a8e01021ede8f5b97e956cd794a046b44 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 07:25:49 -0500 Subject: [PATCH 0390/1054] Add news fragment --- newsfragments/3247.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3247.minor diff --git a/newsfragments/3247.minor b/newsfragments/3247.minor new file mode 100644 index 000000000..e69de29bb From 56e2fa3f2409fc2a87912d8be445212c33ec6dfc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 08:26:44 -0500 Subject: [PATCH 0391/1054] Use to render node nickname and id CI did not like the old way, but it passed in my system. Odd. --- src/allmydata/web/storage_status.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 6e0e26008..8f74c478c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,12 +57,12 @@
                  • Server Nickname: - +
                  • Server Nodeid: - +
                  • Accepting new shares: From 1469af0f3bcad60ea58c008e8da8ccf6c381c03f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 10:10:48 -0500 Subject: [PATCH 0392/1054] Use BeautifulSoup to check favicon in storage page --- src/allmydata/test/web/test_web.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index c3779c0be..1f4f4fdbe 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -954,8 +954,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_storage(self): d = self.GET("/storage") def _check(res): - self.failUnlessIn('Storage Server Status', res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_text(self, soup, 'Storage Server Status') + assert_soup_has_favicon(self, soup) res_u = res.decode('utf-8') self.failUnlessIn(u'
                  • Server Nickname: fake_nickname \u263A
                    • ', res_u) d.addCallback(_check) From 9dd9d98b8ca32b365a828d355a79e05b8012765b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 11:42:42 -0500 Subject: [PATCH 0393/1054] Give the suite the precise string it wants --- src/allmydata/web/storage_status.xhtml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 8f74c478c..d052fbabd 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -55,16 +55,8 @@
                      -
                    • Server Nickname: - - - -
                      • -
                    • Server Nodeid: - - - -
                      • +
                    • Server Nickname:
                      • +
                    • Server Nodeid:
                    • Accepting new shares:
                    • Total buckets: From 0d9ec2b206f14c5473ccb286c793cde149485f09 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Feb 2020 12:52:48 -0500 Subject: [PATCH 0394/1054] Use slots to render storage stats table --- src/allmydata/web/storage.py | 80 ++++++++------------------ src/allmydata/web/storage_status.xhtml | 26 ++++----- 2 files changed, 36 insertions(+), 70 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d670c4231..d26f4a5dd 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -82,64 +82,30 @@ class StorageStatusElement(Element): return u"%d" % size @renderer - def disk_total(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_space(val)) + def storage_stats(self, req, tag): + # Render storage status table that appears near the top of the page. + total = self._get_storage_stat("storage_server.disk_total") + used = self._get_storage_stat("storage_server.disk_used") + free_root = self._get_storage_stat("storage_server.disk_free_for_root") + free_nonroot = self._get_storage_stat("storage_server.disk_free_for_nonroot") + reserved = self._get_storage_stat("storage_server.reserved_space") + available = self._get_storage_stat("storage_server.disk_avail") - @renderer - def disk_total_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_used(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_space(val)) - - @renderer - def disk_used_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_root(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_root_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_nonroot(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_nonroot_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_abbrev_space(val)) - - @renderer - def reserved_space(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_space(val)) - - @renderer - def reserved_space_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_avail(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_space(val)) - - @renderer - def disk_avail_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_abbrev_space(val)) + tag.fillSlots( + disk_total = self.render_space(total), + disk_total_abbrev = self.render_abbrev_space(total), + disk_used = self.render_space(used), + disk_used_abbrev = self.render_abbrev_space(used), + disk_free_for_root = self.render_space(free_root), + disk_free_for_root_abbrev = self.render_abbrev_space(free_root), + disk_free_for_nonroot = self.render_space(free_nonroot), + disk_free_for_nonroot_abbrev = self.render_abbrev_space(free_nonroot), + reserved_space = self.render_space(reserved), + reserved_space_abbrev = self.render_abbrev_space(reserved), + disk_avail = self.render_space(available), + disk_avail_abbrev = self.render_abbrev_space(available) + ) + return tag @renderer def accepting_immutable_shares(self, req, tag): diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d052fbabd..bbf3d2c8a 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -11,15 +11,15 @@

                      Storage Server Status

                      - +
                      - - + + - - + + - - + + - - + + - - + + - - + +
                      Total disk space:()()
                      Disk space used:- ()- ()
                      @@ -28,18 +28,18 @@
                      Disk space free (root):()() [see 1]
                      Disk space free (non-root):()() [see 2]
                      Reserved space:- ()- ()
                      @@ -48,8 +48,8 @@
                      Space Available to Tahoe:()()
                      From 7126d3247ca51bf4a0d36680ee6093d29fce4728 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 25 Feb 2020 08:52:45 -0500 Subject: [PATCH 0395/1054] Prefix member variable with "_" --- src/allmydata/test/test_storage.py | 2 +- src/allmydata/web/storage.py | 40 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 2bedffac2..f4527d8b4 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2981,7 +2981,7 @@ def renderSynchronously(ss): """ :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss.storage, ss.nickname) + elem = StorageStatusElement(ss._storage, ss._nickname) deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d26f4a5dd..9b3daa0cb 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -33,16 +33,16 @@ class StorageStatusElement(Element): :param string nickname: friendly name for storage. """ super(StorageStatusElement, self).__init__() - self.storage = storage - self.nick = nickname + self._storage = storage + self._nickname = nickname @renderer def nickname(self, req, tag): - return self.nick + return self._nickname @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self.storage.my_nodeid) + return idlib.nodeid_b2a(self._storage.my_nodeid) def _get_storage_stat(self, key): """Get storage server statistics. @@ -69,7 +69,7 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - return self.storage.get_stats().get(key) + return self._storage.get_stats().get(key) def render_abbrev_space(self, size): if size is None: @@ -114,7 +114,7 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - s = self.storage.bucket_counter.get_state() + s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" @@ -122,7 +122,7 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - p = self.storage.bucket_counter.get_progress() + p = self._storage.bucket_counter.get_progress() return self.format_crawler_progress(p) def format_crawler_progress(self, p): @@ -152,13 +152,13 @@ class StorageStatusElement(Element): @renderer def storage_running(self, req, tag): - if self.storage: + if self._storage: return tag return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") else: @@ -166,7 +166,7 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: tag("Leases will expire naturally, probably 31 days after " @@ -189,13 +189,13 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: return "" @@ -252,7 +252,7 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker h = lc.get_state()["history"] if not h: return "" @@ -304,18 +304,18 @@ class StorageStatusElement(Element): class StorageStatus(MultiFormatResource): def __init__(self, storage, nickname=""): super(StorageStatus, self).__init__() - self.storage = storage - self.nickname = nickname + self._storage = storage + self._nickname = nickname def render_HTML(self, req): - return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + return renderElement(req, StorageStatusElement(self._storage, self._nickname)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), + d = {"stats": self._storage.get_stats(), + "bucket-counter": self._storage.bucket_counter.get_state(), + "lease-checker": self._storage.lease_checker.get_state(), + "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" From 8d3179194032b398efbea6489dd552ded92dc372 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:45:18 -0500 Subject: [PATCH 0396/1054] Use a helper to exercise render() in storage test cases --- src/allmydata/test/test_storage.py | 50 ++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f4527d8b4..63b341517 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -38,6 +38,7 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin +from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2985,6 +2986,29 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) +def renderDeferred(resource, **kwargs): + """ + Use this to exercise an overridden MultiFormatResource.render(), + usually for output=json or render_GET. It returns a Deferred. + + :param _MultiFormatResource resource: an HTTP resource to be rendered. + + """ + # We should be using twisted.web's DummyRequest here instead of + # nevow's FakeRequest, but right now it is a bit of a problem: see + # web/common.py. MultiFormatResource.render() makes a get_arg() + # call, which does a IRequest(ctx_or_req). IRequest can handle + # FakeRequest, but it can't handle DummyRequest. + req = FakeRequest(**kwargs) + req.fields = None + d = defer.maybeDeferred(resource.render, req) + def _done(res): + if isinstance(res, str): + return res + req.v + return req.v + d.addCallback(_done) + return d + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -3304,7 +3328,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failIfEqual(sr2["configured-diskbytes"], None) self.failIfEqual(sr2["original-sharebytes"], None) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) self.failUnlessIn("So far, this cycle has examined " @@ -3379,7 +3403,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_leases(mutable_si_2), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("recovered: 0 shares, 0 buckets " @@ -3479,7 +3503,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3538,7 +3562,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) @@ -3623,7 +3647,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3684,7 +3708,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled:" @@ -3746,7 +3770,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_3), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: immutable.", s) @@ -3803,7 +3827,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_2), 0) self.failUnlessEqual(count_shares(mutable_si_3), 0) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: mutable.", s) @@ -4034,7 +4058,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): # it also turns all tuples into lists self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4059,7 +4083,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): corrupt_shares = last["corrupt-shares"] self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json_history) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html_history(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4073,7 +4097,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): @@ -4096,7 +4120,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, nodeid) ss.setServiceParent(self.s) w = StorageStatus(ss, "nickname") - d = self.render1(w) + d = renderDeferred(w) def _check_html(html): self.failUnlessIn("

                      Storage Server Status

                      ", html) s = remove_tags(html) @@ -4117,7 +4141,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d def test_status_no_disk_stats(self): From a6e6c5cee03b8675d47bb71be87211e99e7d84a3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:46:09 -0500 Subject: [PATCH 0397/1054] Get rid of WebRenderingMixin in storage test --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 63b341517..0105a5253 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -37,7 +37,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from allmydata.test.common_web import WebRenderingMixin from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( @@ -3198,7 +3197,7 @@ class InstrumentedStorageServer(StorageServer): class No_ST_BLOCKS_StorageServer(StorageServer): LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler -class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() @@ -4100,7 +4099,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d = renderDeferred(page, args={"t": ["json"]}) return d -class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class WebStatus(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() From a056f18ca9c09e3e344e50dd5fbb7ddf3415d26b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:57:12 -0500 Subject: [PATCH 0398/1054] Remove StorageStatus.renderHTTP This was added to please the test suite. Pleased to remove it! --- src/allmydata/web/storage.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9b3daa0cb..63ac47282 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -318,19 +318,3 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" - - # to appease the test suite - def renderHTTP(self, ctx=None): - """Send HTML or JSON formatted data, based on request. - - This function contains a bit of nevow-ism, but since this is - only called from the test suite, the nevow-ism should go away - as we update things. - - :param _nevow.context.WovenContext ctx: context is passed on - from the test suite. We get a request out of this - context, and use the request to render a result. - - """ - from nevow.inevow import IRequest - return self.render(IRequest(ctx)) From de5d04ab79cf325a769f0d4499bb555cc8661fb2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 30 Mar 2020 16:15:44 -0400 Subject: [PATCH 0399/1054] Remove an extraneous directive `t:data` is not really a Twisted template directive. Added my mistake, removing now. --- src/allmydata/web/storage_status.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index bbf3d2c8a..354f9f177 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,7 +57,7 @@
                      • Server Nickname:
                      • Server Nodeid:
                        • -
                      • Accepting new shares: +
                      • Accepting new shares:
                      • Total buckets: From 0af744610f329ccae4621bb8e9099b09cccfd1b6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:00:55 -0400 Subject: [PATCH 0400/1054] Avoid using nevow FakeRequest in storage test. Use twisted.web.server.Request instead, with a DummyChannel. There's still one line of inevitable nevow now, because of code in web/common.py; but that should be easily replaceable once we switch that over. --- src/allmydata/test/test_storage.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0105a5253..e3472e856 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,16 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString + +# We need to use `nevow.inevow.IRequest` for now for compatibility +# with the code in web/common.py. Once nevow bits are gone from +# web/common.py, we can use `twisted.web.iweb.IRequest` here. +from nevow.inevow import IRequest + +from twisted.web.server import Request +from twisted.web.test.test_web import DummyChannel +from zope.interface import implements + from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -37,7 +47,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2985,28 +2994,20 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) -def renderDeferred(resource, **kwargs): - """ - Use this to exercise an overridden MultiFormatResource.render(), - usually for output=json or render_GET. It returns a Deferred. +def renderDeferred(ss): + elem = StorageStatusElement(ss._storage, ss._nickname) + return flattenString(None, elem) - :param _MultiFormatResource resource: an HTTP resource to be rendered. +class JSONRequest(Request): + implements(IRequest) - """ - # We should be using twisted.web's DummyRequest here instead of - # nevow's FakeRequest, but right now it is a bit of a problem: see - # web/common.py. MultiFormatResource.render() makes a get_arg() - # call, which does a IRequest(ctx_or_req). IRequest can handle - # FakeRequest, but it can't handle DummyRequest. - req = FakeRequest(**kwargs) - req.fields = None - d = defer.maybeDeferred(resource.render, req) - def _done(res): - if isinstance(res, str): - return res + req.v - return req.v - d.addCallback(_done) - return d + def __init__(self, **kwargs): + Request.__init__(self, DummyChannel(), **kwargs) + self.args = {"t": ["json"]} + self.fields = {} + +def renderJSON(resource): + return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): @@ -4096,7 +4097,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4140,7 +4141,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d def test_status_no_disk_stats(self): From 6379601722f0ed823eda9d719fc10f563a41149d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:01:22 -0400 Subject: [PATCH 0401/1054] Wrap renderer results in tags --- src/allmydata/web/storage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 63ac47282..ba6609456 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -38,11 +38,11 @@ class StorageStatusElement(Element): @renderer def nickname(self, req, tag): - return self._nickname + return tag(self._nickname) @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self._storage.my_nodeid) + return tag(idlib.nodeid_b2a(self._storage.my_nodeid)) def _get_storage_stat(self, key): """Get storage server statistics. @@ -110,20 +110,20 @@ class StorageStatusElement(Element): @renderer def accepting_immutable_shares(self, req, tag): accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") - return {True: "Yes", False: "No"}[bool(accepting)] + return tag({True: "Yes", False: "No"}[bool(accepting)]) @renderer def last_complete_bucket_count(self, req, tag): s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: - return "Not computed yet" - return str(count) + return tag("Not computed yet") + return tag(str(count)) @renderer def count_crawler_status(self, req, tag): p = self._storage.bucket_counter.get_progress() - return self.format_crawler_progress(p) + return tag(self.format_crawler_progress(p)) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] From 7e26dd42e9260fef4ac7f0da44473d77889c96d2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:05:17 -0400 Subject: [PATCH 0402/1054] Remove redundant render_json() method --- src/allmydata/test/test_storage.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e3472e856..62878bffd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3412,7 +3412,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): "(2 mutable / 2 immutable),", s) self.failUnlessIn("but expiration was not enabled", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(webstatus)) + d.addCallback(lambda ign: renderJSON(webstatus)) def _check_json(raw): data = json.loads(raw) self.failUnlessIn("lease-checker", data) @@ -4049,7 +4049,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) # grr. json turns all dict keys into strings. @@ -4076,7 +4076,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(rec["examined-shares"], 3) self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json_history(raw): data = json.loads(raw) last = data["lease-checker"]["history"]["0"] @@ -4096,9 +4096,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d - def render_json(self, page): - d = renderJSON(page) - return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4129,7 +4126,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): self.failUnlessIn("Accepting new shares: Yes", s) self.failUnlessIn("Reserved space: - 0 B (0)", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) s = data["stats"] @@ -4140,9 +4137,6 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): d.addCallback(_check_json) return d - def render_json(self, page): - d = renderJSON(page) - return d def test_status_no_disk_stats(self): def call_get_disk_stats(whichdir, reserved_space=0): From bdfa667a22d36b57287317c34e6637a44631750a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:13:08 -0400 Subject: [PATCH 0403/1054] Add docstrings to storage test helpers --- src/allmydata/test/test_storage.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 62878bffd..10d74e9ce 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2988,6 +2988,8 @@ def remove_tags(s): def renderSynchronously(ss): """ + Return fully rendered HTML document. + :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss._storage, ss._nickname) @@ -2995,10 +2997,20 @@ def renderSynchronously(ss): return unittest.TestCase().successResultOf(deferred) def renderDeferred(ss): + """ + Return a `Deferred` HTML renderer. + + :param _StorageStatus ss: a StorageStatus instance. + """ elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) class JSONRequest(Request): + """ + A Request with t=json argument added to it. + + This is useful to invoke a Resouce.render_JSON() method. + """ implements(IRequest) def __init__(self, **kwargs): @@ -3007,6 +3019,11 @@ class JSONRequest(Request): self.fields = {} def renderJSON(resource): + """Exercise resouce.render_JSON() + + :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` + that contains a render_JSON() method. + """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From 6b61b75658b2f710d44e7d502de09b0a6e3338e5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:18:15 -0400 Subject: [PATCH 0404/1054] Refactor storage test helpers Rewrite `renderSynchronously()` to use `renderDeferred()` --- src/allmydata/test/test_storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 10d74e9ce..d0df30684 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2992,9 +2992,7 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss._storage, ss._nickname) - deferred = flattenString(None, elem) - return unittest.TestCase().successResultOf(deferred) + return unittest.TestCase().successResultOf(renderDeferred(ss)) def renderDeferred(ss): """ From d449306b1900e954748f6f5183f8a76bba9cb962 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 21:48:49 -0400 Subject: [PATCH 0405/1054] Use DummyRequest in storage tests Using twisted.web.server.Request causes test_new_style_classes to fail like so: Traceback (most recent call last): Failure: testtools.testresult.real._StringException: Traceback (most recent call last): File ".tox/coverage/lib/python2.7/site-packages/allmydata/test/test_python2_regressions.py", line 69, in test_new_style_classes "Expected to find no classic classes.", File ".tox/coverage/lib/python2.7/site-packages/testtools/testcase.py", line 502, in assertThat raise mismatch_error testtools.matchers._impl.MismatchError: !=: reference = set([]) actual = set([]) : Expected to find no classic classes. Seems that `DummyRequest` is an acceptable new style class. --- src/allmydata/test/test_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d0df30684..06ae1708e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,8 +19,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.server import Request -from twisted.web.test.test_web import DummyChannel +# from twisted.web.server import Request +from twisted.web.test.test_web import DummyRequest from zope.interface import implements from foolscap.api import fireEventually @@ -3003,7 +3003,7 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -class JSONRequest(Request): +class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. @@ -3012,7 +3012,7 @@ class JSONRequest(Request): implements(IRequest) def __init__(self, **kwargs): - Request.__init__(self, DummyChannel(), **kwargs) + DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} self.fields = {} From 92e83f268bbb59b9c52a7b56fc692884471460d0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 20 Apr 2020 14:35:32 -0400 Subject: [PATCH 0406/1054] Declare JSON request interface using @implementer "zope.interface.implements(IRequest)" is deprectated in favor of "@zope.interface.implementer(IRequest)" decorator. --- src/allmydata/test/test_storage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 06ae1708e..783debd28 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,9 +19,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -# from twisted.web.server import Request -from twisted.web.test.test_web import DummyRequest -from zope.interface import implements +from twisted.web.test.requesthelper import DummyRequest +from zope.interface import implementer from foolscap.api import fireEventually import itertools @@ -3003,14 +3002,13 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) +@implementer(IRequest) class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. This is useful to invoke a Resouce.render_JSON() method. """ - implements(IRequest) - def __init__(self, **kwargs): DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} From 96fbff684f0bbef122c67c34d3a9f3e2fb58b2cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Apr 2020 10:29:18 -0400 Subject: [PATCH 0407/1054] Use an inner JSONRequest class with renderJSON Once nevow is removed from web/common.py, we can simplify renderJSON(), like so: def renderJSON(): req = Request() req.args = {"t": ["json"]} req.fields = {} return resource.render(req) But for now we have to live with an inner class that implements the nevow.inevow.IRequest interface. --- src/allmydata/test/test_storage.py | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 783debd28..0acf60152 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,7 +19,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.test.requesthelper import DummyRequest +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel from zope.interface import implementer from foolscap.api import fireEventually @@ -3002,24 +3003,20 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -@implementer(IRequest) -class JSONRequest(DummyRequest): - """ - A Request with t=json argument added to it. - - This is useful to invoke a Resouce.render_JSON() method. - """ - def __init__(self, **kwargs): - DummyRequest.__init__(self, b"/", **kwargs) - self.args = {"t": ["json"]} - self.fields = {} - def renderJSON(resource): - """Exercise resouce.render_JSON() + """Render a JSON from the given resource.""" + + @implementer(IRequest) + class JSONRequest(Request): + """ + A Request with t=json argument added to it. This is useful to + invoke a Resouce.render_JSON() method. + """ + def __init__(self): + Request.__init__(self, DummyChannel()) + self.args = {"t": ["json"]} + self.fields = {} - :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` - that contains a render_JSON() method. - """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From d24babe873a09abd6e3aa15cb76b8832cb267a64 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:07:47 -0400 Subject: [PATCH 0408/1054] Use twisted template tags in upload status page template --- src/allmydata/web/upload-status.xhtml | 62 +++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/upload-status.xhtml b/src/allmydata/web/upload-status.xhtml index 8aaecd47a..95a5f7692 100644 --- a/src/allmydata/web/upload-status.xhtml +++ b/src/allmydata/web/upload-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Upload Status @@ -10,46 +10,46 @@

                        File Upload Status

                          -
                        • Started:
                          • -
                        • Storage Index:
                          • -
                        • Helper?:
                          • -
                        • Total Size:
                          • -
                        • Progress (Hash):
                          • -
                        • Progress (Ciphertext):
                          • -
                        • Progress (Encode+Push):
                          • -
                        • Status:
                          • +
                        • Started:
                          • +
                        • Storage Index:
                          • +
                        • Helper?:
                          • +
                        • Total Size:
                          • +
                        • Progress (Hash):
                          • +
                        • Progress (Ciphertext):
                          • +
                        • Progress (Encode+Push):
                          • +
                        • Status:
                        -
                        +

                        Upload Results

                          -
                        • Shares Pushed:
                          • -
                        • Shares Already Present:
                          • -
                        • Sharemap:
                          • -
                        • Servermap:
                          • +
                        • Shares Pushed:
                          • +
                        • Shares Already Present:
                          • +
                        • Sharemap:
                          • +
                        • Servermap:
                        • Timings:
                            • -
                          • File Size: bytes
                            • -
                          • Total: - ()
                            • +
                          • File Size: bytes
                            • +
                          • Total: + ()
                              • -
                            • Storage Index: - ()
                              • -
                            • [Contacting Helper]:
                              • -
                            • [Upload Ciphertext To Helper]: - ()
                              • +
                            • Storage Index: + ()
                              • +
                            • [Contacting Helper]:
                              • +
                            • [Upload Ciphertext To Helper]: + ()
                              • -
                            • Peer Selection:
                              • -
                            • Encode And Push: - ()
                              • +
                            • Peer Selection:
                              • +
                            • Encode And Push: + ()
                                • -
                              • Cumulative Encoding: - ()
                                • -
                              • Cumulative Pushing: - ()
                                • -
                              • Send Hashes And Close:
                                • +
                              • Cumulative Encoding: + ()
                                • +
                              • Cumulative Pushing: + ()
                                • +
                              • Send Hashes And Close:
                              -
                            • [Helper Total]:
                              • +
                            • [Helper Total]:
                        From e6f0f9d03877a8e3241195ba182807cb7230dfbe Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:08:31 -0400 Subject: [PATCH 0409/1054] Make UploadStatusPage a MultiFormatResource --- src/allmydata/web/status.py | 85 +++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0c9e1ebb1..0990f45ab 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -169,62 +169,85 @@ class UploadResultsRendererMixin(RateAndTimeMixin): d.addCallback(_convert) return d -class UploadStatusPage(UploadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("upload-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.upload_status = data +class UploadStatusPage(MultiFormatResource): + + def __init__(self, upload_status): + super(UploadStatusPage, self).__init__() + self._upload_status = upload_status + + def render_HTML(self, req): + elem = UploadStatusElement(self._upload_status) + return renderElement(req, elem) + + +class UploadStatusElement(Element, UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) + + def __init__(self, upload_status): + super(UploadStatusElement, self).__init__() + self._upload_status = upload_status def upload_results(self): - return defer.maybeDeferred(self.upload_status.get_results) + return defer.maybeDeferred(self._upload_status.get_results) - def render_results(self, ctx, data): + @renderer + def results(self, req, tag): d = self.upload_results() def _got_results(results): if results: - return ctx.tag + return tag return "" d.addCallback(_got_results) return d - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + @renderer + def started(self, req, tag): + started_s = render_time(self._upload_status.get_started()) + return tag(started_s) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._upload_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(str(si_s)) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._upload_status.using_helper()]) - def render_total_size(self, ctx, data): - size = data.get_size() + @renderer + def total_size(self, req, tag): + size = self._upload_status.get_size() if size is None: return "(unknown)" - return size + return tag(str(size)) - def render_progress_hash(self, ctx, data): - progress = data.get_progress()[0] + @renderer + def progress_hash(self, req, tag): + progress = self._upload_status.get_progress()[0] + # TODO: make an ascii-art bar + return tag("%.1f%%" % (100.0 * progress)) + + @renderer + def progress_ciphertext(self, req, tag): + progress = self._upload_status.get_progress()[1] # TODO: make an ascii-art bar return "%.1f%%" % (100.0 * progress) - def render_progress_ciphertext(self, ctx, data): - progress = data.get_progress()[1] + @renderer + def progress_encode_push(self, req, tag): + progress = self._upload_status.get_progress()[2] # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_progress_encode_push(self, ctx, data): - progress = data.get_progress()[2] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + @renderer + def status(self, req, tag): + return tag(self._upload_status.get_status()) - def render_status(self, ctx, data): - return data.get_status() class DownloadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'download_results' From 71fd4f5d109af06ab7bd594bf9436da51f010992 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:32 -0400 Subject: [PATCH 0410/1054] Use twisted template tags in upload results page template --- src/allmydata/web/upload-results.xhtml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/upload-results.xhtml b/src/allmydata/web/upload-results.xhtml index fa8894dcd..290c9bd42 100644 --- a/src/allmydata/web/upload-results.xhtml +++ b/src/allmydata/web/upload-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Uploaded @@ -7,37 +7,37 @@ -

                        Uploading File...

                        +

                        Uploading File...

                        Upload Results:

                          -
                        • URI:
                          • -
                        • Download link:
                          • -
                        • Sharemap:
                          • -
                        • Servermap:
                          • +
                        • URI:
                          • +
                        • Download link:
                          • +
                        • Sharemap:
                          • +
                        • Servermap:
                        • Timings:
                            • -
                          • File Size: bytes
                            • -
                          • Total: - ()
                            • +
                          • File Size: bytes
                            • +
                          • Total: + ()
                              • -
                            • Storage Index: - ()
                              • -
                            • [Contacting Helper]:
                              • -
                            • [Upload Ciphertext To Helper]: - ()
                              • +
                            • Storage Index: + ()
                              • +
                            • [Contacting Helper]:
                              • +
                            • [Upload Ciphertext To Helper]: + ()
                              • -
                            • Peer Selection:
                              • -
                            • Encode And Push: - ()
                              • +
                            • Peer Selection:
                              • +
                            • Encode And Push: + ()
                                • -
                              • Cumulative Encoding: - ()
                                • -
                              • Cumulative Pushing: - ()
                                • -
                              • Send Hashes And Close:
                                • +
                              • Cumulative Encoding: + ()
                                • +
                              • Cumulative Pushing: + ()
                                • +
                              • Send Hashes And Close:
                              -
                            • [Helper Total]:
                              • +
                            • [Helper Total]:
                        From 2ab7b15f8b6e7662a4205f3ecd98d80a42003605 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:54 -0400 Subject: [PATCH 0411/1054] Make UploadResultsPage a MultiFormatResource --- src/allmydata/web/unlinked.py | 58 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 8a84a4021..0abf6cc6d 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -2,11 +2,27 @@ import urllib from twisted.web import http from twisted.internet import defer -from nevow import rend, url, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) +from nevow import url, tags as T from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle -from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \ - convert_children_json, WebError, get_format, get_mutable_type +from allmydata.web.common import ( + getxmlfile, + get_arg, + boolean_of_arg, + convert_children_json, + WebError, + get_format, + get_mutable_type, + MultiFormatResource, +) from allmydata.web import status def PUTUnlinkedCHK(req, client): @@ -59,34 +75,50 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(status.UploadResultsRendererMixin, rend.Page): +class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" - docFactory = getxmlfile("upload-results.xhtml") def __init__(self, upload_results): - rend.Page.__init__(self) - self.results = upload_results + super(UploadResultsPage, self).__init__() + self._upload_results = upload_results + + def render_HTML(self, req): + elem = UploadResultsElement(self._upload_results) + return renderElement(req, elem) + + +class UploadResultsElement(Element, status.UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) + + def __init__(self, upload_results): + super(UploadResultsElement, self).__init__() + self._upload_results = upload_results def upload_results(self): - return defer.succeed(self.results) + return defer.succeed(self._upload_results) - def data_done(self, ctx, data): + @renderer + def done(self, req, tag): d = self.upload_results() d.addCallback(lambda res: "done!") return d - def data_uri(self, ctx, data): + @renderer + def uri(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_uri()) return d - def render_download_link(self, ctx, data): + @renderer + def download_link(self, req, tag): d = self.upload_results() d.addCallback(lambda res: - T.a(href="/uri/" + urllib.quote(res.get_uri())) - ["/uri/" + res.get_uri()]) + tags.a("/uri/" + res.get_uri(), + href="/uri/" + urllib.quote(res.get_uri()))) return d + def POSTUnlinkedSSK(req, client, version): # "POST /uri", to create an unlinked file. # SDMF: files are small, and we can only upload data From 6b4d27bec1b0ebd573bb6b9b19db9733290ecebc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:53:38 -0400 Subject: [PATCH 0412/1054] Use twisted web template in UploadResultsRendererMixin --- src/allmydata/web/status.py | 120 +++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0990f45ab..5a9714dd0 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -44,128 +44,150 @@ class RateAndTimeMixin(object): def render_rate(self, ctx, data): return abbreviate_rate(data) + class UploadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'upload_results' - def render_pushed_shares(self, ctx, data): + @renderer + def pushed_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_pushed_shares()) + d.addCallback(lambda res: str(res.get_pushed_shares())) return d - def render_preexisting_shares(self, ctx, data): + @renderer + def preexisting_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_preexisting_shares()) + d.addCallback(lambda res: str(res.get_preexisting_shares())) return d - def render_sharemap(self, ctx, data): + @renderer + def sharemap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_sharemap()) def _render(sharemap): if sharemap is None: return "None" - l = T.ul() + ul = tags.ul() for shnum, servers in sorted(sharemap.items()): server_names = ', '.join([s.get_name() for s in servers]) - l[T.li["%d -> placed on [%s]" % (shnum, server_names)]] - return l + ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) + return ul d.addCallback(_render) return d - def render_servermap(self, ctx, data): + @renderer + def servermap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_servermap()) def _render(servermap): if servermap is None: return "None" - l = T.ul() + ul = tags.ul() for server, shnums in sorted(servermap.items()): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) - l[T.li["[%s] got share%s: %s" % (server.get_name(), - plural(shnums), shares_s)]] - return l + ul(tags.li("[%s] got share%s: %s" % (server.get_name(), + plural(shnums), shares_s))) + return ul d.addCallback(_render) return d - def data_file_size(self, ctx, data): + @renderer + def file_size(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_file_size()) + d.addCallback(lambda res: str(res.get_file_size())) return d def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: res.get_timings().get(name)) + d.addCallback(lambda res: str(res.get_timings().get(name))) return d - def data_time_total(self, ctx, data): - return self._get_time("total") + @renderer + def time_total(self, req, tag): + return tag(self._get_time("total")) - def data_time_storage_index(self, ctx, data): - return self._get_time("storage_index") + @renderer + def time_storage_index(self, req, tag): + return tag(self._get_time("storage_index")) - def data_time_contacting_helper(self, ctx, data): - return self._get_time("contacting_helper") + @renderer + def time_contacting_helper(self, req, tag): + return tag(self._get_time("contacting_helper")) - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") + @renderer + def time_cumulative_fetch(self, req, tag): + return tag(self._get_time("cumulative_fetch")) - def data_time_helper_total(self, ctx, data): - return self._get_time("helper_total") + @renderer + def time_helper_total(self, req, tag): + return tag(self._get_time("helper_total")) - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") + @renderer + def time_peer_selection(self, req, tag): + return tag(self._get_time("peer_selection")) - def data_time_total_encode_and_push(self, ctx, data): - return self._get_time("total_encode_and_push") + @renderer + def time_total_encode_and_push(self, req, tag): + return tag(self._get_time("total_encode_and_push")) - def data_time_cumulative_encoding(self, ctx, data): - return self._get_time("cumulative_encoding") + @renderer + def time_cumulative_encoding(self, req, tag): + return tag(self._get_time("cumulative_encoding")) - def data_time_cumulative_sending(self, ctx, data): - return self._get_time("cumulative_sending") + @renderer + def time_cumulative_sending(self, req, tag): + return tag(self._get_time("cumulative_sending")) - def data_time_hashes_and_close(self, ctx, data): - return self._get_time("hashes_and_close") + @renderer + def time_hashes_and_close(self, req, tag): + return tag(self._get_time("hashes_and_close")) def _get_rate(self, name): d = self.upload_results() def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return compute_rate(file_size, duration) + return str(compute_rate(file_size, duration)) d.addCallback(_convert) return d - def data_rate_total(self, ctx, data): - return self._get_rate("total") + @renderer + def rate_total(self, req, tag): + return tag(self._get_rate("total")) - def data_rate_storage_index(self, ctx, data): - return self._get_rate("storage_index") + @renderer + def rate_storage_index(self, req, tag): + return tag(self._get_rate("storage_index")) - def data_rate_encode(self, ctx, data): - return self._get_rate("cumulative_encoding") + @renderer + def rate_encode(self, req, tag): + return tag(self._get_rate("cumulative_encoding")) - def data_rate_push(self, ctx, data): + @renderer + def rate_push(self, req, tag): return self._get_rate("cumulative_sending") - def data_rate_encode_and_push(self, ctx, data): + @renderer + def rate_encode_and_push(self, req, tag): d = self.upload_results() def _convert(r): file_size = r.get_file_size() time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return None + return str(None) else: - return compute_rate(file_size, time1+time2) + return str(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d - def data_rate_ciphertext_fetch(self, ctx, data): + @renderer + def rate_ciphertext_fetch(self, req, tag): d = self.upload_results() def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return compute_rate(fetch_size, duration) + return str(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From a0655c18d6b3a589885c2b883c058e4f39f55496 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 18:00:20 -0400 Subject: [PATCH 0413/1054] Handle "t=upload" in UploadResultsPage POST handling --- src/allmydata/web/unlinked.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 0abf6cc6d..283d8d586 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -86,6 +86,23 @@ class UploadResultsPage(MultiFormatResource): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) + # This is weird but necessary because: + # + # 1. MultiFormatResource.render() uses argument "t" to figure out + # its output format. + # + # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". + # See URIHandler.render_POST(). + # + # MultiFormatResource.render() looks up "t" argument, which in + # this case has the value "upload", and then it would look for a + # render_UPLOAD() method. + # + # We could change upload request to use more descriptive names + # that do not cause name collisions like this. That should be a + # separate change though. + render_UPLOAD = render_HTML + class UploadResultsElement(Element, status.UploadResultsRendererMixin): From 855668b1885fc87373f83d326b7d1489614944be Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:23:23 -0400 Subject: [PATCH 0414/1054] Remove unused imports --- src/allmydata/web/unlinked.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 283d8d586..33fb7bafa 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -10,11 +10,10 @@ from twisted.web.template import ( renderElement, tags, ) -from nevow import url, tags as T +from nevow import url from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import ( - getxmlfile, get_arg, boolean_of_arg, convert_children_json, From 0af8df72030018f8280a55b7a14eb5b66c44fac1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:25:09 -0400 Subject: [PATCH 0415/1054] Document UploadStatusPage --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 5a9714dd0..ede0ae7be 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -193,8 +193,12 @@ class UploadResultsRendererMixin(RateAndTimeMixin): class UploadStatusPage(MultiFormatResource): + """Renders /status/up-%d.""" def __init__(self, upload_status): + """ + :param IUploadStatus upload_status: stats provider. + """ super(UploadStatusPage, self).__init__() self._upload_status = upload_status From c0d83def22f8a61da6fe0edf38955b193e89b68d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:28:16 -0400 Subject: [PATCH 0416/1054] Document UploadResultsPage parameter --- src/allmydata/web/unlinked.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 33fb7bafa..6ca219c8b 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -78,6 +78,9 @@ class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): + """ + :param IUploadResults upload_results: stats provider. + """ super(UploadResultsPage, self).__init__() self._upload_results = upload_results From 118ea4267cc2c8de3e458988313a51141fcfed72 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 14:53:33 -0400 Subject: [PATCH 0417/1054] Render abbreviated rate and time values --- src/allmydata/web/status.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index ede0ae7be..1bd469555 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -99,7 +99,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: str(res.get_timings().get(name))) + d.addCallback(lambda res: abbreviate_time(res.get_timings().get(name))) return d @renderer @@ -147,7 +147,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return str(compute_rate(file_size, duration)) + return abbreviate_rate(compute_rate(file_size, duration)) d.addCallback(_convert) return d @@ -175,9 +175,9 @@ class UploadResultsRendererMixin(RateAndTimeMixin): time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return str(None) + return abbreviate_rate(None) else: - return str(compute_rate(file_size, time1+time2)) + return abbreviate_rate(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d @@ -187,7 +187,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return str(compute_rate(fetch_size, duration)) + return abbreviate_rate(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From 100c4b1cdedb893f396ff608f119949e2e610eee Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:08:34 -0400 Subject: [PATCH 0418/1054] Add newsfragment --- newsfragments/3287.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3287.minor diff --git a/newsfragments/3287.minor b/newsfragments/3287.minor new file mode 100644 index 000000000..e69de29bb From 9c14eec1e05874b63403b0dec45faff9934d9254 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:22:59 -0400 Subject: [PATCH 0419/1054] Avoid use of RateAndTimeMixin in UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 1bd469555..46805fb99 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -45,7 +45,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(RateAndTimeMixin): +class UploadResultsRendererMixin(object): # this requires a method named 'upload_results' @renderer @@ -392,7 +392,7 @@ class DownloadResultsRendererMixin(RateAndTimeMixin): l = T.ul() for peerid in sorted(per_server.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[peerid]]) l[T.li["[%s]: %s" % (peerid_s, times_s)]] return T.li["Per-Server Segment Fetch Response Times: ", l] From 4b560d10cc37477629c5c49f066ff1cd3b995e55 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:49:34 -0400 Subject: [PATCH 0420/1054] Move time formatter tests UploadResultsRendererMixin no longer has render_time(); moving tests for render_time() as tests for abbreviate_time() to a more appropriate place. --- src/allmydata/test/web/test_util.py | 10 ++++++++++ src/allmydata/test/web/test_web.py | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index 2a6f67abc..a378c1447 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -23,6 +23,16 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us") + self.failUnlessReallyEqual(common.abbreviate_time(None), "") + self.failUnlessReallyEqual(common.abbreviate_time(2.5), "2.50s") + self.failUnlessReallyEqual(common.abbreviate_time(0.25), "250ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.0021), "2.1ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") + self.failUnlessReallyEqual(common.abbreviate_rate(None), "") + self.failUnlessReallyEqual(common.abbreviate_rate(2500000), "2.50MBps") + self.failUnlessReallyEqual(common.abbreviate_rate(30100), "30.1kBps") + self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps") + def test_compute_rate(self): self.failUnlessReallyEqual(common.compute_rate(None, None), None) self.failUnlessReallyEqual(common.compute_rate(None, 1), None) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1f4f4fdbe..bbd44b454 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1052,17 +1052,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps") self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps") - urrm = status.UploadResultsRendererMixin() - self.failUnlessReallyEqual(urrm.render_time(None, None), "") - self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(urrm.render_rate(None, None), "") - self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps") - def test_GET_FILEURL(self): d = self.GET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) From f30dcc7ec6268a01724b3676356a608153b09326 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:11:42 -0400 Subject: [PATCH 0421/1054] Make Element the superclass of UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- src/allmydata/web/unlinked.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 46805fb99..6f3ea3a46 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -45,7 +45,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(object): +class UploadResultsRendererMixin(Element): # this requires a method named 'upload_results' @renderer @@ -207,7 +207,7 @@ class UploadStatusPage(MultiFormatResource): return renderElement(req, elem) -class UploadStatusElement(Element, UploadResultsRendererMixin): +class UploadStatusElement(UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 6ca219c8b..d938691a4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -106,7 +106,7 @@ class UploadResultsPage(MultiFormatResource): render_UPLOAD = render_HTML -class UploadResultsElement(Element, status.UploadResultsRendererMixin): +class UploadResultsElement(status.UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) From 38fe7dd3f59b29ac5f0f8ef0e9dc2456e3c46b91 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:33:30 -0400 Subject: [PATCH 0422/1054] Remove unused import --- src/allmydata/web/unlinked.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index d938691a4..91efff2c4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -4,7 +4,6 @@ from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.web.template import ( - Element, XMLFile, renderer, renderElement, From 62e803db2560e838a679df1627723a460127b586 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:28:31 -0400 Subject: [PATCH 0423/1054] Make UploadStatusPage a Resource --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 6f3ea3a46..39b19cb0b 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -192,7 +192,7 @@ class UploadResultsRendererMixin(Element): return d -class UploadStatusPage(MultiFormatResource): +class UploadStatusPage(Resource, object): """Renders /status/up-%d.""" def __init__(self, upload_status): @@ -202,7 +202,7 @@ class UploadStatusPage(MultiFormatResource): super(UploadStatusPage, self).__init__() self._upload_status = upload_status - def render_HTML(self, req): + def render_GET(self, req): elem = UploadStatusElement(self._upload_status) return renderElement(req, elem) From 44ecfdad0955460d51848f5f6ca505e1b7e930ad Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:53:39 -0400 Subject: [PATCH 0424/1054] Make UploadResultsPage a Resource --- src/allmydata/web/unlinked.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 91efff2c4..09795101a 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -3,6 +3,7 @@ import urllib from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath +from twisted.web.resource import Resource from twisted.web.template import ( XMLFile, renderer, @@ -19,7 +20,6 @@ from allmydata.web.common import ( WebError, get_format, get_mutable_type, - MultiFormatResource, ) from allmydata.web import status @@ -73,7 +73,7 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(MultiFormatResource): +class UploadResultsPage(Resource, object): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): @@ -83,7 +83,7 @@ class UploadResultsPage(MultiFormatResource): super(UploadResultsPage, self).__init__() self._upload_results = upload_results - def render_HTML(self, req): + def render_POST(self, req): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) From 8331e7d8ec11f5080255c46dbdc0c90d43c46e6f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:54:23 -0400 Subject: [PATCH 0425/1054] Remove now-unnecessary UploadResultsPage.render_UPLOAD --- src/allmydata/web/unlinked.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 09795101a..df342d7dd 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -87,23 +87,6 @@ class UploadResultsPage(Resource, object): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) - # This is weird but necessary because: - # - # 1. MultiFormatResource.render() uses argument "t" to figure out - # its output format. - # - # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". - # See URIHandler.render_POST(). - # - # MultiFormatResource.render() looks up "t" argument, which in - # this case has the value "upload", and then it would look for a - # render_UPLOAD() method. - # - # We could change upload request to use more descriptive names - # that do not cause name collisions like this. That should be a - # separate change though. - render_UPLOAD = render_HTML - class UploadResultsElement(status.UploadResultsRendererMixin): From 83f36ab6c22b79996571085ff0781739c7de6aa2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 13:50:06 -0400 Subject: [PATCH 0426/1054] Switch to flake8 as a better way of running pyflakes. --- Makefile | 2 +- newsfragments/3329.other | 0 setup.cfg | 5 +++++ setup.py | 3 ++- tox.ini | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3329.other diff --git a/Makefile b/Makefile index 6e0009ddf..6dd87409e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ default: PYTHON=python export PYTHON -PYFLAKES=pyflakes +PYFLAKES=flake8 export PYFLAKES SOURCES=src/allmydata static misc setup.py diff --git a/newsfragments/3329.other b/newsfragments/3329.other new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index de40c9e23..c2d224225 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,8 @@ install = update_version install develop = update_version develop bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel + +[flake8] +# For now, only use pyflakes errors; flake8 is still helpful because it allows +# ignoring specific errors/warnings when needed. +select = F \ No newline at end of file diff --git a/setup.py b/setup.py index caa37f16e..0d43294cc 100644 --- a/setup.py +++ b/setup.py @@ -353,11 +353,12 @@ setup(name="tahoe-lafs", # also set in __init__.py # discussion. ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ + "flake8", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. - "pyflakes == 2.1.0", + "pyflakes == 2.2.0", # coverage 5.0 breaks the integration tests in some opaque way. # This probably needs to be addressed in a more permanent way # eventually... diff --git a/tox.ini b/tox.ini index 3134a0d9f..44b583baa 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = whitelist_externals = /bin/mv commands = - pyflakes src static misc setup.py + flake8 src static misc setup.py python misc/coding_tools/check-umids.py src python misc/coding_tools/check-debugging.py python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py From 4e2cc6e1fd2cb3eac45f2d1b5a4f99470be6f26e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:41:17 -0400 Subject: [PATCH 0427/1054] Here's a start --- docs/CODE_OF_CONDUCT.md | 53 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 54 insertions(+) create mode 100644 docs/CODE_OF_CONDUCT.md diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..868f59c6a --- /dev/null +++ b/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,53 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer (see below). All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + +The following community members have made themselves available for conduct issues: + +- Jean-Paul Calderone (jean-paul at leastauthority dot com) + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/docs/index.rst b/docs/index.rst index 98b4f241b..b125cdd4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING + ../.github/CODE_OF_CONDUCT servers helper From ff8785acbe93a83e8d631faabd89595dd0866213 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:48:39 -0400 Subject: [PATCH 0428/1054] as far as I can tell there is nothing magic about putting it in .github --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index b125cdd4e..581e74bbb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING - ../.github/CODE_OF_CONDUCT + CODE_OF_CONDUCT servers helper From 07064f096fa580399bcaaff964fbc1a35f6c640a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:50:47 -0400 Subject: [PATCH 0429/1054] news fragment --- newsfragments/2755.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2755.other diff --git a/newsfragments/2755.other b/newsfragments/2755.other new file mode 100644 index 000000000..e3e31465d --- /dev/null +++ b/newsfragments/2755.other @@ -0,0 +1 @@ +The Tahoe-LAFS project has adopted a formal code of conduct. From defac12b192a366996ad809d702d43ead7fd27d7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 6 Jul 2020 10:41:38 -0400 Subject: [PATCH 0430/1054] add a contributor covenant badge --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d53fa0104..274cee41d 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,8 @@ function correctly, preserving your privacy and security. For full documentation, please see http://tahoe-lafs.readthedocs.io/en/latest/ . -|readthedocs| |travis| |circleci| |codecov| +|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov| + INSTALLING ========== @@ -105,3 +106,5 @@ slides. .. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md From 7b8ab9cfa6c9ec70592141f18f3de90ca89be490 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 6 Jul 2020 13:49:07 -0600 Subject: [PATCH 0431/1054] add meejah to code-of-conduct --- docs/CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 868f59c6a..3b14f2d70 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -44,6 +44,7 @@ incident. The following community members have made themselves available for conduct issues: - Jean-Paul Calderone (jean-paul at leastauthority dot com) +- meejah (meejah at meejah dot ca) This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.3.0, available at From 50ce511cf34efa97659643ed305504b3f278a33b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:18:11 -0400 Subject: [PATCH 0432/1054] Perhaps this is the preferred formatting? --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 274cee41d..082925670 100644 --- a/README.rst +++ b/README.rst @@ -107,4 +107,5 @@ slides. :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master -.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :target: docs/CODE_OF_CONDUCT.md From 349d34b099e29be4d1c82365822bfae8a0bb61ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:19:30 -0400 Subject: [PATCH 0433/1054] add some alt text --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 082925670..d3b089186 100644 --- a/README.rst +++ b/README.rst @@ -108,4 +108,5 @@ slides. :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :alt: code of conduct :target: docs/CODE_OF_CONDUCT.md From c4437bd19401a7f9b62111dbebc7e94a585a82ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:07 -0400 Subject: [PATCH 0434/1054] Package Future 0.18.2 for us --- nix/future.nix | 35 +++++++++++++++++++++++++++++++++++ nix/overlays.nix | 5 +++++ nix/tahoe-lafs.nix | 1 + 3 files changed, 41 insertions(+) create mode 100644 nix/future.nix diff --git a/nix/future.nix b/nix/future.nix new file mode 100644 index 000000000..814b7c1b5 --- /dev/null +++ b/nix/future.nix @@ -0,0 +1,35 @@ +{ lib +, buildPythonPackage +, fetchPypi +}: + +buildPythonPackage rec { + pname = "future"; + version = "0.18.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; + }; + + doCheck = false; + + meta = { + description = "Clean single-source support for Python 3 and 2"; + longDescription = '' + python-future is the missing compatibility layer between Python 2 and + Python 3. It allows you to use a single, clean Python 3.x-compatible + codebase to support both Python 2 and Python 3 with minimal overhead. + + It provides future and past packages with backports and forward ports + of features from Python 3 and 2. It also comes with futurize and + pasteurize, customized 2to3-based scripts that helps you to convert + either Py2 or Py3 code easily to support both Python 2 and 3 in a + single clean Py3-style codebase, module by module. + ''; + homepage = https://python-future.org; + downloadPage = https://github.com/PythonCharmers/python-future/releases; + license = with lib.licenses; [ mit ]; + maintainers = with lib.maintainers; [ prikhi ]; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 2a9fe8e5e..7f19a8f1a 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -10,6 +10,11 @@ self: super: { # NixOS autobahn package has trollius as a dependency, although # it is optional. Trollius is unmaintained and fails on CI. autobahn = python-super.callPackage ./autobahn.nix { }; + + # Porting to Python 3 is greatly aided by the future package. A + # slightly newer version than appears in nixos 19.09 is helpful. + future = python-super.callPackage ./future.nix { }; + }; }; } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index cc0eea479..ae5c05f70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,6 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools + future ]; checkInputs = with python.pkgs; [ From 133e1bfb6f83e0f1a98e6aabf30974a2065a52b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:42 -0400 Subject: [PATCH 0435/1054] news fragment --- newsfragments/3338.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3338.minor diff --git a/newsfragments/3338.minor b/newsfragments/3338.minor new file mode 100644 index 000000000..e69de29bb From a56d077d3dfc1b73a0f476354d521a29f6c43265 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 9 Jul 2020 16:59:04 -0400 Subject: [PATCH 0436/1054] Test HTML document generated by web.status.Status --- src/allmydata/test/web/test_status.py | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/allmydata/test/web/test_status.py diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py new file mode 100644 index 000000000..a09edad59 --- /dev/null +++ b/src/allmydata/test/web/test_status.py @@ -0,0 +1,73 @@ +# Tests for code in allmydata.web.status + +from bs4 import BeautifulSoup +from twisted.web.template import flattenString + +from allmydata.web.status import ( + Status, + StatusElement, +) + +from .common import ( + assert_soup_has_favicon, + assert_soup_has_text, + assert_soup_has_tag_with_content, +) +from ..common import TrialTestCase + +from .test_web import FakeHistory + +# Test that status.StatusElement can render HTML. +class StatusTests(TrialTestCase): + + def _render_status_page(self, active, recent): + elem = StatusElement(active, recent) + d = flattenString(None, elem) + return self.successResultOf(d) + + def test_status_page(self): + status = Status(FakeHistory()) + doc = self._render_status_page( + status._get_active_operations(), + status._get_recent_operations() + ) + soup = BeautifulSoup(doc, 'html5lib') + + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Recent and Active Operations" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Active Operations:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"td", + u"retrieve" + ) + + assert_soup_has_tag_with_content( + self, soup, u"td", + u"publish" + ) + + assert_soup_has_tag_with_content( + self, soup, u"td", + u"download" + ) + + assert_soup_has_tag_with_content( + self, soup, u"td", + u"upload" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + "Recent Operations:" + ) + + print doc From 956c2c535d5503291dab5f4541fb5fe2ee8dd75a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 9 Jul 2020 17:09:12 -0400 Subject: [PATCH 0437/1054] Remove unused import --- src/allmydata/test/web/test_status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index a09edad59..e351c13c8 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -10,7 +10,6 @@ from allmydata.web.status import ( from .common import ( assert_soup_has_favicon, - assert_soup_has_text, assert_soup_has_tag_with_content, ) from ..common import TrialTestCase From d4a02aa1cdcdc62a77ebd2a129d3a2d899ab1083 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 9 Jul 2020 17:14:55 -0400 Subject: [PATCH 0438/1054] Remove a print statement --- src/allmydata/test/web/test_status.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index e351c13c8..0779e7d80 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -68,5 +68,3 @@ class StatusTests(TrialTestCase): self, soup, u"h2", "Recent Operations:" ) - - print doc From 427f99621f2e77fb688f50e066b0e3a7496668e2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 10 Jul 2020 06:56:29 -0400 Subject: [PATCH 0439/1054] Do more null checks --- src/allmydata/web/check_results.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 811885ec3..c9c429622 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -671,9 +671,13 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def root_storage_index(self, req, tag): + if not self.monitor.get_status(): + return "" return self.monitor.get_status().get_root_storage_index_string() def _get_counter(self, name): + if not self.monitor.get_status(): + return "" return str(self.monitor.get_status().get_counters().get(name)) @renderer @@ -718,8 +722,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def pre_repair_problems_p(self, req, tag): - c = self.monitor.get_status().get_counters() - if c["count-objects-unhealthy-pre-repair"]: + if self._get_counter("count-objects-unhealthy-pre-repair"): return tag return "" @@ -740,9 +743,8 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def post_repair_problems_p(self, req, tag): - c = self.monitor.get_status().get_counters() - if (c["count-objects-unhealthy-post-repair"] - or c["count-corrupt-shares-post-repair"]): + if (self._get_counter("count-objects-unhealthy-post-repair") + or self._get_counter("count-corrupt-shares-post-repair")): return tag return "" @@ -763,7 +765,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def servers_with_corrupt_shares_p(self, req, tag): - if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]: + if self._get_counter("count-corrupt-shares-pre-repair"): return tag return "" @@ -776,7 +778,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def remaining_corrupt_shares_p(self, req, tag): - if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]: + if self._get_counter("count-corrupt-shares-post-repair"): return tag return "" @@ -796,7 +798,9 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def all_objects(self, req, tag): - results = self.monitor.get_status().get_all_results() + results = {} + if self.monitor.get_status(): + results = self.monitor.get_status().get_all_results() objects = [] for path in sorted(results.keys()): @@ -816,5 +820,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin @renderer def runtime(self, req, tag): - runtime = time.time() - req.processing_started_timestamp + runtime = 'unknown' + if hasattr(req, 'processing_started_timestamp'): + runtime = time.time() - req.processing_started_timestamp return tag("runtime: %s seconds" % runtime) From f9dc2509deed10b0ec516b14595e7fa00f81a63d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 10 Jul 2020 08:00:04 -0400 Subject: [PATCH 0440/1054] Test "deep check and repair" page --- src/allmydata/test/test_checker.py | 105 +++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index bd1bd87c6..b4d904180 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -2,6 +2,9 @@ import re import json import os.path, shutil + +from bs4 import BeautifulSoup + from twisted.trial import unittest from twisted.internet import defer @@ -26,6 +29,11 @@ from .common import ( EMPTY_CLIENT_CONFIG, ) +from .web.common import ( + assert_soup_has_favicon, + assert_soup_has_tag_with_content, +) + class FakeClient(object): def get_storage_broker(self): return self.storage_broker @@ -40,6 +48,7 @@ class TestRequest(object, Request): Request.__init__(self, DummyChannel()) self.args = args or {} self.fields = fields or {} + self.prepath = [b""] class WebResultsRendering(unittest.TestCase): @@ -336,6 +345,102 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessEqual(j["storage-index"], "") _got_lit_results(d) + def test_deep_check_and_repair_renderer(self): + monitor = Monitor() + status = check_results.DeepCheckAndRepairResults("") + monitor.set_status(status) + elem = web_check_results.DeepCheckAndRepairResultsRendererElement(monitor) + doc = self.render_element(elem) + soup = BeautifulSoup(doc, 'html5lib') + + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Deep Check Results" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h1", + u"Deep-Check-And-Repair Results for root SI=" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Checked: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy (before repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy (before repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares (before repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Repairs Attempted: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Repairs Attempted: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Repairs Successful: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + "Repairs Unsuccessful: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy (after repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy (after repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares (after repair): 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Had Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Still Have Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Servers on which corrupt shares were found" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Remaining Corrupt Shares" + ) + + class BalancingAct(GridTestMixin, unittest.TestCase): # test for #1115 regarding the 'count-good-share-hosts' metric From 267b89ae5be0ee0e5659a397b72c19eccf5afe25 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 10 Jul 2020 16:34:16 -0400 Subject: [PATCH 0441/1054] Remove unused import --- src/allmydata/test/web/test_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 822603ade..59a9d6ec6 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -65,7 +65,7 @@ from ..common_web import ( Error, ) from allmydata.client import _Client, SecretHolder -from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP +from .common import unknown_rwcap, unknown_rocap, unknown_immcap # create a fake uploader/downloader, and a couple of fake dirnodes, then # create a webserver that works against them From b49a3eaf3a81f2812d88e2e8e22f81c1e1a11fcc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 10 Jul 2020 21:46:23 -0400 Subject: [PATCH 0442/1054] Check for a non-standard property on request --- src/allmydata/web/check_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index c9c429622..a29ddb188 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -583,7 +583,9 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def runtime(self, req, tag): - runtime = time.time() - req.processing_started_timestamp + runtime = 'unknown' + if hasattr(req, 'processing_started_timestamp'): + runtime = time.time() - req.processing_started_timestamp return tag("runtime: %s seconds" % runtime) From 6eba2fb2e6a122105a8f50f73e5addb64665f9a4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 11 Jul 2020 07:59:28 -0400 Subject: [PATCH 0443/1054] Test "deep check results" page These are some basic tests -- let's see if this improves test coverage, and by how much. --- src/allmydata/test/test_checker.py | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index b4d904180..4c1313793 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -345,6 +345,53 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessEqual(j["storage-index"], "") _got_lit_results(d) + def test_deep_check_renderer(self): + monitor = Monitor() + status = check_results.DeepCheckResults("") + monitor.set_status(status) + + elem = web_check_results.DeepCheckResultsRendererElement(monitor) + + doc = self.render_element(elem) + soup = BeautifulSoup(doc, 'html5lib') + + assert_soup_has_favicon(self, soup) + + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Deep Check Results" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h1", + "Deep-Check Results for root SI=" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Checked: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Healthy: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unhealthy: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Objects Unrecoverable: 0" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"Corrupt Shares: 0" + ) + def test_deep_check_and_repair_renderer(self): monitor = Monitor() status = check_results.DeepCheckAndRepairResults("") From 144001f3808c86bfdc22cfe2b5c28510d3321591 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 11 Jul 2020 08:30:39 -0400 Subject: [PATCH 0444/1054] Use a better assert in status subpath test --- src/allmydata/test/web/test_web.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index bbd44b454..7899c696e 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1038,7 +1038,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_status_path_error(self): # Expect an error, because path is expected to be of the form # "/status/{up,down,..}-%number", with a hyphen. - return self.assertFailure(self.GET("/storage/nodash"), error.Error) + d = self.shouldFail2(error.Error, + "test_status_path_error", + "400 Bad Request", + "no '-' in 'nodash'", + self.GET, + "/status/nodash") + return d def test_status_numbers(self): drrm = status.DownloadResultsRendererMixin() From 1a3a1ffcaa8c8cd71d156cd4bc78110f4359b303 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 14:33:13 -0400 Subject: [PATCH 0445/1054] Restore eliot code. --- src/allmydata/util/eliotutil.py | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 14c867ed2..f6f40945d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -14,8 +14,13 @@ __all__ = [ "eliot_logging_service", "opt_eliot_destination", "opt_help_eliot_destinations", + "validateInstanceOf", + "validateSetMembership", ] +from sys import ( + stdout, +) from functools import wraps from logging import ( INFO, @@ -37,11 +42,15 @@ from attr.validators import ( from eliot import ( ILogger, Message, + FileDestination, add_destinations, remove_destination, write_traceback, start_action, ) +from eliot._validation import ( + ValidationError, +) from eliot.twisted import ( DeferredContext, inline_callbacks, @@ -50,6 +59,12 @@ from eliot.twisted import ( from twisted.python.usage import ( UsageError, ) +from twisted.python.filepath import ( + FilePath, +) +from twisted.python.logfile import ( + LogFile, +) from twisted.logger import ( ILogObserver, eventAsJSON, @@ -60,6 +75,23 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service +def validateInstanceOf(t): + """ + Return an Eliot validator that requires values to be instances of ``t``. + """ + def validator(v): + if not isinstance(v, t): + raise ValidationError("{} not an instance of {}".format(v, t)) + return validator + +def validateSetMembership(s): + """ + Return an Eliot validator that requires values to be elements of ``s``. + """ + def validator(v): + if v not in s: + raise ValidationError("{} not in {}".format(v, s)) + return validator def eliot_logging_service(reactor, destinations): """ @@ -214,6 +246,53 @@ class _DestinationParser(object): else: return parser(kind, args) + def _get_arg(self, arg_name, default, arg_list): + return dict( + arg.split(u"=", 1) + for arg + in arg_list + ).get( + arg_name, + default, + ) + + def _parse_file(self, kind, arg_text): + # Reserve the possibility of an escape character in the future. \ is + # the standard choice but it's the path separator on Windows which + # pretty much ruins it in this context. Most other symbols already + # have some shell-assigned meaning which makes them treacherous to use + # in a CLI interface. Eliminating all such dangerous symbols leaves + # approximately @. + if u"@" in arg_text: + raise ValueError( + u"Unsupported escape character (@) in destination text ({!r}).".format(arg_text), + ) + arg_list = arg_text.split(u",") + path_name = arg_list.pop(0) + if path_name == "-": + get_file = lambda: stdout + else: + path = FilePath(path_name) + rotate_length = int(self._get_arg( + u"rotate_length", + 1024 * 1024 * 1024, + arg_list, + )) + max_rotated_files = int(self._get_arg( + u"max_rotated_files", + 10, + arg_list, + )) + def get_file(): + path.parent().makedirs(ignoreExistingDirectory=True) + return LogFile( + path.basename(), + path.dirname(), + rotateLength=rotate_length, + maxRotatedFiles=max_rotated_files, + ) + return lambda reactor: FileDestination(get_file()) + _parse_destination_description = _DestinationParser().parse From f27cba181e2244a4d596d73160f5915df1c443e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 15:22:03 -0400 Subject: [PATCH 0446/1054] Move existing base32 tests out of test_util.py. --- src/allmydata/test/test_base32.py | 27 +++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 21 +-------------------- 2 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 src/allmydata/test/test_base32.py diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py new file mode 100644 index 000000000..8af0d7d71 --- /dev/null +++ b/src/allmydata/test/test_base32.py @@ -0,0 +1,27 @@ +""" +Tests for allmydata.util.base32. +""" + +import base64 + +from twisted.trial import unittest + +from allmydata.util import base32 + + +class Base32(unittest.TestCase): + def test_b2a_matches_Pythons(self): + y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0" + x = base64.b32encode(y) + while x and x[-1] == '=': + x = x[:-1] + x = x.lower() + self.failUnlessEqual(base32.b2a(y), x) + def test_b2a(self): + self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a") + def test_b2a_or_none(self): + self.failUnlessEqual(base32.b2a_or_none(None), None) + self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a") + def test_a2b(self): + self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34") + self.failUnlessRaises(AssertionError, base32.a2b, "b0gus") diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 14d9f5dd7..b26731d7c 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -14,7 +14,7 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from allmydata.util import base32, idlib, mathutil, hashutil +from allmydata.util import idlib, mathutil, hashutil from allmydata.util import fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil @@ -36,25 +36,6 @@ def sha256(data): return binascii.hexlify(hashlib.sha256(data).digest()) -class Base32(unittest.TestCase): - def test_b2a_matches_Pythons(self): - import base64 - y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0" - x = base64.b32encode(y) - while x and x[-1] == '=': - x = x[:-1] - x = x.lower() - self.failUnlessEqual(base32.b2a(y), x) - def test_b2a(self): - self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a") - def test_b2a_or_none(self): - self.failUnlessEqual(base32.b2a_or_none(None), None) - self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a") - def test_a2b(self): - self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34") - self.failUnlessRaises(AssertionError, base32.a2b, "b0gus") - - class IDLib(unittest.TestCase): def test_nodeid_b2a(self): self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) From a8a7d48c1280d0e3535ede96e72b2e87c6ec695e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 15:25:07 -0400 Subject: [PATCH 0447/1054] Everything must be bytes. --- src/allmydata/test/test_base32.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 8af0d7d71..d24fe8400 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -11,17 +11,21 @@ from allmydata.util import base32 class Base32(unittest.TestCase): def test_b2a_matches_Pythons(self): - y = "\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0" + y = b"\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0" x = base64.b32encode(y) - while x and x[-1] == '=': - x = x[:-1] + x = x.rstrip(b"=") x = x.lower() - self.failUnlessEqual(base32.b2a(y), x) + result = base32.b2a(y) + self.failUnlessEqual(result, x) + self.assertIsInstance(result, bytes) + def test_b2a(self): - self.failUnlessEqual(base32.b2a("\x12\x34"), "ci2a") + self.failUnlessEqual(base32.b2a(b"\x12\x34"), b"ci2a") + def test_b2a_or_none(self): self.failUnlessEqual(base32.b2a_or_none(None), None) - self.failUnlessEqual(base32.b2a_or_none("\x12\x34"), "ci2a") + self.failUnlessEqual(base32.b2a_or_none(b"\x12\x34"), b"ci2a") + def test_a2b(self): - self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34") - self.failUnlessRaises(AssertionError, base32.a2b, "b0gus") + self.failUnlessEqual(base32.a2b(b"ci2a"), b"\x12\x34") + self.failUnlessRaises(AssertionError, base32.a2b, b"b0gus") From bbfd39e6c0c1993f3d2d6068f31287e97b1f87a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 15:30:11 -0400 Subject: [PATCH 0448/1054] Test larger variety of inputs using hypothesis. --- src/allmydata/test/test_base32.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index d24fe8400..7e4f4025f 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -6,18 +6,22 @@ import base64 from twisted.trial import unittest +from hypothesis import ( + strategies as st, + given, +) from allmydata.util import base32 class Base32(unittest.TestCase): - def test_b2a_matches_Pythons(self): - y = b"\x12\x34\x45\x67\x89\x0a\xbc\xde\xf0" - x = base64.b32encode(y) - x = x.rstrip(b"=") - x = x.lower() - result = base32.b2a(y) - self.failUnlessEqual(result, x) - self.assertIsInstance(result, bytes) + + @given(input_bytes=st.binary(max_size=100)) + def test_a2b_b2a_match_Pythons(self, input_bytes): + encoded = base32.b2a(input_bytes) + x = base64.b32encode(input_bytes).rstrip(b"=").lower() + self.failUnlessEqual(encoded, x) + self.assertIsInstance(encoded, bytes) + self.assertEqual(base32.a2b(encoded), input_bytes) def test_b2a(self): self.failUnlessEqual(base32.b2a(b"\x12\x34"), b"ci2a") From e23dbcdf565e72de3d27f0ee599d3e103da73ca5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 15:41:01 -0400 Subject: [PATCH 0449/1054] Document current public API. --- src/allmydata/util/base32.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 2017aaed4..c3b290adf 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -263,3 +263,6 @@ def a2b_l(cs, lengthinbits): res = ''.join(map(chr, octets)) precondition(b2a_l(res, lengthinbits) == cs, "cs is required to be the canonical base-32 encoding of some data.", b2a(res), res=res, cs=cs) return res + + +__all__ = ["b2a", "b2a_l", "a2b", "a2b_l", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"] From 77ae087c77b5ec463fc1be7d5923e7694bdb7d59 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 16:12:53 -0400 Subject: [PATCH 0450/1054] Remove a2b_l and b2a_l from public API, to ease likely future switch to stdlib base64's implementation. --- src/allmydata/immutable/checker.py | 4 ++-- src/allmydata/immutable/downloader/finder.py | 2 +- src/allmydata/immutable/downloader/node.py | 2 +- src/allmydata/storage/immutable.py | 2 +- src/allmydata/util/base32.py | 12 ++++++------ 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 11f4341d4..36bf735dc 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -222,7 +222,7 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin): UEB""" precondition(share_hash_tree[0] is not None, share_hash_tree) prefix = "%d-%s-%s" % (sharenum, bucket, - base32.b2a_l(share_hash_tree[0][:8], 60)) + base32.b2a(share_hash_tree[0][:8])[:12]) log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.download", prefix=prefix) @@ -465,7 +465,7 @@ class Checker(log.PrefixingLogMixin): monitor): assert precondition(isinstance(verifycap, CHKFileVerifierURI), verifycap, type(verifycap)) - prefix = "%s" % base32.b2a_l(verifycap.get_storage_index()[:8], 60) + prefix = "%s" % base32.b2a(verifycap.get_storage_index()[:8])[:12] log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.checker", prefix=prefix) self._verifycap = verifycap diff --git a/src/allmydata/immutable/downloader/finder.py b/src/allmydata/immutable/downloader/finder.py index 3c6788537..a60aee814 100644 --- a/src/allmydata/immutable/downloader/finder.py +++ b/src/allmydata/immutable/downloader/finder.py @@ -43,7 +43,7 @@ class ShareFinder(object): self.overdue_timers = {} self._storage_index = verifycap.storage_index - self._si_prefix = base32.b2a_l(self._storage_index[:8], 60) + self._si_prefix = base32.b2a(self._storage_index[:8])[:12] self._node_logparent = logparent self._download_status = download_status self._lp = log.msg(format="ShareFinder[si=%(si)s] starting", diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index 1c7ecca33..50a8a2ce3 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -44,7 +44,7 @@ class DownloadNode(object): assert isinstance(verifycap, uri.CHKFileVerifierURI) self._verifycap = verifycap self._storage_broker = storage_broker - self._si_prefix = base32.b2a_l(verifycap.storage_index[:8], 60) + self._si_prefix = base32.b2a(verifycap.storage_index[:8])[:12] self.running = True if terminator: terminator.register(self) # calls self.stop() at stopService() diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7f69fb3b8..4ec70e1e7 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -298,7 +298,7 @@ class BucketReader(Referenceable): def __repr__(self): return "<%s %s %s>" % (self.__class__.__name__, - base32.b2a_l(self.storage_index[:8], 60), + base32.b2a(self.storage_index[:8])[:12], self.shnum) def remote_read(self, offset, length): diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index c3b290adf..32fcdbc38 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -52,13 +52,13 @@ def b2a(os): @return the contents of os in base-32 encoded form """ - return b2a_l(os, len(os)*8) + return _b2a_l(os, len(os)*8) def b2a_or_none(os): if os is not None: return b2a(os) -def b2a_l(os, lengthinbits): +def _b2a_l(os, lengthinbits): """ @param os the data to be encoded (a string) @param lengthinbits the number of bits of data in os to be encoded @@ -204,9 +204,9 @@ def a2b(cs): precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) precondition(isinstance(cs, six.binary_type), cs) - return a2b_l(cs, num_octets_that_encode_to_this_many_quintets(len(cs))*8) + return _a2b_l(cs, num_octets_that_encode_to_this_many_quintets(len(cs))*8) -def a2b_l(cs, lengthinbits): +def _a2b_l(cs, lengthinbits): """ @param lengthinbits the number of bits of data in encoded into cs @@ -261,8 +261,8 @@ def a2b_l(cs, lengthinbits): pos = pos * 256 assert len(octets) == numoctets, "len(octets): %s, numoctets: %s, octets: %s" % (len(octets), numoctets, octets,) res = ''.join(map(chr, octets)) - precondition(b2a_l(res, lengthinbits) == cs, "cs is required to be the canonical base-32 encoding of some data.", b2a(res), res=res, cs=cs) + precondition(_b2a_l(res, lengthinbits) == cs, "cs is required to be the canonical base-32 encoding of some data.", b2a(res), res=res, cs=cs) return res -__all__ = ["b2a", "b2a_l", "a2b", "a2b_l", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"] +__all__ = ["b2a", "a2b", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"] From 828e6bc28acabac7c7e50d79cc80c7aa68b68171 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 16:18:13 -0400 Subject: [PATCH 0451/1054] Minimal tests for could_be_base32_encoded(). --- src/allmydata/test/test_base32.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 7e4f4025f..de256c7fa 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -21,6 +21,7 @@ class Base32(unittest.TestCase): x = base64.b32encode(input_bytes).rstrip(b"=").lower() self.failUnlessEqual(encoded, x) self.assertIsInstance(encoded, bytes) + self.assertTrue(base32.could_be_base32_encoded(encoded)) self.assertEqual(base32.a2b(encoded), input_bytes) def test_b2a(self): @@ -33,3 +34,4 @@ class Base32(unittest.TestCase): def test_a2b(self): self.failUnlessEqual(base32.a2b(b"ci2a"), b"\x12\x34") self.failUnlessRaises(AssertionError, base32.a2b, b"b0gus") + self.assertFalse(base32.could_be_base32_encoded(b"b0gus")) From 6464e226c6e0845b8a3dc8f7632a8b43a4d42220 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 16:19:48 -0400 Subject: [PATCH 0452/1054] Hypothesis-based roundtrip test. --- src/allmydata/test/test_base62.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 51df7774d..535156210 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -1,5 +1,10 @@ import random, unittest +from hypothesis import ( + strategies as st, + given, +) + from allmydata.util import base62, mathutil def insecurerandstr(n): @@ -14,6 +19,10 @@ class T(unittest.TestCase): bs2=base62.a2b(ascii) assert bs2 == bs, "bs2: %s:%s, bs: %s:%s, ascii: %s:%s" % (len(bs2), repr(bs2), len(bs), repr(bs), len(ascii), repr(ascii)) + @given(input_bytes=st.binary(max_size=100)) + def test_roundtrip(self, input_bytes): + self._test_ende(input_bytes) + def test_num_octets_that_encode_to_this_many_chars(self): return self._test_num_octets_that_encode_to_this_many_chars(2, 1) return self._test_num_octets_that_encode_to_this_many_chars(3, 2) From ead2a79cf72fdbcd9c6e80bcaca2401f03fe3c64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Jul 2020 16:21:15 -0400 Subject: [PATCH 0453/1054] Everything should be bytes. --- src/allmydata/test/test_base62.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 535156210..0538b9eb6 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -1,5 +1,7 @@ import random, unittest +from past.builtins import chr as byteschr + from hypothesis import ( strategies as st, given, @@ -8,7 +10,7 @@ from hypothesis import ( from allmydata.util import base62, mathutil def insecurerandstr(n): - return ''.join(map(chr, map(random.randrange, [0]*n, [256]*n))) + return b''.join(map(byteschr, map(random.randrange, [0]*n, [256]*n))) class T(unittest.TestCase): def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): @@ -30,19 +32,19 @@ class T(unittest.TestCase): return self._test_num_octets_that_encode_to_this_many_chars(6, 4) def test_ende_0x00(self): - return self._test_ende('\x00') + return self._test_ende(b'\x00') def test_ende_0x01(self): - return self._test_ende('\x01') + return self._test_ende(b'\x01') def test_ende_0x0100(self): - return self._test_ende('\x01\x00') + return self._test_ende(b'\x01\x00') def test_ende_0x000000(self): - return self._test_ende('\x00\x00\x00') + return self._test_ende(b'\x00\x00\x00') def test_ende_0x010000(self): - return self._test_ende('\x01\x00\x00') + return self._test_ende(b'\x01\x00\x00') def test_ende_randstr(self): return self._test_ende(insecurerandstr(2**4)) From b48556c2cb4a4c59ff5448d1035bcfee69724834 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 13 Jul 2020 20:44:30 -0400 Subject: [PATCH 0454/1054] Add tests for status subpages Coverage tests for the coverage gods --- src/allmydata/test/web/test_web.py | 96 ++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 7899c696e..05e423e05 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1038,12 +1038,96 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_status_path_error(self): # Expect an error, because path is expected to be of the form # "/status/{up,down,..}-%number", with a hyphen. - d = self.shouldFail2(error.Error, - "test_status_path_error", - "400 Bad Request", - "no '-' in 'nodash'", - self.GET, - "/status/nodash") + return self.shouldFail2(error.Error, + "test_status_path_error", + "400 Bad Request", + "no '-' in 'nodash'", + self.GET, + "/status/nodash") + + def test_status_up_subpath(self): + # See that "GET /status/up-0" works. + h = self.s.get_history() + ul_num = h.list_all_upload_statuses()[0].get_counter() + d = self.GET("/status/up-{}".format(ul_num), return_response=True) + def _upload_status(result): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - File Upload Status" + ) + d.addCallback(_upload_status) + return d + + def test_status_down_subpath(self): + # See that "GET /status/down-0" works. + h = self.s.get_history() + dl_num = h.list_all_download_statuses()[0].get_counter() + d = self.GET("/status/down-{}".format(dl_num), return_response=True) + def _download_status(result): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - File Download Status" + ) + d.addCallback(_download_status) + return d + + def test_status_mapupdate_subpath(self): + # See that "GET /status/mapupdate-0" works. + h = self.s.get_history() + mu_num = h.list_all_mapupdate_statuses()[0].get_counter() + d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True) + def _mapupdate_status(result): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Mutable File Servermap Update Status" + ) + d.addCallback(_mapupdate_status) + return d + + def test_status_publish_subpath(self): + # See that "GET /status/publish-0" works. + h = self.s.get_history() + pub_num = h.list_all_publish_statuses()[0].get_counter() + d = self.GET("/status/publish-{}".format(pub_num), return_response=True) + def _mapupdate_status(result): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Mutable File Publish Status" + ) + d.addCallback(_mapupdate_status) + return d + + def test_status_retrieve_subpath(self): + # See that "GET /status/retrieve-0" works. + h = self.s.get_history() + ret_num = h.list_all_retrieve_statuses()[0].get_counter() + d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True) + def _mapupdate_status(result): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Mutable File Retrieve Status" + ) + d.addCallback(_mapupdate_status) return d def test_status_numbers(self): From abd539588f63ab77e63b00cd32ae81cd58b0d833 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 13 Jul 2020 20:50:55 -0400 Subject: [PATCH 0455/1054] Rename nested functions for clarity ... and also to remove embarrassing copy-pasta mistakes. --- src/allmydata/test/web/test_web.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 05e423e05..aa53c7d34 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1050,7 +1050,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() ul_num = h.list_all_upload_statuses()[0].get_counter() d = self.GET("/status/up-{}".format(ul_num), return_response=True) - def _upload_status(result): + def _upload_status_check(result): body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1059,7 +1059,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self, soup, u"title", u"Tahoe-LAFS - File Upload Status" ) - d.addCallback(_upload_status) + d.addCallback(_upload_status_check) return d def test_status_down_subpath(self): @@ -1067,7 +1067,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() dl_num = h.list_all_download_statuses()[0].get_counter() d = self.GET("/status/down-{}".format(dl_num), return_response=True) - def _download_status(result): + def _download_status_check(result): body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1076,7 +1076,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self, soup, u"title", u"Tahoe-LAFS - File Download Status" ) - d.addCallback(_download_status) + d.addCallback(_download_status_check) return d def test_status_mapupdate_subpath(self): @@ -1084,7 +1084,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() mu_num = h.list_all_mapupdate_statuses()[0].get_counter() d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True) - def _mapupdate_status(result): + def _mapupdate_status_check(result): body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1093,7 +1093,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self, soup, u"title", u"Tahoe-LAFS - Mutable File Servermap Update Status" ) - d.addCallback(_mapupdate_status) + d.addCallback(_mapupdate_status_check) return d def test_status_publish_subpath(self): @@ -1101,7 +1101,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() pub_num = h.list_all_publish_statuses()[0].get_counter() d = self.GET("/status/publish-{}".format(pub_num), return_response=True) - def _mapupdate_status(result): + def _publish_status_check(result): body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1110,7 +1110,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self, soup, u"title", u"Tahoe-LAFS - Mutable File Publish Status" ) - d.addCallback(_mapupdate_status) + d.addCallback(_publish_status_check) return d def test_status_retrieve_subpath(self): @@ -1118,7 +1118,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() ret_num = h.list_all_retrieve_statuses()[0].get_counter() d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True) - def _mapupdate_status(result): + def _retrieve_status_check(result): body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1127,7 +1127,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self, soup, u"title", u"Tahoe-LAFS - Mutable File Retrieve Status" ) - d.addCallback(_mapupdate_status) + d.addCallback(_retrieve_status_check) return d def test_status_numbers(self): From 73c40e8d2aabe8f6b7dde6aa1ef20be542a50349 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 26 Jun 2020 07:32:53 -0400 Subject: [PATCH 0456/1054] Minimally add py36 to CI --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 44b583baa..259fb199c 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = {py27,pypy27}{-coverage,} +envlist = {py27,pypy27,py36}{-coverage,} minversion = 2.4 [testenv] From 2eab253b0f6b909ef7ba54ff10c48fb5e58c7eda Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 26 Jun 2020 07:50:42 -0400 Subject: [PATCH 0457/1054] Change python_requires in setup.py --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 047d08030..0742a7ce8 100644 --- a/setup.py +++ b/setup.py @@ -353,7 +353,9 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - python_requires="<3.0", + # We support Python 2.7, and we're working on support for 3.6 (the + # highest version that PyPy currently supports). + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <3.7", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From fec1f73bfadaea5ab0cfff685ba13923856e5242 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 1 Jul 2020 07:18:33 -0400 Subject: [PATCH 0458/1054] Ignore eliot.log, some testing artifact --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ee3b02b8f..caf0f2e87 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ zope.interface-*.egg /docs/_build/ /coverage.xml /.hypothesis/ +/eliot.log # This is the plaintext of the private environment needed for some CircleCI # operations. It's never supposed to be checked in. From 329bfe05f942d08184bad3a303d049c82de4459a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 1 Jul 2020 07:49:23 -0400 Subject: [PATCH 0459/1054] Make a crucial relative import explicit --- src/allmydata/version_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index aa42c2eee..7092a422b 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -297,7 +297,7 @@ def _get_platform(): def _get_package_versions_and_locations(): import warnings - from _auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \ + from ._auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \ runtime_warning_messages, warning_imports, ignorable def package_dir(srcfile): From 5a2ee5387a888c60f5c167edb35d48814408f060 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 1 Jul 2020 07:59:39 -0400 Subject: [PATCH 0460/1054] Bump txi2p to unreleased version that supports Py3 The plan is to keep i2p support around, in the hopes that upstream libs (txi2p, foolscap) are ported to Python 3 by the time we _really_ need them. --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0742a7ce8..f7f7b14f5 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,9 @@ tor_requires = [ ] i2p_requires = [ - # See the comment in tor_requires. - "txi2p >= 0.3.2", + # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. + # Also see the comment in tor_requires. + "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From 076c73d22f2179ab01f9087e10294b685f63790e Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 2 Jul 2020 07:35:31 -0400 Subject: [PATCH 0461/1054] Turn on py36 at Travis Drops the old py35 job, per @exarkun: https://github.com/tahoe-lafs/tahoe-lafs/pull/732#discussion_r449615599 Also bumps Ubuntu at Travis so we can get 3.8 eventually. I first went to 3.8 before dropping back to 3.6 as our initial target. Trusty on Travis does include 3.6, but since we want 3.8 "pretty soon," and the OS bump ended up being tricky (see below), let's go ahead and keep the OS bump. Xenial (16.04) is the current default at Travis, and it does have 3.8 available: https://docs.travis-ci.com/user/languages/python/#python-versions The tricky bug is that different versions of virtualenv have different seeding algorithms: https://discuss.python.org/t/-/4146). CI puts us several layers deep in virtualenv-ception and I didn't fully unravel the whole thing, but starting with a modern virtualenv seems to work around the issue. --- .travis.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 490e37b7d..dec435604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python cache: pip -dist: trusty +dist: xenial before_cache: - rm -f $HOME/.cache/pip/log/debug.log git: @@ -16,19 +16,15 @@ install: - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then export PATH=$HOME/Library/Python/2.7/bin:$PATH; fi - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py; fi - pip list - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then pip install --user --upgrade codecov tox setuptools; fi - - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then pip install --upgrade codecov tox setuptools; fi + - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then pip install --user --upgrade codecov tox setuptools virtualenv; fi + - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then pip install --upgrade codecov tox setuptools virtualenv; fi - echo $PATH; which python; which pip; which tox - python misc/build_helpers/show-tool-versions.py script: - | set -eo pipefail - if [ "${T}" = "py35" ]; then - python3 -m compileall -f -x tahoe-depgraph.py . - else - tox -e ${T} - fi + tox -e ${T} # To verify that the resultant PyInstaller-generated binary executes # cleanly (i.e., that it terminates with an exit code of 0 and isn't # failing due to import/packaging-related errors, etc.). @@ -69,9 +65,8 @@ matrix: python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 language: generic # "python" is not available on OS-X - # this is a "lint" job that checks for python3 compatibility - os: linux - python: '3.5' - env: T=py35 + python: '3.6' + env: T=py36 fast_finish: true From b47b4a468b46880d275f1da97fdb3657e2cd7900 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Jul 2020 07:31:43 -0400 Subject: [PATCH 0462/1054] Prevent Nevow from blocking us Per https://github.com/tahoe-lafs/tahoe-lafs/pull/732#issuecomment-653059972 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 259fb199c..af45f5da8 100644 --- a/tox.ini +++ b/tox.ini @@ -45,8 +45,8 @@ usedevelop = False # tests. extras = test commands = - tahoe --version trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} + tahoe --version [testenv:integration] setenv = From d25c8b1a25c8f3b9a117ac04c46c0e098668867d Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Jul 2020 17:35:48 -0400 Subject: [PATCH 0463/1054] Start ratcheting up passing tests under Python 3 --- .gitignore | 1 + misc/python3/passing | 13 ++ misc/python3/ratchet.py | 411 ++++++++++++++++++++++++++++++++++++++++ misc/python3/ratchet.sh | 31 +++ setup.py | 1 + tox.ini | 5 + 6 files changed, 462 insertions(+) create mode 100644 misc/python3/passing create mode 100755 misc/python3/ratchet.py create mode 100755 misc/python3/ratchet.sh diff --git a/.gitignore b/.gitignore index caf0f2e87..b33c1aab8 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ zope.interface-*.egg /coverage.xml /.hypothesis/ /eliot.log +/misc/python3/results.xml # This is the plaintext of the private environment needed for some CircleCI # operations. It's never supposed to be checked in. diff --git a/misc/python3/passing b/misc/python3/passing new file mode 100644 index 000000000..e659c95b6 --- /dev/null +++ b/misc/python3/passing @@ -0,0 +1,13 @@ +allmydata.test.mutable.test_exceptions.Exceptions.test_repr +allmydata.test.test_deferredutil.DeferredUtilTests.test_failure +allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results +allmydata.test.test_deferredutil.DeferredUtilTests.test_success +allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls +allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_observer.Observer.test_lazy_oneshot +allmydata.test.test_observer.Observer.test_observerlist +allmydata.test.test_observer.Observer.test_oneshot +allmydata.test.test_observer.Observer.test_oneshot_fireagain +allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting +allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct +allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist diff --git a/misc/python3/ratchet.py b/misc/python3/ratchet.py new file mode 100755 index 000000000..136fdcf4c --- /dev/null +++ b/misc/python3/ratchet.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +'''Ratchet up passing tests, or ratchet down failing tests. + +Usage: + + ratchet.py <"up" or "down"> + +This script helps when you expect a large test suite to fail spectactularly in +some environment, and you want to gradually improve the situation with minimal +impact to forward development of the same codebase for other environments. The +initial and primary usecase is porting from Python 2 to Python 3. + +The idea is to emit JUnit XML from your test runner, and then invoke ratchet.py +to consume this XML output and operate on a so-called "tracking" file. When +ratcheting up passing tests, the tracking file will contain a list of tests, +one per line, that passed. When ratching down, the tracking file contains a +list of failing tests. On each subsequent run, ratchet.py will compare the +prior results in the tracking file with the new results in the XML, and will +report on both welcome and unwelcome changes. It will modify the tracking file +in the case of welcome changes, and therein lies the ratcheting. + +The exit codes are: + + 0 - no changes observed + 1 - changes observed, whether welcome or unwelcome + 2 - invocation error + +Be sure to call as `./ratchet.py` and not `python ratchet.py` if you want to +see our exit code instead of Python's. + +If does not exist, you'll get a FileNotFoundError: + + >>> _test('up', None, None) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + FileNotFoundError: ... + +If does not exist, that's fine: + + >>> _test('up', '1', None) + Some tests not required to pass did: + c0.t + Conveniently, they have been added to `` for you. Perhaps commit that? + Eep! 0 test(s) were required to pass, but instead 1 did. 🐭 + +Same if you're ratcheting down: + + >>> _test('down', '1', None) + All and only tests expected to fail did. 💃 + +If the test run has the same output as last time, it's all good: + + >>> _test('up', '01001110', '01001110') + All and only tests required to pass did. 💃 + + >>> _test('down', '01001110', '10110001') + All and only tests expected to fail did. 💃 + +If there's a welcome change, that's noted: + + >>> _test('up', '0101', '0100') + Some tests not required to pass did: + c3.t + Conveniently, they have been added to `` for you. Perhaps commit that? + Eep! 1 test(s) were required to pass, but instead 2 did. 🐭 + + >>> _test('down', '0011', '1110') + Some tests expected to fail didn't: + c2.t + Conveniently, they have been removed from `` for you. Perhaps commit that? + Eep! 3 test(s) were expected to fail, but instead 2 did. 🐭 + +And if there is an unwelcome change, that is noted as well: + + >>> _test('up', '1101', '1111') + Some tests required to pass didn't: + c2.t + Eep! 4 test(s) were required to pass, but instead 3 did. 🐭 + + >>> _test('down', '0000', '1101') + Some tests not expected to fail did: + c2.t + Eep! 3 test(s) were expected to fail, but instead 4 did. 🐭 + +And if there are both welcome and unwelcome changes, they are both noted: + + >>> _test('up', '1101', '1011') + Some tests not required to pass did: + c1.t + Conveniently, they have been added to `` for you. Perhaps commit that? + Some tests required to pass didn't: + c2.t + Eep! 3 test(s) were required to pass, but instead 3 did. 🐭 + + >>> _test('down', '0100', '1100') + Some tests not expected to fail did: + c2.t + c3.t + Some tests expected to fail didn't: + c1.t + Conveniently, they have been removed from `` for you. Perhaps commit that? + Eep! 2 test(s) were expected to fail, but instead 3 did. 🐭 + + +To test ratchet.py itself: + + python3 -m doctest ratchet.py + +''' +from __future__ import absolute_import, division, print_function, unicode_literals + +import io +import os +import re +import sys +import tempfile +import xml.etree.ElementTree as Etree + + +class JUnitXMLFile(object): + '''Represent a file containing test results in JUnit XML format. + + >>> eg = _mktemp_junitxml('0100111') + >>> results = JUnitXMLFile(eg.name).parse() + >>> results.failed + ['c0.t', 'c2.t', 'c3.t'] + >>> results.passed + ['c1.t', 'c4.t', 'c5.t', 'c6.t'] + + ''' + + def __init__(self, filepath): + self.filepath = filepath + self.failed = [] + self.failed_aggregates = {} + self.stderr_output = [] + self.passed = [] + self._tree = None + + def parse(self): + if self._tree: + raise RuntimeError('already parsed') + self._tree = Etree.parse(self.filepath) + for testcase in self._tree.findall('testcase'): + self.process_testcase(testcase) + return self + + def process_testcase(self, case): + key = self.case_key(case) + + # look at children but throw away stderr output + nonpassing = [c for c in case if not c.tag == 'system-err'] + n = len(nonpassing) + if n > 1: + raise RuntimeError(f'multiple results for {key}: {nonpassing}') + elif n == 1: + result = nonpassing.pop() + self.failed.append(key) + message = result.get('message') + self.failed_aggregates.setdefault(message, []).append(key) + else: + self.passed.append(key) + + @staticmethod + def case_key(case): + return f'{case.get("classname")}.{case.get("name")}' + + def report(self, details=False): + for k, v in sorted( + self.failed_aggregates.items(), + key = lambda i: len(i[1]), + reverse=True): + print(f'# {k}') + for t in v: + print(f' - {t}') + + +def load_previous_results(txt): + try: + previous_results = open(txt).read() + except FileNotFoundError: + previous_results = '' + parsed = set() + for line in previous_results.splitlines(): + if not line or line.startswith('#'): + continue + parsed.add(line) + return parsed + + +def print_tests(tests): + for test in sorted(tests): + print(' ', test) + + +def ratchet_up_passing(tracking_path, tests): + try: + old = set(open(tracking_path, 'r')) + except FileNotFoundError: + old = set() + new = set(t + '\n' for t in tests) + merged = sorted(old | new) + open(tracking_path, 'w+').writelines(merged) + + +def ratchet_down_failing(tracking_path, tests): + new = set(t + '\n' for t in tests) + open(tracking_path, 'w+').writelines(sorted(new)) + + +def main(direction, junitxml_path, tracking_path): + '''Takes a string indicating which direction to ratchet, "up" or "down," + and two paths, one to test-runner output in JUnit XML format, the other to + a file tracking test results (one test case dotted name per line). Walk the + former looking for the latter, and react appropriately. + + >>> inp = _mktemp_junitxml('0100111') + >>> out = _mktemp_tracking('0000000') + >>> _test_main('up', inp.name, out.name) + Some tests not required to pass did: + c1.t + c4.t + c5.t + c6.t + Conveniently, they have been added to `` for you. Perhaps commit that? + Eep! 0 test(s) were required to pass, but instead 4 did. 🐭 + + ''' + + results = JUnitXMLFile(junitxml_path).parse() + + if tracking_path == '...': + results.report() + return + + previous = load_previous_results(tracking_path) + current = set(results.passed if direction == 'up' else results.failed) + + subjunctive = {'up': 'required to pass', 'down': 'expected to fail'}[direction] + ratchet = None + + too_many = current - previous + if too_many: + print(f'Some tests not {subjunctive} did:') + print_tests(too_many) + if direction == 'up': + # Too many passing tests is good -- let's do more of those! + ratchet_up_passing(tracking_path, current) + print(f'Conveniently, they have been added to `{tracking_path}` for you. Perhaps commit that?') + + not_enough = previous - current + if not_enough: + print(f'Some tests {subjunctive} didn\'t:') + print_tests(not_enough) + if direction == 'down': + # Not enough failing tests is good -- let's do more of those! + ratchet_down_failing(tracking_path, current) + print(f'Conveniently, they have been removed from `{tracking_path}` for you. Perhaps commit that?') + + if too_many or not_enough: + print(f'Eep! {len(previous)} test(s) were {subjunctive}, but instead {len(current)} did. 🐭') + return 1 + + print(f'All and only tests {subjunctive} did. 💃') + return 0 + + +# When called as an executable ... + +if __name__ == '__main__': + try: + direction, junitxml_path, tracking_path = sys.argv[1:4] + if direction not in ('up', 'down'): + raise ValueError + except ValueError: + doc = '\n'.join(__doc__.splitlines()[:6]) + doc = re.sub(' ratchet.py', f' {sys.argv[0]}', doc) + print(doc, file=sys.stderr) + exit_code = 2 + else: + exit_code = main(direction, junitxml_path, tracking_path) + sys.exit(exit_code) + + +# Helpers for when called under doctest ... + +def _test(*a): + return _test_main(*_mk(*a)) + + +def _test_main(direction, junitxml, tracking): + '''Takes a string 'up' or 'down' and paths to (or open file objects for) + the JUnit XML and tracking files to use for this test run. Captures and + emits stdout (slightly modified) for inspection via doctest.''' + junitxml_path = junitxml.name if hasattr(junitxml, 'name') else junitxml + tracking_path = tracking.name if hasattr(tracking, 'name') else tracking + + old_stdout = sys.stdout + sys.stdout = io.StringIO() + try: + main(direction, junitxml_path, tracking_path) + finally: + sys.stdout.seek(0) + out = sys.stdout.read() + out = re.sub('`.*?`', '``', out).strip() + sys.stdout = old_stdout + print(out) + + +class _PotentialFile(object): + '''Represent a file that we are able to create but which doesn't exist yet, + and which, if we create it, will be automatically torn down when the test + run is over.''' + + def __init__(self, filename): + self.d = tempfile.TemporaryDirectory() + self.name = os.path.join(self.d.name, filename) + + +def _mk(direction, spec_junitxml, spec_tracking): + '''Takes a string 'up' or 'down' and two bit strings specifying the state + of the JUnit XML results file and the tracking file to set up for this test + case. Returns the direction (unharmed) and two file-ish objects. + + If a spec string is None the corresponding return value will be a + _PotentialFile object, which has a .name attribute (like a true file + object) that points to a file that does not exist, but could. + + The reason not to simply return the path in all cases is that the file + objects are actually temporary file objects that destroy the underlying + file when they go out of scope, and we want to keep the underlying file + around until the end of the test run.''' + + if None not in(spec_junitxml, spec_tracking): + if len(spec_junitxml) != len(spec_tracking): + raise ValueError('if both given, must be the same length: `{spec_junitxml}` and `{spec_tracking}`') + if spec_junitxml is None: + junitxml_fp = _PotentialFile('results.xml') + else: + junitxml_fp = _mktemp_junitxml(spec_junitxml) + if spec_tracking is None: + tracking_fp = _PotentialFile('tracking') + else: + tracking_fp = _mktemp_tracking(spec_tracking) + return direction, junitxml_fp, tracking_fp + + +def _mktemp_junitxml(spec): + '''Test helper to generate a raw JUnit XML file. + + >>> fp = _mktemp_junitxml('00101') + >>> open(fp.name).read()[:11] + '' + + ''' + fp = tempfile.NamedTemporaryFile() + fp.write(b'') + + passed = '''\ + +''' + failed = '''\ + +Traceback (most recent call last): + File "/foo/bar/baz/buz.py", line 1, in <module> +NameError: name 'heck' is not defined + + +''' + + i = 0 + for c in spec: + if c == '0': + out = failed + elif c == '1': + out = passed + else: + raise ValueError(f'bad c: `{c}`') + fp.write(out.format(i=i).encode('utf8')) + i += 1 + + fp.write(b'') + fp.flush() + return fp + + +def _mktemp_tracking(spec): + '''Test helper to prefabricate a tracking file. + + >>> fp = _mktemp_tracking('01101') + >>> print(open(fp.name).read()[:-1]) + c1.t + c2.t + c4.t + + ''' + fp = tempfile.NamedTemporaryFile() + + i = 0 + for c in spec: + if c == '0': + pass + elif c == '1': + fp.write(f'c{i}.t\n'.encode('utf8')) + else: + raise ValueError(f'bad c: `{c}`') + i += 1 + + fp.flush() + return fp diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh new file mode 100755 index 000000000..00acd60c3 --- /dev/null +++ b/misc/python3/ratchet.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +base="$(dirname $0)" +tracking="passing" + +# trial outputs some things that are only git-ignored in the root, so don't cd quite yet ... +set +e +trial --reporter subunitv2 allmydata | subunit2junitxml > "$base/results.xml" +set -e + +# Okay, now we're clear. +cd "$base" + +# Make sure ratchet.py itself is clean. +python3 -m doctest ratchet.py + +# Now see about Tahoe-LAFS ... +set +e +# P.S. Don't invoke as `python ratchet.py ...` because then Python swallows the +# exit code. +./ratchet.py up results.xml "$tracking" +code=$? +set -e + +# Emit a diff of the tracking file, to aid in the situation where changes are +# not discovered until CI (where TERM might `dumb`). +if [ $TERM = 'dumb' ]; then + export TERM=ansi +fi +git diff "$tracking" + +exit $code diff --git a/setup.py b/setup.py index f7f7b14f5..3a7d8ef8c 100644 --- a/setup.py +++ b/setup.py @@ -385,6 +385,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "fixtures", "beautifulsoup4", "html5lib", + "junitxml", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, diff --git a/tox.ini b/tox.ini index af45f5da8..ab467abd0 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,11 @@ commands = trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} tahoe --version +[testenv:py36] +# git inside of ratchet.sh needs $HOME. +passenv = HOME +commands = {toxinidir}/misc/python3/ratchet.sh + [testenv:integration] setenv = COVERAGE_PROCESS_START=.coveragerc From 8e8a215b83cdcc1a58a4a5086132b442530e8c23 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 3 Jul 2020 17:52:10 -0400 Subject: [PATCH 0464/1054] Is this it? towncrier just wants empty files? --- newsfragments/3325.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3325.other diff --git a/newsfragments/3325.other b/newsfragments/3325.other new file mode 100644 index 000000000..e69de29bb From e2d6b353e82ccbc95c869cf8bcfa436f5c089b1e Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 14 Jul 2020 06:42:45 -0400 Subject: [PATCH 0465/1054] Tighten up condition in depgraph.sh I ran into a circumstance where it appears that there were changes other than to the two files we care about, leading to an empty commit and a CI failure: https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/320/workflows/7d045f5f-1536-4cfa-b232-42837d4c9334/jobs/22127/steps --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index d5ad33bf7..a1ef21142 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -10,7 +10,7 @@ cd tahoe-depgraph # Generate the maybe-changed data. python "${TAHOE}"/misc/python3/tahoe-depgraph.py "${TAHOE}" -if git diff-index --quiet HEAD; then +if git diff-index --quiet HEAD tahoe-deps.json tahoe-ported.json; then echo "Declining to commit without any changes." exit 0 fi From 54e952ceda381a0bef6ee27bc9457b83f10a7537 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 14 Jul 2020 09:01:31 -0400 Subject: [PATCH 0466/1054] Test for 404 responses from status page subpaths --- src/allmydata/test/web/test_web.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index aa53c7d34..f83c44b2f 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1045,6 +1045,27 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.GET, "/status/nodash") + def test_status_path_404(self): + # The test suite (hopefully!) would not have done any setup + # for a very large number of statuses at this point, so these + # all should return 404. + d = self.GET("/status/up-9999999") + d.addBoth(self.should404, "test_status_path_error (up)") + + d = self.GET("/status/down-9999999") + d.addBoth(self.should404, "test_status_path_error (down)") + + d = self.GET("/status/mapupdate-9999999") + d.addBoth(self.should404, "test_status_path_error (mapupdate)") + + d = self.GET("/status/publish-9999999") + d.addBoth(self.should404, "test_status_path_error (publish)") + + d = self.GET("/status/retrieve-9999999") + d.addBoth(self.should404, "test_status_path_error (retrieve)") + + return d + def test_status_up_subpath(self): # See that "GET /status/up-0" works. h = self.s.get_history() From f378edd9d833a9f047ae73ce526da18d1ded5fc5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 10:20:51 -0400 Subject: [PATCH 0467/1054] Fix missing import. --- src/allmydata/test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index b26731d7c..a8b74a3a9 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -14,7 +14,7 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from allmydata.util import idlib, mathutil, hashutil +from allmydata.util import base32, idlib, mathutil, hashutil from allmydata.util import fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil From e34a01d5478b3fa86a283ab43bf5f39810d3b7ef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 10:28:22 -0400 Subject: [PATCH 0468/1054] A couple more asserts. --- src/allmydata/test/test_base62.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 0538b9eb6..4243da7f3 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -20,6 +20,9 @@ class T(unittest.TestCase): ascii=base62.b2a(bs) bs2=base62.a2b(ascii) assert bs2 == bs, "bs2: %s:%s, bs: %s:%s, ascii: %s:%s" % (len(bs2), repr(bs2), len(bs), repr(bs), len(ascii), repr(ascii)) + self.assertIsInstance(ascii, bytes) + self.assertIsInstance(bs, bytes) + self.assertIsInstance(bs2, bytes) @given(input_bytes=st.binary(max_size=100)) def test_roundtrip(self, input_bytes): From b1e732fb9a9b1054ea7d9bbd2a6303cec5d5dfce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 10:34:53 -0400 Subject: [PATCH 0469/1054] Port test_base62 to Python 3. --- src/allmydata/test/test_base62.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 4243da7f3..50d0a37ea 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -1,7 +1,22 @@ -import random, unittest +""" +Tests for allmydata.util.base62. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr +import random, unittest + from hypothesis import ( strategies as st, given, @@ -65,7 +80,7 @@ class T(unittest.TestCase): b=ord(bs[-1]) b = b >> (8 - (lib%8)) b = b << (8 - (lib%8)) - bs = bs[:-1] + chr(b) + bs = bs[:-1] + byteschr(b) asl = base62.b2a_l(bs, lib) assert len(asl) == base62.num_chars_that_this_many_octets_encode_to(numos) # the size of the base-62 encoding must be just right bs2l = base62.a2b_l(asl, lib) From fc73b7c7931911fa38ef462441be66db787b9d34 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 10:51:46 -0400 Subject: [PATCH 0470/1054] News fragment. --- newsfragments/3343.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3343.other diff --git a/newsfragments/3343.other b/newsfragments/3343.other new file mode 100644 index 000000000..e69de29bb From 9c428be0e2cb1a75b43716579fdd66955f105e0a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 10:51:31 -0400 Subject: [PATCH 0471/1054] Port base62 to Python 3. --- src/allmydata/test/test_base62.py | 4 ++-- src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/base62.py | 40 ++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 50d0a37ea..eb3d6f55d 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -25,7 +25,7 @@ from hypothesis import ( from allmydata.util import base62, mathutil def insecurerandstr(n): - return b''.join(map(byteschr, map(random.randrange, [0]*n, [256]*n))) + return bytes(list(map(random.randrange, [0]*n, [256]*n))) class T(unittest.TestCase): def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): @@ -77,7 +77,7 @@ class T(unittest.TestCase): bs = insecurerandstr(numos) # zero-out unused least-sig bits if lib%8: - b=ord(bs[-1]) + b = ord(bs[-1:]) b = b >> (8 - (lib%8)) b = b << (8 - (lib%8)) bs = bs[:-1] + byteschr(b) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa4e5f386..0b85265cf 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -16,6 +16,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.assertutil", + "allmydata.util.base62", "allmydata.util.deferredutil", "allmydata.util.humanreadable", "allmydata.util.mathutil", @@ -25,6 +26,7 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.test_base62", "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_python3", diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index c25ce9688..bd56e4074 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -1,16 +1,24 @@ -# from the Python Standard Library -import string +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from past.builtins import chr as byteschr from allmydata.util.mathutil import log_ceil, log_floor -chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" -BASE62CHAR = '[' + chars + ']' +BASE62CHAR = b'[' + chars + b']' -vals = ''.join([chr(i) for i in range(62)]) -c2vtranstable = string.maketrans(chars, vals) -v2ctranstable = string.maketrans(vals, chars) -identitytranstable = string.maketrans(chars, chars) +vals = b''.join([byteschr(i) for i in range(62)]) +c2vtranstable = bytes.maketrans(chars, vals) +v2ctranstable = bytes.maketrans(vals, chars) +identitytranstable = bytes.maketrans(chars, chars) def b2a(os): """ @@ -47,7 +55,9 @@ def b2a_l(os, lengthinbits): @return the contents of os in base-62 encoded form """ - os = [ord(o) for o in reversed(os)] # treat os as big-endian -- and we want to process the least-significant o first + # We call bytes() again for Python 2, to ensure literals are using future's + # Python 3-compatible variant. + os = [o for o in reversed(bytes(os))] # treat os as big-endian -- and we want to process the least-significant o first value = 0 numvalues = 1 # the number of possible values that value could be @@ -62,7 +72,7 @@ def b2a_l(os, lengthinbits): value //= 62 numvalues //= 62 - return string.translate(''.join([chr(c) for c in reversed(chars)]), v2ctranstable) # make it big-endian + return bytes.translate(bytes([c for c in reversed(chars)]), v2ctranstable) # make it big-endian def num_octets_that_encode_to_this_many_chars(numcs): return log_floor(62**numcs, 256) @@ -91,7 +101,9 @@ def a2b_l(cs, lengthinbits): @return the data encoded in cs """ - cs = [ord(c) for c in reversed(string.translate(cs, c2vtranstable))] # treat cs as big-endian -- and we want to process the least-significant c first + # We call bytes() again for Python 2, to ensure literals are using future's + # Python 3-compatible variant. + cs = [c for c in reversed(bytes(bytes.translate(cs, c2vtranstable)))] # treat cs as big-endian -- and we want to process the least-significant c first value = 0 numvalues = 1 # the number of possible values that value could be @@ -101,10 +113,10 @@ def a2b_l(cs, lengthinbits): numvalues *= 62 numvalues = 2**lengthinbits - bytes = [] + result_bytes = [] while numvalues > 1: - bytes.append(value % 256) + result_bytes.append(value % 256) value //= 256 numvalues //= 256 - return ''.join([chr(b) for b in reversed(bytes)]) # make it big-endian + return bytes([b for b in reversed(result_bytes)]) # make it big-endian From ab3737d6860ca5e103d7b6c2f4b7bb68b370283f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 11:28:51 -0400 Subject: [PATCH 0472/1054] Note this module has been ported to Python 3. --- src/allmydata/util/base62.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index bd56e4074..382e00bb2 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -1,3 +1,8 @@ +""" +Base62 encoding. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function From f3db64ad741630fabdead46d9c2de7fee392e397 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 11:29:09 -0400 Subject: [PATCH 0473/1054] Port to Python 3. --- src/allmydata/test/test_base32.py | 10 ++++++++++ src/allmydata/util/_python3.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index de256c7fa..b10d638b8 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.base32. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import base64 diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 0b85265cf..9bd43b015 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -26,6 +26,8 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.test_assertutil", + "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", From 3abb9e9db43a7ee0f1d09b31b713f726c2d25e37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 11:45:05 -0400 Subject: [PATCH 0474/1054] Rip out a whole bunch of code, either deleting or replacing with stdlib's base32 implementation. --- src/allmydata/util/base32.py | 169 +++-------------------------------- 1 file changed, 12 insertions(+), 157 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 32fcdbc38..7172ed168 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -1,10 +1,15 @@ +""" +Base32 encoding. +""" + +import base64 + # from the Python Standard Library import six import string from allmydata.util.assertutil import precondition -z_base_32_alphabet = "ybndrfg8ejkmcpqxot1uwisza345h769" # Zooko's choice, rationale in "DESIGN" doc rfc3548_alphabet = "abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... chars = rfc3548_alphabet @@ -52,84 +57,12 @@ def b2a(os): @return the contents of os in base-32 encoded form """ - return _b2a_l(os, len(os)*8) + return base64.b32encode(os).rstrip(b"=").lower() def b2a_or_none(os): if os is not None: return b2a(os) -def _b2a_l(os, lengthinbits): - """ - @param os the data to be encoded (a string) - @param lengthinbits the number of bits of data in os to be encoded - - b2a_l() will generate a base-32 encoded string big enough to encode lengthinbits bits. So for - example if os is 2 bytes long and lengthinbits is 15, then b2a_l() will generate a 3-character- - long base-32 encoded string (since 3 quintets is sufficient to encode 15 bits). If os is - 2 bytes long and lengthinbits is 16 (or None), then b2a_l() will generate a 4-character string. - Note that b2a_l() does not mask off unused least-significant bits, so for example if os is - 2 bytes long and lengthinbits is 15, then you must ensure that the unused least-significant bit - of os is a zero bit or you will get the wrong result. This precondition is tested by assertions - if assertions are enabled. - - Warning: if you generate a base-32 encoded string with b2a_l(), and then someone else tries to - decode it by calling a2b() instead of a2b_l(), then they will (probably) get a different - string than the one you encoded! So only use b2a_l() when you are sure that the encoding and - decoding sides know exactly which lengthinbits to use. If you do not have a way for the - encoder and the decoder to agree upon the lengthinbits, then it is best to use b2a() and - a2b(). The only drawback to using b2a() over b2a_l() is that when you have a number of - bits to encode that is not a multiple of 8, b2a() can sometimes generate a base-32 encoded - string that is one or two characters longer than necessary. - - @return the contents of os in base-32 encoded form - """ - precondition(isinstance(lengthinbits, (int, long,)), "lengthinbits is required to be an integer.", lengthinbits=lengthinbits) - precondition((lengthinbits+7)/8 == len(os), "lengthinbits is required to specify a number of bits storable in exactly len(os) octets.", lengthinbits=lengthinbits, lenos=len(os)) - - os = map(ord, os) - - numquintets = (lengthinbits+4)/5 - numoctetsofdata = (lengthinbits+7)/8 - # print "numoctetsofdata: %s, len(os): %s, lengthinbits: %s, numquintets: %s" % (numoctetsofdata, len(os), lengthinbits, numquintets,) - # strip trailing octets that won't be used - del os[numoctetsofdata:] - # zero out any unused bits in the final octet - if lengthinbits % 8 != 0: - os[-1] = os[-1] >> (8-(lengthinbits % 8)) - os[-1] = os[-1] << (8-(lengthinbits % 8)) - # append zero octets for padding if needed - numoctetsneeded = (numquintets*5+7)/8 + 1 - os.extend([0]*(numoctetsneeded-len(os))) - - quintets = [] - cutoff = 256 - num = os[0] - i = 0 - while len(quintets) < numquintets: - i = i + 1 - assert len(os) > i, "len(os): %s, i: %s, len(quintets): %s, numquintets: %s, lengthinbits: %s, numoctetsofdata: %s, numoctetsneeded: %s, os: %s" % (len(os), i, len(quintets), numquintets, lengthinbits, numoctetsofdata, numoctetsneeded, os,) - num = num * 256 - num = num + os[i] - if cutoff == 1: - cutoff = 256 - continue - cutoff = cutoff * 8 - quintet = num / cutoff - quintets.append(quintet) - num = num - (quintet * cutoff) - - cutoff = cutoff / 32 - quintet = num / cutoff - quintets.append(quintet) - num = num - (quintet * cutoff) - - if len(quintets) > numquintets: - assert len(quintets) == (numquintets+1), "len(quintets): %s, numquintets: %s, quintets: %s" % (len(quintets), numquintets, quintets,) - quintets = quintets[:numquintets] - res = string.translate(string.join(map(chr, quintets), ''), v2ctranstable) - assert could_be_base32_encoded_l(res, lengthinbits), "lengthinbits: %s, res: %s" % (lengthinbits, res,) - return res - # b2a() uses the minimal number of quintets sufficient to encode the binary # input. It just so happens that the relation is like this (everything is # modulo 40 bits). @@ -167,36 +100,12 @@ def init_s8(): return tuple(s8) s8 = init_s8() -# A somewhat fast way to determine whether a given string *could* be base-32 encoded data, given a -# lengthinbits. -# The boolean value of s5[lengthinbits%5][ord(s[-1])], where s is the possibly base-32 encoded -# string tells whether the final character is reasonable. -def init_s5(): - s5 = [] - add_check_array(get_trailing_chars_without_lsbs(0), s5) - for lenmod5 in [1,2,3,4]: - add_check_array(get_trailing_chars_without_lsbs(5-lenmod5), s5) - return tuple(s5) -s5 = init_s5() - def could_be_base32_encoded(s, s8=s8, tr=string.translate, identitytranstable=identitytranstable, chars=chars): precondition(isinstance(s, six.binary_type), s) if s == '': return True return s8[len(s)%8][ord(s[-1])] and not tr(s, identitytranstable, chars) -def could_be_base32_encoded_l(s, lengthinbits, s5=s5, tr=string.translate, identitytranstable=identitytranstable, chars=chars): - precondition(isinstance(s, six.binary_type), s) - if s == '': - return True - assert lengthinbits%5 < len(s5), lengthinbits - assert ord(s[-1]) < s5[lengthinbits%5] - return (((lengthinbits+4)/5) == len(s)) and s5[lengthinbits%5][ord(s[-1])] and not string.translate(s, identitytranstable, chars) - -def num_octets_that_encode_to_this_many_quintets(numqs): - # Here is a computation that conveniently expresses this: - return (numqs*5+3)/8 - def a2b(cs): """ @param cs the base-32 encoded data (a string) @@ -204,65 +113,11 @@ def a2b(cs): precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) precondition(isinstance(cs, six.binary_type), cs) - return _a2b_l(cs, num_octets_that_encode_to_this_many_quintets(len(cs))*8) - -def _a2b_l(cs, lengthinbits): - """ - @param lengthinbits the number of bits of data in encoded into cs - - a2b_l() will return a result big enough to hold lengthinbits bits. So for example if cs is - 4 characters long (encoding at least 15 and up to 20 bits) and lengthinbits is 16, then a2b_l() - will return a string of length 2 (since 2 bytes is sufficient to store 16 bits). If cs is 4 - characters long and lengthinbits is 20, then a2b_l() will return a string of length 3 (since - 3 bytes is sufficient to store 20 bits). Note that b2a_l() does not mask off unused least- - significant bits, so for example if cs is 4 characters long and lengthinbits is 17, then you - must ensure that all three of the unused least-significant bits of cs are zero bits or you will - get the wrong result. This precondition is tested by assertions if assertions are enabled. - (Generally you just require the encoder to ensure this consistency property between the least - significant zero bits and value of lengthinbits, and reject strings that have a length-in-bits - which isn't a multiple of 8 and yet don't have trailing zero bits, as improperly encoded.) - - Please see the warning in the docstring of b2a_l() regarding the use of b2a() versus b2a_l(). - - @return the data encoded in cs - """ - precondition(could_be_base32_encoded_l(cs, lengthinbits), "cs is required to be possibly base32 encoded data.", cs=cs, lengthinbits=lengthinbits) - precondition(isinstance(cs, six.binary_type), cs) - if cs == '': - return '' - - qs = map(ord, string.translate(cs, c2vtranstable)) - - numoctets = (lengthinbits+7)/8 - numquintetsofdata = (lengthinbits+4)/5 - # strip trailing quintets that won't be used - del qs[numquintetsofdata:] - # zero out any unused bits in the final quintet - if lengthinbits % 5 != 0: - qs[-1] = qs[-1] >> (5-(lengthinbits % 5)) - qs[-1] = qs[-1] << (5-(lengthinbits % 5)) - # append zero quintets for padding if needed - numquintetsneeded = (numoctets*8+4)/5 - qs.extend([0]*(numquintetsneeded-len(qs))) - - octets = [] - pos = 2048 - num = qs[0] * pos - i = 1 - while len(octets) < numoctets: - while pos > 256: - pos = pos / 32 - num = num + (qs[i] * pos) - i = i + 1 - octet = num / 256 - octets.append(octet) - num = num - (octet * 256) - num = num * 256 - pos = pos * 256 - assert len(octets) == numoctets, "len(octets): %s, numoctets: %s, octets: %s" % (len(octets), numoctets, octets,) - res = ''.join(map(chr, octets)) - precondition(_b2a_l(res, lengthinbits) == cs, "cs is required to be the canonical base-32 encoding of some data.", b2a(res), res=res, cs=cs) - return res + cs = cs.upper() + # Add padding back, to make Python's base64 module happy: + while (len(cs) * 5) % 8 != 0: + cs += b"=" + return base64.b32decode(cs) __all__ = ["b2a", "a2b", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"] From b5db85afbe343755dd9557e6918de53c726f05e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 11:53:40 -0400 Subject: [PATCH 0475/1054] Initial, manual steps of Python 3 port. --- src/allmydata/util/base32.py | 45 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 7172ed168..3221ce262 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -2,21 +2,20 @@ Base32 encoding. """ -import base64 +from builtins import bytes +from past.builtins import chr as byteschr -# from the Python Standard Library -import six -import string +import base64 from allmydata.util.assertutil import precondition -rfc3548_alphabet = "abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... +rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... chars = rfc3548_alphabet -vals = ''.join(map(chr, range(32))) -c2vtranstable = string.maketrans(chars, vals) -v2ctranstable = string.maketrans(vals, chars) -identitytranstable = string.maketrans('', '') +vals = b''.join(map(chr, range(32))) +c2vtranstable = bytes.maketrans(chars, vals) +v2ctranstable = bytes.maketrans(vals, chars) +identitytranstable = bytes.maketrans(b'', b'') def _get_trailing_chars_without_lsbs(N, d): """ @@ -38,18 +37,18 @@ def get_trailing_chars_without_lsbs(N): if N == 0: return chars d = {} - return ''.join(_get_trailing_chars_without_lsbs(N, d=d)) + return b''.join(_get_trailing_chars_without_lsbs(N, d=d)) -BASE32CHAR = '['+get_trailing_chars_without_lsbs(0)+']' -BASE32CHAR_4bits = '['+get_trailing_chars_without_lsbs(1)+']' -BASE32CHAR_3bits = '['+get_trailing_chars_without_lsbs(2)+']' -BASE32CHAR_2bits = '['+get_trailing_chars_without_lsbs(3)+']' -BASE32CHAR_1bits = '['+get_trailing_chars_without_lsbs(4)+']' +BASE32CHAR =b'['+get_trailing_chars_without_lsbs(0)+b']' +BASE32CHAR_4bits =b'['+get_trailing_chars_without_lsbs(1)+b']' +BASE32CHAR_3bits =b'['+get_trailing_chars_without_lsbs(2)+b']' +BASE32CHAR_2bits =b'['+get_trailing_chars_without_lsbs(3)+b']' +BASE32CHAR_1bits =b'['+get_trailing_chars_without_lsbs(4)+b']' BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits -BASE32STR_2bytes = BASE32CHAR+'{3}'+BASE32CHAR_1bits -BASE32STR_3bytes = BASE32CHAR+'{4}'+BASE32CHAR_4bits -BASE32STR_4bytes = BASE32CHAR+'{6}'+BASE32CHAR_2bits -BASE32STR_anybytes = '((?:%s{8})*' % (BASE32CHAR,) + "(?:|%s|%s|%s|%s))" % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) +BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits +BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits +BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits +BASE32STR_anybytes =b'((?:%s{8})*' % (BASE32CHAR,) + b"(?:|%s|%s|%s|%s))" % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) def b2a(os): """ @@ -100,9 +99,9 @@ def init_s8(): return tuple(s8) s8 = init_s8() -def could_be_base32_encoded(s, s8=s8, tr=string.translate, identitytranstable=identitytranstable, chars=chars): - precondition(isinstance(s, six.binary_type), s) - if s == '': +def could_be_base32_encoded(s, s8=s8, tr=bytes.translate, identitytranstable=identitytranstable, chars=chars): + precondition(isinstance(s, bytes), s) + if s == b'': return True return s8[len(s)%8][ord(s[-1])] and not tr(s, identitytranstable, chars) @@ -111,7 +110,7 @@ def a2b(cs): @param cs the base-32 encoded data (a string) """ precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) - precondition(isinstance(cs, six.binary_type), cs) + precondition(isinstance(cs, bytes), cs) cs = cs.upper() # Add padding back, to make Python's base64 module happy: From e0adb3b37db98148e45500bce0395ca5dc82d472 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:35:46 -0400 Subject: [PATCH 0476/1054] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/base32.py | 32 ++++++++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 9bd43b015..76b1d2bbc 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -16,6 +16,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.assertutil", + "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", "allmydata.util.humanreadable", diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 3221ce262..806b04df9 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -1,18 +1,25 @@ """ Base32 encoding. -""" -from builtins import bytes -from past.builtins import chr as byteschr +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import base64 from allmydata.util.assertutil import precondition -rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... +rfc3548_alphabet = bytes(b"abcdefghijklmnopqrstuvwxyz234567") # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... chars = rfc3548_alphabet -vals = b''.join(map(chr, range(32))) +vals = bytes(range(32)) c2vtranstable = bytes.maketrans(chars, vals) v2ctranstable = bytes.maketrans(vals, chars) identitytranstable = bytes.maketrans(b'', b'') @@ -26,9 +33,9 @@ def _get_trailing_chars_without_lsbs(N, d): s.extend(_get_trailing_chars_without_lsbs(N+1, d=d)) i = 0 while i < len(chars): - if not d.has_key(i): + if i not in d: d[i] = None - s.append(chars[i]) + s.append(chars[i:i+1]) i = i + 2**N return s @@ -76,7 +83,7 @@ NUM_OS_TO_NUM_QS=(0, 2, 4, 5, 7,) NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4) NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,) -NUM_QS_TO_NUM_BITS=tuple(map(lambda x: x*8, NUM_QS_TO_NUM_OS)) +NUM_QS_TO_NUM_BITS=tuple([x*8 for x in NUM_QS_TO_NUM_OS]) # A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the # original data had 8K bits for a positive integer K. @@ -84,8 +91,8 @@ NUM_QS_TO_NUM_BITS=tuple(map(lambda x: x*8, NUM_QS_TO_NUM_OS)) # tells whether the final character is reasonable. def add_check_array(cs, sfmap): checka=[0] * 256 - for c in cs: - checka[ord(c)] = 1 + for c in bytes(cs): + checka[c] = 1 sfmap.append(tuple(checka)) def init_s8(): @@ -95,7 +102,7 @@ def init_s8(): if NUM_QS_LEGIT[lenmod8]: add_check_array(get_trailing_chars_without_lsbs(4-(NUM_QS_TO_NUM_BITS[lenmod8]%5)), s8) else: - add_check_array('', s8) + add_check_array(b'', s8) return tuple(s8) s8 = init_s8() @@ -103,7 +110,8 @@ def could_be_base32_encoded(s, s8=s8, tr=bytes.translate, identitytranstable=ide precondition(isinstance(s, bytes), s) if s == b'': return True - return s8[len(s)%8][ord(s[-1])] and not tr(s, identitytranstable, chars) + s = bytes(s) # On Python 2, make sure we're using modern bytes + return s8[len(s)%8][s[-1]] and not tr(s, identitytranstable, chars) def a2b(cs): """ From 148d9a77aed717f073f584923d17033472a173fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:43:52 -0400 Subject: [PATCH 0477/1054] News fragment. --- newsfragments/3342.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3342.other diff --git a/newsfragments/3342.other b/newsfragments/3342.other new file mode 100644 index 000000000..e69de29bb From 27d8eb47fcbc344f42b2b8120e2da8737a2942db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:44:00 -0400 Subject: [PATCH 0478/1054] Test improvements. --- src/allmydata/test/test_base32.py | 4 +++- src/allmydata/test/test_base62.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index b10d638b8..47636a175 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -32,7 +32,9 @@ class Base32(unittest.TestCase): self.failUnlessEqual(encoded, x) self.assertIsInstance(encoded, bytes) self.assertTrue(base32.could_be_base32_encoded(encoded)) - self.assertEqual(base32.a2b(encoded), input_bytes) + decoded = base32.a2b(encoded) + self.assertEqual(decoded, input_bytes) + self.assertIsInstance(decoded, bytes) def test_b2a(self): self.failUnlessEqual(base32.b2a(b"\x12\x34"), b"ci2a") diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index eb3d6f55d..edcd9ee22 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -31,17 +31,17 @@ class T(unittest.TestCase): def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): assert base62.num_octets_that_encode_to_this_many_chars(chars) == octets, "%s != %s <- %s" % (octets, base62.num_octets_that_encode_to_this_many_chars(chars), chars) - def _test_ende(self, bs): + def _test_roundtrip(self, bs): ascii=base62.b2a(bs) bs2=base62.a2b(ascii) - assert bs2 == bs, "bs2: %s:%s, bs: %s:%s, ascii: %s:%s" % (len(bs2), repr(bs2), len(bs), repr(bs), len(ascii), repr(ascii)) + self.assertEqual(bs2, bs) self.assertIsInstance(ascii, bytes) self.assertIsInstance(bs, bytes) self.assertIsInstance(bs2, bytes) @given(input_bytes=st.binary(max_size=100)) def test_roundtrip(self, input_bytes): - self._test_ende(input_bytes) + self._test_roundtrip(input_bytes) def test_num_octets_that_encode_to_this_many_chars(self): return self._test_num_octets_that_encode_to_this_many_chars(2, 1) @@ -50,25 +50,25 @@ class T(unittest.TestCase): return self._test_num_octets_that_encode_to_this_many_chars(6, 4) def test_ende_0x00(self): - return self._test_ende(b'\x00') + return self._test_roundtrip(b'\x00') def test_ende_0x01(self): - return self._test_ende(b'\x01') + return self._test_roundtrip(b'\x01') def test_ende_0x0100(self): - return self._test_ende(b'\x01\x00') + return self._test_roundtrip(b'\x01\x00') def test_ende_0x000000(self): - return self._test_ende(b'\x00\x00\x00') + return self._test_roundtrip(b'\x00\x00\x00') def test_ende_0x010000(self): - return self._test_ende(b'\x01\x00\x00') + return self._test_roundtrip(b'\x01\x00\x00') def test_ende_randstr(self): - return self._test_ende(insecurerandstr(2**4)) + return self._test_roundtrip(insecurerandstr(2**4)) def test_ende_longrandstr(self): - return self._test_ende(insecurerandstr(random.randrange(0, 2**10))) + return self._test_roundtrip(insecurerandstr(random.randrange(0, 2**10))) def test_odd_sizes(self): for j in range(2**6): From a81e859b8292e4c730d562d40a4fd08c992e7d52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:44:56 -0400 Subject: [PATCH 0479/1054] Fix flake error. --- src/allmydata/test/test_base62.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index edcd9ee22..6f3ef8293 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -32,12 +32,12 @@ class T(unittest.TestCase): assert base62.num_octets_that_encode_to_this_many_chars(chars) == octets, "%s != %s <- %s" % (octets, base62.num_octets_that_encode_to_this_many_chars(chars), chars) def _test_roundtrip(self, bs): - ascii=base62.b2a(bs) - bs2=base62.a2b(ascii) - self.assertEqual(bs2, bs) - self.assertIsInstance(ascii, bytes) + encoded = base62.b2a(bs) + decoded = base62.a2b(encoded) + self.assertEqual(decoded, bs) + self.assertIsInstance(encoded, bytes) self.assertIsInstance(bs, bytes) - self.assertIsInstance(bs2, bytes) + self.assertIsInstance(decoded, bytes) @given(input_bytes=st.binary(max_size=100)) def test_roundtrip(self, input_bytes): From 21a9ffef7a9fd8eecc57c774a669eb8a5eaae1a6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:46:12 -0400 Subject: [PATCH 0480/1054] Not strings. --- src/allmydata/util/base32.py | 6 +++--- src/allmydata/util/base62.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 806b04df9..1d1f3ce4b 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -59,9 +59,9 @@ BASE32STR_anybytes =b'((?:%s{8})*' % (BASE32CHAR,) + b"(?:|%s|%s|%s|%s))" % (BAS def b2a(os): """ - @param os the data to be encoded (a string) + @param os the data to be encoded (as bytes) - @return the contents of os in base-32 encoded form + @return the contents of os in base-32 encoded form, as bytes """ return base64.b32encode(os).rstrip(b"=").lower() @@ -115,7 +115,7 @@ def could_be_base32_encoded(s, s8=s8, tr=bytes.translate, identitytranstable=ide def a2b(cs): """ - @param cs the base-32 encoded data (a string) + @param cs the base-32 encoded data (as bytes) """ precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) precondition(isinstance(cs, bytes), cs) diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 382e00bb2..7f349caaa 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -27,9 +27,9 @@ identitytranstable = bytes.maketrans(chars, chars) def b2a(os): """ - @param os the data to be encoded (a string) + @param os the data to be encoded (as bytes) - @return the contents of os in base-62 encoded form + @return the contents of os in base-62 encoded form, as bytes """ cs = b2a_l(os, len(os)*8) assert num_octets_that_encode_to_this_many_chars(len(cs)) == len(os), "%s != %s, numchars: %s" % (num_octets_that_encode_to_this_many_chars(len(cs)), len(os), len(cs)) @@ -37,7 +37,7 @@ def b2a(os): def b2a_l(os, lengthinbits): """ - @param os the data to be encoded (a string) + @param os the data to be encoded (as bytes) @param lengthinbits the number of bits of data in os to be encoded b2a_l() will generate a base-62 encoded string big enough to encode @@ -58,7 +58,7 @@ def b2a_l(os, lengthinbits): bits to encode that is not a multiple of 8, b2a() can sometimes generate a base-62 encoded string that is one or two characters longer than necessary. - @return the contents of os in base-62 encoded form + @return the contents of os in base-62 encoded form, as bytes """ # We call bytes() again for Python 2, to ensure literals are using future's # Python 3-compatible variant. @@ -104,7 +104,7 @@ def a2b_l(cs, lengthinbits): Please see the warning in the docstring of b2a_l() regarding the use of b2a() versus b2a_l(). - @return the data encoded in cs + @return the data encoded in cs, as bytes """ # We call bytes() again for Python 2, to ensure literals are using future's # Python 3-compatible variant. From 5fd01523cfe29b10d44d9b24d60f02959c0c6940 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:53:13 -0400 Subject: [PATCH 0481/1054] This module is not a thing. --- src/allmydata/util/_python3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 76b1d2bbc..5b416cfbc 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -27,7 +27,6 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ - "allmydata.test.test_assertutil", "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", From 3688291df2d92b95284f4716554972c18e9a32b6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Jul 2020 13:53:26 -0400 Subject: [PATCH 0482/1054] Handy way to run Python 3 tests, until the ratchet is going. --- src/allmydata/util/_python3.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5b416cfbc..ea4e0e702 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -33,3 +33,8 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", "allmydata.test.test_python3", ] + + +if __name__ == '__main__': + from subprocess import check_call + check_call(["trial"] + PORTED_TEST_MODULES) From 3f887f92799904c1003d0e2584f9b97048c58aaf Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 14 Jul 2020 20:11:58 -0400 Subject: [PATCH 0483/1054] Respond to review - Use the subunitv2-file reporter to allow for nicer error handling - Undo the depgraph.sh change, see if it really matters - Rename passing -> ratchet-passing to distinguish from ported modules tracker - Misc. documentation and ratchet.sh improvements --- .gitignore | 1 + misc/python3/depgraph.sh | 2 +- misc/python3/{passing => ratchet-passing} | 0 misc/python3/ratchet.py | 4 +--- misc/python3/ratchet.sh | 24 ++++++++++++++--------- setup.py | 1 + 6 files changed, 19 insertions(+), 13 deletions(-) rename misc/python3/{passing => ratchet-passing} (100%) diff --git a/.gitignore b/.gitignore index b33c1aab8..8191c173b 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ zope.interface-*.egg /.hypothesis/ /eliot.log /misc/python3/results.xml +/misc/python3/results.subunit2 # This is the plaintext of the private environment needed for some CircleCI # operations. It's never supposed to be checked in. diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index a1ef21142..d5ad33bf7 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -10,7 +10,7 @@ cd tahoe-depgraph # Generate the maybe-changed data. python "${TAHOE}"/misc/python3/tahoe-depgraph.py "${TAHOE}" -if git diff-index --quiet HEAD tahoe-deps.json tahoe-ported.json; then +if git diff-index --quiet HEAD; then echo "Declining to commit without any changes." exit 0 fi diff --git a/misc/python3/passing b/misc/python3/ratchet-passing similarity index 100% rename from misc/python3/passing rename to misc/python3/ratchet-passing diff --git a/misc/python3/ratchet.py b/misc/python3/ratchet.py index 136fdcf4c..cb672cf67 100755 --- a/misc/python3/ratchet.py +++ b/misc/python3/ratchet.py @@ -26,9 +26,6 @@ The exit codes are: 1 - changes observed, whether welcome or unwelcome 2 - invocation error -Be sure to call as `./ratchet.py` and not `python ratchet.py` if you want to -see our exit code instead of Python's. - If does not exist, you'll get a FileNotFoundError: >>> _test('up', None, None) # doctest: +ELLIPSIS @@ -231,6 +228,7 @@ def main(direction, junitxml_path, tracking_path): results = JUnitXMLFile(junitxml_path).parse() if tracking_path == '...': + # Shortcut to aid in debugging XML parsing issues. results.report() return diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 00acd60c3..a6a2c53c3 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash -base="$(dirname $0)" -tracking="passing" +set -euxo pipefail +tracking_filename="ratchet-passing" -# trial outputs some things that are only git-ignored in the root, so don't cd quite yet ... +# Start somewhere predictable. +cd "$(dirname $0)" +base=$(pwd) + +# Actually, though, trial outputs some things that are only gitignored in the project root. +cd "../.." + +# Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial --reporter subunitv2 allmydata | subunit2junitxml > "$base/results.xml" +SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata +subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" set -e # Okay, now we're clear. @@ -13,11 +21,9 @@ cd "$base" # Make sure ratchet.py itself is clean. python3 -m doctest ratchet.py -# Now see about Tahoe-LAFS ... +# Now see about Tahoe-LAFS (also expected to fail) ... set +e -# P.S. Don't invoke as `python ratchet.py ...` because then Python swallows the -# exit code. -./ratchet.py up results.xml "$tracking" +python3 ratchet.py up results.xml "$tracking_filename" code=$? set -e @@ -26,6 +32,6 @@ set -e if [ $TERM = 'dumb' ]; then export TERM=ansi fi -git diff "$tracking" +git diff "$tracking_filename" exit $code diff --git a/setup.py b/setup.py index 3a7d8ef8c..f22ca0f4c 100644 --- a/setup.py +++ b/setup.py @@ -142,6 +142,7 @@ tor_requires = [ i2p_requires = [ # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. + # URL lookups are in PEP-508 (via https://stackoverflow.com/a/54794506). # Also see the comment in tor_requires. "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p", ] From b4b820ed7bba5af9beb835313120c8d2f69e1630 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 14 Jul 2020 20:47:23 -0400 Subject: [PATCH 0484/1054] Use a helper method to get monitor counter --- src/allmydata/web/check_results.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index a29ddb188..256274141 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -483,8 +483,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def problems_p(self, req, tag): - c = self.monitor.get_status().get_counters() - if c["count-objects-unhealthy"]: + if self._get_monitor_counter("count-objects-unhealthy"): return tag return "" From ef26d66e79883a2fa848b1fac6b399c2aa61f240 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 14 Jul 2020 20:50:12 -0400 Subject: [PATCH 0485/1054] Use correct twisted web template directives --- src/allmydata/web/deep-check-results.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 572e44ff1..abe9bfd63 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -25,8 +25,8 @@

                        Files/Directories That Had Problems:

                          -
                        • -
                        • None
                          • +
                        • +
                        • None
                        From 99ce9ff198961b48486a7428850de1690ecafd8d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 14 Jul 2020 21:42:28 -0400 Subject: [PATCH 0486/1054] Correct argument to SlotsSequenceElement --- src/allmydata/web/check_results.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 256274141..c8165a67b 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -501,8 +501,11 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): if summary: summary_text = ": " + summary summary_text += " [SI: %s]" % cr.get_storage_index_string() - problem = self._join_pathstring(path), ":", self._html(summary_text) - problems.append(problem) + problems.append({ + # TODO: result of _join_pathstring() seems to be + # weird. Might as well use "path" as the key here? + self._join_pathstring(path): self._html(summary_text) + }) return SlotsSequenceElement(tag, problems) From 6322404b1e7735423e0992355896756c64f901a6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 14 Jul 2020 21:44:41 -0400 Subject: [PATCH 0487/1054] Update "deep check results renderer" test --- src/allmydata/test/test_checker.py | 74 +++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 4c1313793..e043a5675 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -16,6 +16,10 @@ from twisted.web.template import flattenString from allmydata import check_results, uri from allmydata import uri as tahoe_uri +from allmydata.interfaces import ( + ICheckResults, + IDeepCheckResults, +) from allmydata.util import base32 from allmydata.web import check_results as web_check_results from allmydata.storage_client import StorageFarmBroker, NativeStorageServer @@ -49,6 +53,61 @@ class TestRequest(object, Request): self.args = args or {} self.fields = fields or {} self.prepath = [b""] + self.postpath = [b""] + + +@implementer(ICheckResults) +class FakeResults(object): + + def __init__(self): + # TODO: figure out a good enough fake SI. + self._storage_index = "fake-si" + + def get_storage_index(self): + return self._storage_index + + def get_storage_index_string(self): + return base32.b2a(self._storage_index) + + def is_healthy(self): + return False + + def is_recoverable(self): + return False + + def get_summary(self): + return "A fake summary" + + +@implementer(IDeepCheckResults) +class FakeDeepCheckResults(object): + + def get_root_storage_index_string(self): + # TODO: figure out a good enough fake root SI. + return "fake-root-si" + + def get_counters(self): + return { + "count-objects-checked": 4, + "count-objects-healthy": 1, + "count-objects-unhealthy": 1, + "count-objects-unrecoverable": 1, + "count-corrupt-shares": 1, + } + + def get_all_results(self): + return { + # TODO: fill this, perhaps with one each of healthy, + # unhealthy, unrecoverable, corrupt. + "fake-result-path": FakeResults() + } + + def get_corrupt_shares(self): + return [ + # TODO: fill this with: + # (IServer, storage_index, sharenum) + ] + class WebResultsRendering(unittest.TestCase): @@ -345,10 +404,11 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessEqual(j["storage-index"], "") _got_lit_results(d) + def test_deep_check_renderer(self): monitor = Monitor() - status = check_results.DeepCheckResults("") - monitor.set_status(status) + result = FakeDeepCheckResults() + monitor.set_status(result) elem = web_check_results.DeepCheckResultsRendererElement(monitor) @@ -369,27 +429,27 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Checked: 0" + u"Objects Checked: 4" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Healthy: 0" + u"Objects Healthy: 1" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy: 0" + u"Objects Unhealthy: 1" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unrecoverable: 0" + u"Objects Unrecoverable: 1" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Corrupt Shares: 0" + u"Corrupt Shares: 1" ) def test_deep_check_and_repair_renderer(self): From 1ed722494ca871bbb95e020b4d70b12cb19f46b6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 10:35:56 -0400 Subject: [PATCH 0488/1054] Add a FakeServer class for testing --- src/allmydata/test/test_checker.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index e043a5675..f975a9a5a 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -17,6 +17,7 @@ from twisted.web.template import flattenString from allmydata import check_results, uri from allmydata import uri as tahoe_uri from allmydata.interfaces import ( + IServer, ICheckResults, IDeepCheckResults, ) @@ -56,6 +57,19 @@ class TestRequest(object, Request): self.postpath = [b""] +@implementer(IServer) +class FakeServer(object): + + def get_name(self): + return "fake name" + + def get_longname(self): + return "fake longname" + + def get_nickname(self): + return "fake nickname" + + @implementer(ICheckResults) class FakeResults(object): @@ -78,7 +92,11 @@ class FakeResults(object): def get_summary(self): return "A fake summary" + def get_corrupt_shares(self): + return (FakeServer(), None, None) + +# TODO: maybe use check_results.DeepCheckResults? @implementer(IDeepCheckResults) class FakeDeepCheckResults(object): @@ -106,6 +124,7 @@ class FakeDeepCheckResults(object): return [ # TODO: fill this with: # (IServer, storage_index, sharenum) + FakeResults().get_corrupt_shares(), ] From 9ed91eabb37331639ed67758a8f9f9f365486ad4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 10:58:05 -0400 Subject: [PATCH 0489/1054] Split out hashutil tests into their own module. --- src/allmydata/test/test_hashutil.py | 113 ++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 107 +------------------------- 2 files changed, 114 insertions(+), 106 deletions(-) create mode 100644 src/allmydata/test/test_hashutil.py diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py new file mode 100644 index 000000000..1df057fb4 --- /dev/null +++ b/src/allmydata/test/test_hashutil.py @@ -0,0 +1,113 @@ +""" +Tests for allmydata.util.hashutil. +""" + +from twisted.trial import unittest + +from allmydata.util import hashutil, base32 + + +class HashUtilTests(unittest.TestCase): + + def test_random_key(self): + k = hashutil.random_key() + self.failUnlessEqual(len(k), hashutil.KEYLEN) + + def test_sha256d(self): + h1 = hashutil.tagged_hash("tag1", "value") + h2 = hashutil.tagged_hasher("tag1") + h2.update("value") + h2a = h2.digest() + h2b = h2.digest() + self.failUnlessEqual(h1, h2a) + self.failUnlessEqual(h2a, h2b) + + def test_sha256d_truncated(self): + h1 = hashutil.tagged_hash("tag1", "value", 16) + h2 = hashutil.tagged_hasher("tag1", 16) + h2.update("value") + h2 = h2.digest() + self.failUnlessEqual(len(h1), 16) + self.failUnlessEqual(len(h2), 16) + self.failUnlessEqual(h1, h2) + + def test_chk(self): + h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret") + h2 = hashutil.convergence_hasher(3, 10, 1000, "secret") + h2.update("data") + h2 = h2.digest() + self.failUnlessEqual(h1, h2) + + def test_hashers(self): + h1 = hashutil.block_hash("foo") + h2 = hashutil.block_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + h1 = hashutil.uri_extension_hash("foo") + h2 = hashutil.uri_extension_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + h1 = hashutil.plaintext_hash("foo") + h2 = hashutil.plaintext_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + h1 = hashutil.crypttext_hash("foo") + h2 = hashutil.crypttext_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + h1 = hashutil.crypttext_segment_hash("foo") + h2 = hashutil.crypttext_segment_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + h1 = hashutil.plaintext_segment_hash("foo") + h2 = hashutil.plaintext_segment_hasher() + h2.update("foo") + self.failUnlessEqual(h1, h2.digest()) + + def test_timing_safe_compare(self): + self.failUnless(hashutil.timing_safe_compare("a", "a")) + self.failUnless(hashutil.timing_safe_compare("ab", "ab")) + self.failIf(hashutil.timing_safe_compare("a", "b")) + self.failIf(hashutil.timing_safe_compare("a", "aa")) + + def _testknown(self, hashf, expected_a, *args): + got = hashf(*args) + got_a = base32.b2a(got) + self.failUnlessEqual(got_a, expected_a) + + def test_known_answers(self): + # assert backwards compatibility + self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "") + self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "") + self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "") + self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "") + self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "") + self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "") + self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "") + self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge") + self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "") + self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "") + self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si") + self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si") + self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20) + self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20) + self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "") + self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk") + self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "") + self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "") + self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20) + self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "") + self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "") + self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk") + self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "") + + self._testknown(hashutil.permute_server_hash, + "kb4354zeeurpo3ze5e275wzbynm6hlap", # b32(expected) + "SI", # peer selection index == storage_index + base32.a2b("u33m4y7klhz3bypswqkozwetvabelhxt"), # seed + ) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index f744feb7e..bdea8c430 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -14,7 +14,7 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from allmydata.util import base32, idlib, mathutil, hashutil +from allmydata.util import idlib, mathutil from allmydata.util import fileutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil @@ -582,111 +582,6 @@ class PollMixinTests(unittest.TestCase): return d -class HashUtilTests(unittest.TestCase): - - def test_random_key(self): - k = hashutil.random_key() - self.failUnlessEqual(len(k), hashutil.KEYLEN) - - def test_sha256d(self): - h1 = hashutil.tagged_hash("tag1", "value") - h2 = hashutil.tagged_hasher("tag1") - h2.update("value") - h2a = h2.digest() - h2b = h2.digest() - self.failUnlessEqual(h1, h2a) - self.failUnlessEqual(h2a, h2b) - - def test_sha256d_truncated(self): - h1 = hashutil.tagged_hash("tag1", "value", 16) - h2 = hashutil.tagged_hasher("tag1", 16) - h2.update("value") - h2 = h2.digest() - self.failUnlessEqual(len(h1), 16) - self.failUnlessEqual(len(h2), 16) - self.failUnlessEqual(h1, h2) - - def test_chk(self): - h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret") - h2 = hashutil.convergence_hasher(3, 10, 1000, "secret") - h2.update("data") - h2 = h2.digest() - self.failUnlessEqual(h1, h2) - - def test_hashers(self): - h1 = hashutil.block_hash("foo") - h2 = hashutil.block_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - h1 = hashutil.uri_extension_hash("foo") - h2 = hashutil.uri_extension_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - h1 = hashutil.plaintext_hash("foo") - h2 = hashutil.plaintext_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - h1 = hashutil.crypttext_hash("foo") - h2 = hashutil.crypttext_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - h1 = hashutil.crypttext_segment_hash("foo") - h2 = hashutil.crypttext_segment_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - h1 = hashutil.plaintext_segment_hash("foo") - h2 = hashutil.plaintext_segment_hasher() - h2.update("foo") - self.failUnlessEqual(h1, h2.digest()) - - def test_timing_safe_compare(self): - self.failUnless(hashutil.timing_safe_compare("a", "a")) - self.failUnless(hashutil.timing_safe_compare("ab", "ab")) - self.failIf(hashutil.timing_safe_compare("a", "b")) - self.failIf(hashutil.timing_safe_compare("a", "aa")) - - def _testknown(self, hashf, expected_a, *args): - got = hashf(*args) - got_a = base32.b2a(got) - self.failUnlessEqual(got_a, expected_a) - - def test_known_answers(self): - # assert backwards compatibility - self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "") - self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "") - self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "") - self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "") - self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "") - self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "") - self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "") - self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge") - self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "") - self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "") - self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si") - self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si") - self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20) - self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20) - self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "") - self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk") - self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "") - self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "") - self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20) - self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "") - self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "") - self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk") - self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "") - - self._testknown(hashutil.permute_server_hash, - "kb4354zeeurpo3ze5e275wzbynm6hlap", # b32(expected) - "SI", # peer selection index == storage_index - base32.a2b("u33m4y7klhz3bypswqkozwetvabelhxt"), # seed - ) - class Abbreviate(unittest.TestCase): def test_abbrev_time_1s(self): diff = timedelta(seconds=1) From 2b896740524cfd465e5b867084974cf363b03d7a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 10:59:14 -0400 Subject: [PATCH 0490/1054] Everything must be bytes. --- src/allmydata/test/test_hashutil.py | 102 +++++++++++++-------------- src/allmydata/test/test_netstring.py | 43 ++++++----- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 1df057fb4..73e558184 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -14,66 +14,66 @@ class HashUtilTests(unittest.TestCase): self.failUnlessEqual(len(k), hashutil.KEYLEN) def test_sha256d(self): - h1 = hashutil.tagged_hash("tag1", "value") - h2 = hashutil.tagged_hasher("tag1") - h2.update("value") + h1 = hashutil.tagged_hash(b"tag1", b"value") + h2 = hashutil.tagged_hasher(b"tag1") + h2.update(b"value") h2a = h2.digest() h2b = h2.digest() self.failUnlessEqual(h1, h2a) self.failUnlessEqual(h2a, h2b) def test_sha256d_truncated(self): - h1 = hashutil.tagged_hash("tag1", "value", 16) - h2 = hashutil.tagged_hasher("tag1", 16) - h2.update("value") + h1 = hashutil.tagged_hash(b"tag1", b"value", 16) + h2 = hashutil.tagged_hasher(b"tag1", 16) + h2.update(b"value") h2 = h2.digest() self.failUnlessEqual(len(h1), 16) self.failUnlessEqual(len(h2), 16) self.failUnlessEqual(h1, h2) def test_chk(self): - h1 = hashutil.convergence_hash(3, 10, 1000, "data", "secret") - h2 = hashutil.convergence_hasher(3, 10, 1000, "secret") - h2.update("data") + h1 = hashutil.convergence_hash(3, 10, 1000, b"data", b"secret") + h2 = hashutil.convergence_hasher(3, 10, 1000, b"secret") + h2.update(b"data") h2 = h2.digest() self.failUnlessEqual(h1, h2) def test_hashers(self): - h1 = hashutil.block_hash("foo") + h1 = hashutil.block_hash(b"foo") h2 = hashutil.block_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) - h1 = hashutil.uri_extension_hash("foo") + h1 = hashutil.uri_extension_hash(b"foo") h2 = hashutil.uri_extension_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) - h1 = hashutil.plaintext_hash("foo") + h1 = hashutil.plaintext_hash(b"foo") h2 = hashutil.plaintext_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) - h1 = hashutil.crypttext_hash("foo") + h1 = hashutil.crypttext_hash(b"foo") h2 = hashutil.crypttext_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) - h1 = hashutil.crypttext_segment_hash("foo") + h1 = hashutil.crypttext_segment_hash(b"foo") h2 = hashutil.crypttext_segment_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) - h1 = hashutil.plaintext_segment_hash("foo") + h1 = hashutil.plaintext_segment_hash(b"foo") h2 = hashutil.plaintext_segment_hasher() - h2.update("foo") + h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) def test_timing_safe_compare(self): - self.failUnless(hashutil.timing_safe_compare("a", "a")) - self.failUnless(hashutil.timing_safe_compare("ab", "ab")) - self.failIf(hashutil.timing_safe_compare("a", "b")) - self.failIf(hashutil.timing_safe_compare("a", "aa")) + self.failUnless(hashutil.timing_safe_compare(b"a", b"a")) + self.failUnless(hashutil.timing_safe_compare(b"ab", b"ab")) + self.failIf(hashutil.timing_safe_compare(b"a", b"b")) + self.failIf(hashutil.timing_safe_compare(b"a", b"aa")) def _testknown(self, hashf, expected_a, *args): got = hashf(*args) @@ -82,32 +82,32 @@ class HashUtilTests(unittest.TestCase): def test_known_answers(self): # assert backwards compatibility - self._testknown(hashutil.storage_index_hash, "qb5igbhcc5esa6lwqorsy7e6am", "") - self._testknown(hashutil.block_hash, "msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", "") - self._testknown(hashutil.uri_extension_hash, "wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", "") - self._testknown(hashutil.plaintext_hash, "5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", "") - self._testknown(hashutil.crypttext_hash, "itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", "") - self._testknown(hashutil.crypttext_segment_hash, "aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", "") - self._testknown(hashutil.plaintext_segment_hash, "4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", "") - self._testknown(hashutil.convergence_hash, "3mo6ni7xweplycin6nowynw2we", 3, 10, 100, "", "converge") - self._testknown(hashutil.my_renewal_secret_hash, "ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", "") - self._testknown(hashutil.my_cancel_secret_hash, "rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", "") - self._testknown(hashutil.file_renewal_secret_hash, "hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", "", "si") - self._testknown(hashutil.file_cancel_secret_hash, "bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", "", "si") - self._testknown(hashutil.bucket_renewal_secret_hash, "e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", "", "\x00"*20) - self._testknown(hashutil.bucket_cancel_secret_hash, "dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", "", "\x00"*20) - self._testknown(hashutil.hmac, "c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", "tag", "") - self._testknown(hashutil.mutable_rwcap_key_hash, "6rvn2iqrghii5n4jbbwwqqsnqu", "iv", "wk") - self._testknown(hashutil.ssk_writekey_hash, "ykpgmdbpgbb6yqz5oluw2q26ye", "") - self._testknown(hashutil.ssk_write_enabler_master_hash, "izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", "") - self._testknown(hashutil.ssk_write_enabler_hash, "fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", "wk", "\x00"*20) - self._testknown(hashutil.ssk_pubkey_fingerprint_hash, "3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", "") - self._testknown(hashutil.ssk_readkey_hash, "vugid4as6qbqgeq2xczvvcedai", "") - self._testknown(hashutil.ssk_readkey_data_hash, "73wsaldnvdzqaf7v4pzbr2ae5a", "iv", "rk") - self._testknown(hashutil.ssk_storage_index_hash, "j7icz6kigb6hxrej3tv4z7ayym", "") + self._testknown(hashutil.storage_index_hash, b"qb5igbhcc5esa6lwqorsy7e6am", b"") + self._testknown(hashutil.block_hash, b"msjr5bh4evuh7fa3zw7uovixfbvlnstr5b65mrerwfnvjxig2jvq", b"") + self._testknown(hashutil.uri_extension_hash, b"wthsu45q7zewac2mnivoaa4ulh5xvbzdmsbuyztq2a5fzxdrnkka", b"") + self._testknown(hashutil.plaintext_hash, b"5lz5hwz3qj3af7n6e3arblw7xzutvnd3p3fjsngqjcb7utf3x3da", b"") + self._testknown(hashutil.crypttext_hash, b"itdj6e4njtkoiavlrmxkvpreosscssklunhwtvxn6ggho4rkqwga", b"") + self._testknown(hashutil.crypttext_segment_hash, b"aovy5aa7jej6ym5ikgwyoi4pxawnoj3wtaludjz7e2nb5xijb7aa", b"") + self._testknown(hashutil.plaintext_segment_hash, b"4fdgf6qruaisyukhqcmoth4t3li6bkolbxvjy4awwcpprdtva7za", b"") + self._testknown(hashutil.convergence_hash, b"3mo6ni7xweplycin6nowynw2we", 3, 10, 100, b"", b"converge") + self._testknown(hashutil.my_renewal_secret_hash, b"ujhr5k5f7ypkp67jkpx6jl4p47pyta7hu5m527cpcgvkafsefm6q", b"") + self._testknown(hashutil.my_cancel_secret_hash, b"rjwzmafe2duixvqy6h47f5wfrokdziry6zhx4smew4cj6iocsfaa", b"") + self._testknown(hashutil.file_renewal_secret_hash, b"hzshk2kf33gzbd5n3a6eszkf6q6o6kixmnag25pniusyaulqjnia", b"", b"si") + self._testknown(hashutil.file_cancel_secret_hash, b"bfciwvr6w7wcavsngxzxsxxaszj72dej54n4tu2idzp6b74g255q", b"", b"si") + self._testknown(hashutil.bucket_renewal_secret_hash, b"e7imrzgzaoashsncacvy3oysdd2m5yvtooo4gmj4mjlopsazmvuq", b"", b"\x00"*20) + self._testknown(hashutil.bucket_cancel_secret_hash, b"dvdujeyxeirj6uux6g7xcf4lvesk632aulwkzjar7srildvtqwma", b"", b"\x00"*20) + self._testknown(hashutil.hmac, b"c54ypfi6pevb3nvo6ba42jtglpkry2kbdopqsi7dgrm4r7tw5sra", b"tag", b"") + self._testknown(hashutil.mutable_rwcap_key_hash, b"6rvn2iqrghii5n4jbbwwqqsnqu", b"iv", b"wk") + self._testknown(hashutil.ssk_writekey_hash, b"ykpgmdbpgbb6yqz5oluw2q26ye", b"") + self._testknown(hashutil.ssk_write_enabler_master_hash, b"izbfbfkoait4dummruol3gy2bnixrrrslgye6ycmkuyujnenzpia", b"") + self._testknown(hashutil.ssk_write_enabler_hash, b"fuu2dvx7g6gqu5x22vfhtyed7p4pd47y5hgxbqzgrlyvxoev62tq", b"wk", b"\x00"*20) + self._testknown(hashutil.ssk_pubkey_fingerprint_hash, b"3opzw4hhm2sgncjx224qmt5ipqgagn7h5zivnfzqycvgqgmgz35q", b"") + self._testknown(hashutil.ssk_readkey_hash, b"vugid4as6qbqgeq2xczvvcedai", b"") + self._testknown(hashutil.ssk_readkey_data_hash, b"73wsaldnvdzqaf7v4pzbr2ae5a", b"iv", b"rk") + self._testknown(hashutil.ssk_storage_index_hash, b"j7icz6kigb6hxrej3tv4z7ayym", b"") self._testknown(hashutil.permute_server_hash, - "kb4354zeeurpo3ze5e275wzbynm6hlap", # b32(expected) - "SI", # peer selection index == storage_index - base32.a2b("u33m4y7klhz3bypswqkozwetvabelhxt"), # seed + b"kb4354zeeurpo3ze5e275wzbynm6hlap", # b32(expected) + b"SI", # peer selection index == storage_index + base32.a2b(b"u33m4y7klhz3bypswqkozwetvabelhxt"), # seed ) diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index 8fecdc49f..339349aa4 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -1,36 +1,41 @@ +""" +Tests for allmydata.util.netstring. +""" from twisted.trial import unittest + from allmydata.util.netstring import netstring, split_netstring + class Netstring(unittest.TestCase): def test_split(self): - a = netstring("hello") + netstring("world") - self.failUnlessEqual(split_netstring(a, 2), (["hello", "world"], len(a))) - self.failUnlessEqual(split_netstring(a, 2, required_trailer=""), (["hello", "world"], len(a))) + a = netstring(b"hello") + netstring(b"world") + self.failUnlessEqual(split_netstring(a, 2), ([b"hello", b"world"], len(a))) + self.failUnlessEqual(split_netstring(a, 2, required_trailer=""), ([b"hello", b"world"], len(a))) self.failUnlessRaises(ValueError, split_netstring, a, 3) - self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, required_trailer="") - self.failUnlessEqual(split_netstring(a+" extra", 2), (["hello", "world"], len(a))) - self.failUnlessEqual(split_netstring(a+"++", 2, required_trailer="++"), - (["hello", "world"], len(a)+2)) + self.failUnlessRaises(ValueError, split_netstring, a+b" extra", 2, required_trailer="") + self.failUnlessEqual(split_netstring(a+b" extra", 2), ([b"hello", b"world"], len(a))) + self.failUnlessEqual(split_netstring(a+b"++", 2, required_trailer=b"++"), + ([b"hello", b"world"], len(a)+2)) self.failUnlessRaises(ValueError, - split_netstring, a+"+", 2, required_trailer="not") + split_netstring, a+b"+", 2, required_trailer=b"not") def test_extra(self): - a = netstring("hello") - self.failUnlessEqual(split_netstring(a, 1), (["hello"], len(a))) - b = netstring("hello") + "extra stuff" + a = netstring(b"hello") + self.failUnlessEqual(split_netstring(a, 1), ([b"hello"], len(a))) + b = netstring(b"hello") + b"extra stuff" self.failUnlessEqual(split_netstring(b, 1), - (["hello"], len(a))) + ([b"hello"], len(a))) def test_nested(self): - a = netstring("hello") + netstring("world") + "extra stuff" - b = netstring("a") + netstring("is") + netstring(a) + netstring(".") + a = netstring(b"hello") + netstring(b"world") + b"extra stuff" + b = netstring(b"a") + netstring(b"is") + netstring(a) + netstring(b".") (top, pos) = split_netstring(b, 4) self.failUnlessEqual(len(top), 4) - self.failUnlessEqual(top[0], "a") - self.failUnlessEqual(top[1], "is") + self.failUnlessEqual(top[0], b"a") + self.failUnlessEqual(top[1], b"is") self.failUnlessEqual(top[2], a) - self.failUnlessEqual(top[3], ".") - self.failUnlessRaises(ValueError, split_netstring, a, 2, required_trailer="") + self.failUnlessEqual(top[3], b".") + self.failUnlessRaises(ValueError, split_netstring, a, 2, required_trailer=b"") bottom = split_netstring(a, 2) - self.failUnlessEqual(bottom, (["hello", "world"], len(netstring("hello")+netstring("world")))) + self.failUnlessEqual(bottom, ([b"hello", b"world"], len(netstring(b"hello")+netstring(b"world")))) From d316ad3ebd69174def810953819db55a506d430d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:02:49 -0400 Subject: [PATCH 0491/1054] A more explicit test. --- src/allmydata/test/test_netstring.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index 339349aa4..e58c72e4e 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -8,6 +8,10 @@ from allmydata.util.netstring import netstring, split_netstring class Netstring(unittest.TestCase): + def test_encode(self): + """netstring() correctly encodes the given bytes.""" + self.assertEqual(netstring(b"abc"), b"3:abc,") + def test_split(self): a = netstring(b"hello") + netstring(b"world") self.failUnlessEqual(split_netstring(a, 2), ([b"hello", b"world"], len(a))) From 093125ca4306744654340fbc0ac7bb243f7adbf4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:05:23 -0400 Subject: [PATCH 0492/1054] Port to Python 3. --- src/allmydata/test/test_hashutil.py | 10 ++++++++++ src/allmydata/test/test_netstring.py | 10 ++++++++++ src/allmydata/util/_python3.py | 2 ++ 3 files changed, 22 insertions(+) diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 73e558184..299f362c1 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.hashutil. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index e58c72e4e..2dc85681c 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.netstring. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ea4e0e702..95e479ea4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -30,7 +30,9 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", + "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", + "allmydata.test.test_netstring", "allmydata.test.test_python3", ] From cfb11f7f4c06379ec906eb651f6414440cad1df2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:08:06 -0400 Subject: [PATCH 0493/1054] More explicit testing of types. --- src/allmydata/test/test_netstring.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index 2dc85681c..1293785db 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -20,10 +20,14 @@ from allmydata.util.netstring import netstring, split_netstring class Netstring(unittest.TestCase): def test_encode(self): """netstring() correctly encodes the given bytes.""" - self.assertEqual(netstring(b"abc"), b"3:abc,") + result = netstring(b"abc") + self.assertEqual(result, b"3:abc,") + self.assertIsInstance(result, bytes) def test_split(self): a = netstring(b"hello") + netstring(b"world") + for s in split_netstring(a, 2)[0]: + self.assertIsInstance(s, bytes) self.failUnlessEqual(split_netstring(a, 2), ([b"hello", b"world"], len(a))) self.failUnlessEqual(split_netstring(a, 2, required_trailer=""), ([b"hello", b"world"], len(a))) self.failUnlessRaises(ValueError, split_netstring, a, 3) From e3e29598e4e3573d9e9857cf7f96bdff085bc7ec Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:13:52 -0400 Subject: [PATCH 0494/1054] A couple of missing explicit byte strings. --- src/allmydata/test/test_netstring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index 1293785db..b00c99bca 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -29,9 +29,9 @@ class Netstring(unittest.TestCase): for s in split_netstring(a, 2)[0]: self.assertIsInstance(s, bytes) self.failUnlessEqual(split_netstring(a, 2), ([b"hello", b"world"], len(a))) - self.failUnlessEqual(split_netstring(a, 2, required_trailer=""), ([b"hello", b"world"], len(a))) + self.failUnlessEqual(split_netstring(a, 2, required_trailer=b""), ([b"hello", b"world"], len(a))) self.failUnlessRaises(ValueError, split_netstring, a, 3) - self.failUnlessRaises(ValueError, split_netstring, a+b" extra", 2, required_trailer="") + self.failUnlessRaises(ValueError, split_netstring, a+b" extra", 2, required_trailer=b"") self.failUnlessEqual(split_netstring(a+b" extra", 2), ([b"hello", b"world"], len(a))) self.failUnlessEqual(split_netstring(a+b"++", 2, required_trailer=b"++"), ([b"hello", b"world"], len(a)+2)) From 60a1a244230ada52ccd101796872ebd59a0e0e7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:13:59 -0400 Subject: [PATCH 0495/1054] Manual portion of port to Python 3. --- src/allmydata/util/netstring.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 73f15658c..71b487f5b 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -1,8 +1,8 @@ - +from past.builtins import long def netstring(s): - assert isinstance(s, str), s # no unicode here - return "%d:%s," % (len(s), s,) + assert isinstance(s, bytes), s # no unicode here + return b"%d:%s," % (len(s), s,) def split_netstring(data, numstrings, position=0, @@ -13,18 +13,19 @@ def split_netstring(data, numstrings, byte which was not consumed (the 'required_trailer', if any, counts as consumed). If 'required_trailer' is not None, throw ValueError if leftover data does not exactly equal 'required_trailer'.""" - + assert type(data) == bytes + assert type(required_trailer) in (type(None), bytes) assert type(position) in (int, long), (repr(position), type(position)) elements = [] assert numstrings >= 0 while position < len(data): - colon = data.index(":", position) + colon = data.index(b":", position) length = int(data[position:colon]) string = data[colon+1:colon+1+length] assert len(string) == length, (len(string), length) elements.append(string) position = colon+1+length - assert data[position] == ",", position + assert data[position] == b","[0], position position += 1 if len(elements) == numstrings: break From bb7d31c5e43a2772c0b3c55a6be6647c1f27893d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 11:19:57 -0400 Subject: [PATCH 0496/1054] Futurize part of port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/netstring.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 95e479ea4..a99a40149 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.humanreadable", "allmydata.util.mathutil", "allmydata.util.namespace", + "allmydata.util.netstring", "allmydata.util.pollmixin", "allmydata.util._python3", ] diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 71b487f5b..7f2bef377 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -1,5 +1,20 @@ +""" +Netstring encoding and decoding. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from past.builtins import long + def netstring(s): assert isinstance(s, bytes), s # no unicode here return b"%d:%s," % (len(s), s,) @@ -13,9 +28,9 @@ def split_netstring(data, numstrings, byte which was not consumed (the 'required_trailer', if any, counts as consumed). If 'required_trailer' is not None, throw ValueError if leftover data does not exactly equal 'required_trailer'.""" - assert type(data) == bytes - assert type(required_trailer) in (type(None), bytes) - assert type(position) in (int, long), (repr(position), type(position)) + assert isinstance(data, bytes) + assert required_trailer is None or isinstance(required_trailer, bytes) + assert isinstance(position, (int, long)), (repr(position), type(position)) elements = [] assert numstrings >= 0 while position < len(data): From ddfe20c8ab90543202b65e9eab6a9ceae0873852 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 13:24:20 -0400 Subject: [PATCH 0497/1054] Use SlotsSequenceElement correctly --- src/allmydata/web/check_results.py | 8 ++++---- src/allmydata/web/deep-check-results.xhtml | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index c8165a67b..f459b1f50 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -502,9 +502,9 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): summary_text = ": " + summary summary_text += " [SI: %s]" % cr.get_storage_index_string() problems.append({ - # TODO: result of _join_pathstring() seems to be - # weird. Might as well use "path" as the key here? - self._join_pathstring(path): self._html(summary_text) + # Not sure self._join_pathstring(path) is the + # right thing to use here. + "problem": self._html(path) + self._html(summary_text), }) return SlotsSequenceElement(tag, problems) @@ -529,7 +529,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): nickname = server.get_nickname() if nickname: name.append(" (%s)" % self._html(nickname)) - problems.append(name) + problems.append({"problem": name}) return SlotsSequenceElement(tag, problems) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index abe9bfd63..f3b2e7d75 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -25,7 +25,9 @@

                        Files/Directories That Had Problems:

                          -
                        • +
                        • + +
                        • None
                        @@ -34,7 +36,9 @@

                        Servers on which corrupt shares were found

                          -
                        • +
                        • + +
                        • None
                        From f547509332854f36de67a8d07379c51d1a4aa8ba Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 13:24:32 -0400 Subject: [PATCH 0498/1054] Use table headers correctly --- src/allmydata/web/deep-check-results.xhtml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index f3b2e7d75..542bfdd75 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -47,11 +47,11 @@

                        Corrupt Shares

                        If repair fails, these shares need to be manually inspected and removed.

                        - - - - - + + + + + @@ -67,12 +67,12 @@

                        All Results

                        ServerServer NicknameStorage IndexShare Number
                        ServerServer NicknameStorage IndexShare Number
                        - - - - - - + + + + + + From 3c079bef7365ef43351b193d2ea39528a8c14caf Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 13:35:22 -0400 Subject: [PATCH 0499/1054] Expand "deep check results" test --- src/allmydata/test/test_checker.py | 33 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index f975a9a5a..538ba0640 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -90,10 +90,11 @@ class FakeResults(object): return False def get_summary(self): - return "A fake summary" + return "fake summary results" def get_corrupt_shares(self): - return (FakeServer(), None, None) + # Returns (IServer, storage_index, sharenum) + return (FakeServer(), "fake-si", 0) # TODO: maybe use check_results.DeepCheckResults? @@ -121,11 +122,8 @@ class FakeDeepCheckResults(object): } def get_corrupt_shares(self): - return [ - # TODO: fill this with: - # (IServer, storage_index, sharenum) - FakeResults().get_corrupt_shares(), - ] + # Returns a set of (IServer, storage_index, sharenum) + return { FakeResults().get_corrupt_shares() } class WebResultsRendering(unittest.TestCase): @@ -430,7 +428,6 @@ class WebResultsRendering(unittest.TestCase): monitor.set_status(result) elem = web_check_results.DeepCheckResultsRendererElement(monitor) - doc = self.render_element(elem) soup = BeautifulSoup(doc, 'html5lib') @@ -471,6 +468,26 @@ class WebResultsRendering(unittest.TestCase): u"Corrupt Shares: 1" ) + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Files/Directories That Had Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Servers on which corrupt shares were found" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"Corrupt Shares" + ) + + assert_soup_has_tag_with_content( + self, soup, u"h2", + u"All Results" + ) + def test_deep_check_and_repair_renderer(self): monitor = Monitor() status = check_results.DeepCheckAndRepairResults("") From 69afd292c2e6c1c453d9da9e9de9c2d5b7bd25ba Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 13:43:50 -0400 Subject: [PATCH 0500/1054] Use a different format for path Seems that _join_pathstring() is the wrong thing to use here. --- src/allmydata/web/check_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index f459b1f50..d6c8fb508 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -573,7 +573,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): result = results.get(path) storage_index = result.get_storage_index() object = { - "path": self._join_pathstring(path), + "path": self._html(path), "healthy": str(result.is_healthy()), "recoverable": str(result.is_recoverable()), "storage_index": self._render_si_link(req, storage_index), From fa5b5fa4365492c144bd5f37b23eb9602e529ca7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 14:55:56 -0400 Subject: [PATCH 0501/1054] Manual part of port to Python 3. --- src/allmydata/util/hashutil.py | 66 +++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index fd8ab4190..f7cc802ca 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -1,3 +1,10 @@ +""" +Hashing utilities. +""" + +from builtins import bytes +from past.builtins import chr as byteschr + import os import hashlib from allmydata.util.netstring import netstring @@ -60,34 +67,34 @@ def tagged_pair_hash(tag, val1, val2, truncate_to=None): # immutable -STORAGE_INDEX_TAG = "allmydata_immutable_key_to_storage_index_v1" -BLOCK_TAG = "allmydata_encoded_subshare_v1" -UEB_TAG = "allmydata_uri_extension_v1" -PLAINTEXT_TAG = "allmydata_plaintext_v1" -CIPHERTEXT_TAG = "allmydata_crypttext_v1" -CIPHERTEXT_SEGMENT_TAG = "allmydata_crypttext_segment_v1" -PLAINTEXT_SEGMENT_TAG = "allmydata_plaintext_segment_v1" -CONVERGENT_ENCRYPTION_TAG = "allmydata_immutable_content_to_key_with_added_secret_v1+" +STORAGE_INDEX_TAG = b"allmydata_immutable_key_to_storage_index_v1" +BLOCK_TAG = b"allmydata_encoded_subshare_v1" +UEB_TAG = b"allmydata_uri_extension_v1" +PLAINTEXT_TAG = b"allmydata_plaintext_v1" +CIPHERTEXT_TAG = b"allmydata_crypttext_v1" +CIPHERTEXT_SEGMENT_TAG = b"allmydata_crypttext_segment_v1" +PLAINTEXT_SEGMENT_TAG = b"allmydata_plaintext_segment_v1" +CONVERGENT_ENCRYPTION_TAG = b"allmydata_immutable_content_to_key_with_added_secret_v1+" -CLIENT_RENEWAL_TAG = "allmydata_client_renewal_secret_v1" -CLIENT_CANCEL_TAG = "allmydata_client_cancel_secret_v1" -FILE_RENEWAL_TAG = "allmydata_file_renewal_secret_v1" -FILE_CANCEL_TAG = "allmydata_file_cancel_secret_v1" -BUCKET_RENEWAL_TAG = "allmydata_bucket_renewal_secret_v1" -BUCKET_CANCEL_TAG = "allmydata_bucket_cancel_secret_v1" +CLIENT_RENEWAL_TAG = b"allmydata_client_renewal_secret_v1" +CLIENT_CANCEL_TAG = b"allmydata_client_cancel_secret_v1" +FILE_RENEWAL_TAG = b"allmydata_file_renewal_secret_v1" +FILE_CANCEL_TAG = b"allmydata_file_cancel_secret_v1" +BUCKET_RENEWAL_TAG = b"allmydata_bucket_renewal_secret_v1" +BUCKET_CANCEL_TAG = b"allmydata_bucket_cancel_secret_v1" # mutable -MUTABLE_WRITEKEY_TAG = "allmydata_mutable_privkey_to_writekey_v1" -MUTABLE_WRITE_ENABLER_MASTER_TAG = "allmydata_mutable_writekey_to_write_enabler_master_v1" -MUTABLE_WRITE_ENABLER_TAG = "allmydata_mutable_write_enabler_master_and_nodeid_to_write_enabler_v1" -MUTABLE_PUBKEY_TAG = "allmydata_mutable_pubkey_to_fingerprint_v1" -MUTABLE_READKEY_TAG = "allmydata_mutable_writekey_to_readkey_v1" -MUTABLE_DATAKEY_TAG = "allmydata_mutable_readkey_to_datakey_v1" -MUTABLE_STORAGEINDEX_TAG = "allmydata_mutable_readkey_to_storage_index_v1" +MUTABLE_WRITEKEY_TAG = b"allmydata_mutable_privkey_to_writekey_v1" +MUTABLE_WRITE_ENABLER_MASTER_TAG = b"allmydata_mutable_writekey_to_write_enabler_master_v1" +MUTABLE_WRITE_ENABLER_TAG = b"allmydata_mutable_write_enabler_master_and_nodeid_to_write_enabler_v1" +MUTABLE_PUBKEY_TAG = b"allmydata_mutable_pubkey_to_fingerprint_v1" +MUTABLE_READKEY_TAG = b"allmydata_mutable_writekey_to_readkey_v1" +MUTABLE_DATAKEY_TAG = b"allmydata_mutable_readkey_to_datakey_v1" +MUTABLE_STORAGEINDEX_TAG = b"allmydata_mutable_readkey_to_storage_index_v1" # dirnodes -DIRNODE_CHILD_WRITECAP_TAG = "allmydata_mutable_writekey_and_salt_to_dirnode_child_capkey_v1" -DIRNODE_CHILD_SALT_TAG = "allmydata_dirnode_child_rwcap_to_salt_v1" +DIRNODE_CHILD_WRITECAP_TAG = b"allmydata_mutable_writekey_and_salt_to_dirnode_child_capkey_v1" +DIRNODE_CHILD_SALT_TAG = b"allmydata_dirnode_child_rwcap_to_salt_v1" def storage_index_hash(key): @@ -158,8 +165,8 @@ def convergence_hash(k, n, segsize, data, convergence): def convergence_hasher(k, n, segsize, convergence): - assert isinstance(convergence, str) - param_tag = netstring("%d,%d,%d" % (k, n, segsize)) + assert isinstance(convergence, bytes) + param_tag = netstring(b"%d,%d,%d" % (k, n, segsize)) tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag return tagged_hasher(tag, KEYLEN) @@ -197,12 +204,13 @@ def bucket_cancel_secret_hash(file_cancel_secret, peerid): def _xor(a, b): - return "".join([chr(ord(c) ^ ord(b)) for c in a]) + return b"".join([byteschr(c ^ b) for c in a]) def hmac(tag, data): - ikey = _xor(tag, "\x36") - okey = _xor(tag, "\x5c") + tag = bytes(tag) # Make sure it matches Python 3 behavior + ikey = _xor(tag, 0x36) + okey = _xor(tag, 0x5c) h1 = hashlib.sha256(ikey + data).digest() h2 = hashlib.sha256(okey + h1).digest() return h2 @@ -251,7 +259,7 @@ def timing_safe_compare(a, b): return bool(tagged_hash(n, a) == tagged_hash(n, b)) -BACKUPDB_DIRHASH_TAG = "allmydata_backupdb_dirhash_v1" +BACKUPDB_DIRHASH_TAG = b"allmydata_backupdb_dirhash_v1" def backupdb_dirhash(contents): From 4800992de6f2461f2c39141b636322a5576915f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 14:58:58 -0400 Subject: [PATCH 0502/1054] Type check assertions. --- src/allmydata/test/test_hashutil.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 299f362c1..cb0967b83 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -22,13 +22,16 @@ class HashUtilTests(unittest.TestCase): def test_random_key(self): k = hashutil.random_key() self.failUnlessEqual(len(k), hashutil.KEYLEN) + self.assertIsInstance(k, bytes) def test_sha256d(self): h1 = hashutil.tagged_hash(b"tag1", b"value") + self.assertIsInstance(h1, bytes) h2 = hashutil.tagged_hasher(b"tag1") h2.update(b"value") h2a = h2.digest() h2b = h2.digest() + self.assertIsInstance(h2a, bytes) self.failUnlessEqual(h1, h2a) self.failUnlessEqual(h2a, h2b) @@ -47,37 +50,45 @@ class HashUtilTests(unittest.TestCase): h2.update(b"data") h2 = h2.digest() self.failUnlessEqual(h1, h2) + self.assertIsInstance(h1, bytes) + self.assertIsInstance(h2, bytes) def test_hashers(self): h1 = hashutil.block_hash(b"foo") h2 = hashutil.block_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) h1 = hashutil.uri_extension_hash(b"foo") h2 = hashutil.uri_extension_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) h1 = hashutil.plaintext_hash(b"foo") h2 = hashutil.plaintext_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) h1 = hashutil.crypttext_hash(b"foo") h2 = hashutil.crypttext_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) h1 = hashutil.crypttext_segment_hash(b"foo") h2 = hashutil.crypttext_segment_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) h1 = hashutil.plaintext_segment_hash(b"foo") h2 = hashutil.plaintext_segment_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) + self.assertIsInstance(h1, bytes) def test_timing_safe_compare(self): self.failUnless(hashutil.timing_safe_compare(b"a", b"a")) @@ -87,6 +98,7 @@ class HashUtilTests(unittest.TestCase): def _testknown(self, hashf, expected_a, *args): got = hashf(*args) + self.assertIsInstance(got, bytes) got_a = base32.b2a(got) self.failUnlessEqual(got_a, expected_a) From ed8b10c12faa13ec8c8fc5df7fc55471aa9be5c6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 15:00:29 -0400 Subject: [PATCH 0503/1054] Finish porting to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/hashutil.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a99a40149..4342b8c04 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -19,6 +19,7 @@ PORTED_MODULES = [ "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", + "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.mathutil", "allmydata.util.namespace", diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index f7cc802ca..d26f5a9b0 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -1,8 +1,17 @@ """ Hashing utilities. -""" -from builtins import bytes +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from past.builtins import chr as byteschr import os From 7e7f77161975f392e4cd6d8c3962424da160bd88 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 15:00:44 -0400 Subject: [PATCH 0504/1054] News file. --- newsfragments/3344.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3344.other diff --git a/newsfragments/3344.other b/newsfragments/3344.other new file mode 100644 index 000000000..e69de29bb From ed6e1cb7ef948a41aa99cfbb167be3bfca3c5f7e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 15:51:09 -0400 Subject: [PATCH 0505/1054] Fix formatting. --- src/allmydata/util/base32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 1d1f3ce4b..3d5b1fc8f 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -55,7 +55,7 @@ BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits -BASE32STR_anybytes =b'((?:%s{8})*' % (BASE32CHAR,) + b"(?:|%s|%s|%s|%s))" % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) +BASE32STR_anybytes = bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) def b2a(os): """ From 6af32fdfa7bbb356f38855281c1ea653a54d930d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Jul 2020 15:59:46 -0400 Subject: [PATCH 0506/1054] Make sure the public API exposes native bytes, not Future bytes. --- src/allmydata/util/base32.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index 3d5b1fc8f..f47960b87 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -12,6 +12,17 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 +if PY2: + def backwardscompat_bytes(b): + """ + Replace Future bytes with native Python 2 bytes, so % works + consistently until other modules are ported. + """ + return getattr(b, "__native__", lambda: b)() +else: + def backwardscompat_bytes(b): + return b + import base64 from allmydata.util.assertutil import precondition @@ -46,16 +57,16 @@ def get_trailing_chars_without_lsbs(N): d = {} return b''.join(_get_trailing_chars_without_lsbs(N, d=d)) -BASE32CHAR =b'['+get_trailing_chars_without_lsbs(0)+b']' -BASE32CHAR_4bits =b'['+get_trailing_chars_without_lsbs(1)+b']' -BASE32CHAR_3bits =b'['+get_trailing_chars_without_lsbs(2)+b']' -BASE32CHAR_2bits =b'['+get_trailing_chars_without_lsbs(3)+b']' -BASE32CHAR_1bits =b'['+get_trailing_chars_without_lsbs(4)+b']' -BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits -BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits -BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits -BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits -BASE32STR_anybytes = bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) +BASE32CHAR = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(0)+b']') +BASE32CHAR_4bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(1)+b']') +BASE32CHAR_3bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(2)+b']') +BASE32CHAR_2bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(3)+b']') +BASE32CHAR_1bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(4)+b']') +BASE32STR_1byte = backwardscompat_bytes(BASE32CHAR+BASE32CHAR_3bits) +BASE32STR_2bytes = backwardscompat_bytes(BASE32CHAR+b'{3}'+BASE32CHAR_1bits) +BASE32STR_3bytes = backwardscompat_bytes(BASE32CHAR+b'{4}'+BASE32CHAR_4bits) +BASE32STR_4bytes = backwardscompat_bytes(BASE32CHAR+b'{6}'+BASE32CHAR_2bits) +BASE32STR_anybytes = backwardscompat_bytes(bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes)) def b2a(os): """ From 07c24dbe5931abaa041118882f3b268b6e6d1df9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:14:10 -0400 Subject: [PATCH 0507/1054] Add a missing super() call --- src/allmydata/web/check_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d6c8fb508..401ec3486 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -600,9 +600,9 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): :param allmydata.interfaces.IStatsProducer client: stats provider. :param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider. """ + super(DeepCheckAndRepairResultsRenderer, self).__init__() self._client = client self.monitor = monitor - self.children = {} def getChild(self, name, req): if not name: From b5a465cc8d8dcab65253b458385f0001e16dd7de Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:20:36 -0400 Subject: [PATCH 0508/1054] Format "all objects" table correctly --- .../web/deep-check-and-repair-results.xhtml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index c9bd998e5..936613c70 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -71,12 +71,12 @@
                        Relative PathHealthyRecoverableStorage IndexSummary
                        Relative PathHealthyRecoverableStorage IndexSummary
                        - - - - - - + + + + + + @@ -86,6 +86,9 @@ + + +
                        Relative PathHealthy Pre-RepairRecoverable Pre-RepairHealthy Post-RepairStorage IndexSummaryRelative PathHealthy Pre-RepairRecoverable Pre-RepairHealthy Post-RepairStorage IndexSummary
                        Nothing to report yet.
                  From e6eb866cef4ffe7693ec97447f3eaf70a1e3e5da Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:21:46 -0400 Subject: [PATCH 0509/1054] Add missing "header" renderer It does not do much here, but it is good to be consistent --- src/allmydata/web/deep-check-results.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 542bfdd75..09532d10d 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -47,7 +47,7 @@

                  Corrupt Shares

                  If repair fails, these shares need to be manually inspected and removed.

                  - + @@ -67,7 +67,7 @@

                  All Results

                  Server Server Nickname Storage Index
                  - + From 589e143a4a2a9306e4f0388fc1b36e8afcdbf61e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:24:11 -0400 Subject: [PATCH 0510/1054] Add an empty renderer --- src/allmydata/web/deep-check-results.xhtml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index 09532d10d..7fb324050 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -81,6 +81,9 @@ + + +
                  Relative Path Healthy Recoverable
                  Nothing to report yet.
                  From 69693663708247015f97271642971102c3426310 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:38:30 -0400 Subject: [PATCH 0511/1054] Document, use, and test "join pathstring" method --- src/allmydata/test/test_checker.py | 7 ++++++- src/allmydata/web/check_results.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 538ba0640..49d38b88c 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -118,7 +118,7 @@ class FakeDeepCheckResults(object): return { # TODO: fill this, perhaps with one each of healthy, # unhealthy, unrecoverable, corrupt. - "fake-result-path": FakeResults() + (u"some", u"fake", u"path"): FakeResults() } def get_corrupt_shares(self): @@ -473,6 +473,11 @@ class WebResultsRendering(unittest.TestCase): u"Files/Directories That Had Problems:" ) + assert_soup_has_tag_with_content( + self, soup, u"li", + u"some/fake/path: fake summary results [SI: mzqwwzjnonuq]" + ) + assert_soup_has_tag_with_content( self, soup, u"h2", u"Servers on which corrupt shares were found" diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 401ec3486..d244d25c8 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -91,6 +91,13 @@ class ResultsBase(object): # determine the permuted peer order def _join_pathstring(self, path): + """ + :param tuple path: a path represented by a tuple, such as + ``(u'some', u'dir', u'file')``. + + :return: a string joined by path separaters, such as + ``u'some/dir/file'``. + """ if path: pathstring = "/".join(self._html(path)) else: @@ -504,7 +511,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): problems.append({ # Not sure self._join_pathstring(path) is the # right thing to use here. - "problem": self._html(path) + self._html(summary_text), + "problem": self._join_pathstring(path) + self._html(summary_text), }) return SlotsSequenceElement(tag, problems) @@ -573,7 +580,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): result = results.get(path) storage_index = result.get_storage_index() object = { - "path": self._html(path), + "path": self._join_pathstring(path), "healthy": str(result.is_healthy()), "recoverable": str(result.is_recoverable()), "storage_index": self._render_si_link(req, storage_index), From 8fedbd52c2fa8bb28dadc4a750722774be7951c4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 16:58:39 -0400 Subject: [PATCH 0512/1054] Expand "deep check and repair" test --- src/allmydata/test/test_checker.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 49d38b88c..0cdee2aa8 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -20,6 +20,7 @@ from allmydata.interfaces import ( IServer, ICheckResults, IDeepCheckResults, + ICheckAndRepairResults, ) from allmydata.util import base32 from allmydata.web import check_results as web_check_results @@ -126,6 +127,25 @@ class FakeDeepCheckResults(object): return { FakeResults().get_corrupt_shares() } +@implementer(ICheckAndRepairResults) +class FakeDeepCheckAndRepairResults(object): + + def get_storage_index(self): + return "" + + def get_pre_repair_results(self): + return FakeResults() + + def get_post_repair_results(self): + return FakeResults() + + def get_repair_attempted(self): + return True + + def get_repair_successful(self): + return False + + class WebResultsRendering(unittest.TestCase): def remove_tags(self, s): @@ -495,8 +515,15 @@ class WebResultsRendering(unittest.TestCase): def test_deep_check_and_repair_renderer(self): monitor = Monitor() + status = check_results.DeepCheckAndRepairResults("") + status.add_check_and_repair( + FakeDeepCheckAndRepairResults(), + (u"some", u"fake", u"path") + ) + monitor.set_status(status) + elem = web_check_results.DeepCheckAndRepairResultsRendererElement(monitor) doc = self.render_element(elem) soup = BeautifulSoup(doc, 'html5lib') From a13ec344b99f41c813902194b22a7b84aee81bf2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 17:07:54 -0400 Subject: [PATCH 0513/1054] Correct usage of SlotsSequenceElement --- src/allmydata/web/check_results.py | 8 ++++---- .../web/deep-check-and-repair-results.xhtml | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d244d25c8..27ca2fed7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -748,7 +748,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin cr = r.get_pre_repair_results() if not cr.is_healthy(): problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) - problems.append(problem) + problems.append({"problem": problem}) return SlotsSequenceElement(tag, problems) @@ -770,7 +770,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin cr = r.get_post_repair_results() if not cr.is_healthy(): problem = self._join_pathstring(path), ": ", self._html(cr.get_summary()) - problems.append(problem) + problems.append({"problem": problem}) return SlotsSequenceElement(tag, problems) @@ -784,7 +784,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin def servers_with_corrupt_shares(self, req, tag): # TODO: this was originally unimplemented before porting to # twisted.web.template; leaving it as such. - corrupt = [] + corrupt = [{"share":"unimplemented"}] return SlotsSequenceElement(tag, corrupt) @renderer @@ -797,7 +797,7 @@ class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin def post_repair_corrupt_shares(self, req, tag): # TODO: this was not implemented before porting to # twisted.web.template; leaving it as such. - corrupt = [] + corrupt = [{"share":"unimplemented"}] return SlotsSequenceElement(tag, corrupt) @renderer diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index 936613c70..c8cf2deea 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -35,7 +35,9 @@

                  Files/Directories That Had Problems:

                    -
                  • +
                  • + +
                  • None
                  @@ -44,7 +46,9 @@

                  Files/Directories That Still Have Problems:

                    -
                  • +
                  • + +
                  • None
                  @@ -52,7 +56,9 @@

                  Servers on which corrupt shares were found

                    -
                  • +
                  • + +
                  • None
                  @@ -61,7 +67,9 @@

                  Remaining Corrupt Shares

                  These shares need to be manually inspected and removed.

                    -
                  • +
                  • + +
                  • None
                @@ -70,7 +78,7 @@
                - + From 2d7095866a3fefd88fb38a386ea3bc59de3ef463 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 17:09:53 -0400 Subject: [PATCH 0514/1054] Expand "deep check and repair results" page test --- src/allmydata/test/test_checker.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 0cdee2aa8..0fa2be7fd 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -542,7 +542,7 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Checked: 0" + u"Objects Checked: 1" ) assert_soup_has_tag_with_content( @@ -552,22 +552,17 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy (before repair): 0" + u"Objects Unhealthy (before repair): 1" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Corrupt Shares (before repair): 0" + u"Corrupt Shares (before repair): 3" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Repairs Attempted: 0" - ) - - assert_soup_has_tag_with_content( - self, soup, u"li", - u"Repairs Attempted: 0" + u"Repairs Attempted: 1" ) assert_soup_has_tag_with_content( @@ -577,7 +572,7 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - "Repairs Unsuccessful: 0" + "Repairs Unsuccessful: 1" ) assert_soup_has_tag_with_content( @@ -587,12 +582,12 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy (after repair): 0" + u"Objects Unhealthy (after repair): 1" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Corrupt Shares (after repair): 0" + u"Corrupt Shares (after repair): 3" ) assert_soup_has_tag_with_content( From 7f93e28a7c9c4ff7ff4c00a7acce0ad3032f9dd5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 18:43:52 -0400 Subject: [PATCH 0515/1054] Parameterize classes used in check/repair tests --- src/allmydata/test/test_checker.py | 134 ++++++++++++++++------------- 1 file changed, 73 insertions(+), 61 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 0fa2be7fd..eb94f5c23 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -72,78 +72,60 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeResults(object): +class FakeCheckResults(object): - def __init__(self): - # TODO: figure out a good enough fake SI. - self._storage_index = "fake-si" + def __init__(self, si=None, + healthy=False, recoverable=False, + summary="fake summary"): + self._storage_index = si + self._is_healthy = healthy + self._is_recoverable = recoverable + self._summary = summary def get_storage_index(self): return self._storage_index def get_storage_index_string(self): - return base32.b2a(self._storage_index) + return base32.b2a_or_none(self._storage_index) def is_healthy(self): - return False + return self._is_healthy def is_recoverable(self): - return False + return self._is_recoverable def get_summary(self): - return "fake summary results" + return self._summary def get_corrupt_shares(self): - # Returns (IServer, storage_index, sharenum) - return (FakeServer(), "fake-si", 0) - - -# TODO: maybe use check_results.DeepCheckResults? -@implementer(IDeepCheckResults) -class FakeDeepCheckResults(object): - - def get_root_storage_index_string(self): - # TODO: figure out a good enough fake root SI. - return "fake-root-si" - - def get_counters(self): - return { - "count-objects-checked": 4, - "count-objects-healthy": 1, - "count-objects-unhealthy": 1, - "count-objects-unrecoverable": 1, - "count-corrupt-shares": 1, - } - - def get_all_results(self): - return { - # TODO: fill this, perhaps with one each of healthy, - # unhealthy, unrecoverable, corrupt. - (u"some", u"fake", u"path"): FakeResults() - } - - def get_corrupt_shares(self): - # Returns a set of (IServer, storage_index, sharenum) - return { FakeResults().get_corrupt_shares() } + # returns a list of (IServer, storage_index, sharenum) + return [(FakeServer(), "", 0)] @implementer(ICheckAndRepairResults) -class FakeDeepCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): + + def __init__(self, si=None, + repair_attempted=False, + repair_success=False): + self._storage_index = si + self._repair_attempted = repair_attempted + self._repair_success = repair_success def get_storage_index(self): - return "" + return self._storage_index def get_pre_repair_results(self): - return FakeResults() + return FakeCheckResults() def get_post_repair_results(self): - return FakeResults() + return FakeCheckResults() def get_repair_attempted(self): - return True + return self._repair_attempted def get_repair_successful(self): - return False + return self._repair_success class WebResultsRendering(unittest.TestCase): @@ -443,9 +425,26 @@ class WebResultsRendering(unittest.TestCase): def test_deep_check_renderer(self): + status = check_results.DeepCheckResults("fake-root-si") + status.add_check( + FakeCheckResults("", False, False), + (u"fake", u"unhealthy", u"unrecoverable") + ) + status.add_check( + FakeCheckResults("", True, True), + (u"fake", u"healthy", u"recoverable") + ) + status.add_check( + FakeCheckResults("", True, False), + (u"fake", u"healthy", u"unrecoverable") + ) + status.add_check( + FakeCheckResults("", False, True), + (u"fake", u"unhealthy", u"recoverable") + ) + monitor = Monitor() - result = FakeDeepCheckResults() - monitor.set_status(result) + monitor.set_status(status) elem = web_check_results.DeepCheckResultsRendererElement(monitor) doc = self.render_element(elem) @@ -470,22 +469,22 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Healthy: 1" + u"Objects Healthy: 2" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy: 1" + u"Objects Unhealthy: 2" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unrecoverable: 1" + u"Objects Unrecoverable: 2" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Corrupt Shares: 1" + u"Corrupt Shares: 4" ) assert_soup_has_tag_with_content( @@ -495,7 +494,12 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"some/fake/path: fake summary results [SI: mzqwwzjnonuq]" + u"fake/unhealthy/recoverable: fake summary" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", + u"fake/unhealthy/unrecoverable: fake summary" ) assert_soup_has_tag_with_content( @@ -514,14 +518,22 @@ class WebResultsRendering(unittest.TestCase): ) def test_deep_check_and_repair_renderer(self): - monitor = Monitor() - status = check_results.DeepCheckAndRepairResults("") + status.add_check_and_repair( - FakeDeepCheckAndRepairResults(), - (u"some", u"fake", u"path") + FakeCheckAndRepairResults("attempted/success", True, True), + (u"attempted", u"success") + ) + status.add_check_and_repair( + FakeCheckAndRepairResults("attempted/failure", True, False), + (u"attempted", u"failure") + ) + status.add_check_and_repair( + FakeCheckAndRepairResults("unattempted/failure", False, False), + (u"unattempted", u"failure") ) + monitor = Monitor() monitor.set_status(status) elem = web_check_results.DeepCheckAndRepairResultsRendererElement(monitor) @@ -542,7 +554,7 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Checked: 1" + u"Objects Checked: 3" ) assert_soup_has_tag_with_content( @@ -552,7 +564,7 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy (before repair): 1" + u"Objects Unhealthy (before repair): 3" ) assert_soup_has_tag_with_content( @@ -562,12 +574,12 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Repairs Attempted: 1" + u"Repairs Attempted: 2" ) assert_soup_has_tag_with_content( self, soup, u"li", - u"Repairs Successful: 0" + u"Repairs Successful: 1" ) assert_soup_has_tag_with_content( @@ -582,7 +594,7 @@ class WebResultsRendering(unittest.TestCase): assert_soup_has_tag_with_content( self, soup, u"li", - u"Objects Unhealthy (after repair): 1" + u"Objects Unhealthy (after repair): 3" ) assert_soup_has_tag_with_content( From 80bf5f0a53c0e7779d9f35a1cbc8a89faecac59f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 15 Jul 2020 19:19:51 -0400 Subject: [PATCH 0516/1054] Remove an unused import --- src/allmydata/test/test_checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index eb94f5c23..758755e9c 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -19,7 +19,6 @@ from allmydata import uri as tahoe_uri from allmydata.interfaces import ( IServer, ICheckResults, - IDeepCheckResults, ICheckAndRepairResults, ) from allmydata.util import base32 From efbae9b3e36777e0bab7dbd1fc53e19a9162603a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:33:53 -0400 Subject: [PATCH 0517/1054] Hard code some known values, generated on the master branch preceding these changes. --- src/allmydata/test/test_base62.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 6f3ef8293..482a968ea 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -27,7 +27,7 @@ from allmydata.util import base62, mathutil def insecurerandstr(n): return bytes(list(map(random.randrange, [0]*n, [256]*n))) -class T(unittest.TestCase): +class Base62(unittest.TestCase): def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): assert base62.num_octets_that_encode_to_this_many_chars(chars) == octets, "%s != %s <- %s" % (octets, base62.num_octets_that_encode_to_this_many_chars(chars), chars) @@ -43,6 +43,22 @@ class T(unittest.TestCase): def test_roundtrip(self, input_bytes): self._test_roundtrip(input_bytes) + def test_known_values(self): + """Known values to ensure the algorithm hasn't changed.""" + + def check_expected(plaintext, encoded): + result1 = base62.b2a(plaintext) + self.assertEqual(encoded, result1) + result2 = base62.a2b(encoded) + self.assertEqual(plaintext, result2) + + check_expected(b"hello", b'7tQLFHz') + check_expected(b"", b'0') + check_expected(b"zzz", b'0Xg7e') + check_expected(b"\x36\xffWAT", b'49pq4mq') + check_expected(b"1234 22323", b'1A0afZe9mxSZpz') + check_expected(b"______", b'0TmAuCHJX') + def test_num_octets_that_encode_to_this_many_chars(self): return self._test_num_octets_that_encode_to_this_many_chars(2, 1) return self._test_num_octets_that_encode_to_this_many_chars(3, 2) From 58c4908d46aff07352f875e351ccb07499dc76db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:36:49 -0400 Subject: [PATCH 0518/1054] More tests are passing. --- misc/python3/ratchet-passing | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index e659c95b6..a6e1de68b 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -1,4 +1,19 @@ allmydata.test.mutable.test_exceptions.Exceptions.test_repr +allmydata.test.test_base32.Base32.test_a2b +allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons +allmydata.test.test_base32.Base32.test_b2a +allmydata.test.test_base32.Base32.test_b2a_or_none +allmydata.test.test_base62.Base62.test_ende_0x00 +allmydata.test.test_base62.Base62.test_ende_0x000000 +allmydata.test.test_base62.Base62.test_ende_0x01 +allmydata.test.test_base62.Base62.test_ende_0x0100 +allmydata.test.test_base62.Base62.test_ende_0x010000 +allmydata.test.test_base62.Base62.test_ende_longrandstr +allmydata.test.test_base62.Base62.test_ende_randstr +allmydata.test.test_base62.Base62.test_known_values +allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars +allmydata.test.test_base62.Base62.test_odd_sizes +allmydata.test.test_base62.Base62.test_roundtrip allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success From 2f693c47f9597f3b7071c7370c44a2850bce5a60 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:37:26 -0400 Subject: [PATCH 0519/1054] Don't leak variable on Python 2. --- src/allmydata/util/base32.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index f47960b87..cb21d1a8a 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -94,7 +94,9 @@ NUM_OS_TO_NUM_QS=(0, 2, 4, 5, 7,) NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4) NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,) -NUM_QS_TO_NUM_BITS=tuple([x*8 for x in NUM_QS_TO_NUM_OS]) +NUM_QS_TO_NUM_BITS=tuple([_x*8 for _x in NUM_QS_TO_NUM_OS]) +if PY2: + del _x # A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the # original data had 8K bits for a positive integer K. From 8d143af43e7855bec8c3e057339d9e377c77c034 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:40:30 -0400 Subject: [PATCH 0520/1054] Another assertion. --- src/allmydata/test/test_base62.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 482a968ea..d1d0c72f3 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -38,6 +38,8 @@ class Base62(unittest.TestCase): self.assertIsInstance(encoded, bytes) self.assertIsInstance(bs, bytes) self.assertIsInstance(decoded, bytes) + # Encoded string only uses values from the base62 allowed characters: + self.assertFalse(set(encoded) - set(base62.chars)) @given(input_bytes=st.binary(max_size=100)) def test_roundtrip(self, input_bytes): From 37fa687d4b54e89d286e640c15fb94f3250b914f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:42:49 -0400 Subject: [PATCH 0521/1054] More passing tests for the ratchet. --- misc/python3/ratchet-passing | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index a6e1de68b..7208bd56e 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -18,7 +18,18 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls +allmydata.test.test_hashutil.HashUtilTests.test_chk +allmydata.test.test_hashutil.HashUtilTests.test_hashers +allmydata.test.test_hashutil.HashUtilTests.test_known_answers +allmydata.test.test_hashutil.HashUtilTests.test_random_key +allmydata.test.test_hashutil.HashUtilTests.test_sha256d +allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated +allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_netstring.Netstring.test_encode +allmydata.test.test_netstring.Netstring.test_extra +allmydata.test.test_netstring.Netstring.test_nested +allmydata.test.test_netstring.Netstring.test_split allmydata.test.test_observer.Observer.test_lazy_oneshot allmydata.test.test_observer.Observer.test_observerlist allmydata.test.test_observer.Observer.test_oneshot From 545848f16474227d3cce139e26dc5b4bb1f577cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 14:57:09 -0400 Subject: [PATCH 0522/1054] Move abbreviate tests into their own module. --- src/allmydata/test/test_util.py | 128 ---------------------------- src/allmydata/test_abbreviate.py | 139 +++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 128 deletions(-) create mode 100644 src/allmydata/test_abbreviate.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index f744feb7e..c6b4dd991 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -687,134 +687,6 @@ class HashUtilTests(unittest.TestCase): base32.a2b("u33m4y7klhz3bypswqkozwetvabelhxt"), # seed ) -class Abbreviate(unittest.TestCase): - def test_abbrev_time_1s(self): - diff = timedelta(seconds=1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('1 second ago', s) - - def test_abbrev_time_25s(self): - diff = timedelta(seconds=25) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('25 seconds ago', s) - - def test_abbrev_time_future_5_minutes(self): - diff = timedelta(minutes=-5) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes in the future', s) - - def test_abbrev_time_hours(self): - diff = timedelta(hours=4) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('4 hours ago', s) - - def test_abbrev_time_day(self): - diff = timedelta(hours=49) # must be more than 2 days - s = abbreviate.abbreviate_time(diff) - self.assertEqual('2 days ago', s) - - def test_abbrev_time_month(self): - diff = timedelta(days=91) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('3 months ago', s) - - def test_abbrev_time_year(self): - diff = timedelta(weeks=(5 * 52) + 1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 years ago', s) - - def test_time(self): - a = abbreviate.abbreviate_time - self.failUnlessEqual(a(None), "unknown") - self.failUnlessEqual(a(0), "0 seconds") - self.failUnlessEqual(a(1), "1 second") - self.failUnlessEqual(a(2), "2 seconds") - self.failUnlessEqual(a(119), "119 seconds") - MIN = 60 - self.failUnlessEqual(a(2*MIN), "2 minutes") - self.failUnlessEqual(a(60*MIN), "60 minutes") - self.failUnlessEqual(a(179*MIN), "179 minutes") - HOUR = 60*MIN - self.failUnlessEqual(a(180*MIN), "3 hours") - self.failUnlessEqual(a(4*HOUR), "4 hours") - DAY = 24*HOUR - MONTH = 30*DAY - self.failUnlessEqual(a(2*DAY), "2 days") - self.failUnlessEqual(a(2*MONTH), "2 months") - YEAR = 365*DAY - self.failUnlessEqual(a(5*YEAR), "5 years") - - def test_space(self): - tests_si = [(None, "unknown"), - (0, "0 B"), - (1, "1 B"), - (999, "999 B"), - (1000, "1000 B"), - (1023, "1023 B"), - (1024, "1.02 kB"), - (20*1000, "20.00 kB"), - (1024*1024, "1.05 MB"), - (1000*1000, "1.00 MB"), - (1000*1000*1000, "1.00 GB"), - (1000*1000*1000*1000, "1.00 TB"), - (1000*1000*1000*1000*1000, "1.00 PB"), - (1000*1000*1000*1000*1000*1000, "1.00 EB"), - (1234567890123456789, "1.23 EB"), - ] - for (x, expected) in tests_si: - got = abbreviate.abbreviate_space(x, SI=True) - self.failUnlessEqual(got, expected) - - tests_base1024 = [(None, "unknown"), - (0, "0 B"), - (1, "1 B"), - (999, "999 B"), - (1000, "1000 B"), - (1023, "1023 B"), - (1024, "1.00 kiB"), - (20*1024, "20.00 kiB"), - (1000*1000, "976.56 kiB"), - (1024*1024, "1.00 MiB"), - (1024*1024*1024, "1.00 GiB"), - (1024*1024*1024*1024, "1.00 TiB"), - (1000*1000*1000*1000*1000, "909.49 TiB"), - (1024*1024*1024*1024*1024, "1.00 PiB"), - (1024*1024*1024*1024*1024*1024, "1.00 EiB"), - (1234567890123456789, "1.07 EiB"), - ] - for (x, expected) in tests_base1024: - got = abbreviate.abbreviate_space(x, SI=False) - self.failUnlessEqual(got, expected) - - self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567), - "(1.23 MB, 1.18 MiB)") - - def test_parse_space(self): - p = abbreviate.parse_abbreviated_size - self.failUnlessEqual(p(""), None) - self.failUnlessEqual(p(None), None) - self.failUnlessEqual(p("123"), 123) - self.failUnlessEqual(p("123B"), 123) - self.failUnlessEqual(p("2K"), 2000) - self.failUnlessEqual(p("2kb"), 2000) - self.failUnlessEqual(p("2KiB"), 2048) - self.failUnlessEqual(p("10MB"), 10*1000*1000) - self.failUnlessEqual(p("10MiB"), 10*1024*1024) - self.failUnlessEqual(p("5G"), 5*1000*1000*1000) - self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024) - self.failUnlessEqual(p("3TB"), 3*1000*1000*1000*1000) - self.failUnlessEqual(p("3TiB"), 3*1024*1024*1024*1024) - self.failUnlessEqual(p("6PB"), 6*1000*1000*1000*1000*1000) - self.failUnlessEqual(p("6PiB"), 6*1024*1024*1024*1024*1024) - self.failUnlessEqual(p("9EB"), 9*1000*1000*1000*1000*1000*1000) - self.failUnlessEqual(p("9EiB"), 9*1024*1024*1024*1024*1024*1024) - - e = self.failUnlessRaises(ValueError, p, "12 cubits") - self.failUnlessIn("12 cubits", str(e)) - e = self.failUnlessRaises(ValueError, p, "1 BB") - self.failUnlessIn("1 BB", str(e)) - e = self.failUnlessRaises(ValueError, p, "fhtagn") - self.failUnlessIn("fhtagn", str(e)) class Limiter(unittest.TestCase): diff --git a/src/allmydata/test_abbreviate.py b/src/allmydata/test_abbreviate.py new file mode 100644 index 000000000..483ccfda0 --- /dev/null +++ b/src/allmydata/test_abbreviate.py @@ -0,0 +1,139 @@ +""" +Tests for allmydata.util.abbreviate. +""" + +from datetime import timedelta + +from twisted.trial import unittest + +from allmydata.util import abbreviate + + +class Abbreviate(unittest.TestCase): + def test_abbrev_time_1s(self): + diff = timedelta(seconds=1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('1 second ago', s) + + def test_abbrev_time_25s(self): + diff = timedelta(seconds=25) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('25 seconds ago', s) + + def test_abbrev_time_future_5_minutes(self): + diff = timedelta(minutes=-5) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 minutes in the future', s) + + def test_abbrev_time_hours(self): + diff = timedelta(hours=4) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('4 hours ago', s) + + def test_abbrev_time_day(self): + diff = timedelta(hours=49) # must be more than 2 days + s = abbreviate.abbreviate_time(diff) + self.assertEqual('2 days ago', s) + + def test_abbrev_time_month(self): + diff = timedelta(days=91) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('3 months ago', s) + + def test_abbrev_time_year(self): + diff = timedelta(weeks=(5 * 52) + 1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 years ago', s) + + def test_time(self): + a = abbreviate.abbreviate_time + self.failUnlessEqual(a(None), "unknown") + self.failUnlessEqual(a(0), "0 seconds") + self.failUnlessEqual(a(1), "1 second") + self.failUnlessEqual(a(2), "2 seconds") + self.failUnlessEqual(a(119), "119 seconds") + MIN = 60 + self.failUnlessEqual(a(2*MIN), "2 minutes") + self.failUnlessEqual(a(60*MIN), "60 minutes") + self.failUnlessEqual(a(179*MIN), "179 minutes") + HOUR = 60*MIN + self.failUnlessEqual(a(180*MIN), "3 hours") + self.failUnlessEqual(a(4*HOUR), "4 hours") + DAY = 24*HOUR + MONTH = 30*DAY + self.failUnlessEqual(a(2*DAY), "2 days") + self.failUnlessEqual(a(2*MONTH), "2 months") + YEAR = 365*DAY + self.failUnlessEqual(a(5*YEAR), "5 years") + + def test_space(self): + tests_si = [(None, "unknown"), + (0, "0 B"), + (1, "1 B"), + (999, "999 B"), + (1000, "1000 B"), + (1023, "1023 B"), + (1024, "1.02 kB"), + (20*1000, "20.00 kB"), + (1024*1024, "1.05 MB"), + (1000*1000, "1.00 MB"), + (1000*1000*1000, "1.00 GB"), + (1000*1000*1000*1000, "1.00 TB"), + (1000*1000*1000*1000*1000, "1.00 PB"), + (1000*1000*1000*1000*1000*1000, "1.00 EB"), + (1234567890123456789, "1.23 EB"), + ] + for (x, expected) in tests_si: + got = abbreviate.abbreviate_space(x, SI=True) + self.failUnlessEqual(got, expected) + + tests_base1024 = [(None, "unknown"), + (0, "0 B"), + (1, "1 B"), + (999, "999 B"), + (1000, "1000 B"), + (1023, "1023 B"), + (1024, "1.00 kiB"), + (20*1024, "20.00 kiB"), + (1000*1000, "976.56 kiB"), + (1024*1024, "1.00 MiB"), + (1024*1024*1024, "1.00 GiB"), + (1024*1024*1024*1024, "1.00 TiB"), + (1000*1000*1000*1000*1000, "909.49 TiB"), + (1024*1024*1024*1024*1024, "1.00 PiB"), + (1024*1024*1024*1024*1024*1024, "1.00 EiB"), + (1234567890123456789, "1.07 EiB"), + ] + for (x, expected) in tests_base1024: + got = abbreviate.abbreviate_space(x, SI=False) + self.failUnlessEqual(got, expected) + + self.failUnlessEqual(abbreviate.abbreviate_space_both(1234567), + "(1.23 MB, 1.18 MiB)") + + def test_parse_space(self): + p = abbreviate.parse_abbreviated_size + self.failUnlessEqual(p(""), None) + self.failUnlessEqual(p(None), None) + self.failUnlessEqual(p("123"), 123) + self.failUnlessEqual(p("123B"), 123) + self.failUnlessEqual(p("2K"), 2000) + self.failUnlessEqual(p("2kb"), 2000) + self.failUnlessEqual(p("2KiB"), 2048) + self.failUnlessEqual(p("10MB"), 10*1000*1000) + self.failUnlessEqual(p("10MiB"), 10*1024*1024) + self.failUnlessEqual(p("5G"), 5*1000*1000*1000) + self.failUnlessEqual(p("4GiB"), 4*1024*1024*1024) + self.failUnlessEqual(p("3TB"), 3*1000*1000*1000*1000) + self.failUnlessEqual(p("3TiB"), 3*1024*1024*1024*1024) + self.failUnlessEqual(p("6PB"), 6*1000*1000*1000*1000*1000) + self.failUnlessEqual(p("6PiB"), 6*1024*1024*1024*1024*1024) + self.failUnlessEqual(p("9EB"), 9*1000*1000*1000*1000*1000*1000) + self.failUnlessEqual(p("9EiB"), 9*1024*1024*1024*1024*1024*1024) + + e = self.failUnlessRaises(ValueError, p, "12 cubits") + self.failUnlessIn("12 cubits", str(e)) + e = self.failUnlessRaises(ValueError, p, "1 BB") + self.failUnlessIn("1 BB", str(e)) + e = self.failUnlessRaises(ValueError, p, "fhtagn") + self.failUnlessIn("fhtagn", str(e)) From 72272cbf0bd0f8d644f1d8debb8fce22df6f6295 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:15:26 -0400 Subject: [PATCH 0523/1054] Move out tests for time_format. --- src/allmydata/test/test_time_format.py | 146 +++++++++++++++++++++++++ src/allmydata/test/test_util.py | 135 ----------------------- 2 files changed, 146 insertions(+), 135 deletions(-) create mode 100644 src/allmydata/test/test_time_format.py diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py new file mode 100644 index 000000000..ae655ff3e --- /dev/null +++ b/src/allmydata/test/test_time_format.py @@ -0,0 +1,146 @@ +""" +Tests for allmydata.util.time_format. +""" + +import time + +from twisted.trial import unittest + +from allmydata.test.common_util import TimezoneMixin +from allmydata.util import time_format + + +class TimeFormat(unittest.TestCase, TimezoneMixin): + def test_epoch(self): + return self._help_test_epoch() + + def test_epoch_in_London(self): + # Europe/London is a particularly troublesome timezone. Nowadays, its + # offset from GMT is 0. But in 1970, its offset from GMT was 1. + # (Apparently in 1970 Britain had redefined standard time to be GMT+1 + # and stayed in standard time all year round, whereas today + # Europe/London standard time is GMT and Europe/London Daylight + # Savings Time is GMT+1.) The current implementation of + # time_format.iso_utc_time_to_localseconds() breaks if the timezone is + # Europe/London. (As soon as this unit test is done then I'll change + # that implementation to something that works even in this case...) + + if not self.have_working_tzset(): + raise unittest.SkipTest("This test can't be run on a platform without time.tzset().") + + self.setTimezone("Europe/London") + return self._help_test_epoch() + + def _help_test_epoch(self): + origtzname = time.tzname + s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01") + self.failUnlessEqual(s, 1.0) + s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01") + self.failUnlessEqual(s, 1.0) + s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01") + self.failUnlessEqual(s, 1.0) + + self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01") + self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "), + "1970-01-01 00:00:01") + + now = time.time() + isostr = time_format.iso_utc(now) + timestamp = time_format.iso_utc_time_to_seconds(isostr) + self.failUnlessEqual(int(timestamp), int(now)) + + def my_time(): + return 1.0 + self.failUnlessEqual(time_format.iso_utc(t=my_time), + "1970-01-01_00:00:01") + e = self.failUnlessRaises(ValueError, + time_format.iso_utc_time_to_seconds, + "invalid timestring") + self.failUnless("not a complete ISO8601 timestamp" in str(e)) + s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500") + self.failUnlessEqual(s, 1.5) + + # Look for daylight-savings-related errors. + thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536") + self.failUnlessEqual(thatmomentinmarch, 1237585742.226536) + self.failUnlessEqual(origtzname, time.tzname) + + def test_iso_utc(self): + when = 1266760143.7841301 + out = time_format.iso_utc_date(when) + self.failUnlessEqual(out, "2010-02-21") + out = time_format.iso_utc_date(t=lambda: when) + self.failUnlessEqual(out, "2010-02-21") + out = time_format.iso_utc(when) + self.failUnlessEqual(out, "2010-02-21_13:49:03.784130") + out = time_format.iso_utc(when, sep="-") + self.failUnlessEqual(out, "2010-02-21-13:49:03.784130") + + def test_parse_duration(self): + p = time_format.parse_duration + DAY = 24*60*60 + self.failUnlessEqual(p("1 day"), DAY) + self.failUnlessEqual(p("2 days"), 2*DAY) + self.failUnlessEqual(p("3 months"), 3*31*DAY) + self.failUnlessEqual(p("4 mo"), 4*31*DAY) + self.failUnlessEqual(p("5 years"), 5*365*DAY) + e = self.failUnlessRaises(ValueError, p, "123") + self.failUnlessIn("no unit (like day, month, or year) in '123'", + str(e)) + + def test_parse_date(self): + self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400) + + def test_format_time(self): + self.failUnlessEqual(time_format.format_time(time.gmtime(0)), '1970-01-01 00:00:00') + self.failUnlessEqual(time_format.format_time(time.gmtime(60)), '1970-01-01 00:01:00') + self.failUnlessEqual(time_format.format_time(time.gmtime(60*60)), '1970-01-01 01:00:00') + seconds_per_day = 60*60*24 + leap_years_1970_to_2014_inclusive = ((2012 - 1968) // 4) + self.failUnlessEqual(time_format.format_time(time.gmtime(seconds_per_day*((2015 - 1970)*365+leap_years_1970_to_2014_inclusive))), '2015-01-01 00:00:00') + + def test_format_time_y2038(self): + seconds_per_day = 60*60*24 + leap_years_1970_to_2047_inclusive = ((2044 - 1968) // 4) + t = (seconds_per_day* + ((2048 - 1970)*365 + leap_years_1970_to_2047_inclusive)) + try: + gm_t = time.gmtime(t) + except ValueError: + raise unittest.SkipTest("Note: this system cannot handle dates after 2037.") + self.failUnlessEqual(time_format.format_time(gm_t), + '2048-01-01 00:00:00') + + def test_format_delta(self): + time_1 = 1389812723 + time_5s_delta = 1389812728 + time_28m7s_delta = 1389814410 + time_1h_delta = 1389816323 + time_1d21h46m49s_delta = 1389977532 + + self.failUnlessEqual( + time_format.format_delta(time_1, time_1), '0s') + + self.failUnlessEqual( + time_format.format_delta(time_1, time_5s_delta), '5s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_28m7s_delta), '28m 7s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_1h_delta), '1h 0m 0s') + self.failUnlessEqual( + time_format.format_delta(time_1, time_1d21h46m49s_delta), '1d 21h 46m 49s') + + self.failUnlessEqual( + time_format.format_delta(time_1d21h46m49s_delta, time_1), '-') + + # time_1 with a decimal fraction will make the delta 1s less + time_1decimal = 1389812723.383963 + + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_5s_delta), '4s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_28m7s_delta), '28m 6s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_1h_delta), '59m 59s') + self.failUnlessEqual( + time_format.format_delta(time_1decimal, time_1d21h46m49s_delta), '1d 21h 46m 48s') diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index c6b4dd991..40a05e9c5 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -762,141 +762,6 @@ class Limiter(unittest.TestCase): d.addCallback(_all_done) return d -class TimeFormat(unittest.TestCase, TimezoneMixin): - def test_epoch(self): - return self._help_test_epoch() - - def test_epoch_in_London(self): - # Europe/London is a particularly troublesome timezone. Nowadays, its - # offset from GMT is 0. But in 1970, its offset from GMT was 1. - # (Apparently in 1970 Britain had redefined standard time to be GMT+1 - # and stayed in standard time all year round, whereas today - # Europe/London standard time is GMT and Europe/London Daylight - # Savings Time is GMT+1.) The current implementation of - # time_format.iso_utc_time_to_localseconds() breaks if the timezone is - # Europe/London. (As soon as this unit test is done then I'll change - # that implementation to something that works even in this case...) - - if not self.have_working_tzset(): - raise unittest.SkipTest("This test can't be run on a platform without time.tzset().") - - self.setTimezone("Europe/London") - return self._help_test_epoch() - - def _help_test_epoch(self): - origtzname = time.tzname - s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01") - self.failUnlessEqual(s, 1.0) - s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01") - self.failUnlessEqual(s, 1.0) - s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01") - self.failUnlessEqual(s, 1.0) - - self.failUnlessEqual(time_format.iso_utc(1.0), "1970-01-01_00:00:01") - self.failUnlessEqual(time_format.iso_utc(1.0, sep=" "), - "1970-01-01 00:00:01") - - now = time.time() - isostr = time_format.iso_utc(now) - timestamp = time_format.iso_utc_time_to_seconds(isostr) - self.failUnlessEqual(int(timestamp), int(now)) - - def my_time(): - return 1.0 - self.failUnlessEqual(time_format.iso_utc(t=my_time), - "1970-01-01_00:00:01") - e = self.failUnlessRaises(ValueError, - time_format.iso_utc_time_to_seconds, - "invalid timestring") - self.failUnless("not a complete ISO8601 timestamp" in str(e)) - s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01.500") - self.failUnlessEqual(s, 1.5) - - # Look for daylight-savings-related errors. - thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536") - self.failUnlessEqual(thatmomentinmarch, 1237585742.226536) - self.failUnlessEqual(origtzname, time.tzname) - - def test_iso_utc(self): - when = 1266760143.7841301 - out = time_format.iso_utc_date(when) - self.failUnlessEqual(out, "2010-02-21") - out = time_format.iso_utc_date(t=lambda: when) - self.failUnlessEqual(out, "2010-02-21") - out = time_format.iso_utc(when) - self.failUnlessEqual(out, "2010-02-21_13:49:03.784130") - out = time_format.iso_utc(when, sep="-") - self.failUnlessEqual(out, "2010-02-21-13:49:03.784130") - - def test_parse_duration(self): - p = time_format.parse_duration - DAY = 24*60*60 - self.failUnlessEqual(p("1 day"), DAY) - self.failUnlessEqual(p("2 days"), 2*DAY) - self.failUnlessEqual(p("3 months"), 3*31*DAY) - self.failUnlessEqual(p("4 mo"), 4*31*DAY) - self.failUnlessEqual(p("5 years"), 5*365*DAY) - e = self.failUnlessRaises(ValueError, p, "123") - self.failUnlessIn("no unit (like day, month, or year) in '123'", - str(e)) - - def test_parse_date(self): - self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400) - - def test_format_time(self): - self.failUnlessEqual(time_format.format_time(time.gmtime(0)), '1970-01-01 00:00:00') - self.failUnlessEqual(time_format.format_time(time.gmtime(60)), '1970-01-01 00:01:00') - self.failUnlessEqual(time_format.format_time(time.gmtime(60*60)), '1970-01-01 01:00:00') - seconds_per_day = 60*60*24 - leap_years_1970_to_2014_inclusive = ((2012 - 1968) // 4) - self.failUnlessEqual(time_format.format_time(time.gmtime(seconds_per_day*((2015 - 1970)*365+leap_years_1970_to_2014_inclusive))), '2015-01-01 00:00:00') - - def test_format_time_y2038(self): - seconds_per_day = 60*60*24 - leap_years_1970_to_2047_inclusive = ((2044 - 1968) // 4) - t = (seconds_per_day* - ((2048 - 1970)*365 + leap_years_1970_to_2047_inclusive)) - try: - gm_t = time.gmtime(t) - except ValueError: - raise unittest.SkipTest("Note: this system cannot handle dates after 2037.") - self.failUnlessEqual(time_format.format_time(gm_t), - '2048-01-01 00:00:00') - - def test_format_delta(self): - time_1 = 1389812723 - time_5s_delta = 1389812728 - time_28m7s_delta = 1389814410 - time_1h_delta = 1389816323 - time_1d21h46m49s_delta = 1389977532 - - self.failUnlessEqual( - time_format.format_delta(time_1, time_1), '0s') - - self.failUnlessEqual( - time_format.format_delta(time_1, time_5s_delta), '5s') - self.failUnlessEqual( - time_format.format_delta(time_1, time_28m7s_delta), '28m 7s') - self.failUnlessEqual( - time_format.format_delta(time_1, time_1h_delta), '1h 0m 0s') - self.failUnlessEqual( - time_format.format_delta(time_1, time_1d21h46m49s_delta), '1d 21h 46m 49s') - - self.failUnlessEqual( - time_format.format_delta(time_1d21h46m49s_delta, time_1), '-') - - # time_1 with a decimal fraction will make the delta 1s less - time_1decimal = 1389812723.383963 - - self.failUnlessEqual( - time_format.format_delta(time_1decimal, time_5s_delta), '4s') - self.failUnlessEqual( - time_format.format_delta(time_1decimal, time_28m7s_delta), '28m 6s') - self.failUnlessEqual( - time_format.format_delta(time_1decimal, time_1h_delta), '59m 59s') - self.failUnlessEqual( - time_format.format_delta(time_1decimal, time_1d21h46m49s_delta), '1d 21h 46m 48s') - ctr = [0] class EqButNotIs(object): From 1f4cf403c6403f3470de3693e3ddb8b079e9e353 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:25:32 -0400 Subject: [PATCH 0524/1054] Use minor for these sort of tickets. --- newsfragments/{3325.other => 3325.minor} | 0 newsfragments/{3329.other => 3329.minor} | 0 newsfragments/{3339.other => 3339.minor} | 0 newsfragments/{3340.other => 3340.minor} | 0 newsfragments/{3341.other => 3341.minor} | 0 newsfragments/{3342.other => 3342.minor} | 0 newsfragments/{3343.other => 3343.minor} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3325.other => 3325.minor} (100%) rename newsfragments/{3329.other => 3329.minor} (100%) rename newsfragments/{3339.other => 3339.minor} (100%) rename newsfragments/{3340.other => 3340.minor} (100%) rename newsfragments/{3341.other => 3341.minor} (100%) rename newsfragments/{3342.other => 3342.minor} (100%) rename newsfragments/{3343.other => 3343.minor} (100%) diff --git a/newsfragments/3325.other b/newsfragments/3325.minor similarity index 100% rename from newsfragments/3325.other rename to newsfragments/3325.minor diff --git a/newsfragments/3329.other b/newsfragments/3329.minor similarity index 100% rename from newsfragments/3329.other rename to newsfragments/3329.minor diff --git a/newsfragments/3339.other b/newsfragments/3339.minor similarity index 100% rename from newsfragments/3339.other rename to newsfragments/3339.minor diff --git a/newsfragments/3340.other b/newsfragments/3340.minor similarity index 100% rename from newsfragments/3340.other rename to newsfragments/3340.minor diff --git a/newsfragments/3341.other b/newsfragments/3341.minor similarity index 100% rename from newsfragments/3341.other rename to newsfragments/3341.minor diff --git a/newsfragments/3342.other b/newsfragments/3342.minor similarity index 100% rename from newsfragments/3342.other rename to newsfragments/3342.minor diff --git a/newsfragments/3343.other b/newsfragments/3343.minor similarity index 100% rename from newsfragments/3343.other rename to newsfragments/3343.minor From 554be90b7e1e97cc3afef887708b60696346f3c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:27:36 -0400 Subject: [PATCH 0525/1054] Unnecessary code, and was triggering flake8. --- src/allmydata/test/test_base62.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index d1d0c72f3..e26532e9d 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -104,10 +104,3 @@ class Base62(unittest.TestCase): bs2l = base62.a2b_l(asl, lib) assert len(bs2l) == numos # the size of the result must be just right assert bs == bs2l - -def suite(): - suite = unittest.makeSuite(T, 'test') - return suite - -if __name__ == "__main__": - unittest.main() From e77ab46549af4de09e6bb0f65769f41b46b7c354 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:28:39 -0400 Subject: [PATCH 0526/1054] Correct type. --- newsfragments/{3344.other => 3344.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3344.other => 3344.minor} (100%) diff --git a/newsfragments/3344.other b/newsfragments/3344.minor similarity index 100% rename from newsfragments/3344.other rename to newsfragments/3344.minor From e90d1f38d22bb3e8d5550c4d03d083cc61f91d9b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:44:41 -0400 Subject: [PATCH 0527/1054] Make TimezoneMixin importable on Python 3. --- src/allmydata/test/common_util.py | 25 ------------------------- src/allmydata/test/test_time_format.py | 2 +- src/allmydata/test/test_util.py | 2 +- src/allmydata/test/web/test_web.py | 3 ++- src/allmydata/util/_python3.py | 1 + 5 files changed, 5 insertions(+), 28 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index bf44785a5..2e8bbd006 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -185,31 +185,6 @@ class TestMixin(SignalMixin): self.fail("Reactor was still active when it was required to be quiescent.") -class TimezoneMixin(object): - - def setTimezone(self, timezone): - def tzset_if_possible(): - # Windows doesn't have time.tzset(). - if hasattr(time, 'tzset'): - time.tzset() - - unset = object() - originalTimezone = os.environ.get('TZ', unset) - def restoreTimezone(): - if originalTimezone is unset: - del os.environ['TZ'] - else: - os.environ['TZ'] = originalTimezone - tzset_if_possible() - - os.environ['TZ'] = timezone - self.addCleanup(restoreTimezone) - tzset_if_possible() - - def have_working_tzset(self): - return hasattr(time, 'tzset') - - try: import win32file import win32con diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index ae655ff3e..7edf9f12d 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -6,7 +6,7 @@ import time from twisted.trial import unittest -from allmydata.test.common_util import TimezoneMixin +from allmydata.test.common_py3 import TimezoneMixin from allmydata.util import time_format diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 40a05e9c5..c290903a7 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -21,7 +21,7 @@ from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans from allmydata.util.fileutil import EncryptedTemporaryFile -from allmydata.test.common_util import ReallyEqualMixin, TimezoneMixin +from allmydata.test.common_util import ReallyEqualMixin if six.PY3: long = int diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b3c8c7f46..5678a98f8 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -60,6 +60,7 @@ from .common import ( from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION from allmydata.mutable import servermap, publish, retrieve from .. import common_util as testutil +from ..common_py3 import TimezoneMixin from ..common_web import ( do_http, Error, @@ -311,7 +312,7 @@ class FakeClient(_Client): MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT -class WebMixin(testutil.TimezoneMixin): +class WebMixin(TimezoneMixin): def setUp(self): self.setTimezone('UTC-13:00') self.s = FakeClient() diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa4e5f386..19bd35890 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.namespace", "allmydata.util.pollmixin", "allmydata.util._python3", + "allmydata.test.common_py3", ] PORTED_TEST_MODULES = [ From 4c047b90e5fa6e0e2b14384b2c0016cfa4463716 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:46:23 -0400 Subject: [PATCH 0528/1054] Manual steps of port to Python 3. --- src/allmydata/util/time_format.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index eb3434c47..27b2e0976 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -1,5 +1,9 @@ -# ISO-8601: -# http://www.cl.cam.ac.uk/~mgk25/iso-time.html +""" +Time formatting utilities. + +ISO-8601: +http://www.cl.cam.ac.uk/~mgk25/iso-time.html +""" import calendar, datetime, re, time @@ -74,11 +78,11 @@ def format_delta(time_1, time_2): delta = int(time_2 - time_1) seconds = delta % 60 delta -= seconds - minutes = (delta / 60) % 60 + minutes = (delta // 60) % 60 delta -= minutes * 60 - hours = delta / (60*60) % 24 + hours = delta // (60*60) % 24 delta -= hours * 24 - days = delta / (24*60*60) + days = delta // (24*60*60) if not days: if not hours: if not minutes: From a4620bf17675ca9bc902e5f46856ebfcabcfaa8b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Jul 2020 15:58:39 -0400 Subject: [PATCH 0529/1054] Automated port of the test module. --- src/allmydata/test/test_time_format.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 7edf9f12d..54593e5b9 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -1,6 +1,14 @@ """ Tests for allmydata.util.time_format. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import time From 6a2ce9efc6d897348d3eaac1209d8862e1ad65aa Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 16:55:12 -0400 Subject: [PATCH 0530/1054] Use docstrings --- src/allmydata/test/web/test_status.py | 4 +++- src/allmydata/test/web/test_web.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 0779e7d80..7f4227f17 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -1,4 +1,6 @@ -# Tests for code in allmydata.web.status +""" +Tests for ```allmydata.web.status```. +""" from bs4 import BeautifulSoup from twisted.web.template import flattenString diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f83c44b2f..f0d5c10dc 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1036,8 +1036,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_status_path_error(self): - # Expect an error, because path is expected to be of the form - # "/status/{up,down,..}-%number", with a hyphen. + """ + Expect an error, because path is expected to be of the form + "/status/{up,down,..}-%number", with a hyphen. + """ return self.shouldFail2(error.Error, "test_status_path_error", "400 Bad Request", From e48d9306217007f5d7901a5ceac9539866963856 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 16:56:41 -0400 Subject: [PATCH 0531/1054] Clarify intention of a test case --- src/allmydata/test/web/test_web.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f0d5c10dc..11b024f4d 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1048,9 +1048,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi "/status/nodash") def test_status_path_404(self): - # The test suite (hopefully!) would not have done any setup - # for a very large number of statuses at this point, so these - # all should return 404. + """ + Looking for non-existent statuses under child paths should + exercises all the iterators in web.status.Status.getChild(). + + The test suite (hopefully!) would not have done any setup for + a very large number of statuses at this point, now or in the + future, so these all should always return 404. + """ d = self.GET("/status/up-9999999") d.addBoth(self.should404, "test_status_path_error (up)") From 95642ae02c36236ea21004f0ea0e1ec3b804002a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 17:13:36 -0400 Subject: [PATCH 0532/1054] Use a helper to check status subpath results --- src/allmydata/test/web/test_web.py | 69 ++++++++---------------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 11b024f4d..45bc7034f 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1073,21 +1073,22 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d + def _check_status_subpath_result(self, result, expected_title): + body, status, _ = result + self.failUnlessReallyEqual(int(status), 200) + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", expected_title + ) + def test_status_up_subpath(self): # See that "GET /status/up-0" works. h = self.s.get_history() ul_num = h.list_all_upload_statuses()[0].get_counter() d = self.GET("/status/up-{}".format(ul_num), return_response=True) - def _upload_status_check(result): - body, status, _ = result - self.failUnlessReallyEqual(int(status), 200) - soup = BeautifulSoup(body, 'html5lib') - assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content( - self, soup, u"title", - u"Tahoe-LAFS - File Upload Status" - ) - d.addCallback(_upload_status_check) + d.addCallback(self._check_status_subpath_result, + u"Tahoe-LAFS - File Upload Status") return d def test_status_down_subpath(self): @@ -1095,16 +1096,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() dl_num = h.list_all_download_statuses()[0].get_counter() d = self.GET("/status/down-{}".format(dl_num), return_response=True) - def _download_status_check(result): - body, status, _ = result - self.failUnlessReallyEqual(int(status), 200) - soup = BeautifulSoup(body, 'html5lib') - assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content( - self, soup, u"title", - u"Tahoe-LAFS - File Download Status" - ) - d.addCallback(_download_status_check) + d.addCallback(self._check_status_subpath_result, + u"Tahoe-LAFS - File Download Status") return d def test_status_mapupdate_subpath(self): @@ -1112,16 +1105,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() mu_num = h.list_all_mapupdate_statuses()[0].get_counter() d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True) - def _mapupdate_status_check(result): - body, status, _ = result - self.failUnlessReallyEqual(int(status), 200) - soup = BeautifulSoup(body, 'html5lib') - assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content( - self, soup, u"title", - u"Tahoe-LAFS - Mutable File Servermap Update Status" - ) - d.addCallback(_mapupdate_status_check) + d.addCallback(self._check_status_subpath_result, + u"Tahoe-LAFS - Mutable File Servermap Update Status") return d def test_status_publish_subpath(self): @@ -1129,16 +1114,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() pub_num = h.list_all_publish_statuses()[0].get_counter() d = self.GET("/status/publish-{}".format(pub_num), return_response=True) - def _publish_status_check(result): - body, status, _ = result - self.failUnlessReallyEqual(int(status), 200) - soup = BeautifulSoup(body, 'html5lib') - assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content( - self, soup, u"title", - u"Tahoe-LAFS - Mutable File Publish Status" - ) - d.addCallback(_publish_status_check) + d.addCallback(self._check_status_subpath_result, + u"Tahoe-LAFS - Mutable File Publish Status") return d def test_status_retrieve_subpath(self): @@ -1146,16 +1123,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi h = self.s.get_history() ret_num = h.list_all_retrieve_statuses()[0].get_counter() d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True) - def _retrieve_status_check(result): - body, status, _ = result - self.failUnlessReallyEqual(int(status), 200) - soup = BeautifulSoup(body, 'html5lib') - assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content( - self, soup, u"title", - u"Tahoe-LAFS - Mutable File Retrieve Status" - ) - d.addCallback(_retrieve_status_check) + d.addCallback(self._check_status_subpath_result, + u"Tahoe-LAFS - Mutable File Retrieve Status") return d def test_status_numbers(self): From 8db60e3486d78e1b628faaf8b408942251471d4a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 17:16:18 -0400 Subject: [PATCH 0533/1054] Do the simple thing when testing status subpaths --- src/allmydata/test/web/test_web.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 45bc7034f..c8f954d01 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1084,45 +1084,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_status_up_subpath(self): # See that "GET /status/up-0" works. - h = self.s.get_history() - ul_num = h.list_all_upload_statuses()[0].get_counter() - d = self.GET("/status/up-{}".format(ul_num), return_response=True) + d = self.GET("/status/up-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Upload Status") return d def test_status_down_subpath(self): # See that "GET /status/down-0" works. - h = self.s.get_history() - dl_num = h.list_all_download_statuses()[0].get_counter() - d = self.GET("/status/down-{}".format(dl_num), return_response=True) + d = self.GET("/status/down-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Download Status") return d def test_status_mapupdate_subpath(self): # See that "GET /status/mapupdate-0" works. - h = self.s.get_history() - mu_num = h.list_all_mapupdate_statuses()[0].get_counter() - d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True) + d = self.GET("/status/mapupdate-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Servermap Update Status") return d def test_status_publish_subpath(self): # See that "GET /status/publish-0" works. - h = self.s.get_history() - pub_num = h.list_all_publish_statuses()[0].get_counter() - d = self.GET("/status/publish-{}".format(pub_num), return_response=True) + d = self.GET("/status/publish-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Publish Status") return d def test_status_retrieve_subpath(self): # See that "GET /status/retrieve-0" works. - h = self.s.get_history() - ret_num = h.list_all_retrieve_statuses()[0].get_counter() - d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True) + d = self.GET("/status/retrieve-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Retrieve Status") return d From 66dd2d365829685edaa09f0c11fba99da0bf5b25 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 17:19:50 -0400 Subject: [PATCH 0534/1054] Use docstrings --- src/allmydata/test/web/test_web.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index c8f954d01..e7832c81c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1074,6 +1074,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def _check_status_subpath_result(self, result, expected_title): + """ + Helper to verify that results of "GET /status/up-0" and + similar are as expected. + """ body, status, _ = result self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') @@ -1083,35 +1087,45 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi ) def test_status_up_subpath(self): - # See that "GET /status/up-0" works. + """ + See that "GET /status/up-0" works. + """ d = self.GET("/status/up-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Upload Status") return d def test_status_down_subpath(self): - # See that "GET /status/down-0" works. + """ + See that "GET /status/down-0" works. + """ d = self.GET("/status/down-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Download Status") return d def test_status_mapupdate_subpath(self): - # See that "GET /status/mapupdate-0" works. + """ + See that "GET /status/mapupdate-0" works. + """ d = self.GET("/status/mapupdate-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Servermap Update Status") return d def test_status_publish_subpath(self): - # See that "GET /status/publish-0" works. + """ + See that "GET /status/publish-0" works. + """ d = self.GET("/status/publish-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Publish Status") return d def test_status_retrieve_subpath(self): - # See that "GET /status/retrieve-0" works. + """ + See that "GET /status/retrieve-0" works. + """ d = self.GET("/status/retrieve-0", return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Retrieve Status") From 06c8bbfb19bb0194a6007305dca313322e2c652b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 17:25:25 -0400 Subject: [PATCH 0535/1054] Use clearer names for tests --- src/allmydata/test/web/test_web.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e7832c81c..97a997aa5 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1035,19 +1035,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d - def test_status_path_error(self): + def test_status_path_nodash_error(self): """ Expect an error, because path is expected to be of the form "/status/{up,down,..}-%number", with a hyphen. """ return self.shouldFail2(error.Error, - "test_status_path_error", + "test_status_path_nodash", "400 Bad Request", "no '-' in 'nodash'", self.GET, "/status/nodash") - def test_status_path_404(self): + def test_status_path_404_error(self): """ Looking for non-existent statuses under child paths should exercises all the iterators in web.status.Status.getChild(). @@ -1057,19 +1057,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi future, so these all should always return 404. """ d = self.GET("/status/up-9999999") - d.addBoth(self.should404, "test_status_path_error (up)") + d.addBoth(self.should404, "test_status_path_404_error (up)") d = self.GET("/status/down-9999999") - d.addBoth(self.should404, "test_status_path_error (down)") + d.addBoth(self.should404, "test_status_path_404_error (down)") d = self.GET("/status/mapupdate-9999999") - d.addBoth(self.should404, "test_status_path_error (mapupdate)") + d.addBoth(self.should404, "test_status_path_404_error (mapupdate)") d = self.GET("/status/publish-9999999") - d.addBoth(self.should404, "test_status_path_error (publish)") + d.addBoth(self.should404, "test_status_path_404_error (publish)") d = self.GET("/status/retrieve-9999999") - d.addBoth(self.should404, "test_status_path_error (retrieve)") + d.addBoth(self.should404, "test_status_path_404_error (retrieve)") return d From fe165cc99d7534f0fc139dc9429bf0757ba901ad Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 16 Jul 2020 19:07:51 -0400 Subject: [PATCH 0536/1054] Do the right thing, not the simple thing Seems that NixOS CI is saying that the simple thing is not the right thing: "GET /status/down-0" etc all resulted in 404 responses on NixOS. So we perhaps should use the actual counter value. --- src/allmydata/test/web/test_web.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 97a997aa5..f7158918c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1090,7 +1090,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ See that "GET /status/up-0" works. """ - d = self.GET("/status/up-0", return_response=True) + h = self.s.get_history() + ul_num = h.list_all_upload_statuses()[0].get_counter() + d = self.GET("/status/up-{}".format(ul_num), return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Upload Status") return d @@ -1099,7 +1101,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ See that "GET /status/down-0" works. """ - d = self.GET("/status/down-0", return_response=True) + h = self.s.get_history() + dl_num = h.list_all_download_statuses()[0].get_counter() + d = self.GET("/status/down-{}".format(dl_num), return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - File Download Status") return d @@ -1108,7 +1112,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ See that "GET /status/mapupdate-0" works. """ - d = self.GET("/status/mapupdate-0", return_response=True) + h = self.s.get_history() + mu_num = h.list_all_mapupdate_statuses()[0].get_counter() + d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Servermap Update Status") return d @@ -1117,7 +1123,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ See that "GET /status/publish-0" works. """ - d = self.GET("/status/publish-0", return_response=True) + h = self.s.get_history() + pub_num = h.list_all_publish_statuses()[0].get_counter() + d = self.GET("/status/publish-{}".format(pub_num), return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Publish Status") return d @@ -1126,7 +1134,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ See that "GET /status/retrieve-0" works. """ - d = self.GET("/status/retrieve-0", return_response=True) + h = self.s.get_history() + ret_num = h.list_all_retrieve_statuses()[0].get_counter() + d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True) d.addCallback(self._check_status_subpath_result, u"Tahoe-LAFS - Mutable File Retrieve Status") return d From 372cc00a25d91c0e4076092d86da7d615b03645d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 09:46:21 -0400 Subject: [PATCH 0537/1054] Reject non-existent child nodes of /status Requests on both `/status` and `/status/` will be served with the same status page, but `/status//`, `/status///` and so on will be processed further down, resulting in a 40x response. --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 39b19cb0b..2c19c45dd 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1214,7 +1214,7 @@ class Status(MultiFormatResource): # final URL segment will be an empty string. Resources can # thus know if they were requested with or without a final # slash." - if not path: + if not path and request.postpath != ['']: return self h = self.history From c942064ee05219ecb068a1b7864bca05ccc043ef Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:42:41 -0400 Subject: [PATCH 0538/1054] Stop using TravisCI for Python 2.7/Linux test coverage --- .travis.yml | 3 --- newsfragments/3330.minor | 0 2 files changed, 3 deletions(-) create mode 100644 newsfragments/3330.minor diff --git a/.travis.yml b/.travis.yml index dec435604..28519b573 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,9 +54,6 @@ matrix: - os: linux python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 - - os: linux - python: '2.7' - env: T=py27 LANG=C - os: osx python: '2.7' env: T=py27 LANG=en_US.UTF-8 diff --git a/newsfragments/3330.minor b/newsfragments/3330.minor new file mode 100644 index 000000000..e69de29bb From f3780cd4cbb3144d706641b26b5b4ab6959d73b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:43:39 -0400 Subject: [PATCH 0539/1054] Stop using TravisCI for Python 2.7/macOS test coverage --- .travis.yml | 4 ---- newsfragments/3331.minor | 0 2 files changed, 4 deletions(-) create mode 100644 newsfragments/3331.minor diff --git a/.travis.yml b/.travis.yml index 28519b573..0a0a28a9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,10 +54,6 @@ matrix: - os: linux python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 - - os: osx - python: '2.7' - env: T=py27 LANG=en_US.UTF-8 - language: generic # "python" is not available on OS-X - os: osx python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 diff --git a/newsfragments/3331.minor b/newsfragments/3331.minor new file mode 100644 index 000000000..e69de29bb From 0803aef1cb8574d33fac8c161ca44d2962041c9e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:44:19 -0400 Subject: [PATCH 0540/1054] Stop using TravisCI for pyinstaller/macOS test coverage --- .travis.yml | 4 ---- newsfragments/3332.minor | 0 2 files changed, 4 deletions(-) create mode 100644 newsfragments/3332.minor diff --git a/.travis.yml b/.travis.yml index 0a0a28a9e..e40a4506d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,10 +54,6 @@ matrix: - os: linux python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 - - os: osx - python: '2.7' - env: T=pyinstaller LANG=en_US.UTF-8 - language: generic # "python" is not available on OS-X - os: linux python: '3.6' env: T=py36 diff --git a/newsfragments/3332.minor b/newsfragments/3332.minor new file mode 100644 index 000000000..e69de29bb From 82a4fca4161b14d38dcb2c56f20b97aefba52144 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:45:05 -0400 Subject: [PATCH 0541/1054] Stop using TravisCI for coverage/Linux test coverage --- .travis.yml | 3 --- newsfragments/3333.minor | 0 2 files changed, 3 deletions(-) create mode 100644 newsfragments/3333.minor diff --git a/.travis.yml b/.travis.yml index e40a4506d..edb17c9c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,9 +45,6 @@ notifications: matrix: include: - - os: linux - python: '2.7' - env: T=coverage LANG=en_US.UTF-8 - os: linux python: '2.7' env: T=codechecks LANG=en_US.UTF-8 diff --git a/newsfragments/3333.minor b/newsfragments/3333.minor new file mode 100644 index 000000000..e69de29bb From 1b491f293a208f6931dc053bd25c0df9e6fe45af Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:45:50 -0400 Subject: [PATCH 0542/1054] Stop using TravisCI for codechecks/Linux test coverage --- .travis.yml | 3 --- newsfragments/3334.minor | 0 2 files changed, 3 deletions(-) create mode 100644 newsfragments/3334.minor diff --git a/.travis.yml b/.travis.yml index edb17c9c9..be02a8502 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,9 +45,6 @@ notifications: matrix: include: - - os: linux - python: '2.7' - env: T=codechecks LANG=en_US.UTF-8 - os: linux python: '2.7' env: T=pyinstaller LANG=en_US.UTF-8 diff --git a/newsfragments/3334.minor b/newsfragments/3334.minor new file mode 100644 index 000000000..e69de29bb From 16c8026b6dacde78b756683433e574941ac08f00 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:46:23 -0400 Subject: [PATCH 0543/1054] Stop using TravisCI for pyinstaller/Linux test coverage --- .travis.yml | 3 --- newsfragments/3335.minor | 0 2 files changed, 3 deletions(-) create mode 100644 newsfragments/3335.minor diff --git a/.travis.yml b/.travis.yml index be02a8502..73741586d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,9 +45,6 @@ notifications: matrix: include: - - os: linux - python: '2.7' - env: T=pyinstaller LANG=en_US.UTF-8 - os: linux python: '3.6' env: T=py36 diff --git a/newsfragments/3335.minor b/newsfragments/3335.minor new file mode 100644 index 000000000..e69de29bb From cfe70335683f0be269058da8c331516de65199be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 12:49:12 -0400 Subject: [PATCH 0544/1054] Remove obsolete supporting configuration There are no longer any macOS jobs. Delete the macOS checks. There are no longer any coverage jobs. Delete codecov hook. There are no longer any pyinstaller jobs. Delete extra pyinstaller check. --- .travis.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 73741586d..40f351ae6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,7 @@ env: - TAHOE_LAFS_HYPOTHESIS_PROFILE=ci install: -# ~/.local/bin is on $PATH by default, but on OS-X, --user puts it elsewhere - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then export PATH=$HOME/Library/Python/2.7/bin:$PATH; fi - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py; fi - - pip list - - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then pip install --user --upgrade codecov tox setuptools virtualenv; fi - - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then pip install --upgrade codecov tox setuptools virtualenv; fi + - pip install --upgrade tox setuptools virtualenv - echo $PATH; which python; which pip; which tox - python misc/build_helpers/show-tool-versions.py @@ -25,13 +20,6 @@ script: - | set -eo pipefail tox -e ${T} - # To verify that the resultant PyInstaller-generated binary executes - # cleanly (i.e., that it terminates with an exit code of 0 and isn't - # failing due to import/packaging-related errors, etc.). - if [ "${T}" = "pyinstaller" ]; then dist/Tahoe-LAFS/tahoe --version; fi - -after_success: - - if [ "${T}" = "coverage" ]; then codecov; fi notifications: email: false From 02d8bd061afcfa85d3cf7d5e986ca4cba056de5a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 10:49:30 -0400 Subject: [PATCH 0545/1054] Test for trailing slashes after `/status/` Both `/status` and `/status/` are treated alike; any additional trailing slashes will be rejected. --- src/allmydata/test/web/test_web.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f7158918c..850d494ab 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1047,6 +1047,46 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.GET, "/status/nodash") + def test_status_path_trailing_slashes(self): + """ + Test that both `GET /status` and `GET /status/` are treated + alike, but reject any additional trailing slashes and other + non-existent child nodes. + """ + def _check_status(response): + (body, status, _) = response + + self.failUnlessReallyEqual(int(status), 200) + + soup = BeautifulSoup(body, 'html5lib') + assert_soup_has_favicon(self, soup) + assert_soup_has_tag_with_content( + self, soup, u"title", + u"Tahoe-LAFS - Recent and Active Operations" + ) + + d = self.GET("/status", return_response=True) + d.addCallback(_check_status) + + d = self.GET("/status/", return_response=True) + d.addCallback(_check_status) + + d = self.shouldFail2(error.Error, + "test_status_path_trailing_slashes", + "400 Bad Request", + "no '-' in ''", + self.GET, + "/status//") + + d = self.shouldFail2(error.Error, + "test_status_path_trailing_slashes", + "400 Bad Request", + "no '-' in ''", + self.GET, + "/status////////") + + return d + def test_status_path_404_error(self): """ Looking for non-existent statuses under child paths should From 2d19c383ca2ea78967537c54e98da6bb71907b4c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 10:58:01 -0400 Subject: [PATCH 0546/1054] Rename a variable for clarity --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2c19c45dd..202fbaa61 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1290,8 +1290,8 @@ class StatusElement(Element): @renderer def recent_operations(self, req, tag): - active = [self.get_op_state(op) for op in self._recent] - return SlotsSequenceElement(tag, active) + recent = [self.get_op_state(op) for op in self._recent] + return SlotsSequenceElement(tag, recent) @staticmethod def get_op_state(op): From 5c886b1b2d6e3b6929b12d5723afd8039e6cbcb5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 10:58:34 -0400 Subject: [PATCH 0547/1054] Format link to status page child node correctly Some Nevow-ism accidentally escaped earlier: link to `/status/up-0` and so on were actually rendered as `/up-0` and so on. --- src/allmydata/web/status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 202fbaa61..6ff619504 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1344,7 +1344,8 @@ class StatusElement(Element): link = "mapupdate-%d" % op.get_counter() result["progress"] = "%.1f%%" % (100.0 * progress) - result["status"] = tags.a(op.get_status(), href=link) + result["status"] = tags.a(op.get_status(), + href="/status/{}".format(link)) return result From 828ec53e24fb77c70e22550f2c15eb395a558ae1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 11:28:19 -0400 Subject: [PATCH 0548/1054] Update status page test case Link to upload status 0 should contain `/status/` prefix, or it will navigate to `/up-0` instead of `/status/up-0`, resulting in a 404 not found message. Same with other statuses too. --- src/allmydata/test/web/test_web.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 850d494ab..1c155f6ce 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -972,11 +972,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d = self.GET("/status", followRedirect=True) def _check(res): self.failUnlessIn('Recent and Active Operations', res) - self.failUnlessIn('"down-%d"' % dl_num, res) - self.failUnlessIn('"up-%d"' % ul_num, res) - self.failUnlessIn('"mapupdate-%d"' % mu_num, res) - self.failUnlessIn('"publish-%d"' % pub_num, res) - self.failUnlessIn('"retrieve-%d"' % ret_num, res) + self.failUnlessIn('"/status/down-%d"' % dl_num, res) + self.failUnlessIn('"/status/up-%d"' % ul_num, res) + self.failUnlessIn('"/status/mapupdate-%d"' % mu_num, res) + self.failUnlessIn('"/status/publish-%d"' % pub_num, res) + self.failUnlessIn('"/status/retrieve-%d"' % ret_num, res) d.addCallback(_check) d.addCallback(lambda res: self.GET("/status/?t=json")) def _check_json(res): From 8fd0cf7eb6de6b57ad0e659f7694d89c1fa93ed3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 11:40:18 -0400 Subject: [PATCH 0549/1054] Add a test for checking status page links --- src/allmydata/test/web/test_web.py | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1c155f6ce..d787124eb 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1047,6 +1047,58 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.GET, "/status/nodash") + def test_status_page_contains_links(self): + """ + Check that the rendered `/status` page contains all the + expected links. + """ + def _check_status_page_links(response): + (body, status, _) = response + + self.failUnlessReallyEqual(int(status), 200) + + soup = BeautifulSoup(body, 'html5lib') + + # Check for `Not started` + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"a", + u"Not started", + {u"href": u"/status/retrieve-0"} + ) + + # Check for `Not started` + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"a", + u"Not started", + {u"href": u"/status/publish-0"} + ) + + # Check for `Not started` + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"a", + u"Not started", + {u"href": u"/status/mapupdate-0"} + ) + + # Check for `fetching segments + # 2,3; errors on segment 1`: see build_one_ds() above. + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"a", + u"fetching segments 2,3; errors on segment 1", + {u"href": u"/status/down-0"} + ) + + # Check for `Not started` + assert_soup_has_tag_with_attributes_and_content( + self, soup, u"a", + u"Not started", + {u"href": u"/status/up-0"} + ) + + d = self.GET("/status", return_response=True) + d.addCallback(_check_status_page_links) + return d + def test_status_path_trailing_slashes(self): """ Test that both `GET /status` and `GET /status/` are treated From bd2f43884358b37259861870098538eea35b4eff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 13:48:08 -0400 Subject: [PATCH 0550/1054] Replace the TravisCI Python 3.6 config with CircleCI Python 3.6 config --- .circleci/config.yml | 12 +++++++++++- .travis.yml | 40 ---------------------------------------- 2 files changed, 11 insertions(+), 41 deletions(-) delete mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..9654905d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,9 @@ workflows: # Test against PyPy 2.7 - "pypy2.7-buster" + # Just one Python 3.6 configuration while the port is in-progress. + - "python3.6" + # Other assorted tasks and configurations - "lint" - "pyinstaller" @@ -261,13 +264,20 @@ jobs: user: "nobody" - ubuntu-18.04: + ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - image: "tahoelafsci/ubuntu:18.04" user: "nobody" + python3.6: + <<: *UBUNTU_18_04 + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + + centos-8: &RHEL_DERIV docker: - image: "tahoelafsci/centos:8" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 40f351ae6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: false -language: python -cache: pip -dist: xenial -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log -git: - depth: 1000 - -env: - global: - - TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - -install: - - pip install --upgrade tox setuptools virtualenv - - echo $PATH; which python; which pip; which tox - - python misc/build_helpers/show-tool-versions.py - -script: - - | - set -eo pipefail - tox -e ${T} - -notifications: - email: false - irc: - channels: "chat.freenode.net#tahoe-lafs" - on_success: always # for testing - on_failure: always - template: - - "%{repository}#%{build_number} [%{branch}: %{commit} by %{author}] %{message}" - - "Changes: %{compare_url} | Details: %{build_url}" - -matrix: - include: - - os: linux - python: '3.6' - env: T=py36 - - fast_finish: true From 293efd7c9fe929e3e6fd029e1f1f1da91440c387 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 13:50:06 -0400 Subject: [PATCH 0551/1054] Use actual status counter number when testing --- src/allmydata/test/web/test_web.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index d787124eb..1d235af63 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1058,41 +1058,47 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessReallyEqual(int(status), 200) soup = BeautifulSoup(body, 'html5lib') + h = self.s.get_history() # Check for `Not started` + ret_num = h.list_all_retrieve_statuses()[0].get_counter() assert_soup_has_tag_with_attributes_and_content( self, soup, u"a", u"Not started", - {u"href": u"/status/retrieve-0"} + {u"href": u"/status/retrieve-{}".format(ret_num)} ) # Check for `Not started` + pub_num = h.list_all_publish_statuses()[0].get_counter() assert_soup_has_tag_with_attributes_and_content( self, soup, u"a", u"Not started", - {u"href": u"/status/publish-0"} + {u"href": u"/status/publish-{}".format(pub_num)} ) # Check for `Not started` + mu_num = h.list_all_mapupdate_statuses()[0].get_counter() assert_soup_has_tag_with_attributes_and_content( self, soup, u"a", u"Not started", - {u"href": u"/status/mapupdate-0"} + {u"href": u"/status/mapupdate-{}".format(mu_num)} ) # Check for `fetching segments # 2,3; errors on segment 1`: see build_one_ds() above. + dl_num = h.list_all_download_statuses()[0].get_counter() assert_soup_has_tag_with_attributes_and_content( self, soup, u"a", u"fetching segments 2,3; errors on segment 1", - {u"href": u"/status/down-0"} + {u"href": u"/status/down-{}".format(dl_num)} ) # Check for `Not started` + ul_num = h.list_all_upload_statuses()[0].get_counter() assert_soup_has_tag_with_attributes_and_content( self, soup, u"a", u"Not started", - {u"href": u"/status/up-0"} + {u"href": u"/status/up-{}".format(ul_num)} ) d = self.GET("/status", return_response=True) From aab598c2f54b0beb3373400250ad8babff13149a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 14:30:17 -0400 Subject: [PATCH 0552/1054] Update integration test for status page As a fallout of transition from nevow to twisted web, list of hrefs in the parsed status page will be [u'/status/down-0', u'/status/up-0', u'/'] (with an added '/status/' prefix) so we need to update our assumptions. See 5c886b1b2 for the change that necessitated this. --- integration/test_web.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index 4ba0a6fd1..575e4fc1a 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -219,23 +219,21 @@ def test_status(alice): found_upload = False found_download = False for href in hrefs: - if href.startswith(u"/") or not href: + if href == u"/" or not href: continue - resp = requests.get( - util.node_url(alice.node_dir, u"status/{}".format(href)), - ) - if href.startswith(u'up'): + resp = requests.get(util.node_url(alice.node_dir, href)) + if href.startswith(u"/status/up"): assert "File Upload Status" in resp.content if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: found_upload = True - elif href.startswith(u'down'): + elif href.startswith(u"/status/down"): assert "File Download Status" in resp.content if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: found_download = True # download the specialized event information resp = requests.get( - util.node_url(alice.node_dir, u"status/{}/event_json".format(href)), + util.node_url(alice.node_dir, u"{}/event_json".format(href)), ) js = json.loads(resp.content) # there's usually just one "read" operation, but this can handle many .. From c14197e1bc409f323e3a8eb41977f0726079f3aa Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 16:13:07 -0400 Subject: [PATCH 0553/1054] Make FakeDownloadStatus an instance of DownloadStatus The real DownloadStatus is easy enough to instantiate. Might as well use that, cutting some crufty code here. --- src/allmydata/test/web/test_status.py | 43 ++++----------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 42f2b9794..c52a50bfb 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -4,11 +4,9 @@ from bs4 import BeautifulSoup from twisted.web.template import flattenString from zope.interface import implementer -from allmydata.interfaces import ( - IDownloadResults, - IDownloadStatus, -) +from allmydata.interfaces import IDownloadResults from allmydata.web.status import DownloadStatusElement +from allmydata.immutable.downloader.status import DownloadStatus from .common import ( assert_soup_has_favicon, @@ -37,8 +35,7 @@ class FakeDownloadResults(object): self.timings = timings -@implementer(IDownloadStatus) -class FakeDownloadStatus(object): +class FakeDownloadStatus(DownloadStatus): def __init__(self, storage_index = None, @@ -50,49 +47,21 @@ class FakeDownloadStatus(object): """ See IDownloadStatus and IDownloadResults for parameters. """ - self.storage_index = storage_index - self.file_size = file_size - self.dyhb_requests = [] - self.read_events = [] - self.segment_events = [] - self.block_requests = [] + super(FakeDownloadStatus, self).__init__(storage_index, file_size) self.servers_used = servers_used self.server_problems = server_problems self.servermap = servermap self.timings = timings - def get_started(self): - return None - - def get_storage_index(self): - return self.storage_index - - def get_size(self): - return self.file_size - - def using_helper(self): - return False - - def get_status(self): - return "FakeDownloadStatus" - - def get_progress(self): - return 0 - - def get_active(): - return False - - def get_counter(): - return 0 - def get_results(self): - return FakeDownloadResults(self.file_size, + return FakeDownloadResults(self.size, self.servers_used, self.server_problems, self.servermap, self.timings) + # Tests for code in allmydata.web.status.DownloadStatusElement class DownloadStatusElementTests(TrialTestCase): From fe07ab39da8cabe6b43f1d8403375b92e7d5939a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 16:14:35 -0400 Subject: [PATCH 0554/1054] Use docstrings --- src/allmydata/test/web/test_status.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index c52a50bfb..6de1b7a35 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -1,4 +1,6 @@ -# Tests for code in allmydata.web.status +""" +Tests for ```allmydata.web.status```. +""" from bs4 import BeautifulSoup from twisted.web.template import flattenString @@ -62,8 +64,10 @@ class FakeDownloadStatus(DownloadStatus): self.timings) -# Tests for code in allmydata.web.status.DownloadStatusElement class DownloadStatusElementTests(TrialTestCase): + """ + Tests for ```allmydata.web.status.DownloadStatusElement```. + """ def _render_download_status_element(self, status): """ From b0a5a65f4f8494c8d8d19014a31ae67fb4d9fcc9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:23:42 -0400 Subject: [PATCH 0555/1054] Attempt to enable a Python 3 CircleCI job --- .circleci/Dockerfile.centos | 7 ++++--- .circleci/Dockerfile.debian | 7 ++++--- .circleci/Dockerfile.fedora | 7 ++++--- .circleci/Dockerfile.slackware | 5 +++-- .circleci/Dockerfile.ubuntu | 7 ++++--- .circleci/config.yml | 22 ++++++++++++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index febb61545..632e13137 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM centos:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python2 \ - python2-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index de16148e0..394495e17 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM debian:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -12,8 +13,8 @@ RUN apt-get --quiet update && \ lsb-release \ sudo \ build-essential \ - python2.7 \ - python2.7-dev \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ libffi-dev \ libssl-dev \ libyaml-dev \ @@ -23,7 +24,7 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" # Only the integration tests currently need this but it doesn't hurt to always # have it present and it's simpler than building a whole extra image just for diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index 6ad22d676..f17140f3e 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM fedora:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python \ - python-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml-devel \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware index 73ba6b32d..97ee07609 100644 --- a/.circleci/Dockerfile.slackware +++ b/.circleci/Dockerfile.slackware @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM vbatts/slackware:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -23,7 +24,7 @@ RUN slackpkg update && \ binutils-2 \ gcc-5 \ gcc-g++-5 \ - python-2 \ + python-${PYTHON_VERSION} \ libffi-3 \ libyaml-0 \ sqlite-3 \ @@ -46,4 +47,4 @@ RUN slackpkg install \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 882dfe308..4ca2319a9 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM ubuntu:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -13,8 +14,8 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install \ sudo \ build-essential \ - python2.7 \ - python2.7-dev \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ libffi-dev \ libssl-dev \ libyaml-dev \ @@ -26,4 +27,4 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/config.yml b/.circleci/config.yml index 9654905d7..dd423e2af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,7 @@ workflows: - "build-image-centos-8" - "build-image-slackware-14.2" - "build-image-pypy-2.7-buster" + - "build-image-python36-ubuntu" jobs: @@ -449,6 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION:-python2} -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ @@ -464,6 +466,7 @@ jobs: environment: DISTRO: "debian" TAG: "8" + PYTHON_VERSION: "2.7" build-image-debian-9: @@ -472,6 +475,7 @@ jobs: environment: DISTRO: "debian" TAG: "9" + PYTHON_VERSION: "2.7" build-image-ubuntu-16.04: @@ -480,6 +484,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "16.04" + PYTHON_VERSION: "2.7" build-image-ubuntu-18.04: @@ -488,6 +493,16 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" + PYTHON_VERSION: "2.7" + + + build-image-python36-ubuntu: + <<: *BUILD_IMAGE + + environment: + DISTRO: "ubuntu" + TAG: "18.04" + PYTHON_VERSION: "3" build-image-centos-8: @@ -496,6 +511,7 @@ jobs: environment: DISTRO: "centos" TAG: "8" + PYTHON_VERSION: "2" build-image-fedora-28: @@ -504,6 +520,8 @@ jobs: environment: DISTRO: "fedora" TAG: "28" + # The default on Fedora (this version anyway) is still Python 2. + PYTHON_VERSION: "" build-image-fedora-29: @@ -520,6 +538,7 @@ jobs: environment: DISTRO: "slackware" TAG: "14.2" + PYTHON_VERSION: "2" build-image-pypy-2.7-buster: @@ -528,3 +547,6 @@ jobs: environment: DISTRO: "pypy" TAG: "2.7-buster" + # We only have Python 2 for PyPy right now so there's no support for + # setting up PyPy 3 in the image building toolchain.. + PYTHON_VERSION: "" From c198c006e34c2fe8e4d562628a91caa196863b8b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 17 Jul 2020 21:02:24 -0400 Subject: [PATCH 0556/1054] Avoid over-long lines --- src/allmydata/test/web/test_status.py | 87 ++++++++++++++++++++------- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 6de1b7a35..c5cba18ec 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -82,35 +82,72 @@ class DownloadStatusElementTests(TrialTestCase): """ See if we can render the page almost fully. """ - status = FakeDownloadStatus("si-1", 123, - ["s-1", "s-2", "s-3"], - {"s-1": "unknown problem"}, - {"s-1": [1], "s-2": [1,2], "s-3": [2,3]}, - {"fetch_per_server": {"s-1": [1], "s-2": [2,3], "s-3": [3,2]}}) + status = FakeDownloadStatus( + "si-1", 123, + ["s-1", "s-2", "s-3"], + {"s-1": "unknown problem"}, + {"s-1": [1], "s-2": [1,2], "s-3": [2,3]}, + {"fetch_per_server": + {"s-1": [1], "s-2": [2,3], "s-3": [3,2]}} + ) result = self._render_download_status_element(status) soup = BeautifulSoup(result, 'html5lib') - assert_soup_has_text(self, soup, u"Tahoe-LAFS - File Download Status") assert_soup_has_favicon(self, soup) - assert_soup_has_tag_with_content(self, soup, u"li", u"File Size: 123 bytes") - assert_soup_has_tag_with_content(self, soup, u"li", u"Progress: 0.0%") + assert_soup_has_tag_with_content( + self, soup, u"title", u"Tahoe-LAFS - File Download Status" + ) - assert_soup_has_tag_with_content(self, soup, u"li", u"Servers Used: [omwtc], [omwte], [omwtg]") + assert_soup_has_tag_with_content( + self, soup, u"li", u"File Size: 123 bytes" + ) + assert_soup_has_tag_with_content( + self, soup, u"li", u"Progress: 0.0%" + ) - assert_soup_has_tag_with_content(self, soup, u"li", u"Server Problems:") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: unknown problem") + assert_soup_has_tag_with_content( + self, soup, u"li", u"Servers Used: [omwtc], [omwte], [omwtg]" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"Server Problems:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwtc]: unknown problem" + ) assert_soup_has_tag_with_content(self, soup, u"li", u"Servermap:") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc] has share: #1") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte] has shares: #1,#2") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg] has shares: #2,#3") - assert_soup_has_tag_with_content(self, soup, u"li", u"Per-Server Segment Fetch Response Times:") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtc]: 1.00s") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwte]: 2.00s, 3.00s") - assert_soup_has_tag_with_content(self, soup, u"li", u"[omwtg]: 3.00s, 2.00s") + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwtc] has share: #1" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwte] has shares: #1,#2" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwtg] has shares: #2,#3" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"Per-Server Segment Fetch Response Times:" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwtc]: 1.00s" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwte]: 2.00s, 3.00s" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"[omwtg]: 3.00s, 2.00s" + ) def test_download_status_element_partial(self): """ @@ -120,6 +157,14 @@ class DownloadStatusElementTests(TrialTestCase): result = self._render_download_status_element(status) soup = BeautifulSoup(result, 'html5lib') - assert_soup_has_tag_with_content(self, soup, u"li", u"Servermap: None") - assert_soup_has_tag_with_content(self, soup, u"li", u"File Size: 0 bytes") - assert_soup_has_tag_with_content(self, soup, u"li", u"Total: None (None)") + assert_soup_has_tag_with_content( + self, soup, u"li", u"Servermap: None" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"File Size: 0 bytes" + ) + + assert_soup_has_tag_with_content( + self, soup, u"li", u"Total: None (None)" + ) From 879a8012a0f384252f18df898bd276b3eca6e6d9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 18 Jul 2020 06:48:36 -0400 Subject: [PATCH 0557/1054] Remove unused import --- src/allmydata/test/web/test_status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index c5cba18ec..c6573ab96 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -12,7 +12,6 @@ from allmydata.immutable.downloader.status import DownloadStatus from .common import ( assert_soup_has_favicon, - assert_soup_has_text, assert_soup_has_tag_with_content, ) from ..common import TrialTestCase From c58cf56e83569f46d845bd90d0ea66e2b1643196 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 18 Jul 2020 08:34:25 -0400 Subject: [PATCH 0558/1054] Remove unused imports --- src/allmydata/web/status.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 9637e2403..3a89e8f5e 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -13,10 +13,8 @@ from twisted.web.template import ( renderElement, tags, ) -from nevow import rend, tags as T from allmydata.util import base32, idlib from allmydata.web.common import ( - getxmlfile, abbreviate_time, abbreviate_rate, abbreviate_size, From 9e82df4fa72d613fe2dc6ccd6b23bfb1102d561d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sun, 19 Jul 2020 09:19:19 -0400 Subject: [PATCH 0559/1054] Use skipif decorator to omit Tor integration test setup on Windows --- integration/conftest.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 35e550416..bb104b464 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -358,11 +358,10 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques @pytest.fixture(scope='session') +@pytest.mark.skipif(sys.platform.startswith('win'), + 'Tor tests are unstable on Windows') def chutney(reactor, temp_dir): - if sys.platform.startswith('win'): - pytest.skip('Tor tests are unstable on Windows') - chutney_dir = join(temp_dir, 'chutney') mkdir(chutney_dir) @@ -392,11 +391,10 @@ def chutney(reactor, temp_dir): @pytest.fixture(scope='session') +@pytest.mark.skipif(sys.platform.startswith('win'), + reason='Tor tests are unstable on Windows') def tor_network(reactor, temp_dir, chutney, request): - if sys.platform.startswith('win'): - pytest.skip('Tor tests are unstable on Windows') - # this is the actual "chutney" script at the root of a chutney checkout chutney_dir = chutney chut = join(chutney_dir, 'chutney') From 30a91a1106728a14ddabc92b56d0defd25a41051 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Jul 2020 00:29:35 -0600 Subject: [PATCH 0560/1054] public RSA exponent 17 -> 65537 --- src/allmydata/crypto/rsa.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 74f564ded..e82bf12d1 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -46,18 +46,8 @@ def create_signing_keypair(key_size): :returns: 2-tuple of (private_key, public_key) """ - # Tahoe's original use of pycryptopp would use cryptopp's default - # public_exponent, which is 17 - # - # Thus, we are using 17 here as well. However, there are other - # choices; see this for more discussion: - # https://security.stackexchange.com/questions/2335/should-rsa-public-exponent-be-only-in-3-5-17-257-or-65537-due-to-security-c - # - # Another popular choice is 65537. See: - # https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/#cryptography.hazmat.primitives.asymmetric.rsa.generate_private_key - # https://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html priv_key = rsa.generate_private_key( - public_exponent=17, + public_exponent=65537, key_size=key_size, backend=default_backend() ) From e4e59c7aab7ad9a7f55229996ca7403bf38c8854 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Jul 2020 07:35:04 -0400 Subject: [PATCH 0561/1054] Use a Chutney revision that works with Python 2 --- integration/conftest.py | 20 +++++++++++++++++++- newsfragments/3348.bugfix | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3348.bugfix diff --git a/integration/conftest.py b/integration/conftest.py index 5395d7c5f..48be2f26d 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -377,13 +377,31 @@ def chutney(reactor, temp_dir): proto, 'git', ( - 'git', 'clone', '--depth=1', + 'git', 'clone', 'https://git.torproject.org/chutney.git', chutney_dir, ), env=environ, ) pytest_twisted.blockon(proto.done) + + # XXX: Here we reset Chutney to the last revision known to work + # with Python 2, as a workaround for Chunteny moving to Python 3. + # When this is no longer necessary, we will have to drop this and + # add '--depth=1' back to the above 'git clone' subprocess. + proto = _DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + 'git', + ( + 'git', '-C', chutney_dir, + 'reset', '--hard', + '99bd06c7554b9113af8c0877b6eca4ceb95dcbaa' + ), + env=environ, + ) + pytest_twisted.blockon(proto.done) + return chutney_dir diff --git a/newsfragments/3348.bugfix b/newsfragments/3348.bugfix new file mode 100644 index 000000000..e0f1f6c5e --- /dev/null +++ b/newsfragments/3348.bugfix @@ -0,0 +1 @@ +Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests. From 8cdfcb1c2440b6175a40af6caf73d5ae650de5b0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Jul 2020 09:40:06 -0600 Subject: [PATCH 0562/1054] news --- newsfragments/3349.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3349.bugfix diff --git a/newsfragments/3349.bugfix b/newsfragments/3349.bugfix new file mode 100644 index 000000000..08f2d7314 --- /dev/null +++ b/newsfragments/3349.bugfix @@ -0,0 +1 @@ +Mutable files now use RSA exponent 65537 From 957e0897152e583d7a43b3f4e2108f3db8844e00 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Jul 2020 11:56:44 -0400 Subject: [PATCH 0563/1054] Fix typo --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 48be2f26d..9c4c7433c 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -386,7 +386,7 @@ def chutney(reactor, temp_dir): pytest_twisted.blockon(proto.done) # XXX: Here we reset Chutney to the last revision known to work - # with Python 2, as a workaround for Chunteny moving to Python 3. + # with Python 2, as a workaround for Chutney moving to Python 3. # When this is no longer necessary, we will have to drop this and # add '--depth=1' back to the above 'git clone' subprocess. proto = _DumpOutputProtocol(None) From c229ba71967996852f26082c526e3a65742aef48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 13:55:53 -0400 Subject: [PATCH 0564/1054] Workaround future/pypy incompatibility. --- src/allmydata/util/base32.py | 13 ++++++++----- src/allmydata/util/base62.py | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index cb21d1a8a..ba5f80d67 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -19,21 +19,24 @@ if PY2: consistently until other modules are ported. """ return getattr(b, "__native__", lambda: b)() + import string + maketrans = string.maketrans else: def backwardscompat_bytes(b): return b + maketrans = bytes.maketrans import base64 from allmydata.util.assertutil import precondition -rfc3548_alphabet = bytes(b"abcdefghijklmnopqrstuvwxyz234567") # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... +rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... chars = rfc3548_alphabet -vals = bytes(range(32)) -c2vtranstable = bytes.maketrans(chars, vals) -v2ctranstable = bytes.maketrans(vals, chars) -identitytranstable = bytes.maketrans(b'', b'') +vals = backwardscompat_bytes(bytes(range(32))) +c2vtranstable = maketrans(chars, vals) +v2ctranstable = maketrans(vals, chars) +identitytranstable = maketrans(b'', b'') def _get_trailing_chars_without_lsbs(N, d): """ diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 7f349caaa..3d80671ba 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -12,6 +12,14 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 +if PY2: + import string + maketrans = string.maketrans + translate = string.translate +else: + maketrans = bytes.maketrans + translate = bytes.translate + from past.builtins import chr as byteschr from allmydata.util.mathutil import log_ceil, log_floor @@ -21,9 +29,9 @@ chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" BASE62CHAR = b'[' + chars + b']' vals = b''.join([byteschr(i) for i in range(62)]) -c2vtranstable = bytes.maketrans(chars, vals) -v2ctranstable = bytes.maketrans(vals, chars) -identitytranstable = bytes.maketrans(chars, chars) +c2vtranstable = maketrans(chars, vals) +v2ctranstable = maketrans(vals, chars) +identitytranstable = maketrans(chars, chars) def b2a(os): """ @@ -77,7 +85,7 @@ def b2a_l(os, lengthinbits): value //= 62 numvalues //= 62 - return bytes.translate(bytes([c for c in reversed(chars)]), v2ctranstable) # make it big-endian + return translate(bytes([c for c in reversed(chars)]), v2ctranstable) # make it big-endian def num_octets_that_encode_to_this_many_chars(numcs): return log_floor(62**numcs, 256) @@ -108,7 +116,7 @@ def a2b_l(cs, lengthinbits): """ # We call bytes() again for Python 2, to ensure literals are using future's # Python 3-compatible variant. - cs = [c for c in reversed(bytes(bytes.translate(cs, c2vtranstable)))] # treat cs as big-endian -- and we want to process the least-significant c first + cs = [c for c in reversed(bytes(translate(cs, c2vtranstable)))] # treat cs as big-endian -- and we want to process the least-significant c first value = 0 numvalues = 1 # the number of possible values that value could be From 116f232e802c3c7c371755eeb058f4d5c29c1d6e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 14:08:31 -0400 Subject: [PATCH 0565/1054] Port time_format to Python 3. --- src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/time_format.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 19bd35890..c1cfeb9ba 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.namespace", "allmydata.util.pollmixin", "allmydata.util._python3", + "allmydata.util.time_format", "allmydata.test.common_py3", ] @@ -29,4 +30,5 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_python3", + "allmydata.test.test_time_format", ] diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index 27b2e0976..5807791e4 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -4,6 +4,15 @@ Time formatting utilities. ISO-8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 +from future.utils import native_str import calendar, datetime, re, time @@ -18,6 +27,7 @@ def iso_utc_date(now=None, t=time.time): def iso_utc(now=None, sep='_', t=time.time): if now is None: now = t() + sep = native_str(sep) # Python 2 doesn't allow unicode input to isoformat return datetime.datetime.utcfromtimestamp(now).isoformat(sep) def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})[T_ ](?P\d{2}):(?P\d{2}):(?P\d{2})(?P\.\d+)?")): From 4018b772a35aba728af5c041fa50b474a12f2a43 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 14:15:39 -0400 Subject: [PATCH 0566/1054] Port abbreviate.py. As far as I can tell, all division is already float division. --- src/allmydata/{ => test}/test_abbreviate.py | 8 ++++++++ src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/abbreviate.py | 8 ++++++++ 3 files changed, 18 insertions(+) rename src/allmydata/{ => test}/test_abbreviate.py (93%) diff --git a/src/allmydata/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py similarity index 93% rename from src/allmydata/test_abbreviate.py rename to src/allmydata/test/test_abbreviate.py index 483ccfda0..7a0cbbec9 100644 --- a/src/allmydata/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -1,6 +1,14 @@ """ Tests for allmydata.util.abbreviate. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from datetime import timedelta diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c1cfeb9ba..c06f5c58b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.deferredutil", "allmydata.util.humanreadable", @@ -27,6 +28,7 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.test_abbreviate", "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_python3", diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 6fdf894ce..7361ce06d 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -1,3 +1,11 @@ +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re from datetime import timedelta From 6de05941b9a6c028ac446f2bc0d45c5d1dfdd789 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 14:27:08 -0400 Subject: [PATCH 0567/1054] Remove unnecessary imports. --- src/allmydata/test/common_util.py | 2 +- src/allmydata/test/test_util.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 2e8bbd006..996052692 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -import os, signal, time +import os, signal from random import randrange from six.moves import StringIO diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index c290903a7..f041f8cc0 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -8,15 +8,14 @@ import yaml import gc # support PyPy from six.moves import StringIO -from datetime import timedelta from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil -from allmydata.util import fileutil, abbreviate -from allmydata.util import limiter, time_format, pollmixin +from allmydata.util import fileutil +from allmydata.util import limiter, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans From 556869366666b924390551d6bb69ff6fc48b72db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 14:38:01 -0400 Subject: [PATCH 0568/1054] More passing tests. --- misc/python3/ratchet-passing | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index e659c95b6..5796c0cf8 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -1,4 +1,14 @@ allmydata.test.mutable.test_exceptions.Exceptions.test_repr +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_1s +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_25s +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_day +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_future_5_minutes +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_hours +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_month +allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year +allmydata.test.test_abbreviate.Abbreviate.test_parse_space +allmydata.test.test_abbreviate.Abbreviate.test_space +allmydata.test.test_abbreviate.Abbreviate.test_time allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success @@ -11,3 +21,11 @@ allmydata.test.test_observer.Observer.test_oneshot_fireagain allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist +allmydata.test.test_time_format.TimeFormat.test_epoch +allmydata.test.test_time_format.TimeFormat.test_epoch_in_London +allmydata.test.test_time_format.TimeFormat.test_format_delta +allmydata.test.test_time_format.TimeFormat.test_format_time +allmydata.test.test_time_format.TimeFormat.test_format_time_y2038 +allmydata.test.test_time_format.TimeFormat.test_iso_utc +allmydata.test.test_time_format.TimeFormat.test_parse_date +allmydata.test.test_time_format.TimeFormat.test_parse_duration From 7f5643a096c227f3457482fba9194a9565572721 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Jul 2020 14:56:55 -0400 Subject: [PATCH 0569/1054] Add note on Tor integration test failures on Windows --- integration/test_tor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/test_tor.py b/integration/test_tor.py index cec88a483..28360207a 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -18,6 +18,10 @@ import util # see "conftest.py" for the fixtures (e.g. "tor_network") +# XXX: Integration tests that involve Tor do not run reliably on +# Windows. They are skipped for now, in order to reduce CI noise. +# +# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3347 if sys.platform.startswith('win'): pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) From d6f24af424ac5975d0d52a4a987cd555da61b53b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 15:10:43 -0400 Subject: [PATCH 0570/1054] Move span tests into their own module. --- src/allmydata/test/test_spans.py | 605 +++++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 595 ------------------------------ 2 files changed, 605 insertions(+), 595 deletions(-) create mode 100644 src/allmydata/test/test_spans.py diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py new file mode 100644 index 000000000..9901a24be --- /dev/null +++ b/src/allmydata/test/test_spans.py @@ -0,0 +1,605 @@ +""" +Tests for allmydata.util.spans. +""" + +from __future__ import print_function + +import binascii +import hashlib + +from twisted.trial import unittest + +from allmydata.util.spans import Spans, overlap, DataSpans + + +def sha256(data): + """ + :param bytes data: data to hash + + :returns: a hex-encoded SHA256 hash of the data + """ + return binascii.hexlify(hashlib.sha256(data).digest()) + + +class SimpleSpans(object): + # this is a simple+inefficient form of util.spans.Spans . We compare the + # behavior of this reference model against the real (efficient) form. + + def __init__(self, _span_or_start=None, length=None): + self._have = set() + if length is not None: + for i in range(_span_or_start, _span_or_start+length): + self._have.add(i) + elif _span_or_start: + for (start,length) in _span_or_start: + self.add(start, length) + + def add(self, start, length): + for i in range(start, start+length): + self._have.add(i) + return self + + def remove(self, start, length): + for i in range(start, start+length): + self._have.discard(i) + return self + + def each(self): + return sorted(self._have) + + def __iter__(self): + items = sorted(self._have) + prevstart = None + prevend = None + for i in items: + if prevstart is None: + prevstart = prevend = i + continue + if i == prevend+1: + prevend = i + continue + yield (prevstart, prevend-prevstart+1) + prevstart = prevend = i + if prevstart is not None: + yield (prevstart, prevend-prevstart+1) + + def __nonzero__(self): # this gets us bool() + return self.len() + + def len(self): + return len(self._have) + + def __add__(self, other): + s = self.__class__(self) + for (start, length) in other: + s.add(start, length) + return s + + def __sub__(self, other): + s = self.__class__(self) + for (start, length) in other: + s.remove(start, length) + return s + + def __iadd__(self, other): + for (start, length) in other: + self.add(start, length) + return self + + def __isub__(self, other): + for (start, length) in other: + self.remove(start, length) + return self + + def __and__(self, other): + s = self.__class__() + for i in other.each(): + if i in self._have: + s.add(i, 1) + return s + + def __contains__(self, start_and_length): + (start, length) = start_and_length + for i in range(start, start+length): + if i not in self._have: + return False + return True + +class ByteSpans(unittest.TestCase): + def test_basic(self): + s = Spans() + self.failUnlessEqual(list(s), []) + self.failIf(s) + self.failIf((0,1) in s) + self.failUnlessEqual(s.len(), 0) + + s1 = Spans(3, 4) # 3,4,5,6 + self._check1(s1) + + s1 = Spans(long(3), long(4)) # 3,4,5,6 + self._check1(s1) + + s2 = Spans(s1) + self._check1(s2) + + s2.add(10,2) # 10,11 + self._check1(s1) + self.failUnless((10,1) in s2) + self.failIf((10,1) in s1) + self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11]) + self.failUnlessEqual(s2.len(), 6) + + s2.add(15,2).add(20,2) + self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11,15,16,20,21]) + self.failUnlessEqual(s2.len(), 10) + + s2.remove(4,3).remove(15,1) + self.failUnlessEqual(list(s2.each()), [3,10,11,16,20,21]) + self.failUnlessEqual(s2.len(), 6) + + s1 = SimpleSpans(3, 4) # 3 4 5 6 + s2 = SimpleSpans(5, 4) # 5 6 7 8 + i = s1 & s2 + self.failUnlessEqual(list(i.each()), [5, 6]) + + def _check1(self, s): + self.failUnlessEqual(list(s), [(3,4)]) + self.failUnless(s) + self.failUnlessEqual(s.len(), 4) + self.failIf((0,1) in s) + self.failUnless((3,4) in s) + self.failUnless((3,1) in s) + self.failUnless((5,2) in s) + self.failUnless((6,1) in s) + self.failIf((6,2) in s) + self.failIf((7,1) in s) + self.failUnlessEqual(list(s.each()), [3,4,5,6]) + + def test_large(self): + s = Spans(4, 2**65) # don't do this with a SimpleSpans + self.failUnlessEqual(list(s), [(4, 2**65)]) + self.failUnless(s) + self.failUnlessEqual(s.len(), 2**65) + self.failIf((0,1) in s) + self.failUnless((4,2) in s) + self.failUnless((2**65,2) in s) + + def test_math(self): + s1 = Spans(0, 10) # 0,1,2,3,4,5,6,7,8,9 + s2 = Spans(5, 3) # 5,6,7 + s3 = Spans(8, 4) # 8,9,10,11 + + s = s1 - s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,8,9]) + s = s1 - s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7]) + s = s2 - s3 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s1 & s2 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s2 & s1 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s1 & s3 + self.failUnlessEqual(list(s.each()), [8,9]) + s = s3 & s1 + self.failUnlessEqual(list(s.each()), [8,9]) + s = s2 & s3 + self.failUnlessEqual(list(s.each()), []) + s = s3 & s2 + self.failUnlessEqual(list(s.each()), []) + s = Spans() & s3 + self.failUnlessEqual(list(s.each()), []) + s = s3 & Spans() + self.failUnlessEqual(list(s.each()), []) + + s = s1 + s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9]) + s = s1 + s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9,10,11]) + s = s2 + s3 + self.failUnlessEqual(list(s.each()), [5,6,7,8,9,10,11]) + + s = Spans(s1) + s -= s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,8,9]) + s = Spans(s1) + s -= s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7]) + s = Spans(s2) + s -= s3 + self.failUnlessEqual(list(s.each()), [5,6,7]) + + s = Spans(s1) + s += s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9]) + s = Spans(s1) + s += s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9,10,11]) + s = Spans(s2) + s += s3 + self.failUnlessEqual(list(s.each()), [5,6,7,8,9,10,11]) + + def test_random(self): + # attempt to increase coverage of corner cases by comparing behavior + # of a simple-but-slow model implementation against the + # complex-but-fast actual implementation, in a large number of random + # operations + S1 = SimpleSpans + S2 = Spans + s1 = S1(); s2 = S2() + seed = "" + def _create(subseed): + ns1 = S1(); ns2 = S2() + for i in range(10): + what = sha256(subseed+str(i)) + start = int(what[2:4], 16) + length = max(1,int(what[5:6], 16)) + ns1.add(start, length); ns2.add(start, length) + return ns1, ns2 + + #print + for i in range(1000): + what = sha256(seed+str(i)) + op = what[0] + subop = what[1] + start = int(what[2:4], 16) + length = max(1,int(what[5:6], 16)) + #print what + if op in "0": + if subop in "01234": + s1 = S1(); s2 = S2() + elif subop in "5678": + s1 = S1(start, length); s2 = S2(start, length) + else: + s1 = S1(s1); s2 = S2(s2) + #print "s2 = %s" % s2.dump() + elif op in "123": + #print "s2.add(%d,%d)" % (start, length) + s1.add(start, length); s2.add(start, length) + elif op in "456": + #print "s2.remove(%d,%d)" % (start, length) + s1.remove(start, length); s2.remove(start, length) + elif op in "78": + ns1, ns2 = _create(what[7:11]) + #print "s2 + %s" % ns2.dump() + s1 = s1 + ns1; s2 = s2 + ns2 + elif op in "9a": + ns1, ns2 = _create(what[7:11]) + #print "%s - %s" % (s2.dump(), ns2.dump()) + s1 = s1 - ns1; s2 = s2 - ns2 + elif op in "bc": + ns1, ns2 = _create(what[7:11]) + #print "s2 += %s" % ns2.dump() + s1 += ns1; s2 += ns2 + elif op in "de": + ns1, ns2 = _create(what[7:11]) + #print "%s -= %s" % (s2.dump(), ns2.dump()) + s1 -= ns1; s2 -= ns2 + else: + ns1, ns2 = _create(what[7:11]) + #print "%s &= %s" % (s2.dump(), ns2.dump()) + s1 = s1 & ns1; s2 = s2 & ns2 + #print "s2 now %s" % s2.dump() + self.failUnlessEqual(list(s1.each()), list(s2.each())) + self.failUnlessEqual(s1.len(), s2.len()) + self.failUnlessEqual(bool(s1), bool(s2)) + self.failUnlessEqual(list(s1), list(s2)) + for j in range(10): + what = sha256(what[12:14]+str(j)) + start = int(what[2:4], 16) + length = max(1, int(what[5:6], 16)) + span = (start, length) + self.failUnlessEqual(bool(span in s1), bool(span in s2)) + + + # s() + # s(start,length) + # s(s0) + # s.add(start,length) : returns s + # s.remove(start,length) + # s.each() -> list of byte offsets, mostly for testing + # list(s) -> list of (start,length) tuples, one per span + # (start,length) in s -> True if (start..start+length-1) are all members + # NOT equivalent to x in list(s) + # s.len() -> number of bytes, for testing, bool(), and accounting/limiting + # bool(s) (__nonzeron__) + # s = s1+s2, s1-s2, +=s1, -=s1 + + def test_overlap(self): + for a in range(20): + for b in range(10): + for c in range(20): + for d in range(10): + self._test_overlap(a,b,c,d) + + def _test_overlap(self, a, b, c, d): + s1 = set(range(a,a+b)) + s2 = set(range(c,c+d)) + #print "---" + #self._show_overlap(s1, "1") + #self._show_overlap(s2, "2") + o = overlap(a,b,c,d) + expected = s1.intersection(s2) + if not expected: + self.failUnlessEqual(o, None) + else: + start,length = o + so = set(range(start,start+length)) + #self._show(so, "o") + self.failUnlessEqual(so, expected) + + def _show_overlap(self, s, c): + import sys + out = sys.stdout + if s: + for i in range(max(s)): + if i in s: + out.write(c) + else: + out.write(" ") + out.write("\n") + +def extend(s, start, length, fill): + if len(s) >= start+length: + return s + assert len(fill) == 1 + return s + fill*(start+length-len(s)) + +def replace(s, start, data): + assert len(s) >= start+len(data) + return s[:start] + data + s[start+len(data):] + +class SimpleDataSpans(object): + def __init__(self, other=None): + self.missing = "" # "1" where missing, "0" where found + self.data = "" + if other: + for (start, data) in other.get_chunks(): + self.add(start, data) + + def __nonzero__(self): # this gets us bool() + return self.len() + def len(self): + return len(self.missing.replace("1", "")) + def _dump(self): + return [i for (i,c) in enumerate(self.missing) if c == "0"] + def _have(self, start, length): + m = self.missing[start:start+length] + if not m or len(m) list of byte offsets, mostly for testing - # list(s) -> list of (start,length) tuples, one per span - # (start,length) in s -> True if (start..start+length-1) are all members - # NOT equivalent to x in list(s) - # s.len() -> number of bytes, for testing, bool(), and accounting/limiting - # bool(s) (__nonzeron__) - # s = s1+s2, s1-s2, +=s1, -=s1 - - def test_overlap(self): - for a in range(20): - for b in range(10): - for c in range(20): - for d in range(10): - self._test_overlap(a,b,c,d) - - def _test_overlap(self, a, b, c, d): - s1 = set(range(a,a+b)) - s2 = set(range(c,c+d)) - #print "---" - #self._show_overlap(s1, "1") - #self._show_overlap(s2, "2") - o = overlap(a,b,c,d) - expected = s1.intersection(s2) - if not expected: - self.failUnlessEqual(o, None) - else: - start,length = o - so = set(range(start,start+length)) - #self._show(so, "o") - self.failUnlessEqual(so, expected) - - def _show_overlap(self, s, c): - import sys - out = sys.stdout - if s: - for i in range(max(s)): - if i in s: - out.write(c) - else: - out.write(" ") - out.write("\n") - -def extend(s, start, length, fill): - if len(s) >= start+length: - return s - assert len(fill) == 1 - return s + fill*(start+length-len(s)) - -def replace(s, start, data): - assert len(s) >= start+len(data) - return s[:start] + data + s[start+len(data):] - -class SimpleDataSpans(object): - def __init__(self, other=None): - self.missing = "" # "1" where missing, "0" where found - self.data = "" - if other: - for (start, data) in other.get_chunks(): - self.add(start, data) - - def __nonzero__(self): # this gets us bool() - return self.len() - def len(self): - return len(self.missing.replace("1", "")) - def _dump(self): - return [i for (i,c) in enumerate(self.missing) if c == "0"] - def _have(self, start, length): - m = self.missing[start:start+length] - if not m or len(m) Date: Tue, 21 Jul 2020 15:14:52 -0400 Subject: [PATCH 0571/1054] Manual porting to Python 3. --- src/allmydata/test/test_spans.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 9901a24be..1766bf353 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -231,7 +231,7 @@ class ByteSpans(unittest.TestCase): def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = sha256(subseed+str(i)) + what = sha256(subseed+bytes(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, length); ns2.add(start, length) @@ -239,7 +239,7 @@ class ByteSpans(unittest.TestCase): #print for i in range(1000): - what = sha256(seed+str(i)) + what = sha256(seed+bytes(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -285,7 +285,7 @@ class ByteSpans(unittest.TestCase): self.failUnlessEqual(bool(s1), bool(s2)) self.failUnlessEqual(list(s1), list(s2)) for j in range(10): - what = sha256(what[12:14]+str(j)) + what = sha256(what[12:14]+bytes(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) span = (start, length) @@ -549,19 +549,19 @@ class StringSpans(unittest.TestCase): S1 = SimpleDataSpans S2 = DataSpans s1 = S1(); s2 = S2() - seed = "" + seed = b"" def _randstr(length, seed): created = 0 pieces = [] while created < length: - piece = sha256(seed + str(created)) + piece = sha256(seed + bytes(created)) pieces.append(piece) created += len(piece) return "".join(pieces)[:length] def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = sha256(subseed+str(i)) + what = sha256(subseed+bytes(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, _randstr(length, what[7:9])); @@ -570,7 +570,7 @@ class StringSpans(unittest.TestCase): #print for i in range(1000): - what = sha256(seed+str(i)) + what = sha256(seed+bytes(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -598,7 +598,7 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(s1.len(), s2.len()) self.failUnlessEqual(list(s1._dump()), list(s2._dump())) for j in range(100): - what = sha256(what[12:14]+str(j)) + what = sha256(what[12:14]+bytes(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) d1 = s1.get(start, length); d2 = s2.get(start, length) From e9eb93468f03586b33ccb831807e71c6742212b8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 15:25:23 -0400 Subject: [PATCH 0572/1054] Finish port to Python 3. --- src/allmydata/test/test_spans.py | 114 +++++++++++++++++-------------- src/allmydata/util/_python3.py | 2 + src/allmydata/util/spans.py | 13 +++- 3 files changed, 76 insertions(+), 53 deletions(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 1766bf353..1814e3382 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -3,6 +3,15 @@ Tests for allmydata.util.spans. """ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from past.builtins import long import binascii import hashlib @@ -63,8 +72,8 @@ class SimpleSpans(object): if prevstart is not None: yield (prevstart, prevend-prevstart+1) - def __nonzero__(self): # this gets us bool() - return self.len() + def __bool__(self): # this gets us bool() + return bool(self.len()) def len(self): return len(self._have) @@ -227,7 +236,7 @@ class ByteSpans(unittest.TestCase): S1 = SimpleSpans S2 = Spans s1 = S1(); s2 = S2() - seed = "" + seed = b"" def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): @@ -240,38 +249,38 @@ class ByteSpans(unittest.TestCase): #print for i in range(1000): what = sha256(seed+bytes(i)) - op = what[0] - subop = what[1] + op = what[0:1] + subop = what[1:2] start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) #print what - if op in "0": - if subop in "01234": + if op in b"0": + if subop in b"01234": s1 = S1(); s2 = S2() - elif subop in "5678": + elif subop in b"5678": s1 = S1(start, length); s2 = S2(start, length) else: s1 = S1(s1); s2 = S2(s2) #print "s2 = %s" % s2.dump() - elif op in "123": + elif op in b"123": #print "s2.add(%d,%d)" % (start, length) s1.add(start, length); s2.add(start, length) - elif op in "456": + elif op in b"456": #print "s2.remove(%d,%d)" % (start, length) s1.remove(start, length); s2.remove(start, length) - elif op in "78": + elif op in b"78": ns1, ns2 = _create(what[7:11]) #print "s2 + %s" % ns2.dump() s1 = s1 + ns1; s2 = s2 + ns2 - elif op in "9a": + elif op in b"9a": ns1, ns2 = _create(what[7:11]) #print "%s - %s" % (s2.dump(), ns2.dump()) s1 = s1 - ns1; s2 = s2 - ns2 - elif op in "bc": + elif op in b"bc": ns1, ns2 = _create(what[7:11]) #print "s2 += %s" % ns2.dump() s1 += ns1; s2 += ns2 - elif op in "de": + elif op in b"de": ns1, ns2 = _create(what[7:11]) #print "%s -= %s" % (s2.dump(), ns2.dump()) s1 -= ns1; s2 -= ns2 @@ -352,17 +361,20 @@ def replace(s, start, data): class SimpleDataSpans(object): def __init__(self, other=None): self.missing = "" # "1" where missing, "0" where found - self.data = "" + self.data = b"" if other: for (start, data) in other.get_chunks(): self.add(start, data) - def __nonzero__(self): # this gets us bool() - return self.len() + def __bool__(self): # this gets us bool() + return bool(self.len()) + def len(self): return len(self.missing.replace("1", "")) + def _dump(self): return [i for (i,c) in enumerate(self.missing) if c == "0"] + def _have(self, start, length): m = self.missing[start:start+length] if not m or len(m) Date: Wed, 22 Jul 2020 09:59:26 -0400 Subject: [PATCH 0573/1054] Increase the Python 3 ratchet. --- misc/python3/ratchet-passing | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index a6e1de68b..f678ec619 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -26,3 +26,11 @@ allmydata.test.test_observer.Observer.test_oneshot_fireagain allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist +allmydata.test.test_spans.ByteSpans.test_basic +allmydata.test.test_spans.ByteSpans.test_large +allmydata.test.test_spans.ByteSpans.test_math +allmydata.test.test_spans.ByteSpans.test_overlap +allmydata.test.test_spans.ByteSpans.test_random +allmydata.test.test_spans.StringSpans.test_basic +allmydata.test.test_spans.StringSpans.test_random +allmydata.test.test_spans.StringSpans.test_test From 51e9c2183cf15268985412481b01112f8b2a5b77 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:00:15 -0400 Subject: [PATCH 0574/1054] Add missing file. --- src/allmydata/test/common_py3.py | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/allmydata/test/common_py3.py diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py new file mode 100644 index 000000000..e6303d2f2 --- /dev/null +++ b/src/allmydata/test/common_py3.py @@ -0,0 +1,42 @@ +""" +Common utilities that have been ported to Python 3. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +import os +import time + + +class TimezoneMixin(object): + + def setTimezone(self, timezone): + def tzset_if_possible(): + # Windows doesn't have time.tzset(). + if hasattr(time, 'tzset'): + time.tzset() + + unset = object() + originalTimezone = os.environ.get('TZ', unset) + def restoreTimezone(): + if originalTimezone is unset: + del os.environ['TZ'] + else: + os.environ['TZ'] = originalTimezone + tzset_if_possible() + + os.environ['TZ'] = timezone + self.addCleanup(restoreTimezone) + tzset_if_possible() + + def have_working_tzset(self): + return hasattr(time, 'tzset') From 13306f70e74ff89382e39ebe87ab6784315a4f79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:01:26 -0400 Subject: [PATCH 0575/1054] News file. --- newsfragments/3351.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3351.minor diff --git a/newsfragments/3351.minor b/newsfragments/3351.minor new file mode 100644 index 000000000..e69de29bb From b4e6686211cb105f1b61154115868b2efa361fd1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:29:44 -0400 Subject: [PATCH 0576/1054] Port to Python 3. --- src/allmydata/test/test_observer.py | 14 ++++++++++++++ src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/observer.py | 15 ++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index 32fd0a395..b37f0d3e1 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -1,3 +1,17 @@ +""" +Tests for allmydata.util.observer. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ea4e0e702..92ad34174 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.humanreadable", "allmydata.util.mathutil", "allmydata.util.namespace", + "allmydata.util.observer", "allmydata.util.pollmixin", "allmydata.util._python3", ] @@ -31,6 +32,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base62", "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", + "allmydata.test.test_observer", "allmydata.test.test_python3", ] diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index 30eb92329..d5003dfb3 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -1,4 +1,17 @@ -# -*- test-case-name: allmydata.test.test_observer -*- +""" +Observer for Twisted code. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import weakref from twisted.internet import defer From 04bf9aeffcdf023c2d9a79c11bb8225f85df8eff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:30:13 -0400 Subject: [PATCH 0577/1054] News file. --- newsfragments/3353.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3353.minor diff --git a/newsfragments/3353.minor b/newsfragments/3353.minor new file mode 100644 index 000000000..e69de29bb From e427163ec84bcf4e665461e7a530a1da987c7d95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:33:23 -0400 Subject: [PATCH 0578/1054] Move pipeline tests into their own module. --- src/allmydata/test/test_pipeline.py | 190 ++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 178 +------------------------- 2 files changed, 191 insertions(+), 177 deletions(-) create mode 100644 src/allmydata/test/test_pipeline.py diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py new file mode 100644 index 000000000..b5c0f535a --- /dev/null +++ b/src/allmydata/test/test_pipeline.py @@ -0,0 +1,190 @@ +""" +Tests for allmydata.util.pipeline. + +Ported to Python 3. +""" + +import gc + +from twisted.internet import defer +from twisted.trial import unittest +from twisted.python import log +from twisted.python.failure import Failure + +from allmydata.util import pipeline + + +class Pipeline(unittest.TestCase): + def pause(self, *args, **kwargs): + d = defer.Deferred() + self.calls.append( (d, args, kwargs) ) + return d + + def failUnlessCallsAre(self, expected): + #print self.calls + #print expected + self.failUnlessEqual(len(self.calls), len(expected), self.calls) + for i,c in enumerate(self.calls): + self.failUnlessEqual(c[1:], expected[i], str(i)) + + def test_basic(self): + self.calls = [] + finished = [] + p = pipeline.Pipeline(100) + + d = p.flush() # fires immediately + d.addCallbacks(finished.append, log.err) + self.failUnlessEqual(len(finished), 1) + finished = [] + + d = p.add(10, self.pause, "one") + # the call should start right away, and our return Deferred should + # fire right away + d.addCallbacks(finished.append, log.err) + self.failUnlessEqual(len(finished), 1) + self.failUnlessEqual(finished[0], None) + self.failUnlessCallsAre([ ( ("one",) , {} ) ]) + self.failUnlessEqual(p.gauge, 10) + + # pipeline: [one] + + finished = [] + d = p.add(20, self.pause, "two", kw=2) + # pipeline: [one, two] + + # the call and the Deferred should fire right away + d.addCallbacks(finished.append, log.err) + self.failUnlessEqual(len(finished), 1) + self.failUnlessEqual(finished[0], None) + self.failUnlessCallsAre([ ( ("one",) , {} ), + ( ("two",) , {"kw": 2} ), + ]) + self.failUnlessEqual(p.gauge, 30) + + self.calls[0][0].callback("one-result") + # pipeline: [two] + self.failUnlessEqual(p.gauge, 20) + + finished = [] + d = p.add(90, self.pause, "three", "posarg1") + # pipeline: [two, three] + flushed = [] + fd = p.flush() + fd.addCallbacks(flushed.append, log.err) + self.failUnlessEqual(flushed, []) + + # the call will be made right away, but the return Deferred will not, + # because the pipeline is now full. + d.addCallbacks(finished.append, log.err) + self.failUnlessEqual(len(finished), 0) + self.failUnlessCallsAre([ ( ("one",) , {} ), + ( ("two",) , {"kw": 2} ), + ( ("three", "posarg1"), {} ), + ]) + self.failUnlessEqual(p.gauge, 110) + + self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause) + + # retiring either call will unblock the pipeline, causing the #3 + # Deferred to fire + self.calls[2][0].callback("three-result") + # pipeline: [two] + + self.failUnlessEqual(len(finished), 1) + self.failUnlessEqual(finished[0], None) + self.failUnlessEqual(flushed, []) + + # retiring call#2 will finally allow the flush() Deferred to fire + self.calls[1][0].callback("two-result") + self.failUnlessEqual(len(flushed), 1) + + def test_errors(self): + self.calls = [] + p = pipeline.Pipeline(100) + + d1 = p.add(200, self.pause, "one") + d2 = p.flush() + + finished = [] + d1.addBoth(finished.append) + self.failUnlessEqual(finished, []) + + flushed = [] + d2.addBoth(flushed.append) + self.failUnlessEqual(flushed, []) + + self.calls[0][0].errback(ValueError("oops")) + + self.failUnlessEqual(len(finished), 1) + f = finished[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(pipeline.PipelineError)) + self.failUnlessIn("PipelineError", str(f.value)) + self.failUnlessIn("ValueError", str(f.value)) + r = repr(f.value) + self.failUnless("ValueError" in r, r) + f2 = f.value.error + self.failUnless(f2.check(ValueError)) + + self.failUnlessEqual(len(flushed), 1) + f = flushed[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(pipeline.PipelineError)) + f2 = f.value.error + self.failUnless(f2.check(ValueError)) + + # now that the pipeline is in the failed state, any new calls will + # fail immediately + + d3 = p.add(20, self.pause, "two") + + finished = [] + d3.addBoth(finished.append) + self.failUnlessEqual(len(finished), 1) + f = finished[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(pipeline.PipelineError)) + r = repr(f.value) + self.failUnless("ValueError" in r, r) + f2 = f.value.error + self.failUnless(f2.check(ValueError)) + + d4 = p.flush() + flushed = [] + d4.addBoth(flushed.append) + self.failUnlessEqual(len(flushed), 1) + f = flushed[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(pipeline.PipelineError)) + f2 = f.value.error + self.failUnless(f2.check(ValueError)) + + def test_errors2(self): + self.calls = [] + p = pipeline.Pipeline(100) + + d1 = p.add(10, self.pause, "one") + d2 = p.add(20, self.pause, "two") + d3 = p.add(30, self.pause, "three") + d4 = p.flush() + + # one call fails, then the second one succeeds: make sure + # ExpandableDeferredList tolerates the second one + + flushed = [] + d4.addBoth(flushed.append) + self.failUnlessEqual(flushed, []) + + self.calls[0][0].errback(ValueError("oops")) + self.failUnlessEqual(len(flushed), 1) + f = flushed[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(pipeline.PipelineError)) + f2 = f.value.error + self.failUnless(f2.check(ValueError)) + + self.calls[1][0].callback("two-result") + self.calls[2][0].errback(ValueError("three-error")) + + del d1,d2,d3,d4 + gc.collect() # for PyPy diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index f744feb7e..a71600627 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -5,19 +5,17 @@ import six import hashlib import os, time, sys import yaml -import gc # support PyPy from six.moves import StringIO from datetime import timedelta from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure -from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil from allmydata.util import fileutil, abbreviate from allmydata.util import limiter, time_format, pollmixin -from allmydata.util import statistics, dictutil, pipeline, yamlutil +from allmydata.util import statistics, dictutil, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.spans import Spans, overlap, DataSpans from allmydata.util.fileutil import EncryptedTemporaryFile @@ -1121,180 +1119,6 @@ class DictUtil(unittest.TestCase): self.failUnlessEqual(d["one"], 1) self.failUnlessEqual(d.get_aux("one"), None) -class Pipeline(unittest.TestCase): - def pause(self, *args, **kwargs): - d = defer.Deferred() - self.calls.append( (d, args, kwargs) ) - return d - - def failUnlessCallsAre(self, expected): - #print self.calls - #print expected - self.failUnlessEqual(len(self.calls), len(expected), self.calls) - for i,c in enumerate(self.calls): - self.failUnlessEqual(c[1:], expected[i], str(i)) - - def test_basic(self): - self.calls = [] - finished = [] - p = pipeline.Pipeline(100) - - d = p.flush() # fires immediately - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - finished = [] - - d = p.add(10, self.pause, "one") - # the call should start right away, and our return Deferred should - # fire right away - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessCallsAre([ ( ("one",) , {} ) ]) - self.failUnlessEqual(p.gauge, 10) - - # pipeline: [one] - - finished = [] - d = p.add(20, self.pause, "two", kw=2) - # pipeline: [one, two] - - # the call and the Deferred should fire right away - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessCallsAre([ ( ("one",) , {} ), - ( ("two",) , {"kw": 2} ), - ]) - self.failUnlessEqual(p.gauge, 30) - - self.calls[0][0].callback("one-result") - # pipeline: [two] - self.failUnlessEqual(p.gauge, 20) - - finished = [] - d = p.add(90, self.pause, "three", "posarg1") - # pipeline: [two, three] - flushed = [] - fd = p.flush() - fd.addCallbacks(flushed.append, log.err) - self.failUnlessEqual(flushed, []) - - # the call will be made right away, but the return Deferred will not, - # because the pipeline is now full. - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 0) - self.failUnlessCallsAre([ ( ("one",) , {} ), - ( ("two",) , {"kw": 2} ), - ( ("three", "posarg1"), {} ), - ]) - self.failUnlessEqual(p.gauge, 110) - - self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause) - - # retiring either call will unblock the pipeline, causing the #3 - # Deferred to fire - self.calls[2][0].callback("three-result") - # pipeline: [two] - - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessEqual(flushed, []) - - # retiring call#2 will finally allow the flush() Deferred to fire - self.calls[1][0].callback("two-result") - self.failUnlessEqual(len(flushed), 1) - - def test_errors(self): - self.calls = [] - p = pipeline.Pipeline(100) - - d1 = p.add(200, self.pause, "one") - d2 = p.flush() - - finished = [] - d1.addBoth(finished.append) - self.failUnlessEqual(finished, []) - - flushed = [] - d2.addBoth(flushed.append) - self.failUnlessEqual(flushed, []) - - self.calls[0][0].errback(ValueError("oops")) - - self.failUnlessEqual(len(finished), 1) - f = finished[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - self.failUnlessIn("PipelineError", str(f.value)) - self.failUnlessIn("ValueError", str(f.value)) - r = repr(f.value) - self.failUnless("ValueError" in r, r) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - # now that the pipeline is in the failed state, any new calls will - # fail immediately - - d3 = p.add(20, self.pause, "two") - - finished = [] - d3.addBoth(finished.append) - self.failUnlessEqual(len(finished), 1) - f = finished[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - r = repr(f.value) - self.failUnless("ValueError" in r, r) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - d4 = p.flush() - flushed = [] - d4.addBoth(flushed.append) - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - def test_errors2(self): - self.calls = [] - p = pipeline.Pipeline(100) - - d1 = p.add(10, self.pause, "one") - d2 = p.add(20, self.pause, "two") - d3 = p.add(30, self.pause, "three") - d4 = p.flush() - - # one call fails, then the second one succeeds: make sure - # ExpandableDeferredList tolerates the second one - - flushed = [] - d4.addBoth(flushed.append) - self.failUnlessEqual(flushed, []) - - self.calls[0][0].errback(ValueError("oops")) - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - self.calls[1][0].callback("two-result") - self.calls[2][0].errback(ValueError("three-error")) - - del d1,d2,d3,d4 - gc.collect() # for PyPy class SampleError(Exception): pass From 0763f9f90b4ae7549efac5d6752d3a1e7bb058f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:36:50 -0400 Subject: [PATCH 0579/1054] Port to Python 3. --- src/allmydata/test/test_pipeline.py | 8 ++++++++ src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/pipeline.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py index b5c0f535a..ab7059521 100644 --- a/src/allmydata/test/test_pipeline.py +++ b/src/allmydata/test/test_pipeline.py @@ -3,6 +3,14 @@ Tests for allmydata.util.pipeline. Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import gc diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 92ad34174..72e56581b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -23,6 +23,7 @@ PORTED_MODULES = [ "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.observer", + "allmydata.util.pipeline", "allmydata.util.pollmixin", "allmydata.util._python3", ] @@ -33,6 +34,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_observer", + "allmydata.test.test_pipeline", "allmydata.test.test_python3", ] diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py index 285a06b98..df80e2c6c 100644 --- a/src/allmydata/util/pipeline.py +++ b/src/allmydata/util/pipeline.py @@ -1,9 +1,24 @@ +""" +A pipeline of Deferreds. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.internet import defer from twisted.python.failure import Failure from twisted.python import log from allmydata.util.assertutil import precondition + class PipelineError(Exception): """One of the pipelined messages returned an error. The received Failure object is stored in my .error attribute.""" From 691322764d3d71a4beba842dcf6ea95868f9ae6a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:37:30 -0400 Subject: [PATCH 0580/1054] Ratchet tests. --- misc/python3/ratchet-passing | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index a6e1de68b..08f9b3fe9 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -23,6 +23,9 @@ allmydata.test.test_observer.Observer.test_lazy_oneshot allmydata.test.test_observer.Observer.test_observerlist allmydata.test.test_observer.Observer.test_oneshot allmydata.test.test_observer.Observer.test_oneshot_fireagain +allmydata.test.test_pipeline.Pipeline.test_basic +allmydata.test.test_pipeline.Pipeline.test_errors +allmydata.test.test_pipeline.Pipeline.test_errors2 allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist From e48aecfa1a346c1e344bf5fa0ac67df388dda503 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 12:51:18 -0400 Subject: [PATCH 0581/1054] Move parsing tests to better location, and fix them. --- src/allmydata/test/test_storage.py | 19 ------------------- src/allmydata/test/test_time_format.py | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0acf60152..6867adc3b 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3852,25 +3852,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): expiration_mode="bogus") self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e)) - def test_parse_duration(self): - DAY = 24*60*60 - MONTH = 31*DAY - YEAR = 365*DAY - p = time_format.parse_duration - self.failUnlessEqual(p("7days"), 7*DAY) - self.failUnlessEqual(p("31day"), 31*DAY) - self.failUnlessEqual(p("60 days"), 60*DAY) - self.failUnlessEqual(p("2mo"), 2*MONTH) - self.failUnlessEqual(p("3 month"), 3*MONTH) - self.failUnlessEqual(p("2years"), 2*YEAR) - e = self.failUnlessRaises(ValueError, p, "2kumquats") - self.failUnlessIn("no unit (like day, month, or year) in '2kumquats'", str(e)) - - def test_parse_date(self): - p = time_format.parse_date - self.failUnless(isinstance(p("2009-03-18"), int), p("2009-03-18")) - self.failUnlessEqual(p("2009-03-18"), 1237334400) - def test_limited_history(self): basedir = "storage/LeaseCrawler/limited_history" fileutil.make_dirs(basedir) diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 54593e5b9..5f2c3ad46 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -10,6 +10,8 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 +from past.builtins import long + import time from twisted.trial import unittest @@ -87,6 +89,8 @@ class TimeFormat(unittest.TestCase, TimezoneMixin): def test_parse_duration(self): p = time_format.parse_duration DAY = 24*60*60 + MONTH = 31*DAY + YEAR = 365*DAY self.failUnlessEqual(p("1 day"), DAY) self.failUnlessEqual(p("2 days"), 2*DAY) self.failUnlessEqual(p("3 months"), 3*31*DAY) @@ -95,9 +99,20 @@ class TimeFormat(unittest.TestCase, TimezoneMixin): e = self.failUnlessRaises(ValueError, p, "123") self.failUnlessIn("no unit (like day, month, or year) in '123'", str(e)) + self.failUnlessEqual(p("7days"), 7*DAY) + self.failUnlessEqual(p("31day"), 31*DAY) + self.failUnlessEqual(p("60 days"), 60*DAY) + self.failUnlessEqual(p("2mo"), 2*MONTH) + self.failUnlessEqual(p("3 month"), 3*MONTH) + self.failUnlessEqual(p("2years"), 2*YEAR) + e = self.failUnlessRaises(ValueError, p, "2kumquats") + self.failUnlessIn("no unit (like day, month, or year) in '2kumquats'", str(e)) def test_parse_date(self): - self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400) + p = time_format.parse_date + self.failUnlessEqual(p("2010-02-21"), 1266710400) + self.failUnless(isinstance(p("2009-03-18"), (int, long)), p("2009-03-18")) + self.failUnlessEqual(p("2009-03-18"), 1237334400) def test_format_time(self): self.failUnlessEqual(time_format.format_time(time.gmtime(0)), '1970-01-01 00:00:00') From 5129e1ef14797618de4c36bff02088e85712e11a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 13:08:24 -0400 Subject: [PATCH 0582/1054] Explicitly mark bytestrings. --- src/allmydata/test/test_hashtree.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index d96e8ebd7..f9d38dfff 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -7,8 +7,8 @@ from allmydata import hashtree def make_tree(numleaves): - leaves = ["%d" % i for i in range(numleaves)] - leaf_hashes = [tagged_hash("tag", leaf) for leaf in leaves] + leaves = [b"%d" % i for i in range(numleaves)] + leaf_hashes = [tagged_hash(b"tag", leaf) for leaf in leaves] ht = hashtree.HashTree(leaf_hashes) return ht @@ -20,7 +20,7 @@ class Complete(unittest.TestCase): ht = make_tree(8) root = ht[0] self.failUnlessEqual(len(root), 32) - self.failUnlessEqual(ht.get_leaf(0), tagged_hash("tag", "0")) + self.failUnlessEqual(ht.get_leaf(0), tagged_hash(b"tag", b"0")) self.failUnlessRaises(IndexError, ht.get_leaf, 8) self.failUnlessEqual(ht.get_leaf_index(0), 7) self.failUnlessRaises(IndexError, ht.parent, 0) @@ -157,7 +157,7 @@ class Incomplete(unittest.TestCase): chain = {0: ht[0], 2: ht[2], 4: ht[4], 8: ht[8]} # this should fail because the leaf hash is just plain wrong try: - iht.set_hashes(chain, leaves={0: tagged_hash("bad tag", "0")}) + iht.set_hashes(chain, leaves={0: tagged_hash(b"bad tag", b"0")}) except hashtree.BadHashError: pass else: @@ -166,7 +166,7 @@ class Incomplete(unittest.TestCase): # this should fail because we give it conflicting hashes: one as an # internal node, another as a leaf try: - iht.set_hashes(chain, leaves={1: tagged_hash("bad tag", "1")}) + iht.set_hashes(chain, leaves={1: tagged_hash(b"bad tag", b"1")}) except hashtree.BadHashError: pass else: @@ -177,7 +177,7 @@ class Incomplete(unittest.TestCase): # this should fail because the internal hash is wrong try: - iht.set_hashes(bad_chain, leaves={0: tagged_hash("tag", "0")}) + iht.set_hashes(bad_chain, leaves={0: tagged_hash(b"tag", b"0")}) except hashtree.BadHashError: pass else: @@ -185,23 +185,23 @@ class Incomplete(unittest.TestCase): # this should succeed try: - iht.set_hashes(chain, leaves={0: tagged_hash("tag", "0")}) + iht.set_hashes(chain, leaves={0: tagged_hash(b"tag", b"0")}) except hashtree.BadHashError as e: self.fail("bad hash: %s" % e) - self.failUnlessEqual(ht.get_leaf(0), tagged_hash("tag", "0")) + self.failUnlessEqual(ht.get_leaf(0), tagged_hash(b"tag", b"0")) self.failUnlessRaises(IndexError, ht.get_leaf, 8) # this should succeed too try: - iht.set_hashes(leaves={1: tagged_hash("tag", "1")}) + iht.set_hashes(leaves={1: tagged_hash(b"tag", b"1")}) except hashtree.BadHashError: self.fail("bad hash") # this should fail because we give it hashes that conflict with some # that we added successfully before try: - iht.set_hashes(leaves={1: tagged_hash("bad tag", "1")}) + iht.set_hashes(leaves={1: tagged_hash(b"bad tag", b"1")}) except hashtree.BadHashError: pass else: @@ -214,6 +214,6 @@ class Incomplete(unittest.TestCase): # this should succeed try: - iht.set_hashes(chain, leaves={4: tagged_hash("tag", "4")}) + iht.set_hashes(chain, leaves={4: tagged_hash(b"tag", b"4")}) except hashtree.BadHashError as e: self.fail("bad hash: %s" % e) From 15254d08fc8867bd05f97f19b7a0cfecbb276b50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 13:11:05 -0400 Subject: [PATCH 0583/1054] A few more explicit byte strings. --- src/allmydata/test/test_hashtree.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index f9d38dfff..47aa97b96 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -1,5 +1,15 @@ # -*- test-case-name: allmydata.test.test_hashtree -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + + from twisted.trial import unittest from allmydata.util.hashutil import tagged_hash @@ -143,7 +153,7 @@ class Incomplete(unittest.TestCase): current_hashes = list(iht) # this should fail because there aren't enough hashes known try: - iht.set_hashes(leaves={0: tagged_hash("tag", "0")}) + iht.set_hashes(leaves={0: tagged_hash(b"tag", b"0")}) except hashtree.NotEnoughHashesError: pass else: @@ -173,7 +183,7 @@ class Incomplete(unittest.TestCase): self.fail("didn't catch bad hash") bad_chain = chain.copy() - bad_chain[2] = ht[2] + "BOGUS" + bad_chain[2] = ht[2] + b"BOGUS" # this should fail because the internal hash is wrong try: From 64e4af2d05612a5db0f71d4dfe2bfe65c7c5aacf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 13:17:49 -0400 Subject: [PATCH 0584/1054] Port to Python 3. --- misc/python3/ratchet-passing | 8 ++++++++ newsfragments/3354.minor | 1 + src/allmydata/hashtree.py | 31 +++++++++++++++++++---------- src/allmydata/test/test_hashtree.py | 6 +++++- src/allmydata/util/_python3.py | 2 ++ 5 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3354.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 7208bd56e..293733b2b 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -18,6 +18,14 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls +allmydata.test.test_hashtree.Complete.test_create +allmydata.test.test_hashtree.Complete.test_dump +allmydata.test.test_hashtree.Complete.test_needed_hashes +allmydata.test.test_hashtree.Incomplete.test_check +allmydata.test.test_hashtree.Incomplete.test_create +allmydata.test.test_hashtree.Incomplete.test_depth_of +allmydata.test.test_hashtree.Incomplete.test_large +allmydata.test.test_hashtree.Incomplete.test_needed_hashes allmydata.test.test_hashutil.HashUtilTests.test_chk allmydata.test.test_hashutil.HashUtilTests.test_hashers allmydata.test.test_hashutil.HashUtilTests.test_known_answers diff --git a/newsfragments/3354.minor b/newsfragments/3354.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3354.minor @@ -0,0 +1 @@ + diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 576d72a0e..77798b3c2 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -1,7 +1,4 @@ # -*- test-case-name: allmydata.test.test_hashtree -*- - -from allmydata.util import mathutil # from the pyutil library - """ Read and write chunks from files. @@ -50,6 +47,17 @@ or implied. It probably won't make your computer catch on fire, or eat your children, but it might. Use at your own risk. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from allmydata.util import mathutil # from the pyutil library + from allmydata.util import base32 from allmydata.util.hashutil import tagged_hash, tagged_pair_hash @@ -170,9 +178,10 @@ def depth_of(i): return mathutil.log_floor(i+1, 2) def empty_leaf_hash(i): - return tagged_hash('Merkle tree empty leaf', "%d" % i) + return tagged_hash(b'Merkle tree empty leaf', b"%d" % i) + def pair_hash(a, b): - return tagged_pair_hash('Merkle tree internal node', a, b) + return tagged_pair_hash(b'Merkle tree internal node', a, b) class HashTree(CompleteBinaryTreeMixin, list): """ @@ -215,7 +224,7 @@ class HashTree(CompleteBinaryTreeMixin, list): while len(rows[-1]) != 1: last = rows[-1] rows += [[pair_hash(last[2*i], last[2*i+1]) - for i in xrange(len(last)//2)]] + for i in range(len(last)//2)]] # Flatten the list of rows into a single list. rows.reverse() self[:] = sum(rows, []) @@ -289,7 +298,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): rows = [L] while len(rows[-1]) != 1: last = rows[-1] - rows += [[None for i in xrange(len(last)//2)]] + rows += [[None for i in range(len(last)//2)]] # Flatten the list of rows into a single list. rows.reverse() self[:] = sum(rows, []) @@ -372,12 +381,12 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): assert isinstance(hashes, dict) for h in hashes.values(): - assert isinstance(h, str) + assert isinstance(h, bytes) assert isinstance(leaves, dict) for h in leaves.values(): - assert isinstance(h, str) + assert isinstance(h, bytes) new_hashes = hashes.copy() - for leafnum,leafhash in leaves.iteritems(): + for leafnum,leafhash in leaves.items(): hashnum = self.first_leaf_num + leafnum if hashnum in new_hashes: if new_hashes[hashnum] != leafhash: @@ -416,7 +425,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): # first we provisionally add all hashes to the tree, comparing # any duplicates - for i,h in new_hashes.iteritems(): + for i,h in new_hashes.items(): if self[i]: if self[i] != h: raise BadHashError("new hash %s does not match " diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index 47aa97b96..b96f4abfb 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -1,4 +1,8 @@ -# -*- test-case-name: allmydata.test.test_hashtree -*- +""" +Tests for allmydata.hashtree. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4342b8c04..a79f20cd0 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.hashtree", "allmydata.util.assertutil", "allmydata.util.base32", "allmydata.util.base62", @@ -32,6 +33,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", + "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", "allmydata.test.test_netstring", From eb688dfd7e491b405a85491321161d234e74501a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 13:22:50 -0400 Subject: [PATCH 0585/1054] Fix lint. --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 6867adc3b..ccf9fc5cb 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -26,7 +26,7 @@ from zope.interface import implementer from foolscap.api import fireEventually import itertools from allmydata import interfaces -from allmydata.util import fileutil, hashutil, base32, pollmixin, time_format +from allmydata.util import fileutil, hashutil, base32, pollmixin from allmydata.storage.server import StorageServer from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader From 31ed6b0a4fa722e4810689f1d2dad604b7184cf6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jul 2020 11:37:56 -0400 Subject: [PATCH 0586/1054] News fragment. --- newsfragments/3346.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3346.minor diff --git a/newsfragments/3346.minor b/newsfragments/3346.minor new file mode 100644 index 000000000..e69de29bb From b0083e342b3095473fd1de23f643c50c50fdbf01 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jul 2020 11:47:06 -0400 Subject: [PATCH 0587/1054] Document being ported. --- src/allmydata/test/test_abbreviate.py | 2 ++ src/allmydata/util/abbreviate.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 7a0cbbec9..958c36742 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -1,5 +1,7 @@ """ Tests for allmydata.util.abbreviate. + +Ported to Python 3. """ from __future__ import absolute_import from __future__ import division diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 7361ce06d..95d547836 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -1,3 +1,8 @@ +""" +Convert timestamps to abbreviated English text. + +Ported to Python 3. +""" from __future__ import division from __future__ import absolute_import from __future__ import print_function From cab1b02ba2b01b3dec53edc5fe408eecce15d552 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jul 2020 11:47:12 -0400 Subject: [PATCH 0588/1054] Use constants, since we have them. --- src/allmydata/test/test_time_format.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 5f2c3ad46..fa68ccc36 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -93,9 +93,9 @@ class TimeFormat(unittest.TestCase, TimezoneMixin): YEAR = 365*DAY self.failUnlessEqual(p("1 day"), DAY) self.failUnlessEqual(p("2 days"), 2*DAY) - self.failUnlessEqual(p("3 months"), 3*31*DAY) - self.failUnlessEqual(p("4 mo"), 4*31*DAY) - self.failUnlessEqual(p("5 years"), 5*365*DAY) + self.failUnlessEqual(p("3 months"), 3*MONTH) + self.failUnlessEqual(p("4 mo"), 4*MONTH) + self.failUnlessEqual(p("5 years"), 5*YEAR) e = self.failUnlessRaises(ValueError, p, "123") self.failUnlessIn("no unit (like day, month, or year) in '123'", str(e)) From ab4393b50ecc3fc9a1c81804fc71144f2eee44bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jul 2020 11:49:02 -0400 Subject: [PATCH 0589/1054] Document unicode. --- src/allmydata/util/abbreviate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 95d547836..e7bdd8410 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -22,6 +22,10 @@ MONTH = 30*DAY YEAR = 365*DAY def abbreviate_time(s): + """ + Given time in seconds (float or int) or timedelta, summarize as English by + returning unicode string. + """ postfix = '' if isinstance(s, timedelta): # this feels counter-intuitive that positive numbers in a @@ -58,6 +62,9 @@ def abbreviate_time(s): return _plural(s / YEAR, "year") def abbreviate_space(s, SI=True): + """ + Given size in bytes summarize as English by returning unicode string. + """ if s is None: return "unknown" if SI: From 38648c0f8fb9e7bb6e56d2a5aa4cc0ba5382f04b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 11:09:08 -0400 Subject: [PATCH 0590/1054] Fix indentation --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8c59978c6..08c11b3a6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -38,7 +38,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", - "allmydata.test.test_netstring", + "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", "allmydata.test.test_python3", From 135ade02b159a8bde5c9083797d10720baf5c2f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 13:47:56 -0400 Subject: [PATCH 0591/1054] Start porting. --- src/allmydata/test/test_iputil.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index ebb77c3c8..7f26a53fb 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -10,7 +10,9 @@ from allmydata.util.namespace import Namespace DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") -MOCK_IPADDR_OUTPUT = """\ +# Mock output from subprocesses should be bytes, that's what happens on both +# Python 2 and Python 3: +MOCK_IPADDR_OUTPUT = b"""\ 1: lo: mtu 16436 qdisc noqueue state UNKNOWN \n\ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo @@ -28,7 +30,7 @@ MOCK_IPADDR_OUTPUT = """\ valid_lft forever preferred_lft forever """ -MOCK_IFCONFIG_OUTPUT = """\ +MOCK_IFCONFIG_OUTPUT = b"""\ eth1 Link encap:Ethernet HWaddr d4:3d:7e:01:b4:3e \n\ inet addr:192.168.0.6 Bcast:192.168.0.255 Mask:255.255.255.0 inet6 addr: fe80::d63d:7eff:fe01:b43e/64 Scope:Link @@ -59,7 +61,7 @@ wlan0 Link encap:Ethernet HWaddr 90:f6:52:27:15:0a \n\ """ # This is actually from a VirtualBox VM running XP. -MOCK_ROUTE_OUTPUT = """\ +MOCK_ROUTE_OUTPUT = b"""\ =========================================================================== Interface List 0x1 ........................... MS TCP Loopback interface @@ -98,6 +100,11 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): def test_get_local_ip_for(self): addr = iputil.get_local_ip_for('127.0.0.1') self.failUnless(DOTTED_QUAD_RE.match(addr)) + # Bytes can be taken as input: + bytes_addr = iputil.get_local_ip_for(b'127.0.0.1') + self.assertEqual(addr, bytes_addr) + # The output is a native string: + self.assertIsInstance(addr, str) def test_list_async(self): d = iputil.get_local_addresses_async() From 45a891114b033306dcbb66b3310d5d601970f628 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 13:48:11 -0400 Subject: [PATCH 0592/1054] Fix some things that prevent Python 3 imports. --- src/allmydata/util/encodingutil.py | 3 ++- src/allmydata/util/fileutil.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 568267b35..b31fd1205 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -4,7 +4,6 @@ unicode and back. """ import sys, os, re, locale -from types import NoneType from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -12,6 +11,8 @@ from twisted.python.filepath import FilePath from allmydata.util import log from allmydata.util.fileutil import abspath_expanduser_unicode +NoneType = type(None) + def canonical_encoding(encoding): if encoding is None: diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 269a8a356..36946ff02 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -4,7 +4,7 @@ from __future__ import print_function Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time, binascii +import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple from errno import ENOENT @@ -190,7 +190,7 @@ def make_dirs(dirname, mode=0o777): if not os.path.isdir(dirname): if tx: raise tx - raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rm_dir(dirname): """ From 04db52b9e4a1881d03c946e40485c92464147273 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:01:14 -0400 Subject: [PATCH 0593/1054] Some manual tweaks to be more likely to work with Python 3. The Linux distribution porting code was obsolete (using Python APIs not present in Python 3, /etc/lsb-release isn't a thing on Fedora 31, for example), so replaced it with maintained third-party library. --- nix/tahoe-lafs.nix | 4 +- setup.py | 3 ++ src/allmydata/version_checks.py | 89 +++------------------------------ 3 files changed, 12 insertions(+), 84 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 097a463d1..49bf3075a 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, pyutil +, html5lib, pyutil, nix }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future pyutil + future pyutil nix ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index f22ca0f4c..9bb385cb4 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,9 @@ install_requires = [ # Utility code: "pyutil >= 3.3.0", + + # Linux distribution detection: + "distro >= 1.4.0", ] setup_requires = [ diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 7092a422b..c7f4d6c8a 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -10,10 +10,12 @@ __all__ = [ "normalized_version", ] -import os, platform, re, subprocess, sys, traceback, pkg_resources +import os, platform, re, sys, traceback, pkg_resources import six +import distro + from . import ( __appname__, full_version, @@ -80,7 +82,7 @@ def normalized_version(verstr, what=None): return verlib.NormalizedVersion(suggested) except verlib.IrrationalVersionError: raise - except StandardError: + except Exception: cls, value, trace = sys.exc_info() new_exc = PackagingError("could not parse %s due to %s: %s" % (what or repr(verstr), cls.__name__, value)) @@ -201,83 +203,6 @@ def _extract_openssl_version(ssl_module): return (version, None, comment if comment else None) -def _get_linux_distro(): - """ Tries to determine the name of the Linux OS distribution name. - - First, try to parse a file named "/etc/lsb-release". If it exists, and - contains the "DISTRIB_ID=" line and the "DISTRIB_RELEASE=" line, then return - the strings parsed from that file. - - If that doesn't work, then invoke platform.dist(). - - If that doesn't work, then try to execute "lsb_release", as standardized in - 2001: - - http://refspecs.freestandards.org/LSB_1.0.0/gLSB/lsbrelease.html - - The current version of the standard is here: - - http://refspecs.freestandards.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/lsbrelease.html - - that lsb_release emitted, as strings. - - Returns a tuple (distname,version). Distname is what LSB calls a - "distributor id", e.g. "Ubuntu". Version is what LSB calls a "release", - e.g. "8.04". - - A version of this has been submitted to python as a patch for the standard - library module "platform": - - http://bugs.python.org/issue3937 - """ - global _distname,_version - if _distname and _version: - return (_distname, _version) - - try: - with open("/etc/lsb-release", "rU") as etclsbrel: - for line in etclsbrel: - m = _distributor_id_file_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - m = _release_file_re.search(line) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - (_distname, _version) = platform.dist()[:2] - if _distname and _version: - return (_distname, _version) - - if os.path.isfile("/usr/bin/lsb_release") or os.path.isfile("/bin/lsb_release"): - try: - p = subprocess.Popen(["lsb_release", "--all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - rc = p.wait() - if rc == 0: - for line in p.stdout.readlines(): - m = _distributor_id_cmdline_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - - m = _release_cmdline_re.search(p.stdout.read()) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - if os.path.exists("/etc/arch-release"): - return ("Arch_Linux", "") - - return (_distname,_version) def _get_platform(): # Our version of platform.platform(), telling us both less and more than the @@ -288,7 +213,7 @@ def _get_platform(): if "linux" in platform.system().lower(): return ( platform.system() + "-" + - "_".join(_get_linux_distro()) + "-" + + "_".join(distro.linux_distribution()[:2]) + "-" + platform.machine() + "-" + "_".join([x for x in platform.architecture() if x]) ) @@ -321,7 +246,7 @@ def _get_package_versions_and_locations(): for modulename in warning_imports: try: __import__(modulename) - except ImportError: + except (ImportError, SyntaxError): pass finally: # Leave suppressions for UserWarnings and global_deprecation_messages active. @@ -355,7 +280,7 @@ def _get_package_versions_and_locations(): try: __import__(modulename) module = sys.modules[modulename] - except ImportError: + except (ImportError, SyntaxError): etype, emsg, etrace = sys.exc_info() trace_info = (etype, str(emsg), ([None] + traceback.extract_tb(etrace))[-1]) packages.append( (pkgname, (None, None, trace_info)) ) From f84d51d79528c87c118e254ee48d9e34b5c19aad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:12:10 -0400 Subject: [PATCH 0594/1054] Port to Python 3. --- src/allmydata/test/test_version.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index fd0cb0e20..5f397d9f9 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -1,3 +1,12 @@ +""" +Tests for allmydata.util.verlib and allmydata.version_checks. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import sys import pkg_resources diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a79f20cd0..4bb639996 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -38,6 +38,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", "allmydata.test.test_netstring", "allmydata.test.test_python3", + "allmydata.test.test_version", ] From 337a4381bba90f38f4630eeabf7eaa4a34d90764 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:31:36 -0400 Subject: [PATCH 0595/1054] Finish port to Python 3. --- misc/python3/ratchet-passing | 10 ++++++++++ newsfragments/3357.minor | 1 + src/allmydata/test/test_version.py | 8 ++++++-- src/allmydata/util/verlib.py | 12 +++++++++++- src/allmydata/version_checks.py | 14 ++++++++++++-- 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3357.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 293733b2b..522a9fc6d 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -45,3 +45,13 @@ allmydata.test.test_observer.Observer.test_oneshot_fireagain allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist +allmydata.test.test_version.CheckRequirement.test_cross_check +allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions +allmydata.test.test_version.CheckRequirement.test_extract_openssl_version +allmydata.test.test_version.CheckRequirement.test_packages_from_pkg_resources +allmydata.test.test_version.T.test_report_import_error +allmydata.test.test_version.VersionTestCase.test_basic_versions +allmydata.test.test_version.VersionTestCase.test_comparison +allmydata.test.test_version.VersionTestCase.test_from_parts +allmydata.test.test_version.VersionTestCase.test_irrational_versions +allmydata.test.test_version.VersionTestCase.test_suggest_normalized_version diff --git a/newsfragments/3357.minor b/newsfragments/3357.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3357.minor @@ -0,0 +1 @@ + diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 5f397d9f9..1b447e346 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -8,6 +8,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import sys import pkg_resources from operator import ( @@ -88,7 +92,7 @@ class CheckRequirement(unittest.TestCase): res = cross_check({}, [("foo", ("unparseable", "", None))]) self.failUnlessEqual(len(res), 1) - self.failUnlessIn("version 'unparseable'", res[0]) + self.assertTrue(("version 'unparseable'" in res[0]) or ("version u'unparseable'" in res[0])) self.failUnlessIn("was not found by pkg_resources", res[0]) res = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere", "distribute"))]) @@ -129,7 +133,7 @@ class CheckRequirement(unittest.TestCase): res = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("2.0", "/somewhere_different", None))]) self.failUnlessEqual(len(res), 1) - self.failUnlessIn("but version '2.0'", res[0]) + self.assertTrue(("but version '2.0'" in res[0]) or ("but version u'2.0'" in res[0])) def test_extract_openssl_version(self): self.failUnlessEqual(extract_openssl_version(MockSSL("")), diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py index 619f1a845..f69e34e3d 100644 --- a/src/allmydata/util/verlib.py +++ b/src/allmydata/util/verlib.py @@ -1,11 +1,21 @@ """ "Rational" version definition and parsing for DistutilsVersionFight discussion at PyCon 2009. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re + class IrrationalVersionError(Exception): """This is an irrational version.""" pass diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index c7f4d6c8a..51a49d78a 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -1,7 +1,17 @@ """ Produce reports about the versions of Python software in use by Tahoe-LAFS for debugging and auditing purposes. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 __all__ = [ "PackagingError", @@ -111,7 +121,7 @@ def _get_error_string(errors, debug=False): def _cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): """This function returns a list of errors due to any failed cross-checks.""" - from _auto_deps import not_import_versionable + from ._auto_deps import not_import_versionable errors = [] not_pkg_resourceable = ['python', 'platform', __appname__.lower(), 'openssl'] @@ -311,7 +321,7 @@ def _get_package_versions_and_locations(): imported_packages = set([p.lower() for (p, _) in packages]) extra_packages = [] - for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): + for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.items(): if pr_name not in imported_packages and pr_name not in ignorable: extra_packages.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) From 4a52061377a6e85f1aab1597ccd35539a23cdbe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:34:20 -0400 Subject: [PATCH 0596/1054] Fixing braino. --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 49bf3075a..0e941bce1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, pyutil, nix +, html5lib, pyutil, distro }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future pyutil nix + future pyutil distro ]; checkInputs = with python.pkgs; [ From e80f1388683505531338c3a56df506c769f5c56d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:48:56 -0400 Subject: [PATCH 0597/1054] Fix indent. --- src/allmydata/test/test_spans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 1814e3382..f62d6e684 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -530,7 +530,7 @@ class StringSpans(unittest.TestCase): d = b.get(t_start, t_len) if d is not None: which2 = "%s+(%d-%d)" % (which, t_start, - t_start+t_len-1) + t_start+t_len-1) self.failUnlessEqual(d, S[t_start:t_start+t_len], which2) # check that removing a subspan gives the right value From bf134019797b2fadc8f92405720b1956e4b2fbc5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:58:08 -0400 Subject: [PATCH 0598/1054] Move SignalMixin to common_py3.py. --- src/allmydata/test/common_py3.py | 25 +++++++++++++++++++++++++ src/allmydata/test/common_util.py | 23 ++--------------------- src/allmydata/test/test_iputil.py | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index e6303d2f2..97745e293 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -15,6 +15,9 @@ if PY2: import os import time +import signal + +from twisted.internet import reactor class TimezoneMixin(object): @@ -40,3 +43,25 @@ class TimezoneMixin(object): def have_working_tzset(self): return hasattr(time, 'tzset') + + +class SignalMixin(object): + # This class is necessary for any code which wants to use Processes + # outside the usual reactor.run() environment. It is copied from + # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses + # something rather different. + sigchldHandler = None + + def setUp(self): + # make sure SIGCHLD handler is installed, as it should be on + # reactor.run(). problem is reactor may not have been run when this + # test runs. + if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): + self.sigchldHandler = signal.signal(signal.SIGCHLD, + reactor._handleSigchld) + return super(SignalMixin, self).setUp() + + def tearDown(self): + if self.sigchldHandler: + signal.signal(signal.SIGCHLD, self.sigchldHandler) + return super(SignalMixin, self).tearDown() diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 996052692..83f37a11d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -12,6 +12,8 @@ from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) from ..scripts import runner +from .common_py3 import SignalMixin + def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) @@ -88,27 +90,6 @@ class ReallyEqualMixin(object): self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) -class SignalMixin(object): - # This class is necessary for any code which wants to use Processes - # outside the usual reactor.run() environment. It is copied from - # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses - # something rather different. - sigchldHandler = None - - def setUp(self): - # make sure SIGCHLD handler is installed, as it should be on - # reactor.run(). problem is reactor may not have been run when this - # test runs. - if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): - self.sigchldHandler = signal.signal(signal.SIGCHLD, - reactor._handleSigchld) - return super(SignalMixin, self).setUp() - - def tearDown(self): - if self.sigchldHandler: - signal.signal(signal.SIGCHLD, self.sigchldHandler) - return super(SignalMixin, self).tearDown() - class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 7f26a53fb..0c0992430 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -4,7 +4,7 @@ import re, errno, subprocess, os from twisted.trial import unittest from allmydata.util import iputil -import allmydata.test.common_util as testutil +import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace From 361de059800a5f150775340dee273bde8e3ce11e Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Sat, 25 Jul 2020 11:12:43 +0200 Subject: [PATCH 0599/1054] futurize and make the tests pass back under py2 --- src/allmydata/test/common_util.py | 2 +- src/allmydata/test/test_util.py | 47 +++++++++++++++++------------- src/allmydata/util/encodingutil.py | 3 +- src/allmydata/util/fileutil.py | 21 +++++++++---- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 996052692..510994d12 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -11,7 +11,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) -from ..scripts import runner +#from ..scripts import runner def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 8c0e937f3..042875c6a 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,5 +1,11 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import zip, str, range, object import binascii import six import hashlib @@ -238,9 +244,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, "one") + fileutil.write_atomically(fn, u"one") self.failUnlessEqual(fileutil.read(fn), "one") - fileutil.write_atomically(fn, "two", mode="") # non-binary + fileutil.write_atomically(fn, u"two", mode="") # non-binary self.failUnlessEqual(fileutil.read(fn), "two") def test_rename(self): @@ -323,13 +329,13 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(10+11+12+13, used) def test_abspath_expanduser_unicode(self): - self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring") + self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") - saved_cwd = os.path.normpath(os.getcwdu()) + saved_cwd = os.path.normpath(os.getcwd()).decode("utf8") abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) - self.failUnless(isinstance(saved_cwd, unicode), saved_cwd) - self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd) + self.failUnless(isinstance(saved_cwd, str), saved_cwd) + self.failUnless(isinstance(abspath_cwd, str), abspath_cwd) if sys.platform == "win32": self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd)) else: @@ -390,10 +396,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): os.chdir(cwd) for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'): uabspath = fileutil.abspath_expanduser_unicode(upath) - self.failUnless(isinstance(uabspath, unicode), uabspath) + self.failUnless(isinstance(uabspath, str), uabspath) uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False) - self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong) + self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong) finally: os.chdir(saved_cwd) @@ -547,7 +553,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): def test_encrypted_tempfile(self): f = EncryptedTemporaryFile() - f.write("foobar") + f.write(b"foobar") f.close() @@ -707,7 +713,7 @@ class DictUtil(unittest.TestCase): # we put the serialized form in the auxdata d.set_with_aux("key", ("filecap", "metadata"), "serialized") - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failUnlessEqual(d["key"], ("filecap", "metadata")) self.failUnlessEqual(d.get_aux("key"), "serialized") def _get_missing(key): @@ -725,7 +731,7 @@ class DictUtil(unittest.TestCase): d.set_with_aux("key2", "value2", "aux2") self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failIf("key2" in d) self.failUnlessRaises(KeyError, _get_missing, "key2") self.failUnlessEqual(d.get("key2"), None) @@ -806,7 +812,7 @@ class SimpleSpans(object): if prevstart is not None: yield (prevstart, prevend-prevstart+1) - def __nonzero__(self): # this gets us bool() + def __bool__(self): # this gets us bool() return self.len() def len(self): @@ -859,7 +865,7 @@ class ByteSpans(unittest.TestCase): s1 = Spans(3, 4) # 3,4,5,6 self._check1(s1) - s1 = Spans(long(3), long(4)) # 3,4,5,6 + s1 = Spans(int(3), int(4)) # 3,4,5,6 self._check1(s1) s2 = Spans(s1) @@ -1100,7 +1106,7 @@ class SimpleDataSpans(object): for (start, data) in other.get_chunks(): self.add(start, data) - def __nonzero__(self): # this gets us bool() + def __bool__(self): # this gets us bool() return self.len() def len(self): return len(self.missing.replace("1", "")) @@ -1186,9 +1192,9 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(ds.get(2, 4), "fear") ds = klass() - ds.add(long(2), "four") - ds.add(long(3), "ea") - self.failUnlessEqual(ds.get(long(2), long(4)), "fear") + ds.add(int(2), "four") + ds.add(int(3), "ea") + self.failUnlessEqual(ds.get(int(2), int(4)), "fear") def do_scan(self, klass): @@ -1351,6 +1357,7 @@ class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) back = yamlutil.safe_load(data) - self.failUnlessEqual(type(back[0]), unicode) - self.failUnlessEqual(type(back[1]), unicode) - self.failUnlessEqual(type(back[2]), unicode) + str_type = type("x") # workaround for future.builtins.str + self.failUnlessEqual(type(back[0]), str_type) + self.failUnlessEqual(type(back[1]), str_type) + self.failUnlessEqual(type(back[2]), str_type) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 568267b35..491e5d61d 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -4,7 +4,6 @@ unicode and back. """ import sys, os, re, locale -from types import NoneType from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -139,7 +138,7 @@ def to_str(s): return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, (NoneType, str)), s) + precondition(isinstance(s, str) or s is None, s) if s is None: return s return s.decode('utf-8') diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 269a8a356..bdabe5aeb 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,10 +1,16 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range """ Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time, binascii +import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple from errno import ENOENT @@ -190,7 +196,7 @@ def make_dirs(dirname, mode=0o777): if not os.path.isdir(dirname): if tx: raise tx - raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rm_dir(dirname): """ @@ -277,7 +283,7 @@ def put_file(path, inf): outf.write(data) def precondition_abspath(path): - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("an abspath must be a Unicode string") if sys.platform == "win32": @@ -309,7 +315,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): abspath_expanduser_unicode. On Windows, the result will be a long path unless long_path is given as False. """ - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("paths must be Unicode strings") if base is not None and long_path: precondition_abspath(base) @@ -330,7 +336,10 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): if not os.path.isabs(path): if base is None: - path = os.path.join(os.getcwdu(), path) + cwd = os.getcwd() + if PY2: + cwd = cwd.decode('utf8') + path = os.path.join(cwd, path) else: path = os.path.join(base, path) @@ -415,7 +424,7 @@ ERROR_ENVVAR_NOT_FOUND = 203 def windows_getenv(name): # Based on , # with improved error handling. Returns None if there is no enivronment variable of the given name. - if not isinstance(name, unicode): + if not isinstance(name, str): raise AssertionError("name must be Unicode") n = GetEnvironmentVariableW(name, None, 0) From 13e292295eeaa9d4b1682243ca1679c724f11f6b Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Sat, 25 Jul 2020 11:22:03 +0200 Subject: [PATCH 0600/1054] Revert "futurize and make the tests pass back under py2" This reverts commit 361de059800a5f150775340dee273bde8e3ce11e. --- src/allmydata/test/common_util.py | 2 +- src/allmydata/test/test_util.py | 47 +++++++++++++----------------- src/allmydata/util/encodingutil.py | 3 +- src/allmydata/util/fileutil.py | 21 ++++--------- 4 files changed, 29 insertions(+), 44 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 510994d12..996052692 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -11,7 +11,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) -#from ..scripts import runner +from ..scripts import runner def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 042875c6a..8c0e937f3 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,11 +1,5 @@ from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from builtins import zip, str, range, object import binascii import six import hashlib @@ -244,9 +238,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, u"one") + fileutil.write_atomically(fn, "one") self.failUnlessEqual(fileutil.read(fn), "one") - fileutil.write_atomically(fn, u"two", mode="") # non-binary + fileutil.write_atomically(fn, "two", mode="") # non-binary self.failUnlessEqual(fileutil.read(fn), "two") def test_rename(self): @@ -329,13 +323,13 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(10+11+12+13, used) def test_abspath_expanduser_unicode(self): - self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") + self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring") - saved_cwd = os.path.normpath(os.getcwd()).decode("utf8") + saved_cwd = os.path.normpath(os.getcwdu()) abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) - self.failUnless(isinstance(saved_cwd, str), saved_cwd) - self.failUnless(isinstance(abspath_cwd, str), abspath_cwd) + self.failUnless(isinstance(saved_cwd, unicode), saved_cwd) + self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd) if sys.platform == "win32": self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd)) else: @@ -396,10 +390,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): os.chdir(cwd) for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'): uabspath = fileutil.abspath_expanduser_unicode(upath) - self.failUnless(isinstance(uabspath, str), uabspath) + self.failUnless(isinstance(uabspath, unicode), uabspath) uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False) - self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong) + self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong) finally: os.chdir(saved_cwd) @@ -553,7 +547,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): def test_encrypted_tempfile(self): f = EncryptedTemporaryFile() - f.write(b"foobar") + f.write("foobar") f.close() @@ -713,7 +707,7 @@ class DictUtil(unittest.TestCase): # we put the serialized form in the auxdata d.set_with_aux("key", ("filecap", "metadata"), "serialized") - self.failUnlessEqual(list(d.keys()), ["key"]) + self.failUnlessEqual(d.keys(), ["key"]) self.failUnlessEqual(d["key"], ("filecap", "metadata")) self.failUnlessEqual(d.get_aux("key"), "serialized") def _get_missing(key): @@ -731,7 +725,7 @@ class DictUtil(unittest.TestCase): d.set_with_aux("key2", "value2", "aux2") self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) del d["key2"] - self.failUnlessEqual(list(d.keys()), ["key"]) + self.failUnlessEqual(d.keys(), ["key"]) self.failIf("key2" in d) self.failUnlessRaises(KeyError, _get_missing, "key2") self.failUnlessEqual(d.get("key2"), None) @@ -812,7 +806,7 @@ class SimpleSpans(object): if prevstart is not None: yield (prevstart, prevend-prevstart+1) - def __bool__(self): # this gets us bool() + def __nonzero__(self): # this gets us bool() return self.len() def len(self): @@ -865,7 +859,7 @@ class ByteSpans(unittest.TestCase): s1 = Spans(3, 4) # 3,4,5,6 self._check1(s1) - s1 = Spans(int(3), int(4)) # 3,4,5,6 + s1 = Spans(long(3), long(4)) # 3,4,5,6 self._check1(s1) s2 = Spans(s1) @@ -1106,7 +1100,7 @@ class SimpleDataSpans(object): for (start, data) in other.get_chunks(): self.add(start, data) - def __bool__(self): # this gets us bool() + def __nonzero__(self): # this gets us bool() return self.len() def len(self): return len(self.missing.replace("1", "")) @@ -1192,9 +1186,9 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(ds.get(2, 4), "fear") ds = klass() - ds.add(int(2), "four") - ds.add(int(3), "ea") - self.failUnlessEqual(ds.get(int(2), int(4)), "fear") + ds.add(long(2), "four") + ds.add(long(3), "ea") + self.failUnlessEqual(ds.get(long(2), long(4)), "fear") def do_scan(self, klass): @@ -1357,7 +1351,6 @@ class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) back = yamlutil.safe_load(data) - str_type = type("x") # workaround for future.builtins.str - self.failUnlessEqual(type(back[0]), str_type) - self.failUnlessEqual(type(back[1]), str_type) - self.failUnlessEqual(type(back[2]), str_type) + self.failUnlessEqual(type(back[0]), unicode) + self.failUnlessEqual(type(back[1]), unicode) + self.failUnlessEqual(type(back[2]), unicode) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 491e5d61d..568267b35 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -4,6 +4,7 @@ unicode and back. """ import sys, os, re, locale +from types import NoneType from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -138,7 +139,7 @@ def to_str(s): return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, str) or s is None, s) + precondition(isinstance(s, (NoneType, str)), s) if s is None: return s return s.decode('utf-8') diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index bdabe5aeb..269a8a356 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,16 +1,10 @@ from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range """ Futz with files like a pro. """ -import sys, os, stat, tempfile, time, binascii +import sys, exceptions, os, stat, tempfile, time, binascii import six from collections import namedtuple from errno import ENOENT @@ -196,7 +190,7 @@ def make_dirs(dirname, mode=0o777): if not os.path.isdir(dirname): if tx: raise tx - raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rm_dir(dirname): """ @@ -283,7 +277,7 @@ def put_file(path, inf): outf.write(data) def precondition_abspath(path): - if not isinstance(path, str): + if not isinstance(path, unicode): raise AssertionError("an abspath must be a Unicode string") if sys.platform == "win32": @@ -315,7 +309,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): abspath_expanduser_unicode. On Windows, the result will be a long path unless long_path is given as False. """ - if not isinstance(path, str): + if not isinstance(path, unicode): raise AssertionError("paths must be Unicode strings") if base is not None and long_path: precondition_abspath(base) @@ -336,10 +330,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): if not os.path.isabs(path): if base is None: - cwd = os.getcwd() - if PY2: - cwd = cwd.decode('utf8') - path = os.path.join(cwd, path) + path = os.path.join(os.getcwdu(), path) else: path = os.path.join(base, path) @@ -424,7 +415,7 @@ ERROR_ENVVAR_NOT_FOUND = 203 def windows_getenv(name): # Based on , # with improved error handling. Returns None if there is no enivronment variable of the given name. - if not isinstance(name, str): + if not isinstance(name, unicode): raise AssertionError("name must be Unicode") n = GetEnvironmentVariableW(name, None, 0) From 1f34e6298790c6ff9db53743a3a0b441ffaa1313 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:14:01 -0400 Subject: [PATCH 0601/1054] Port test module to Python 3. --- src/allmydata/test/test_iputil.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 0c0992430..1113e3540 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -1,3 +1,11 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os @@ -8,7 +16,7 @@ import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace -DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") +DOTTED_QUAD_RE=re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") # Mock output from subprocesses should be bytes, that's what happens on both # Python 2 and Python 3: @@ -104,7 +112,7 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): bytes_addr = iputil.get_local_ip_for(b'127.0.0.1') self.assertEqual(addr, bytes_addr) # The output is a native string: - self.assertIsInstance(addr, str) + self.assertIsInstance(addr, native_str) def test_list_async(self): d = iputil.get_local_addresses_async() From fce7221481fd582a904bdb16ea3b81b57dfac970 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:42:20 -0400 Subject: [PATCH 0602/1054] Some steps to manually port to Python 3. --- src/allmydata/util/iputil.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 57a7e790a..71dded28f 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -171,11 +171,11 @@ def get_local_ip_for(target): # ... thus wrote Greg Smith in time immemorial... # Also, the Win32 APIs for this are really klunky and error-prone. --Daira -_win32_re = re.compile(r'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P
                \d+\.\d+\.\d+\.\d+)\s+(?P\d+)\s*$', flags=re.M|re.I|re.S) +_win32_re = re.compile(br'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P
                \d+\.\d+\.\d+\.\d+)\s+(?P\d+)\s*$', flags=re.M|re.I|re.S) _win32_commands = (('route.exe', ('print',), _win32_re),) # These work in most Unices. -_addr_re = re.compile(r'^\s*inet [a-zA-Z]*:?(?P
                \d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S) +_addr_re = re.compile(br'^\s*inet [a-zA-Z]*:?(?P
                \d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S) _unix_commands = (('/bin/ip', ('addr',), _addr_re), ('/sbin/ip', ('addr',), _addr_re), ('/sbin/ifconfig', ('-a',), _addr_re), @@ -209,10 +209,13 @@ def _synchronously_find_addresses_via_config(): else: exes_to_try = which(pathtotool) + subprocess_error = getattr( + subprocess, "SubprocessError", subprocess.CalledProcessError + ) for exe in exes_to_try: try: addresses = _query(exe, args, regex) - except Exception: + except (IOError, OSError, ValueError, subprocess_error): addresses = [] if addresses: return addresses @@ -224,7 +227,7 @@ def _query(path, args, regex): return [] env = {'LANG': 'en_US.UTF-8'} TRIES = 5 - for trial in xrange(TRIES): + for trial in range(TRIES): try: p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) (output, err) = p.communicate() @@ -235,13 +238,13 @@ def _query(path, args, regex): raise addresses = [] - outputsplit = output.split('\n') + outputsplit = output.split(b'\n') for outline in outputsplit: m = regex.match(outline) if m: addr = m.group('address') if addr not in addresses: - addresses.append(addr) + addresses.append(addr.decode("utf-8")) return addresses From 436c2e77e422370c81ccdb650816ca8cd48ecca0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:44:58 -0400 Subject: [PATCH 0603/1054] Automated port to Python 3. --- src/allmydata/util/iputil.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 71dded28f..4623dbba6 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -1,4 +1,12 @@ -# from the Python Standard Library +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import os, re, socket, subprocess, errno from sys import platform From 9919d2c9a7fcb2887541d887d94daf2db8b08275 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:46:03 -0400 Subject: [PATCH 0604/1054] Record port status. --- src/allmydata/test/test_iputil.py | 6 ++++++ src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/iputil.py | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 1113e3540..3fac776c2 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -1,3 +1,9 @@ +""" +Tests for allmydata.util.iputil. + +Ported to Python 3. +""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ff6916a72..d939d08b3 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -23,6 +23,7 @@ PORTED_MODULES = [ "allmydata.util.deferredutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", + "allmydata.util.iputil", "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.netstring", @@ -42,6 +43,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", + "allmydata.test.test_iputil", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 4623dbba6..947430e35 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -1,3 +1,9 @@ +""" +Utilities for getting IP addresses. + +Ported to Python 3. +""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function From f9bda5bbd3975746dda392b636b80eb372d56370 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:46:22 -0400 Subject: [PATCH 0605/1054] News file. --- newsfragments/3356.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3356.minor diff --git a/newsfragments/3356.minor b/newsfragments/3356.minor new file mode 100644 index 000000000..e69de29bb From 817355d17e98fffe6bb5ee9a6e8cc03e6081de93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 13:06:41 -0400 Subject: [PATCH 0606/1054] Minimal testing for listenOnUnused. --- src/allmydata/test/test_iputil.py | 33 ++++++++++++++++++++++++++++++- src/allmydata/util/iputil.py | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 3fac776c2..a058da299 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -13,10 +13,12 @@ from future.utils import PY2, native_str if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -import re, errno, subprocess, os +import re, errno, subprocess, os, socket from twisted.trial import unittest +from foolscap.api import Tub + from allmydata.util import iputil import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace @@ -183,3 +185,32 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): def test_list_async_mock_cygwin(self): self.patch(iputil, 'platform', "cygwin") return self._test_list_async_mock(None, None, CYGWIN_TEST_ADDRESSES) + + +class ListenOnUsed(unittest.TestCase): + """Tests for listenOnUnused.""" + + def create_tub(self, basedir): + os.makedirs(basedir) + tubfile = os.path.join(basedir, "tub.pem") + tub = Tub(certFile=tubfile) + tub.setOption("expose-remote-exception-types", False) + tub.startService() + self.addCleanup(tub.stopService) + return tub + + def test_random_port(self): + """A random port is selected if none is given.""" + tub = self.create_tub("utils/ListenOnUsed/test_randomport") + self.assertEqual(len(tub.getListeners()), 0) + portnum = iputil.listenOnUnused(tub) + # We can connect to this port: + s = socket.socket() + s.connect(("127.0.0.1", portnum)) + s.close() + self.assertEqual(len(tub.getListeners()), 1) + + # Listen on another port: + tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2") + portnum2 = iputil.listenOnUnused(tub2) + self.assertNotEqual(portnum, portnum2) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 947430e35..6a9639280 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -379,4 +379,5 @@ __all__ = ["allocate_tcp_port", "get_local_addresses_sync", "get_local_addresses_async", "get_local_ip_for", + "listenOnUnused", ] From 0071c1a48d0fa7ade416216456ff9f1c5446ef14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 13:08:25 -0400 Subject: [PATCH 0607/1054] More passing tests. --- misc/python3/ratchet-passing | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 6832b7fa7..bbe594998 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -44,6 +44,13 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for +allmydata.test.test_iputil.ListAddresses.test_list_async +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route +allmydata.test.test_iputil.ListenOnUsed.test_random_port allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested From d91b4f0e40ef6359213c73927e0102b8ce338f53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:12:09 -0400 Subject: [PATCH 0608/1054] Fix some tests and flake issues. --- src/allmydata/test/common_util.py | 2 +- src/allmydata/util/iputil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 83f37a11d..261d39af3 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -import os, signal +import os from random import randrange from six.moves import StringIO diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 6a9639280..940f215db 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -370,7 +370,7 @@ def listenOnUnused(tub, portnum=None): """ portnum, endpoint = _foolscapEndpointForPortNumber(portnum) tub.listenOn(endpoint) - tub.setLocation("localhost:%d" % (portnum,)) + tub.setLocation(native_str("localhost:%d" % (portnum,))) return portnum From ce8a57580846d659eb680c1f9efa429cabe3402d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:16:25 -0400 Subject: [PATCH 0609/1054] Test for specific ports. --- src/allmydata/test/test_iputil.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index a058da299..42677af2f 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -214,3 +214,13 @@ class ListenOnUsed(unittest.TestCase): tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2") portnum2 = iputil.listenOnUnused(tub2) self.assertNotEqual(portnum, portnum2) + + def test_specific_port(self): + """The given port is used.""" + tub = self.create_tub("utils/ListenOnUsed/test_givenport") + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + port2 = iputil.listenOnUnused(tub, port) + self.assertEqual(port, port2) From 74fe9ccf56a36902ff884bc298b4a85d4d4fc545 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:16:34 -0400 Subject: [PATCH 0610/1054] Foolscap expects a native string. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 940f215db..af982e908 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -321,7 +321,7 @@ def _foolscapEndpointForPortNumber(portnum): # approach is error prone for the reasons described on # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787 portnum = allocate_tcp_port() - return (portnum, "tcp:%d" % (portnum,)) + return (portnum, native_str("tcp:%d" % (portnum,))) @implementer(IStreamServerEndpoint) From d8bf811b92acaef6510e3e7503fb8424c2f57d5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:20:35 -0400 Subject: [PATCH 0611/1054] Native string all the things. --- src/allmydata/util/iputil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index af982e908..921c481ff 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -102,7 +102,7 @@ except ImportError: increase_rlimits = _increase_rlimits def get_local_addresses_sync(): - return _synchronously_find_addresses_via_config() + return [native_str(a) for a in _synchronously_find_addresses_via_config()] def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ @@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET addresses.append(addr) return addresses d.addCallback(_collect) - + d.addCallback(lambda addresses: [native_str(s) for s in addresses]) return d def get_local_ip_for(target): @@ -176,7 +176,7 @@ def get_local_ip_for(target): except (socket.error, CannotListenError): # no route to that host localip = None - return localip + return native_str(localip) # Wow, I'm really amazed at home much mileage we've gotten out of calling From b140d1f1afa837ff4c8cd064faf0176b2c4893c6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:29:01 -0400 Subject: [PATCH 0612/1054] Move statistics tests out. --- src/allmydata/test/test_statistics.py | 147 ++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 140 +----------------------- 2 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 src/allmydata/test/test_statistics.py diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py new file mode 100644 index 000000000..7fe23f88f --- /dev/null +++ b/src/allmydata/test/test_statistics.py @@ -0,0 +1,147 @@ +""" +Tests for allmydata.util.statistics. +""" + +from io import StringIO + +from twisted.trial import unittest + +from allmydata.util import statistics + + +class Statistics(unittest.TestCase): + def should_assert(self, msg, func, *args, **kwargs): + try: + func(*args, **kwargs) + self.fail(msg) + except AssertionError: + pass + + def failUnlessListEqual(self, a, b, msg = None): + self.failUnlessEqual(len(a), len(b)) + for i in range(len(a)): + self.failUnlessEqual(a[i], b[i], msg) + + def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None): + self.failUnlessEqual(len(a), len(b)) + for i in range(len(a)): + self.failUnlessAlmostEqual(a[i], b[i], places, msg) + + def test_binomial_coeff(self): + f = statistics.binomial_coeff + self.failUnlessEqual(f(20, 0), 1) + self.failUnlessEqual(f(20, 1), 20) + self.failUnlessEqual(f(20, 2), 190) + self.failUnlessEqual(f(20, 8), f(20, 12)) + self.should_assert("Should assert if n < k", f, 2, 3) + + def test_binomial_distribution_pmf(self): + f = statistics.binomial_distribution_pmf + + pmf_comp = f(2, .1) + pmf_stat = [0.81, 0.18, 0.01] + self.failUnlessListAlmostEqual(pmf_comp, pmf_stat) + + # Summing across a PMF should give the total probability 1 + self.failUnlessAlmostEqual(sum(pmf_comp), 1) + self.should_assert("Should assert if not 0<=p<=1", f, 1, -1) + self.should_assert("Should assert if n < 1", f, 0, .1) + + out = StringIO() + statistics.print_pmf(pmf_comp, out=out) + lines = out.getvalue().splitlines() + self.failUnlessEqual(lines[0], "i=0: 0.81") + self.failUnlessEqual(lines[1], "i=1: 0.18") + self.failUnlessEqual(lines[2], "i=2: 0.01") + + def test_survival_pmf(self): + f = statistics.survival_pmf + # Cross-check binomial-distribution method against convolution + # method. + p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20 + pmf1 = statistics.survival_pmf_via_conv(p_list) + pmf2 = statistics.survival_pmf_via_bd(p_list) + self.failUnlessListAlmostEqual(pmf1, pmf2) + self.failUnlessTrue(statistics.valid_pmf(pmf1)) + self.should_assert("Should assert if p_i > 1", f, [1.1]); + self.should_assert("Should assert if p_i < 0", f, [-.1]); + + def test_repair_count_pmf(self): + survival_pmf = statistics.binomial_distribution_pmf(5, .9) + repair_pmf = statistics.repair_count_pmf(survival_pmf, 3) + # repair_pmf[0] == sum(survival_pmf[0,1,2,5]) + # repair_pmf[1] == survival_pmf[4] + # repair_pmf[2] = survival_pmf[3] + self.failUnlessListAlmostEqual(repair_pmf, + [0.00001 + 0.00045 + 0.0081 + 0.59049, + .32805, + .0729, + 0, 0, 0]) + + def test_repair_cost(self): + survival_pmf = statistics.binomial_distribution_pmf(5, .9) + bwcost = statistics.bandwidth_cost_function + cost = statistics.mean_repair_cost(bwcost, 1000, + survival_pmf, 3, ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 558.90) + cost = statistics.mean_repair_cost(bwcost, 1000, + survival_pmf, 3, ul_dl_ratio=8.0) + self.failUnlessAlmostEqual(cost, 1664.55) + + # I haven't manually checked the math beyond here -warner + cost = statistics.eternal_repair_cost(bwcost, 1000, + survival_pmf, 3, + discount_rate=0, ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 65292.056074766246) + cost = statistics.eternal_repair_cost(bwcost, 1000, + survival_pmf, 3, + discount_rate=0.05, + ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 9133.6097158191551) + + def test_convolve(self): + f = statistics.convolve + v1 = [ 1, 2, 3 ] + v2 = [ 4, 5, 6 ] + v3 = [ 7, 8 ] + v1v2result = [ 4, 13, 28, 27, 18 ] + # Convolution is commutative + r1 = f(v1, v2) + r2 = f(v2, v1) + self.failUnlessListEqual(r1, r2, "Convolution should be commutative") + self.failUnlessListEqual(r1, v1v2result, "Didn't match known result") + # Convolution is associative + r1 = f(f(v1, v2), v3) + r2 = f(v1, f(v2, v3)) + self.failUnlessListEqual(r1, r2, "Convolution should be associative") + # Convolution is distributive + r1 = f(v3, [ a + b for a, b in zip(v1, v2) ]) + tmp1 = f(v3, v1) + tmp2 = f(v3, v2) + r2 = [ a + b for a, b in zip(tmp1, tmp2) ] + self.failUnlessListEqual(r1, r2, "Convolution should be distributive") + # Convolution is scalar multiplication associative + tmp1 = f(v1, v2) + r1 = [ a * 4 for a in tmp1 ] + tmp2 = [ a * 4 for a in v1 ] + r2 = f(tmp2, v2) + self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative") + + def test_find_k(self): + f = statistics.find_k + g = statistics.pr_file_loss + plist = [.9] * 10 + [.8] * 10 # N=20 + t = .0001 + k = f(plist, t) + self.failUnlessEqual(k, 10) + self.failUnless(g(plist, k) < t) + + def test_pr_file_loss(self): + f = statistics.pr_file_loss + plist = [.5] * 10 + self.failUnlessEqual(f(plist, 3), .0546875) + + def test_pr_backup_file_loss(self): + f = statistics.pr_backup_file_loss + plist = [.5] * 10 + self.failUnlessEqual(f(plist, .5, 3), .02734375) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1519c42da..ff6a1650c 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -4,7 +4,6 @@ import six import os, time, sys import yaml -from six.moves import StringIO from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -12,7 +11,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin -from allmydata.util import statistics, dictutil, yamlutil +from allmydata.util import dictutil, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -34,143 +33,6 @@ class Math(unittest.TestCase): f = mathutil.round_sigfigs self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002) -class Statistics(unittest.TestCase): - def should_assert(self, msg, func, *args, **kwargs): - try: - func(*args, **kwargs) - self.fail(msg) - except AssertionError: - pass - - def failUnlessListEqual(self, a, b, msg = None): - self.failUnlessEqual(len(a), len(b)) - for i in range(len(a)): - self.failUnlessEqual(a[i], b[i], msg) - - def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None): - self.failUnlessEqual(len(a), len(b)) - for i in range(len(a)): - self.failUnlessAlmostEqual(a[i], b[i], places, msg) - - def test_binomial_coeff(self): - f = statistics.binomial_coeff - self.failUnlessEqual(f(20, 0), 1) - self.failUnlessEqual(f(20, 1), 20) - self.failUnlessEqual(f(20, 2), 190) - self.failUnlessEqual(f(20, 8), f(20, 12)) - self.should_assert("Should assert if n < k", f, 2, 3) - - def test_binomial_distribution_pmf(self): - f = statistics.binomial_distribution_pmf - - pmf_comp = f(2, .1) - pmf_stat = [0.81, 0.18, 0.01] - self.failUnlessListAlmostEqual(pmf_comp, pmf_stat) - - # Summing across a PMF should give the total probability 1 - self.failUnlessAlmostEqual(sum(pmf_comp), 1) - self.should_assert("Should assert if not 0<=p<=1", f, 1, -1) - self.should_assert("Should assert if n < 1", f, 0, .1) - - out = StringIO() - statistics.print_pmf(pmf_comp, out=out) - lines = out.getvalue().splitlines() - self.failUnlessEqual(lines[0], "i=0: 0.81") - self.failUnlessEqual(lines[1], "i=1: 0.18") - self.failUnlessEqual(lines[2], "i=2: 0.01") - - def test_survival_pmf(self): - f = statistics.survival_pmf - # Cross-check binomial-distribution method against convolution - # method. - p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20 - pmf1 = statistics.survival_pmf_via_conv(p_list) - pmf2 = statistics.survival_pmf_via_bd(p_list) - self.failUnlessListAlmostEqual(pmf1, pmf2) - self.failUnlessTrue(statistics.valid_pmf(pmf1)) - self.should_assert("Should assert if p_i > 1", f, [1.1]); - self.should_assert("Should assert if p_i < 0", f, [-.1]); - - def test_repair_count_pmf(self): - survival_pmf = statistics.binomial_distribution_pmf(5, .9) - repair_pmf = statistics.repair_count_pmf(survival_pmf, 3) - # repair_pmf[0] == sum(survival_pmf[0,1,2,5]) - # repair_pmf[1] == survival_pmf[4] - # repair_pmf[2] = survival_pmf[3] - self.failUnlessListAlmostEqual(repair_pmf, - [0.00001 + 0.00045 + 0.0081 + 0.59049, - .32805, - .0729, - 0, 0, 0]) - - def test_repair_cost(self): - survival_pmf = statistics.binomial_distribution_pmf(5, .9) - bwcost = statistics.bandwidth_cost_function - cost = statistics.mean_repair_cost(bwcost, 1000, - survival_pmf, 3, ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 558.90) - cost = statistics.mean_repair_cost(bwcost, 1000, - survival_pmf, 3, ul_dl_ratio=8.0) - self.failUnlessAlmostEqual(cost, 1664.55) - - # I haven't manually checked the math beyond here -warner - cost = statistics.eternal_repair_cost(bwcost, 1000, - survival_pmf, 3, - discount_rate=0, ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 65292.056074766246) - cost = statistics.eternal_repair_cost(bwcost, 1000, - survival_pmf, 3, - discount_rate=0.05, - ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 9133.6097158191551) - - def test_convolve(self): - f = statistics.convolve - v1 = [ 1, 2, 3 ] - v2 = [ 4, 5, 6 ] - v3 = [ 7, 8 ] - v1v2result = [ 4, 13, 28, 27, 18 ] - # Convolution is commutative - r1 = f(v1, v2) - r2 = f(v2, v1) - self.failUnlessListEqual(r1, r2, "Convolution should be commutative") - self.failUnlessListEqual(r1, v1v2result, "Didn't match known result") - # Convolution is associative - r1 = f(f(v1, v2), v3) - r2 = f(v1, f(v2, v3)) - self.failUnlessListEqual(r1, r2, "Convolution should be associative") - # Convolution is distributive - r1 = f(v3, [ a + b for a, b in zip(v1, v2) ]) - tmp1 = f(v3, v1) - tmp2 = f(v3, v2) - r2 = [ a + b for a, b in zip(tmp1, tmp2) ] - self.failUnlessListEqual(r1, r2, "Convolution should be distributive") - # Convolution is scalar multiplication associative - tmp1 = f(v1, v2) - r1 = [ a * 4 for a in tmp1 ] - tmp2 = [ a * 4 for a in v1 ] - r2 = f(tmp2, v2) - self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative") - - def test_find_k(self): - f = statistics.find_k - g = statistics.pr_file_loss - plist = [.9] * 10 + [.8] * 10 # N=20 - t = .0001 - k = f(plist, t) - self.failUnlessEqual(k, 10) - self.failUnless(g(plist, k) < t) - - def test_pr_file_loss(self): - f = statistics.pr_file_loss - plist = [.5] * 10 - self.failUnlessEqual(f(plist, 3), .0546875) - - def test_pr_backup_file_loss(self): - f = statistics.pr_backup_file_loss - plist = [.5] * 10 - self.failUnlessEqual(f(plist, .5, 3), .02734375) - class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0o777): From 8d84be77d86f8af6e3025b237e3e681219ea2b7e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:31:31 -0400 Subject: [PATCH 0613/1054] Port to Python 3. --- src/allmydata/test/test_statistics.py | 10 +++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 7fe23f88f..a14fb55ab 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -1,8 +1,16 @@ """ Tests for allmydata.util.statistics. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from io import StringIO +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from six.moves import StringIO # native string StringIO from twisted.trial import unittest diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 53ccc6f63..f06f5bf3b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -48,6 +48,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_pipeline", "allmydata.test.test_python3", "allmydata.test.test_spans", + "allmydata.test.test_statistics", "allmydata.test.test_time_format", "allmydata.test.test_version", ] From 2ca223a67ca243890711846e039e3cd0f9259e93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:34:28 -0400 Subject: [PATCH 0614/1054] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/statistics.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f06f5bf3b..d5bdae2f4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -31,6 +31,7 @@ PORTED_MODULES = [ "allmydata.util.pollmixin", "allmydata.util._python3", "allmydata.util.spans", + "allmydata.util.statistics", "allmydata.util.time_format", "allmydata.test.common_py3", ] diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 3e50b39dc..a690eb235 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -1,3 +1,8 @@ +""" +Statistical utilities. + +Ported to Python 3. +""" # Copyright (c) 2009 Shawn Willden # mailto:shawn@willden.org # I hereby license all patches I have contributed or will contribute to the @@ -5,7 +10,18 @@ # either the GNU General Public License, version 2 or later, or under the # Transitive Grace Period Public License, version 1 or later. -from __future__ import division, print_function + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from functools import reduce + from allmydata.util.mathutil import round_sigfigs import math import sys @@ -78,7 +94,7 @@ def survival_pmf_via_bd(p_list): """ pmf_list = [ binomial_distribution_pmf(p_list.count(p), p) for p in set(p_list) ] - return reduce(convolve, pmf_list) + return list(reduce(convolve, pmf_list)) def survival_pmf_via_conv(p_list): """ @@ -89,7 +105,7 @@ def survival_pmf_via_conv(p_list): intended for internal use and testing only. """ pmf_list = [ [1 - p, p] for p in p_list ]; - return reduce(convolve, pmf_list) + return list(reduce(convolve, pmf_list)) def print_pmf(pmf, n=4, out=sys.stdout): """ From bde2f1394dae44d87b131f84335c2564f6710c46 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:53:26 -0400 Subject: [PATCH 0615/1054] Move dictutil tests out. --- src/allmydata/test/test_dictutil.py | 80 +++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 74 +------------------------- 2 files changed, 81 insertions(+), 73 deletions(-) create mode 100644 src/allmydata/test/test_dictutil.py diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py new file mode 100644 index 000000000..c22b2dfe3 --- /dev/null +++ b/src/allmydata/test/test_dictutil.py @@ -0,0 +1,80 @@ +""" +Tests for allmydata.util.dictutil. +""" + +from twisted.trial import unittest + +from allmydata.util import dictutil + + +class DictUtil(unittest.TestCase): + def test_dict_of_sets(self): + ds = dictutil.DictOfSets() + ds.add(1, "a") + ds.add(2, "b") + ds.add(2, "b") + ds.add(2, "c") + self.failUnlessEqual(ds[1], set(["a"])) + self.failUnlessEqual(ds[2], set(["b", "c"])) + ds.discard(3, "d") # should not raise an exception + ds.discard(2, "b") + self.failUnlessEqual(ds[2], set(["c"])) + ds.discard(2, "c") + self.failIf(2 in ds) + + ds.add(3, "f") + ds2 = dictutil.DictOfSets() + ds2.add(3, "f") + ds2.add(3, "g") + ds2.add(4, "h") + ds.update(ds2) + self.failUnlessEqual(ds[1], set(["a"])) + self.failUnlessEqual(ds[3], set(["f", "g"])) + self.failUnlessEqual(ds[4], set(["h"])) + + def test_auxdict(self): + d = dictutil.AuxValueDict() + # we put the serialized form in the auxdata + d.set_with_aux("key", ("filecap", "metadata"), "serialized") + + self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(d["key"], ("filecap", "metadata")) + self.failUnlessEqual(d.get_aux("key"), "serialized") + def _get_missing(key): + return d[key] + self.failUnlessRaises(KeyError, _get_missing, "nonkey") + self.failUnlessEqual(d.get("nonkey"), None) + self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue") + self.failUnlessEqual(d.get_aux("nonkey"), None) + self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue") + + d["key"] = ("filecap2", "metadata2") + self.failUnlessEqual(d["key"], ("filecap2", "metadata2")) + self.failUnlessEqual(d.get_aux("key"), None) + + d.set_with_aux("key2", "value2", "aux2") + self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) + del d["key2"] + self.failUnlessEqual(d.keys(), ["key"]) + self.failIf("key2" in d) + self.failUnlessRaises(KeyError, _get_missing, "key2") + self.failUnlessEqual(d.get("key2"), None) + self.failUnlessEqual(d.get_aux("key2"), None) + d["key2"] = "newvalue2" + self.failUnlessEqual(d.get("key2"), "newvalue2") + self.failUnlessEqual(d.get_aux("key2"), None) + + d = dictutil.AuxValueDict({1:2,3:4}) + self.failUnlessEqual(sorted(d.keys()), [1,3]) + self.failUnlessEqual(d[1], 2) + self.failUnlessEqual(d.get_aux(1), None) + + d = dictutil.AuxValueDict([ (1,2), (3,4) ]) + self.failUnlessEqual(sorted(d.keys()), [1,3]) + self.failUnlessEqual(d[1], 2) + self.failUnlessEqual(d.get_aux(1), None) + + d = dictutil.AuxValueDict(one=1, two=2) + self.failUnlessEqual(sorted(d.keys()), ["one","two"]) + self.failUnlessEqual(d["one"], 1) + self.failUnlessEqual(d.get_aux("one"), None) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index ff6a1650c..1e9dd0f49 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -11,7 +11,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin -from allmydata.util import dictutil, yamlutil +from allmydata.util import yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -527,78 +527,6 @@ class EqButNotIs(object): def __eq__(self, other): return self.x == other -class DictUtil(unittest.TestCase): - def test_dict_of_sets(self): - ds = dictutil.DictOfSets() - ds.add(1, "a") - ds.add(2, "b") - ds.add(2, "b") - ds.add(2, "c") - self.failUnlessEqual(ds[1], set(["a"])) - self.failUnlessEqual(ds[2], set(["b", "c"])) - ds.discard(3, "d") # should not raise an exception - ds.discard(2, "b") - self.failUnlessEqual(ds[2], set(["c"])) - ds.discard(2, "c") - self.failIf(2 in ds) - - ds.add(3, "f") - ds2 = dictutil.DictOfSets() - ds2.add(3, "f") - ds2.add(3, "g") - ds2.add(4, "h") - ds.update(ds2) - self.failUnlessEqual(ds[1], set(["a"])) - self.failUnlessEqual(ds[3], set(["f", "g"])) - self.failUnlessEqual(ds[4], set(["h"])) - - def test_auxdict(self): - d = dictutil.AuxValueDict() - # we put the serialized form in the auxdata - d.set_with_aux("key", ("filecap", "metadata"), "serialized") - - self.failUnlessEqual(d.keys(), ["key"]) - self.failUnlessEqual(d["key"], ("filecap", "metadata")) - self.failUnlessEqual(d.get_aux("key"), "serialized") - def _get_missing(key): - return d[key] - self.failUnlessRaises(KeyError, _get_missing, "nonkey") - self.failUnlessEqual(d.get("nonkey"), None) - self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue") - self.failUnlessEqual(d.get_aux("nonkey"), None) - self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue") - - d["key"] = ("filecap2", "metadata2") - self.failUnlessEqual(d["key"], ("filecap2", "metadata2")) - self.failUnlessEqual(d.get_aux("key"), None) - - d.set_with_aux("key2", "value2", "aux2") - self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) - del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) - self.failIf("key2" in d) - self.failUnlessRaises(KeyError, _get_missing, "key2") - self.failUnlessEqual(d.get("key2"), None) - self.failUnlessEqual(d.get_aux("key2"), None) - d["key2"] = "newvalue2" - self.failUnlessEqual(d.get("key2"), "newvalue2") - self.failUnlessEqual(d.get_aux("key2"), None) - - d = dictutil.AuxValueDict({1:2,3:4}) - self.failUnlessEqual(sorted(d.keys()), [1,3]) - self.failUnlessEqual(d[1], 2) - self.failUnlessEqual(d.get_aux(1), None) - - d = dictutil.AuxValueDict([ (1,2), (3,4) ]) - self.failUnlessEqual(sorted(d.keys()), [1,3]) - self.failUnlessEqual(d[1], 2) - self.failUnlessEqual(d.get_aux(1), None) - - d = dictutil.AuxValueDict(one=1, two=2) - self.failUnlessEqual(sorted(d.keys()), ["one","two"]) - self.failUnlessEqual(d["one"], 1) - self.failUnlessEqual(d.get_aux("one"), None) - class SampleError(Exception): pass From 79ae478a4806502f882641b4b51b018f7c32700a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:56:31 -0400 Subject: [PATCH 0616/1054] Port to Python 3. --- src/allmydata/test/test_dictutil.py | 14 ++++++++++++-- src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/dictutil.py | 13 ++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index c22b2dfe3..0868db124 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.dictutil. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest @@ -37,7 +47,7 @@ class DictUtil(unittest.TestCase): # we put the serialized form in the auxdata d.set_with_aux("key", ("filecap", "metadata"), "serialized") - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failUnlessEqual(d["key"], ("filecap", "metadata")) self.failUnlessEqual(d.get_aux("key"), "serialized") def _get_missing(key): @@ -55,7 +65,7 @@ class DictUtil(unittest.TestCase): d.set_with_aux("key2", "value2", "aux2") self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failIf("key2" in d) self.failUnlessRaises(KeyError, _get_missing, "key2") self.failUnlessEqual(d.get("key2"), None) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d5bdae2f4..66c0172ea 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ PORTED_MODULES = [ "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", + "allmydata.util.dictutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.mathutil", @@ -41,6 +42,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", + "allmydata.test.test_dictutil", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d59e60dd6..9ed99d7c4 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -1,6 +1,17 @@ """ Tools to mess with dicts. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + class DictOfSets(dict): def add(self, key, value): @@ -10,7 +21,7 @@ class DictOfSets(dict): self[key] = set([value]) def update(self, otherdictofsets): - for key, values in otherdictofsets.iteritems(): + for key, values in otherdictofsets.items(): if key in self: self[key].update(values) else: From 058a76dc857545080fd81a98c8b147a73de7aeac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:58:28 -0400 Subject: [PATCH 0617/1054] Ratchet, and news file. --- misc/python3/ratchet-passing | 11 +++++++++++ newsfragments/3359.minor | 0 2 files changed, 11 insertions(+) create mode 100644 newsfragments/3359.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 6832b7fa7..dd0f8a7b0 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -28,6 +28,8 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls +allmydata.test.test_dictutil.DictUtil.test_auxdict +allmydata.test.test_dictutil.DictUtil.test_dict_of_sets allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes @@ -66,6 +68,15 @@ allmydata.test.test_spans.ByteSpans.test_random allmydata.test.test_spans.StringSpans.test_basic allmydata.test.test_spans.StringSpans.test_random allmydata.test.test_spans.StringSpans.test_test +allmydata.test.test_statistics.Statistics.test_binomial_coeff +allmydata.test.test_statistics.Statistics.test_binomial_distribution_pmf +allmydata.test.test_statistics.Statistics.test_convolve +allmydata.test.test_statistics.Statistics.test_find_k +allmydata.test.test_statistics.Statistics.test_pr_backup_file_loss +allmydata.test.test_statistics.Statistics.test_pr_file_loss +allmydata.test.test_statistics.Statistics.test_repair_cost +allmydata.test.test_statistics.Statistics.test_repair_count_pmf +allmydata.test.test_statistics.Statistics.test_survival_pmf allmydata.test.test_time_format.TimeFormat.test_epoch allmydata.test.test_time_format.TimeFormat.test_epoch_in_London allmydata.test.test_time_format.TimeFormat.test_format_delta diff --git a/newsfragments/3359.minor b/newsfragments/3359.minor new file mode 100644 index 000000000..e69de29bb From 94d489ff46883279f2994c03ca316f0c82f490d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 16:01:12 -0400 Subject: [PATCH 0618/1054] Fix trailing whitespace. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 921c481ff..ac5367cfc 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET addresses.append(addr) return addresses d.addCallback(_collect) - d.addCallback(lambda addresses: [native_str(s) for s in addresses]) + d.addCallback(lambda addresses: [native_str(s) for s in addresses]) return d def get_local_ip_for(target): From cbbe260a4eafef224f3bf06f599030a463f0c7c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 16:19:36 -0400 Subject: [PATCH 0619/1054] Windows doesn't like Unicode strings in os.environ in Python 2.7. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index ac5367cfc..36f4bb79b 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -239,7 +239,7 @@ def _synchronously_find_addresses_via_config(): def _query(path, args, regex): if not os.path.isfile(path): return [] - env = {'LANG': 'en_US.UTF-8'} + env = {native_str('LANG'): native_str('en_US.UTF-8')} TRIES = 5 for trial in range(TRIES): try: From 9f5c58c173705df87d7bb8847ca3adaa31d28268 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 06:27:40 -0400 Subject: [PATCH 0620/1054] Use super() --- src/allmydata/test/test_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 758755e9c..e02343a5f 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -48,9 +48,9 @@ class FakeClient(object): # Also, the request needs to have `args` and `fields` properties so # that `allmydata.web.common.get_arg()` won't complain. @implementer(IRequest) -class TestRequest(object, Request): +class TestRequest(Request, object): def __init__(self, args=None, fields=None): - Request.__init__(self, DummyChannel()) + super(TestRequest, self).__init__(DummyChannel()) self.args = args or {} self.fields = fields or {} self.prepath = [b""] From 80802a74cdf8b6ed1a971cf086d36508fc91d903 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 06:53:52 -0400 Subject: [PATCH 0621/1054] Use BeautifulSoup to remove tags --- src/allmydata/test/test_checker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index e02343a5f..ee9be5abc 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -129,10 +129,9 @@ class FakeCheckAndRepairResults(object): class WebResultsRendering(unittest.TestCase): - def remove_tags(self, s): - s = re.sub(r'<[^>]*>', ' ', s) - s = re.sub(r'\s+', ' ', s) - return s + @staticmethod + def remove_tags(html): + return BeautifulSoup(html).get_text() def create_fake_client(self): sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) From adca146a1b2b66f9f69477150a29f238f11c1e6d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 07:10:44 -0400 Subject: [PATCH 0622/1054] Use space as separator when removing tags --- src/allmydata/test/test_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index ee9be5abc..252938143 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -131,7 +131,7 @@ class WebResultsRendering(unittest.TestCase): @staticmethod def remove_tags(html): - return BeautifulSoup(html).get_text() + return BeautifulSoup(html).get_text(separator=" ") def create_fake_client(self): sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) @@ -227,7 +227,7 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessIn("Wrong Shares: 0", s) self.failUnlessIn("Recoverable Versions: 1", s) self.failUnlessIn("Unrecoverable Versions: 0", s) - self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s) + self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s) cr = check_results.CheckResults(u, u.get_storage_index(), healthy=False, recoverable=True, @@ -250,7 +250,7 @@ class WebResultsRendering(unittest.TestCase): s = self.remove_tags(html) self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) - self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) + self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) html = self.render_element(w) s = self.remove_tags(html) From e6867a1355e101216ca40ed78c2829ad0ca8e648 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 07:15:39 -0400 Subject: [PATCH 0623/1054] Use docstring to describe a class --- src/allmydata/test/test_checker.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 252938143..bc0dc0482 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -43,12 +43,17 @@ class FakeClient(object): def get_storage_broker(self): return self.storage_broker -# XXX: We have to have this class because `common.get_arg()` expects a -# `nevow.inevow.IRequest`, which `twisted.web.server.Request` isn't. -# Also, the request needs to have `args` and `fields` properties so -# that `allmydata.web.common.get_arg()` won't complain. @implementer(IRequest) class TestRequest(Request, object): + """ + A minimal Request class to use in tests. + + XXX: We have to have this class because `common.get_arg()` expects + a `nevow.inevow.IRequest`, which `twisted.web.server.Request` + isn't. The request needs to have `args`, `fields`, `prepath`, and + `postpath` properties so that `allmydata.web.common.get_arg()` + won't complain. + """ def __init__(self, args=None, fields=None): super(TestRequest, self).__init__(DummyChannel()) self.args = args or {} From 1b7a35e7730eaab6ed0bf516b1c9635561067036 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 07:18:01 -0400 Subject: [PATCH 0624/1054] Do not strip tags where it is not necessary --- src/allmydata/test/test_checker.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index bc0dc0482..c22d42914 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -175,12 +175,10 @@ class WebResultsRendering(unittest.TestCase): lcr = web_check_results.LiteralCheckResultsRendererElement() html = self.render_element(lcr) - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) + self.failUnlessIn("Literal files are always healthy", html) html = self.render_element(lcr, args={"return_to": ["FOOURL"]}) - s = self.remove_tags(html) - self.failUnlessIn("Literal files are always healthy", s) + self.failUnlessIn("Literal files are always healthy", html) self.failUnlessIn('Return to file.', html) c = self.create_fake_client() From ca6cf887f757b9497b5a8f8ebc49c71d931554a5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 07:26:11 -0400 Subject: [PATCH 0625/1054] Test JSON rendering synchronously --- src/allmydata/test/test_checker.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index c22d42914..779c7fa66 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -405,24 +405,18 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessIn("Repair unsuccessful", s) self.failUnlessIn("Post-Repair Checker Results:", s) - w2 = web_check_results.CheckAndRepairResultsRenderer(c, crr) - d = self.render_json(w2) - def _got_json(data): - j = json.loads(data) - self.failUnlessEqual(j["repair-attempted"], True) - self.failUnlessEqual(j["storage-index"], - "2k6avpjga3dho3zsjo6nnkt7n4") - self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") - self.failUnlessEqual(j["post-repair-results"]["summary"], "worse") - _got_json(d) + w = web_check_results.CheckAndRepairResultsRenderer(c, crr) + j = json.loads(self.render_json(w)) + self.failUnlessEqual(j["repair-attempted"], True) + self.failUnlessEqual(j["storage-index"], + "2k6avpjga3dho3zsjo6nnkt7n4") + self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") + self.failUnlessEqual(j["post-repair-results"]["summary"], "worse") - w3 = web_check_results.CheckAndRepairResultsRenderer(c, None) - d = self.render_json(w3) - def _got_lit_results(data): - j = json.loads(data) - self.failUnlessEqual(j["repair-attempted"], False) - self.failUnlessEqual(j["storage-index"], "") - _got_lit_results(d) + w = web_check_results.CheckAndRepairResultsRenderer(c, None) + j = json.loads(self.render_json(w)) + self.failUnlessEqual(j["repair-attempted"], False) + self.failUnlessEqual(j["storage-index"], "") def test_deep_check_renderer(self): From 0f28a9387982ad957a8f7b84ab1e2f9b0c4dbb1f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 08:08:48 -0400 Subject: [PATCH 0626/1054] Remove unused import --- src/allmydata/test/test_checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 779c7fa66..2296194f0 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -1,5 +1,4 @@ -import re import json import os.path, shutil From 4fdcd06fbdb4ce1e2dc7d3ea10ccccb70ca5eea2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 10:48:17 -0400 Subject: [PATCH 0627/1054] Another passing test for the ratchet. --- misc/python3/ratchet-passing | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bbe594998..68f1a968b 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -51,6 +51,7 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route allmydata.test.test_iputil.ListenOnUsed.test_random_port +allmydata.test.test_iputil.ListenOnUsed.test_specific_port allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested From 4dae5d867c8dd44d9ceb68b6732030f00b391c6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:08:38 -0400 Subject: [PATCH 0628/1054] Add retry logic, listenOnUnused is potentially flaky. --- setup.py | 1 + src/allmydata/test/test_iputil.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 9bb385cb4..a35023b8b 100644 --- a/setup.py +++ b/setup.py @@ -390,6 +390,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "beautifulsoup4", "html5lib", "junitxml", + "tenacity", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 42677af2f..c6caed7f9 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -17,6 +17,8 @@ import re, errno, subprocess, os, socket from twisted.trial import unittest +from tenacity import retry, stop_after_attempt + from foolscap.api import Tub from allmydata.util import iputil @@ -199,6 +201,7 @@ class ListenOnUsed(unittest.TestCase): self.addCleanup(tub.stopService) return tub + @retry(stop=stop_after_attempt(7)) def test_random_port(self): """A random port is selected if none is given.""" tub = self.create_tub("utils/ListenOnUsed/test_randomport") @@ -215,6 +218,7 @@ class ListenOnUsed(unittest.TestCase): portnum2 = iputil.listenOnUnused(tub2) self.assertNotEqual(portnum, portnum2) + @retry(stop=stop_after_attempt(7)) def test_specific_port(self): """The given port is used.""" tub = self.create_tub("utils/ListenOnUsed/test_givenport") From cc494e3d3978dd79eb67b3397bc8abcb29b7d1c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:11:05 -0400 Subject: [PATCH 0629/1054] Document that API returns native strings. --- src/allmydata/util/iputil.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 36f4bb79b..890ff98e7 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -102,13 +102,18 @@ except ImportError: increase_rlimits = _increase_rlimits def get_local_addresses_sync(): + """ + Return a list of IPv4 addresses (as dotted-quad native strings) that are + currently configured on this host, sorted in descending order of how likely + we think they are to work. + """ return [native_str(a) for a in _synchronously_find_addresses_via_config()] def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad - strings) that are currently configured on this host, sorted in descending - order of how likely we think they are to work. + native strings) that are currently configured on this host, sorted in + descending order of how likely we think they are to work. @param target: we want to learn an IP address they could try using to connect to us; The default value is fine, but it might help if you @@ -137,7 +142,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET def get_local_ip_for(target): """Find out what our IP address is for use by a given target. - @return: the IP address as a dotted-quad string which could be used by + @return: the IP address as a dotted-quad native string which could be used to connect to us. It might work for them, it might not. If there is no suitable address (perhaps we don't currently have an externally-visible interface), this will return None. @@ -172,7 +177,7 @@ def get_local_ip_for(target): return localip finally: d = port.stopListening() - d.addErrback(log.err) + 3 d.addErrback(log.err) except (socket.error, CannotListenError): # no route to that host localip = None From 8c9c691c02c420253a79393669856d0bea07f4e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:11:27 -0400 Subject: [PATCH 0630/1054] Fix typo. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 890ff98e7..8754fca53 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -177,7 +177,7 @@ def get_local_ip_for(target): return localip finally: d = port.stopListening() - 3 d.addErrback(log.err) + d.addErrback(log.err) except (socket.error, CannotListenError): # no route to that host localip = None From 9ca1fdef816263b61cdb51e801dd26aa7b992d28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:12:17 -0400 Subject: [PATCH 0631/1054] Note it's ported. --- src/allmydata/test/test_statistics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index a14fb55ab..cfbbc1dc0 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -1,5 +1,7 @@ """ Tests for allmydata.util.statistics. + +Ported to Python 3. """ from __future__ import absolute_import from __future__ import division From e954314fe52a6b3e40549b27f794aa00d0fd4375 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:23:40 -0400 Subject: [PATCH 0632/1054] Don't expose Python 3 dicts to innocent, unsuspecting Python 2 code. --- src/allmydata/util/dictutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 9ed99d7c4..d5f0da29b 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -10,7 +10,10 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + # IMPORTANT: We deliberately don't import dict. The issue is that we're + # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of + # code that doesn't support it. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 class DictOfSets(dict): From 17b15ae08577241df493aa972984b19a39fe12f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:25:34 -0400 Subject: [PATCH 0633/1054] Add another test. --- src/allmydata/test/test_statistics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index cfbbc1dc0..8d5837fc5 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -44,6 +44,7 @@ class Statistics(unittest.TestCase): self.failUnlessEqual(f(20, 2), 190) self.failUnlessEqual(f(20, 8), f(20, 12)) self.should_assert("Should assert if n < k", f, 2, 3) + self.assertEqual(f(5, 3), f(5, 2)) def test_binomial_distribution_pmf(self): f = statistics.binomial_distribution_pmf From 199c4f6acd350885e28e30b91216c77d1b739b0e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 13:00:44 -0400 Subject: [PATCH 0634/1054] Try to fix nix. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 0e941bce1..f2e61d6c2 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -59,6 +59,7 @@ python.pkgs.buildPythonPackage rec { fixtures beautifulsoup4 html5lib + tenacity ]; checkPhase = '' From 4007fb4de7e0ab40c8f2aadc985ddb2770a975cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 16:12:48 -0400 Subject: [PATCH 0635/1054] Rearrange imports --- src/allmydata/test/web/test_web.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1ada50433..0b975f388 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -54,6 +54,9 @@ from .common import ( assert_soup_has_tag_with_attributes, assert_soup_has_tag_with_content, assert_soup_has_tag_with_attributes_and_content, + unknown_rwcap, + unknown_rocap, + unknown_immcap, ) from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION @@ -64,7 +67,6 @@ from ..common_web import ( Error, ) from allmydata.client import _Client, SecretHolder -from .common import unknown_rwcap, unknown_rocap, unknown_immcap # create a fake uploader/downloader, and a couple of fake dirnodes, then # create a webserver that works against them From 951250f9ff327bf00291b2cae301515d71024ead Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 28 Jul 2020 16:13:58 -0400 Subject: [PATCH 0636/1054] Remove unused FAVICON_MARKUP --- src/allmydata/test/web/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 871cdeb26..1f568ad8d 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -5,8 +5,6 @@ unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8') unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8') unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8') -FAVICON_MARKUP = '' - def assert_soup_has_favicon(testcase, soup): """ From 93c474c39bd640f0702a47693f11236ae34d6f0f Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 29 Jul 2020 09:00:50 +0200 Subject: [PATCH 0637/1054] make test_util pass under both pythons --- src/allmydata/client.py | 2 +- src/allmydata/test/common_util.py | 7 ++++- src/allmydata/test/test_util.py | 52 ++++++++++++++++--------------- src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/dictutil.py | 2 +- src/allmydata/util/spans.py | 8 +++-- src/allmydata/util/statistics.py | 1 + 7 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 5e8bf0d6d..f731c3163 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -741,7 +741,7 @@ class _Client(node.Node, pollmixin.PollMixin): private_key_str = self.config.get_or_create_private_config("node.privkey", _make_key) private_key, public_key = ed25519.signing_keypair_from_string(private_key_str) public_key_str = ed25519.string_from_verifying_key(public_key) - self.config.write_config_file("node.pubkey", public_key_str + "\n") + self.config.write_config_file("node.pubkey", public_key_str + "\n", "w") self._node_private_key = private_key self._node_public_key = public_key diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 510994d12..64cc59ffa 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,7 @@ from __future__ import print_function import os, signal +from future.utils import PY2 from random import randrange from six.moves import StringIO @@ -10,8 +11,12 @@ from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, + get_io_encoding) -#from ..scripts import runner + +if PY2: # XXX this is a hack that makes some tests pass on Python3, remove + # in the future + from ..scripts import runner def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 042875c6a..c95ddaafe 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -36,12 +36,12 @@ def sha256(data): :returns: a hex-encoded SHA256 hash of the data """ - return binascii.hexlify(hashlib.sha256(data).digest()) + return binascii.hexlify(hashlib.sha256(data.encode('utf8')).digest()).decode("utf8") class IDLib(unittest.TestCase): def test_nodeid_b2a(self): - self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) + self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32) class MyList(list): @@ -244,10 +244,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, u"one") - self.failUnlessEqual(fileutil.read(fn), "one") + fileutil.write_atomically(fn, b"one") + self.failUnlessEqual(fileutil.read(fn), b"one") fileutil.write_atomically(fn, u"two", mode="") # non-binary - self.failUnlessEqual(fileutil.read(fn), "two") + self.failUnlessEqual(fileutil.read(fn), b"two") def test_rename(self): basedir = "util/FileUtil/test_rename" @@ -270,20 +270,20 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) # when only dest exists - fileutil.write(dest_path, "dest") + fileutil.write(dest_path, b"dest") self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) - self.failUnlessEqual(fileutil.read(dest_path), "dest") + self.failUnlessEqual(fileutil.read(dest_path), b"dest") # when both exist - fileutil.write(source_path, "source") + fileutil.write(source_path, b"source") self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path) - self.failUnlessEqual(fileutil.read(source_path), "source") - self.failUnlessEqual(fileutil.read(dest_path), "dest") + self.failUnlessEqual(fileutil.read(source_path), b"source") + self.failUnlessEqual(fileutil.read(dest_path), b"dest") # when only source exists os.remove(dest_path) fileutil.rename_no_overwrite(source_path, dest_path) - self.failUnlessEqual(fileutil.read(dest_path), "source") + self.failUnlessEqual(fileutil.read(dest_path), b"source") self.failIf(os.path.exists(source_path)) def test_replace_file(self): @@ -297,21 +297,21 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path) # when only replaced exists - fileutil.write(replaced_path, "foo") + fileutil.write(replaced_path, b"foo") self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "foo") + self.failUnlessEqual(fileutil.read(replaced_path), b"foo") # when both replaced and replacement exist - fileutil.write(replacement_path, "bar") + fileutil.write(replacement_path, b"bar") fileutil.replace_file(replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "bar") + self.failUnlessEqual(fileutil.read(replaced_path), b"bar") self.failIf(os.path.exists(replacement_path)) # when only replacement exists os.remove(replaced_path) - fileutil.write(replacement_path, "bar") + fileutil.write(replacement_path, b"bar") fileutil.replace_file(replaced_path, replacement_path) - self.failUnlessEqual(fileutil.read(replaced_path), "bar") + self.failUnlessEqual(fileutil.read(replaced_path), b"bar") self.failIf(os.path.exists(replacement_path)) def test_du(self): @@ -331,7 +331,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): def test_abspath_expanduser_unicode(self): self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") - saved_cwd = os.path.normpath(os.getcwd()).decode("utf8") + saved_cwd = os.path.normpath(os.getcwd()) + if PY2: + saved_cwd = saved_cwd.decode("utf8") abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) self.failUnless(isinstance(saved_cwd, str), saved_cwd) @@ -452,9 +454,9 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): fileutil.remove(long_path) self.addCleanup(_cleanup) - fileutil.write(long_path, "test") + fileutil.write(long_path, b"test") self.failUnless(os.path.exists(long_path)) - self.failUnlessEqual(fileutil.read(long_path), "test") + self.failUnlessEqual(fileutil.read(long_path), b"test") _cleanup() self.failIf(os.path.exists(long_path)) @@ -512,7 +514,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): # create a file f = os.path.join(basedir, "1.txt") - fileutil.write(f, "a"*10) + fileutil.write(f, b"a"*10) fileinfo = fileutil.get_pathinfo(f) self.failUnlessTrue(fileinfo.isfile) self.failUnlessTrue(fileinfo.exists) @@ -540,7 +542,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): fileutil.make_dirs(basedir) f = os.path.join(basedir, "1.txt") - fileutil.write(f, "a"*10) + fileutil.write(f, b"a"*10) # create a symlink pointing to 1.txt slname = os.path.join(basedir, "linkto1.txt") @@ -568,7 +570,7 @@ class PollMixinTests(unittest.TestCase): def test_PollMixin_False_then_True(self): i = iter([False, True]) - d = self.pm.poll(check_f=i.next, + d = self.pm.poll(check_f=lambda: next(i), pollinterval=0.1) return d @@ -813,7 +815,7 @@ class SimpleSpans(object): yield (prevstart, prevend-prevstart+1) def __bool__(self): # this gets us bool() - return self.len() + return bool(self.len()) def len(self): return len(self._have) @@ -1107,7 +1109,7 @@ class SimpleDataSpans(object): self.add(start, data) def __bool__(self): # this gets us bool() - return self.len() + return bool(self.len()) def len(self): return len(self.missing.replace("1", "")) def _dump(self): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6873c5c08..6ccf4161b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ PORTED_MODULES = [ "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", + "allmydata.util.fileutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.mathutil", @@ -47,6 +48,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_pipeline", "allmydata.test.test_python3", "allmydata.test.test_time_format", + "allmydata.test.test_util", ] diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d59e60dd6..da6a20e37 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -10,7 +10,7 @@ class DictOfSets(dict): self[key] = set([value]) def update(self, otherdictofsets): - for key, values in otherdictofsets.iteritems(): + for key, values in otherdictofsets.items(): if key in self: self[key].update(values) else: diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 1576f059a..4af91d033 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -1,5 +1,9 @@ from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import object + class Spans(object): """I represent a compressed list of booleans, one per index (an integer). @@ -155,7 +159,7 @@ class Spans(object): for s in self._spans: yield s - def __nonzero__(self): # this gets us bool() + def __bool__(self): # this gets us bool() return bool(self.len()) def len(self): @@ -235,7 +239,7 @@ class DataSpans(object): for (start, data) in other.get_chunks(): self.add(start, data) - def __nonzero__(self): # this gets us bool() + def __bool__(self): # this gets us bool() return bool(self.len()) def len(self): diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 3e50b39dc..d877d1275 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -8,6 +8,7 @@ from __future__ import division, print_function from allmydata.util.mathutil import round_sigfigs import math +from functools import reduce import sys def pr_file_loss(p_list, k): From 724bf7de75236428378438cd2912348a7274234b Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 29 Jul 2020 19:56:01 +0200 Subject: [PATCH 0638/1054] remove open from imported builtins --- src/allmydata/util/fileutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index bdabe5aeb..b590b264f 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range """ Futz with files like a pro. """ From bfead0630d0a8ff25c51f0957eface42ce424dec Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 29 Jul 2020 19:56:29 +0200 Subject: [PATCH 0639/1054] leave a comment explaining why we removed open --- src/allmydata/util/fileutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index b590b264f..31d3cecb7 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: + # open is not here because we want to use native strings on Py2 from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range """ Futz with files like a pro. From 9c16a0554e7ee0f67f3ffcd757d3c475da84ef26 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:38:58 -0400 Subject: [PATCH 0640/1054] Get rid of unused code. --- src/allmydata/test/test_util.py | 77 +-------------------------------- src/allmydata/util/limiter.py | 40 ----------------- 2 files changed, 1 insertion(+), 116 deletions(-) delete mode 100644 src/allmydata/util/limiter.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1e9dd0f49..330abebe5 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -10,7 +10,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil -from allmydata.util import limiter, pollmixin +from allmydata.util import pollmixin from allmydata.util import yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile @@ -429,81 +429,6 @@ class PollMixinTests(unittest.TestCase): return d -class Limiter(unittest.TestCase): - - def job(self, i, foo): - self.calls.append( (i, foo) ) - self.simultaneous += 1 - self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous) - d = defer.Deferred() - def _done(): - self.simultaneous -= 1 - d.callback("done %d" % i) - reactor.callLater(1.0, _done) - return d - - def bad_job(self, i, foo): - raise ValueError("bad_job %d" % i) - - def test_limiter(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_done) - return d - - def test_errors(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d2 = l.add(self.bad_job, 21, "21") - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _most_done(res): - results = [] - for (success, result) in res: - self.failUnlessEqual(success, True) - results.append(result) - results.sort() - expected_results = ["done %d" % i for i in range(20)] - expected_results.sort() - self.failUnlessEqual(results, expected_results) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - def _good(res): - self.fail("should have failed, not got %s" % (res,)) - def _err(f): - f.trap(ValueError) - self.failUnless("bad_job 21" in str(f)) - d2.addCallbacks(_good, _err) - return d2 - d.addCallback(_most_done) - def _all_done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_all_done) - return d - - ctr = [0] class EqButNotIs(object): def __init__(self, x): diff --git a/src/allmydata/util/limiter.py b/src/allmydata/util/limiter.py deleted file mode 100644 index 0391ede11..000000000 --- a/src/allmydata/util/limiter.py +++ /dev/null @@ -1,40 +0,0 @@ - -from twisted.internet import defer -from foolscap.api import eventually - -class ConcurrencyLimiter(object): - """I implement a basic concurrency limiter. Add work to it in the form of - (callable, args, kwargs) tuples. No more than LIMIT callables will be - outstanding at any one time. - """ - - def __init__(self, limit=10): - self.limit = limit - self.pending = [] - self.active = 0 - - def __repr__(self): - return "" % (self.active, len(self.pending), - self.limit) - - def add(self, cb, *args, **kwargs): - d = defer.Deferred() - task = (cb, args, kwargs, d) - self.pending.append(task) - self.maybe_start_task() - return d - - def maybe_start_task(self): - if self.active >= self.limit: - return - if not self.pending: - return - (cb, args, kwargs, done_d) = self.pending.pop(0) - self.active += 1 - d = defer.maybeDeferred(cb, *args, **kwargs) - d.addBoth(self._done, done_d) - - def _done(self, res, done_d): - self.active -= 1 - eventually(done_d.callback, res) - eventually(self.maybe_start_task) From 0a1e365ed1ea1832dcf58e0b3fc934ad2d4c757a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:46:49 -0400 Subject: [PATCH 0641/1054] Tests for gcutil. --- src/allmydata/test/test_iputil.py | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index c6caed7f9..06ab13f39 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -14,6 +14,7 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os, socket +import gc from twisted.trial import unittest @@ -21,7 +22,7 @@ from tenacity import retry, stop_after_attempt from foolscap.api import Tub -from allmydata.util import iputil +from allmydata.util import iputil, gcutil import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace @@ -228,3 +229,33 @@ class ListenOnUsed(unittest.TestCase): s.close() port2 = iputil.listenOnUnused(tub, port) self.assertEqual(port, port2) + + +class GcUtil(unittest.TestCase): + """Tests for allmydata.util.gcutil, which is used only by listenOnUnused.""" + + def test_gc_after_allocations(self): + """The resource tracker triggers allocations every 26 allocations.""" + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + for _ in range(25): + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 0) + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 1) + del collections[:] + + def test_release_delays_gc(self): + """Releasing a file descriptor resource delays GC collection.""" + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + gcutil.fileDescriptorResource.allocate() + for _ in range(3): + gcutil.fileDescriptorResource.release() + for _ in range(25): + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 0) + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 1) From 5e308260b105865db1f472ed12524ccf7b18ae14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:49:48 -0400 Subject: [PATCH 0642/1054] Port to Python 3. --- newsfragments/3361.minor | 0 src/allmydata/util/_python3.py | 1 + src/allmydata/util/gcutil.py | 10 ++++++++++ 3 files changed, 11 insertions(+) create mode 100644 newsfragments/3361.minor diff --git a/newsfragments/3361.minor b/newsfragments/3361.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..b91e079f8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.base62", "allmydata.util.deferredutil", "allmydata.util.dictutil", + "allmydata.util.gcutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index 8fb3a64c9..ade95341f 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -7,7 +7,17 @@ Helpers for managing garbage collection. a result. Register allocation and release of *bare* file descriptors with this object (file objects, socket objects, etc, have their own integration with the garbage collector and don't need to bother with this). + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 __all__ = [ "fileDescriptorResource", From 3a6587971aed1656671939ae745fa4a20897807f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:51:11 -0400 Subject: [PATCH 0643/1054] Ratchet. --- misc/python3/ratchet-passing | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..d1ea2dfa2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -46,6 +46,8 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_iputil.GcUtil.test_gc_after_allocations +allmydata.test.test_iputil.GcUtil.test_release_delays_gc allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for allmydata.test.test_iputil.ListAddresses.test_list_async allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin From da87bd7e7fadab6f4c4a1459482864e98e2dc92f Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 29 Jul 2020 21:03:05 +0200 Subject: [PATCH 0644/1054] news fragment --- newsfragments/3358.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3358.minor diff --git a/newsfragments/3358.minor b/newsfragments/3358.minor new file mode 100644 index 000000000..e69de29bb From 7618b295cb8fa8176acae07e343287ac29947ffe Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Thu, 30 Jul 2020 09:29:20 +0200 Subject: [PATCH 0645/1054] Revert "merge master (I think properly)" This reverts commit 6140b5ff52aea8905500e54aa0c8e7cef5a4fe80, reversing changes made to da87bd7e7fadab6f4c4a1459482864e98e2dc92f. --- src/allmydata/test/common_util.py | 5 ++- src/allmydata/test/test_util.py | 50 +++++++++++++++++------------- src/allmydata/util/encodingutil.py | 3 +- src/allmydata/util/fileutil.py | 17 +++++----- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index a4478254b..64cc59ffa 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -13,7 +13,10 @@ from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) -from ..scripts import runner + +if PY2: # XXX this is a hack that makes some tests pass on Python3, remove + # in the future + from ..scripts import runner def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 662c4662a..c95ddaafe 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,5 +1,11 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import zip, str, range, object import binascii import six import hashlib @@ -238,10 +244,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, "one") - self.failUnlessEqual(fileutil.read(fn), "one") - fileutil.write_atomically(fn, "two", mode="") # non-binary - self.failUnlessEqual(fileutil.read(fn), "two") + fileutil.write_atomically(fn, b"one") + self.failUnlessEqual(fileutil.read(fn), b"one") + fileutil.write_atomically(fn, u"two", mode="") # non-binary + self.failUnlessEqual(fileutil.read(fn), b"two") def test_rename(self): basedir = "util/FileUtil/test_rename" @@ -323,13 +329,15 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(10+11+12+13, used) def test_abspath_expanduser_unicode(self): - self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring") + self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") - saved_cwd = os.path.normpath(os.getcwdu()) + saved_cwd = os.path.normpath(os.getcwd()) + if PY2: + saved_cwd = saved_cwd.decode("utf8") abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) - self.failUnless(isinstance(saved_cwd, unicode), saved_cwd) - self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd) + self.failUnless(isinstance(saved_cwd, str), saved_cwd) + self.failUnless(isinstance(abspath_cwd, str), abspath_cwd) if sys.platform == "win32": self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd)) else: @@ -390,10 +398,10 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): os.chdir(cwd) for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'): uabspath = fileutil.abspath_expanduser_unicode(upath) - self.failUnless(isinstance(uabspath, unicode), uabspath) + self.failUnless(isinstance(uabspath, str), uabspath) uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False) - self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong) + self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong) finally: os.chdir(saved_cwd) @@ -547,7 +555,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): def test_encrypted_tempfile(self): f = EncryptedTemporaryFile() - f.write("foobar") + f.write(b"foobar") f.close() @@ -707,7 +715,7 @@ class DictUtil(unittest.TestCase): # we put the serialized form in the auxdata d.set_with_aux("key", ("filecap", "metadata"), "serialized") - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failUnlessEqual(d["key"], ("filecap", "metadata")) self.failUnlessEqual(d.get_aux("key"), "serialized") def _get_missing(key): @@ -725,7 +733,7 @@ class DictUtil(unittest.TestCase): d.set_with_aux("key2", "value2", "aux2") self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failIf("key2" in d) self.failUnlessRaises(KeyError, _get_missing, "key2") self.failUnlessEqual(d.get("key2"), None) @@ -859,7 +867,7 @@ class ByteSpans(unittest.TestCase): s1 = Spans(3, 4) # 3,4,5,6 self._check1(s1) - s1 = Spans(long(3), long(4)) # 3,4,5,6 + s1 = Spans(int(3), int(4)) # 3,4,5,6 self._check1(s1) s2 = Spans(s1) @@ -1102,7 +1110,6 @@ class SimpleDataSpans(object): def __bool__(self): # this gets us bool() return bool(self.len()) - def len(self): return len(self.missing.replace("1", "")) def _dump(self): @@ -1187,9 +1194,9 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(ds.get(2, 4), "fear") ds = klass() - ds.add(long(2), "four") - ds.add(long(3), "ea") - self.failUnlessEqual(ds.get(long(2), long(4)), "fear") + ds.add(int(2), "four") + ds.add(int(3), "ea") + self.failUnlessEqual(ds.get(int(2), int(4)), "fear") def do_scan(self, klass): @@ -1352,6 +1359,7 @@ class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) back = yamlutil.safe_load(data) - self.failUnlessEqual(type(back[0]), unicode) - self.failUnlessEqual(type(back[1]), unicode) - self.failUnlessEqual(type(back[2]), unicode) + str_type = type("x") # workaround for future.builtins.str + self.failUnlessEqual(type(back[0]), str_type) + self.failUnlessEqual(type(back[1]), str_type) + self.failUnlessEqual(type(back[2]), str_type) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 568267b35..491e5d61d 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -4,7 +4,6 @@ unicode and back. """ import sys, os, re, locale -from types import NoneType from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -139,7 +138,7 @@ def to_str(s): return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, (NoneType, str)), s) + precondition(isinstance(s, str) or s is None, s) if s is None: return s return s.decode('utf-8') diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 73bc9cfbe..31d3cecb7 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,7 +1,7 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -from __future__ import unicode_literals +from __future__ import unicode_literals from future.utils import PY2 if PY2: @@ -11,7 +11,7 @@ if PY2: Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time, binascii +import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple from errno import ENOENT @@ -197,7 +197,7 @@ def make_dirs(dirname, mode=0o777): if not os.path.isdir(dirname): if tx: raise tx - raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rm_dir(dirname): """ @@ -284,7 +284,7 @@ def put_file(path, inf): outf.write(data) def precondition_abspath(path): - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("an abspath must be a Unicode string") if sys.platform == "win32": @@ -316,7 +316,7 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): abspath_expanduser_unicode. On Windows, the result will be a long path unless long_path is given as False. """ - if not isinstance(path, unicode): + if not isinstance(path, str): raise AssertionError("paths must be Unicode strings") if base is not None and long_path: precondition_abspath(base) @@ -337,7 +337,10 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): if not os.path.isabs(path): if base is None: - path = os.path.join(os.getcwdu(), path) + cwd = os.getcwd() + if PY2: + cwd = cwd.decode('utf8') + path = os.path.join(cwd, path) else: path = os.path.join(base, path) @@ -422,7 +425,7 @@ ERROR_ENVVAR_NOT_FOUND = 203 def windows_getenv(name): # Based on , # with improved error handling. Returns None if there is no enivronment variable of the given name. - if not isinstance(name, unicode): + if not isinstance(name, str): raise AssertionError("name must be Unicode") n = GetEnvironmentVariableW(name, None, 0) From ecb3ee023df259dd39ce298fe9d0b545a2286552 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Thu, 30 Jul 2020 09:36:11 +0200 Subject: [PATCH 0646/1054] add a dummy commit --- src/allmydata/util/fileutil.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 31d3cecb7..9a638a83a 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -8,6 +8,8 @@ if PY2: # open is not here because we want to use native strings on Py2 from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range """ +Ported to Python3 + Futz with files like a pro. """ From bf37413e0590f14e172b839e10906f991975cebb Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Sun, 2 Aug 2020 11:16:18 +0200 Subject: [PATCH 0647/1054] address the review (mostly) --- src/allmydata/test/common_util.py | 2 +- src/allmydata/test/test_util.py | 2 +- src/allmydata/util/fileutil.py | 11 ++++++----- src/allmydata/util/spans.py | 4 ---- src/allmydata/util/statistics.py | 2 -- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 06676a5f9..4e25e897e 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -import os, signal +import os from random import randrange from six.moves import StringIO diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index c66431dde..19e50d9f1 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import zip, str, range, object + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range import binascii import six import os, time, sys diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 9a638a83a..d57464d6f 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,3 +1,9 @@ +""" +Ported to Python3 + +Futz with files like a pro. +""" + from __future__ import print_function from __future__ import absolute_import from __future__ import division @@ -7,11 +13,6 @@ from future.utils import PY2 if PY2: # open is not here because we want to use native strings on Py2 from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range -""" -Ported to Python3 - -Futz with files like a pro. -""" import sys, os, stat, tempfile, time, binascii import six diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index c8bcb9314..814904d68 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -7,10 +7,6 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -from future.utils import PY2 -if PY2: - from future.builtins import object - class Spans(object): """I represent a compressed list of booleans, one per index (an integer). diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 71e9a96ec..c5fb5c6bf 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -20,8 +20,6 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -from functools import reduce - from allmydata.util.mathutil import round_sigfigs import math from functools import reduce From c178c55178df373316cb82e2acdf87fceddc3713 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Sun, 2 Aug 2020 12:03:59 +0200 Subject: [PATCH 0648/1054] remove this import --- src/allmydata/test/test_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 19e50d9f1..1a4d044c3 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -6,7 +6,6 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range -import binascii import six import os, time, sys import yaml From 34e3052814ca409ffef339dd801ea9816bacdab8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:43:21 -0400 Subject: [PATCH 0649/1054] Don't use global state for testing. --- src/allmydata/test/test_iputil.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 06ab13f39..d3262a9ea 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -236,26 +236,28 @@ class GcUtil(unittest.TestCase): def test_gc_after_allocations(self): """The resource tracker triggers allocations every 26 allocations.""" + tracker = gcutil._ResourceTracker() collections = [] self.patch(gc, "collect", lambda: collections.append(1)) for _ in range(2): for _ in range(25): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 0) - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 1) del collections[:] def test_release_delays_gc(self): """Releasing a file descriptor resource delays GC collection.""" + tracker = gcutil._ResourceTracker() collections = [] self.patch(gc, "collect", lambda: collections.append(1)) for _ in range(2): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() for _ in range(3): - gcutil.fileDescriptorResource.release() + tracker.release() for _ in range(25): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 0) - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 1) From 3330737656ce6cbe79e4210774429d125aa15fef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:56:47 -0400 Subject: [PATCH 0650/1054] Be a bit stricter about expected outputs. --- src/allmydata/test/test_crypto.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 53ba344a4..1cf55ca0f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,3 +1,5 @@ +from future.utils import native_bytes + import six import unittest @@ -283,7 +285,7 @@ class TestEd25519(unittest.TestCase): private_key, public_key = ed25519.create_signing_keypair() private_key_str = ed25519.string_from_signing_key(private_key) - self.assertIsInstance(private_key_str, six.string_types) + self.assertIsInstance(private_key_str, native_bytes) private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) @@ -299,7 +301,7 @@ class TestEd25519(unittest.TestCase): # ditto, but for the verifying keys public_key_str = ed25519.string_from_verifying_key(public_key) - self.assertIsInstance(public_key_str, six.string_types) + self.assertIsInstance(public_key_str, native_bytes) public_key2 = ed25519.verifying_key_from_string(public_key_str) self.assertEqual( @@ -403,7 +405,7 @@ class TestRsa(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair(2048) priv_key_str = rsa.der_string_from_signing_key(priv_key) - self.assertIsInstance(priv_key_str, six.string_types) + self.assertIsInstance(priv_key_str, native_bytes) priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) From 5815547661247ec021103a9a559c3de1fbc78728 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:58:37 -0400 Subject: [PATCH 0651/1054] Remove an unnecessary conversion that breaks Python 3. --- src/allmydata/test/test_crypto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 1cf55ca0f..1e9402370 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -39,17 +39,18 @@ class TestRegression(unittest.TestCase): # priv = rsa.generate(2048) # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) - RSA_2048_PRIV_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_2048_PRIV_KEY, native_bytes) with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: # Signature created using `RSA_2048_PRIV_KEY` via: # # sig = priv.sign(b'test') - RSA_2048_SIG = six.b(b64decode(f.read().strip())) + RSA_2048_SIG = b64decode(f.read().strip()) with RESOURCE_DIR.child('pycryptopp-rsa-2048-pub.txt').open('r') as f: # The public key corresponding to `RSA_2048_PRIV_KEY`. - RSA_2048_PUB_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PUB_KEY = b64decode(f.read().strip()) def test_old_start_up_test(self): """ From cd50ac434a556c3784d726051bb460b400f58fde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:59:51 -0400 Subject: [PATCH 0652/1054] Port to Python 3. --- src/allmydata/test/test_crypto.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 1e9402370..7f6271ef3 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,3 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from future.utils import native_bytes import six diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..f96a6f1e4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -42,6 +42,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", "allmydata.test.test_hashtree", From d71951ecf1313926efc291664c182af5d421343b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:00:09 -0400 Subject: [PATCH 0653/1054] News fragment. --- newsfragments/3364.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3364.minor diff --git a/newsfragments/3364.minor b/newsfragments/3364.minor new file mode 100644 index 000000000..e69de29bb From 24729dbd610d4a7a15855c4934c627e4ead97b59 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:06:46 -0400 Subject: [PATCH 0654/1054] Port to Python 3. --- src/allmydata/crypto/__init__.py | 11 +++++++++++ src/allmydata/crypto/aes.py | 10 ++++++++++ src/allmydata/crypto/ed25519.py | 11 +++++++++++ src/allmydata/crypto/error.py | 10 ++++++++++ src/allmydata/crypto/rsa.py | 11 ++++++++++- src/allmydata/crypto/util.py | 10 ++++++++++ src/allmydata/util/_python3.py | 6 ++++++ 7 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index ee92f223a..4eb753b57 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -5,4 +5,15 @@ For the most part, these functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use these functions and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 4194c63df..196d751bd 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -6,7 +6,17 @@ These functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use functions from allmydata.crypto.aes and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 37e305d19..424dc23fa 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -13,7 +13,18 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form, but the minor usability improvement is not worth the documentation and specification confusion of using a non-standard encoding. So we stick with base32. + +Ported to Python 3. ''' +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index 62c0b3e5b..b105805a5 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -1,6 +1,16 @@ """ Exceptions raise by allmydata.crypto.* modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 class BadSignature(Exception): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index e82bf12d1..95c39bcc6 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -9,8 +9,17 @@ features of any objects that `cryptography` documents. That is, the public and private keys are opaque objects; DO NOT depend on any of their methods. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 6aa1f0973..6c6ed1059 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -1,6 +1,16 @@ """ Utilities used by allmydata.crypto modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f96a6f1e4..c29184b6c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,12 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.crypto", + "allmydata.crypto.aes", + "allmydata.crypto.ed25519", + "allmydata.crypto.error", + "allmydata.crypto.rsa", + "allmydata.crypto.util", "allmydata.hashtree", "allmydata.util.abbreviate", "allmydata.util.assertutil", From c0cabeaca0277ccf36e5fea902ebf5a95f5599bf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:08:16 -0400 Subject: [PATCH 0655/1054] Update ratchet. --- misc/python3/ratchet-passing | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..d214396ff 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -24,6 +24,33 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes +allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes +allmydata.test.test_crypto.TestEd25519.test_key_serialization +allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair +allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair +allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes +allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size +allmydata.test.test_crypto.TestRegression.test_iv_not_bytes +allmydata.test.test_crypto.TestRegression.test_key_incorrect_size +allmydata.test.test_crypto.TestRegression.test_old_start_up_test +allmydata.test.test_crypto.TestRsa.test_keys +allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad +allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string +allmydata.test.test_crypto.TestUtil.test_remove_prefix_good +allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial +allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success From bfbca87f265d00241cd9892ac5fb5de19c3c4ce7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:31:02 -0400 Subject: [PATCH 0656/1054] Missing from previous PR. --- src/allmydata/hashtree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 77798b3c2..c692e3597 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -45,6 +45,8 @@ Written by Connelly Barnes in 2005 and released into the public domain with no warranty of any kind, either expressed or implied. It probably won't make your computer catch on fire, or eat your children, but it might. Use at your own risk. + +Ported to Python 3. """ from __future__ import absolute_import From 1703230a2a5bc48278581e558e9ca9091d7d5647 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:33:21 -0400 Subject: [PATCH 0657/1054] Use nummedobj from pyutil. --- src/allmydata/util/log.py | 2 +- src/allmydata/util/nummedobj.py | 42 --------------------------------- 2 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 src/allmydata/util/nummedobj.py diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 454002000..8ebf1262b 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -1,4 +1,4 @@ -from allmydata.util import nummedobj +from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log diff --git a/src/allmydata/util/nummedobj.py b/src/allmydata/util/nummedobj.py deleted file mode 100644 index 50d7c6454..000000000 --- a/src/allmydata/util/nummedobj.py +++ /dev/null @@ -1,42 +0,0 @@ -import collections, itertools, functools - -objnums = collections.defaultdict(itertools.count) - - -@functools.total_ordering -class NummedObj(object): - """ - This is useful for nicer debug printouts. Instead of objects of the same class being - distinguished from one another by their memory address, they each get a unique number, which - can be read as "the first object of this class", "the second object of this class", etc. This - is especially useful because separate runs of a program will yield identical debug output, - (assuming that the objects get created in the same order in each run). This makes it possible - to diff outputs from separate runs to see what changed, without having to ignore a difference - on every line due to different memory addresses of objects. - """ - - def __init__(self, klass=None): - """ - @param klass: in which class are you counted? If default value of `None', then self.__class__ will be used. - """ - if klass is None: - klass = self.__class__ - self._classname = klass.__name__ - - self._objid = objnums[self._classname].next() - - def __repr__(self): - return "<%s #%d>" % (self._classname, self._objid,) - - def __lt__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) < (other._objid, other._classname,) - return NotImplemented - - def __eq__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) == (other._objid, other._classname,) - return NotImplemented - - def __hash__(self): - return id(self) From 02daa12031105f36649828b5e024bbb27ed5d295 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:33:29 -0400 Subject: [PATCH 0658/1054] Move logging tests into their own, Python 3-compatible, module. --- src/allmydata/test/test_util.py | 16 ---------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1e9dd0f49..bd0e16df1 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -6,13 +6,11 @@ import yaml from twisted.trial import unittest from twisted.internet import defer, reactor -from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin from allmydata.util import yamlutil -from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -528,20 +526,6 @@ class EqButNotIs(object): return self.x == other -class SampleError(Exception): - pass - -class Log(unittest.TestCase): - def test_err(self): - try: - raise SampleError("simple sample") - except: - f = Failure() - tahoe_log.err(format="intentional sample error", - failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") - self.flushLoggedErrors(SampleError) - - class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..046468eeb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -48,6 +48,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", "allmydata.test.test_iputil", + "allmydata.test.test_log", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", From 622ed2f971a94f54de19f2cdb1ee2459723155a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 14:09:28 -0400 Subject: [PATCH 0659/1054] Standalone logging tests, most of them new, and an attempt to fix the parent msgid logic so it's a little less broken. --- src/allmydata/test/test_log.py | 113 +++++++++++++++++++++++++++++++++ src/allmydata/util/log.py | 4 +- 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 src/allmydata/test/test_log.py diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py new file mode 100644 index 000000000..00ba03013 --- /dev/null +++ b/src/allmydata/test/test_log.py @@ -0,0 +1,113 @@ +""" +Tests for allmydata.util.log. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest +from twisted.python.failure import Failure + +from foolscap.logging import log + +from allmydata.util import log as tahoe_log + + +class SampleError(Exception): + pass + + +class LoggingObject0(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject2(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject3(tahoe_log.PrefixingLogMixin): + pass + + +class Log(unittest.TestCase): + def setUp(self): + self.messages = [] + + def msg(msg, facility, parent, *args, **kwargs): + self.messages.append((msg, facility, parent, args, kwargs)) + return "msg{}".format(len(self.messages)) + + self.patch(log, "msg", msg) + + def test_err(self): + """Logging with log.err() causes tests to fail.""" + try: + raise SampleError("simple sample") + except: + f = Failure() + tahoe_log.err(format="intentional sample error", + failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") + self.flushLoggedErrors(SampleError) + + def test_default_facility(self): + obj = LoggingObject1(facility="defaultfac") + obj.log("hello") + obj.log("world", facility="override") + self.assertEqual(self.messages[-2][1], "defaultfac") + self.assertEqual(self.messages[-1][1], "override") + + def test_with_prefix(self): + obj = LoggingObject0("fac", prefix="pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + + def test_no_prefix(self): + obj = LoggingObject2() + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_numming(self): + obj = LoggingObject3() + obj2 = LoggingObject3() + obj.log("hello") + obj2.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_parent_id(self): + obj = LoggingObject1() + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + [None, "par1", "par2", "msg1", "msg1"]) + + def test_grandparent_id(self): + obj = LoggingObject1(grandparentmsgid="grand") + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + ["grand", "par1", "par2", "msg1", "msg1"]) diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 8ebf1262b..c1966aadb 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -36,8 +36,8 @@ class LogMixin(object): def log(self, msg, facility=None, parent=None, *args, **kwargs): if facility is None: facility = self._facility - pmsgid = None - if parent is None: + pmsgid = parent + if pmsgid is None: pmsgid = self._parentmsgid if pmsgid is None: pmsgid = self._grandparentmsgid From 2a623e0b05e0162d9332eea40429b4115c5984af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 14:13:16 -0400 Subject: [PATCH 0660/1054] Port to Python 3. --- misc/python3/ratchet-passing | 7 +++++++ newsfragments/3365.minor | 0 src/allmydata/util/_python3.py | 1 + src/allmydata/util/log.py | 14 ++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 newsfragments/3365.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..2edba65c0 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -54,6 +54,13 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route allmydata.test.test_iputil.ListenOnUsed.test_random_port allmydata.test.test_iputil.ListenOnUsed.test_specific_port +allmydata.test.test_log.Log.test_default_facility +allmydata.test.test_log.Log.test_err +allmydata.test.test_log.Log.test_grandparent_id +allmydata.test.test_log.Log.test_no_prefix +allmydata.test.test_log.Log.test_numming +allmydata.test.test_log.Log.test_parent_id +allmydata.test.test_log.Log.test_with_prefix allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested diff --git a/newsfragments/3365.minor b/newsfragments/3365.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 046468eeb..a1c727697 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -25,6 +25,7 @@ PORTED_MODULES = [ "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", + "allmydata.util.log", "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.netstring", diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index c1966aadb..cd20c764d 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -1,3 +1,17 @@ +""" +Logging utilities. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from pyutil import nummedobj from foolscap.logging import log From a2f78ca7034c8cc231c7cc9f1c39c1fa6859233b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:26:49 -0400 Subject: [PATCH 0661/1054] don't have a default --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd423e2af..04b208e34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,7 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION:-python2} + --build-arg PYTHON_VERSION=${PYTHON_VERSION} -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ From 95050fefd6f5e66ef17c98ef98f7d02b1848196c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:26:56 -0400 Subject: [PATCH 0662/1054] do have line continuation --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04b208e34..dcaf6cf14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,7 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION} + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ From 704f751b06204ff7794771cb71c0b62950a7a13d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:45:10 -0400 Subject: [PATCH 0663/1054] untabify --- .circleci/Dockerfile.debian | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 394495e17..87d5487d6 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -9,16 +9,16 @@ ENV BUILD_SRC_ROOT /tmp/project RUN apt-get --quiet update && \ apt-get --quiet --yes install \ - git \ - lsb-release \ + git \ + lsb-release \ sudo \ - build-essential \ - python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-dev \ - libffi-dev \ - libssl-dev \ - libyaml-dev \ - virtualenv + build-essential \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + virtualenv # Get the project source. This is better than it seems. CircleCI will # *update* this checkout on each job run, saving us more time per-job. From ee32b3555ac8ee7880709e26e65dfa47a1737cb4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 20:00:13 -0400 Subject: [PATCH 0664/1054] Burn the Python version into the Docker image tag --- .circleci/config.yml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dcaf6cf14..77b661ab9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ jobs: debian-9: &DEBIAN docker: - - image: "tahoelafsci/debian:9" + - image: "tahoelafsci/debian:9-py2" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -198,14 +198,14 @@ jobs: debian-8: <<: *DEBIAN docker: - - image: "tahoelafsci/debian:8" + - image: "tahoelafsci/debian:8-py2" user: "nobody" pypy2.7-buster: <<: *DEBIAN docker: - - image: "tahoelafsci/pypy:2.7-buster" + - image: "tahoelafsci/pypy:buster-py2" user: "nobody" environment: @@ -261,19 +261,23 @@ jobs: ubuntu-16.04: <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:16.04" + - image: "tahoelafsci/ubuntu:16.04-py2" user: "nobody" ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:18.04" + - image: "tahoelafsci/ubuntu:18.04-py2" user: "nobody" python3.6: <<: *UBUNTU_18_04 + docker: + - image: "tahoelafsci/ubuntu:18.04-py3" + user: "nobody" + environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" @@ -281,7 +285,7 @@ jobs: centos-8: &RHEL_DERIV docker: - - image: "tahoelafsci/centos:8" + - image: "tahoelafsci/centos:8-py2" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -303,20 +307,20 @@ jobs: fedora-28: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:28" + - image: "tahoelafsci/fedora:28-py2" user: "nobody" fedora-29: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:29" + - image: "tahoelafsci/fedora:29-py2" user: "nobody" slackware-14.2: docker: - - image: "tahoelafsci/slackware:14.2" + - image: "tahoelafsci/slackware:14.2-py2" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -397,8 +401,9 @@ jobs: - image: "docker:17.05.0-ce-git" environment: - DISTRO: "tahoelafsci/:foo" - TAG: "tahoelafsci/distro:" + DISTRO: "tahoelafsci/:foo-py2" + TAG: "tahoelafsci/distro:-py2" + PYTHON_VERSION: "tahoelafsci/distro:tag-py Date: Fri, 17 Jul 2020 20:36:17 -0400 Subject: [PATCH 0665/1054] FROM creates a new build stage which discards all prior ARGs --- .circleci/Dockerfile.centos | 2 +- .circleci/Dockerfile.debian | 2 +- .circleci/Dockerfile.fedora | 2 +- .circleci/Dockerfile.slackware | 2 +- .circleci/Dockerfile.ubuntu | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index 632e13137..9070d71d9 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM centos:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 87d5487d6..96c54736c 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM debian:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index f17140f3e..e60dbb85d 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM fedora:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware index 97ee07609..4b6d96f95 100644 --- a/.circleci/Dockerfile.slackware +++ b/.circleci/Dockerfile.slackware @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM vbatts/slackware:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 4ca2319a9..2fcc60f5a 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM ubuntu:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv From b99ec195cdb86d51a8d1fbb532d289715c735269 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:29:08 -0400 Subject: [PATCH 0666/1054] See if this helps the py36 zfec installation --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77b661ab9..6dbdf8ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,8 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + # zfec 1.5.3 is uninstallable without a utf-8 locale + LC_ALL: "en_US.utf-8" centos-8: &RHEL_DERIV From ec7531f1138d3ccedb07027afb9592f510d73b2c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:40:04 -0400 Subject: [PATCH 0667/1054] maybe this will let it get to the right place --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab467abd0..205fe1531 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME +passenv = HOME LC_ALL commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 8348e7032b24b01dda7270f73613a792f0a15fcc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:53:27 -0400 Subject: [PATCH 0668/1054] this is basically already set (LANG in UTF_8_ENVIRONMENT) --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6dbdf8ff4..77b661ab9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,8 +281,6 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" - # zfec 1.5.3 is uninstallable without a utf-8 locale - LC_ALL: "en_US.utf-8" centos-8: &RHEL_DERIV From a513c7ab28573b73bc65b56a573b9588b2d99ffc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:53:40 -0400 Subject: [PATCH 0669/1054] try setting it in here --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 205fe1531..9a11df98c 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,9 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME LC_ALL +passenv = HOME +setenv = + LANG=en_US.UTF-8 commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 612afdcf7c572a509227f98bb276fcd1f9b07404 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:58:25 -0400 Subject: [PATCH 0670/1054] that is useless, the error is in the docker image build --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9a11df98c..ab467abd0 100644 --- a/tox.ini +++ b/tox.ini @@ -51,8 +51,6 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. passenv = HOME -setenv = - LANG=en_US.UTF-8 commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From ab9ccae1d19ca76bbd9f109f11533546693581c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:58:36 -0400 Subject: [PATCH 0671/1054] try doing the `pip wheel` with the right LANG --- .circleci/populate-wheelhouse.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 75afb6f6f..519a80cac 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -36,8 +36,9 @@ PIP="${BOOTSTRAP_VENV}/bin/pip" # Tell pip where it can find any existing wheels. export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" -# Populate the wheelhouse, if necessary. -"${PIP}" \ +# Populate the wheelhouse, if necessary. zfec 1.5.3 can only be built with a +# UTF-8 environment so make sure we have one, at least for this invocation. +LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[test] \ From 1b0f83599d196c84da2f4ad9961ee767e2ae4ff8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 17:08:45 -0400 Subject: [PATCH 0672/1054] See if we can get the image names right --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77b661ab9..203993f4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ jobs: debian-9: &DEBIAN docker: - - image: "tahoelafsci/debian:9-py2" + - image: "tahoelafsci/debian:9-py2.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -198,7 +198,7 @@ jobs: debian-8: <<: *DEBIAN docker: - - image: "tahoelafsci/debian:8-py2" + - image: "tahoelafsci/debian:8-py2.7" user: "nobody" @@ -261,14 +261,14 @@ jobs: ubuntu-16.04: <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:16.04-py2" + - image: "tahoelafsci/ubuntu:16.04-py2.7" user: "nobody" ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:18.04-py2" + - image: "tahoelafsci/ubuntu:18.04-py2.7" user: "nobody" @@ -307,7 +307,7 @@ jobs: fedora-28: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:28-py2" + - image: "tahoelafsci/fedora:28-py" user: "nobody" From a47bd93fd045551b7a716eb284987862b917c5ff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 17:12:15 -0400 Subject: [PATCH 0673/1054] news fragment --- newsfragments/3336.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3336.minor diff --git a/newsfragments/3336.minor b/newsfragments/3336.minor new file mode 100644 index 000000000..e69de29bb From f0ddda966533f10ca341a53e0efc0435dcac4edb Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 31 Jul 2020 08:14:39 -0400 Subject: [PATCH 0674/1054] Add a codecheck for tabs in indentation --- newsfragments/3326.minor | 0 setup.cfg | 8 +++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3326.minor diff --git a/newsfragments/3326.minor b/newsfragments/3326.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index c2d224225..f4539279e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,8 @@ bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel [flake8] -# For now, only use pyflakes errors; flake8 is still helpful because it allows -# ignoring specific errors/warnings when needed. -select = F \ No newline at end of file +# Enforce all pyflakes constraints, and also prohibit tabs for indentation. +# Reference: +# https://flake8.pycqa.org/en/latest/user/error-codes.html +# https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +select = F, W191 From 2a6cd8eb58f273c721725bb8eeb265fb6425a472 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 31 Jul 2020 07:41:33 -0400 Subject: [PATCH 0675/1054] Get `tox -e codechecks` to work (on macOS?) Without this, git fails underneath towncrier with an "error: Could not expand include path '~/.gitcinclude'". See: https://stackoverflow.com/q/36908041 I had added similar for `tox -e py36` in d25c8b1a. --- newsfragments/3366.minor | 0 tox.ini | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3366.minor diff --git a/newsfragments/3366.minor b/newsfragments/3366.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index ab467abd0..4356cfd24 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ commands = tahoe --version [testenv:py36] -# git inside of ratchet.sh needs $HOME. +# On macOS, git inside of ratchet.sh needs $HOME. passenv = HOME commands = {toxinidir}/misc/python3/ratchet.sh @@ -77,6 +77,8 @@ commands = coverage xml [testenv:codechecks] +# On macOS, git inside of towncrier needs $HOME. +passenv = HOME whitelist_externals = /bin/mv commands = From d53f67bc4111d73db3788035cae2ef766ceb7d24 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Tue, 4 Aug 2020 11:15:32 +0200 Subject: [PATCH 0676/1054] add assert for write_atomically --- src/allmydata/util/fileutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index d57464d6f..5c60b167a 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -263,6 +263,9 @@ def move_into_place(source, dest): os.rename(source, dest) def write_atomically(target, contents, mode="b"): + assert ( + isinstance(contents, bytes) and ("b" in mode or mode == "") or + isinstance(contents, unicode) and "t" in mode), (type(contents), mode) with open(target+".tmp", "w"+mode) as f: f.write(contents) move_into_place(target+".tmp", target) From 5fcc8c65dbdda99f9dae6525c6a540b5e836d3cb Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Tue, 4 Aug 2020 11:15:44 +0200 Subject: [PATCH 0677/1054] add passing ratchet tests --- misc/python3/ratchet-passing | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..3fc9d3fad 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -9,6 +9,9 @@ allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year allmydata.test.test_abbreviate.Abbreviate.test_parse_space allmydata.test.test_abbreviate.Abbreviate.test_space allmydata.test.test_abbreviate.Abbreviate.test_time +allmydata.test.test_backupdb.BackupDB.test_basic +allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2 +allmydata.test.test_backupdb.BackupDB.test_wrong_version allmydata.test.test_base32.Base32.test_a2b allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons allmydata.test.test_base32.Base32.test_b2a @@ -93,6 +96,26 @@ allmydata.test.test_time_format.TimeFormat.test_format_time_y2038 allmydata.test.test_time_format.TimeFormat.test_iso_utc allmydata.test.test_time_format.TimeFormat.test_parse_date allmydata.test.test_time_format.TimeFormat.test_parse_duration +allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode +allmydata.test.test_util.FileUtil.test_create_long_path +allmydata.test.test_util.FileUtil.test_disk_stats +allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative +allmydata.test.test_util.FileUtil.test_du +allmydata.test.test_util.FileUtil.test_encrypted_tempfile +allmydata.test.test_util.FileUtil.test_get_pathinfo +allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink +allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode +allmydata.test.test_util.FileUtil.test_remove_if_possible +allmydata.test.test_util.FileUtil.test_rename +allmydata.test.test_util.FileUtil.test_rename_no_overwrite +allmydata.test.test_util.FileUtil.test_replace_file +allmydata.test.test_util.FileUtil.test_rm_dir +allmydata.test.test_util.FileUtil.test_windows_expanduser_win7 +allmydata.test.test_util.FileUtil.test_windows_expanduser_xp +allmydata.test.test_util.FileUtil.test_write_atomically +allmydata.test.test_util.IDLib.test_nodeid_b2a +allmydata.test.test_util.Limiter.test_errors +allmydata.test.test_util.Limiter.test_limiter allmydata.test.test_version.CheckRequirement.test_cross_check allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions allmydata.test.test_version.CheckRequirement.test_extract_openssl_version From d8bcbb901e039c97dd37587979ba9a29b926caf0 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Tue, 4 Aug 2020 12:26:43 +0200 Subject: [PATCH 0678/1054] I think default mode is text --- src/allmydata/test/test_util.py | 2 +- src/allmydata/util/fileutil.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1a4d044c3..afcdda60c 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -94,7 +94,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): basedir = "util/FileUtil/test_write_atomically" fileutil.make_dirs(basedir) fn = os.path.join(basedir, "here") - fileutil.write_atomically(fn, b"one") + fileutil.write_atomically(fn, b"one", "b") self.failUnlessEqual(fileutil.read(fn), b"one") fileutil.write_atomically(fn, u"two", mode="") # non-binary self.failUnlessEqual(fileutil.read(fn), b"two") diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 5c60b167a..288875e0a 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -264,8 +264,8 @@ def move_into_place(source, dest): def write_atomically(target, contents, mode="b"): assert ( - isinstance(contents, bytes) and ("b" in mode or mode == "") or - isinstance(contents, unicode) and "t" in mode), (type(contents), mode) + isinstance(contents, bytes) and "b" in mode or + isinstance(contents, unicode) and "t" in mode or mode == ""), (type(contents), mode) with open(target+".tmp", "w"+mode) as f: f.write(contents) move_into_place(target+".tmp", target) From aeb681a854ec0f3ef450dd2783b477adb7012a06 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Tue, 4 Aug 2020 15:49:55 +0200 Subject: [PATCH 0679/1054] hopefully fix the ratchet files --- misc/python3/ratchet-passing | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 3fc9d3fad..62cd5d0cc 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -116,6 +116,12 @@ allmydata.test.test_util.FileUtil.test_write_atomically allmydata.test.test_util.IDLib.test_nodeid_b2a allmydata.test.test_util.Limiter.test_errors allmydata.test.test_util.Limiter.test_limiter +allmydata.test.test_util.Log.test_err +allmydata.test.test_util.Math.test_round_sigfigs +allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True +allmydata.test.test_util.PollMixinTests.test_PollMixin_True +allmydata.test.test_util.PollMixinTests.test_timeout +allmydata.test.test_util.YAML.test_convert allmydata.test.test_version.CheckRequirement.test_cross_check allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions allmydata.test.test_version.CheckRequirement.test_extract_openssl_version From ce33101d080c21e62653b12385480ccc894b2177 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Aug 2020 15:03:06 -0400 Subject: [PATCH 0680/1054] Remove unused import. --- src/allmydata/test/test_crypto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 7f6271ef3..498723cf4 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -9,7 +9,6 @@ if PY2: from future.utils import native_bytes -import six import unittest from base64 import b64decode From ff7cf4d731242744c2b971f485b48e50a9cce02f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Aug 2020 15:15:32 -0400 Subject: [PATCH 0681/1054] Test improvements. --- src/allmydata/test/test_log.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 00ba03013..1d24d7f7b 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -59,9 +59,14 @@ class Log(unittest.TestCase): f = Failure() tahoe_log.err(format="intentional sample error", failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") - self.flushLoggedErrors(SampleError) + result = self.flushLoggedErrors(SampleError) + self.assertEqual(len(result), 1) def test_default_facility(self): + """ + If facility is passed to PrefixingLogMixin.__init__, it is used as + default facility. + """ obj = LoggingObject1(facility="defaultfac") obj.log("hello") obj.log("world", facility="override") @@ -69,6 +74,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][1], "override") def test_with_prefix(self): + """ + If prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ obj = LoggingObject0("fac", prefix="pre1") obj.log("hello") obj.log("world") @@ -76,6 +85,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], '(pre1): world') def test_no_prefix(self): + """ + If no prefix is passed to PrefixingLogMixin.__init__, it is not used in + message rendering. + """ obj = LoggingObject2() obj.log("hello") obj.log("world") @@ -83,6 +96,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], ': world') def test_numming(self): + """ + Objects inheriting from PrefixingLogMixin get a unique number from a + class-specific counter. + """ obj = LoggingObject3() obj2 = LoggingObject3() obj.log("hello") @@ -91,6 +108,12 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], ': world') def test_parent_id(self): + """ + The parent message id can be passed in, otherwise the first message's + id is used as the parent. + + This logic is pretty bogus, but that's what the code does. + """ obj = LoggingObject1() result = obj.log("zero") self.assertEqual(result, "msg1") @@ -102,6 +125,10 @@ class Log(unittest.TestCase): [None, "par1", "par2", "msg1", "msg1"]) def test_grandparent_id(self): + """ + If grandparent message id is given, it's used as parent id of the first + message. + """ obj = LoggingObject1(grandparentmsgid="grand") result = obj.log("zero") self.assertEqual(result, "msg1") From 0bef1eb4e2c6f76c82df67cc6d990e9f8f285b0d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:25:40 -0400 Subject: [PATCH 0682/1054] Support prefix that is bytes. --- src/allmydata/test/test_log.py | 11 +++++++++++ src/allmydata/util/log.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 1d24d7f7b..2c26e8ff4 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -84,6 +84,17 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-2][0], '(pre1): hello') self.assertEqual(self.messages[-1][0], '(pre1): world') + def test_with_bytes_prefix(self): + """ + If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ + obj = LoggingObject0("fac", prefix=b"pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + def test_no_prefix(self): """ If no prefix is passed to PrefixingLogMixin.__init__, it is not used in diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index cd20c764d..dcb5838a3 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -68,6 +68,8 @@ class PrefixingLogMixin(nummedobj.NummedObj, LogMixin): LogMixin.__init__(self, facility, grandparentmsgid) if prefix: + if isinstance(prefix, bytes): + prefix = prefix.decode("utf-8", errors="replace") self._prefix = "%s(%s): " % (self.__repr__(), prefix) else: self._prefix = "%s: " % (self.__repr__(),) From 70851fe75384ab3d7015480329ab6995372b7896 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:53:23 -0400 Subject: [PATCH 0683/1054] Death to newint. --- src/allmydata/crypto/__init__.py | 2 +- src/allmydata/crypto/aes.py | 2 +- src/allmydata/crypto/ed25519.py | 2 +- src/allmydata/crypto/error.py | 2 +- src/allmydata/crypto/rsa.py | 2 +- src/allmydata/crypto/util.py | 2 +- src/allmydata/hashtree.py | 2 +- src/allmydata/test/common_py3.py | 2 +- src/allmydata/test/test_abbreviate.py | 2 +- src/allmydata/test/test_base32.py | 2 +- src/allmydata/test/test_base62.py | 2 +- src/allmydata/test/test_crypto.py | 2 +- src/allmydata/test/test_deferredutil.py | 2 +- src/allmydata/test/test_dictutil.py | 2 +- src/allmydata/test/test_hashtree.py | 2 +- src/allmydata/test/test_hashutil.py | 2 +- src/allmydata/test/test_humanreadable.py | 2 +- src/allmydata/test/test_iputil.py | 2 +- src/allmydata/test/test_netstring.py | 2 +- src/allmydata/test/test_observer.py | 2 +- src/allmydata/test/test_pipeline.py | 2 +- src/allmydata/test/test_python3.py | 2 +- src/allmydata/test/test_spans.py | 2 +- src/allmydata/test/test_statistics.py | 2 +- src/allmydata/test/test_time_format.py | 2 +- src/allmydata/test/test_version.py | 2 +- src/allmydata/util/_python3.py | 2 +- src/allmydata/util/abbreviate.py | 2 +- src/allmydata/util/assertutil.py | 2 +- src/allmydata/util/base32.py | 2 +- src/allmydata/util/base62.py | 2 +- src/allmydata/util/deferredutil.py | 2 +- src/allmydata/util/gcutil.py | 2 +- src/allmydata/util/hashutil.py | 2 +- src/allmydata/util/humanreadable.py | 2 +- src/allmydata/util/iputil.py | 2 +- src/allmydata/util/mathutil.py | 2 +- src/allmydata/util/netstring.py | 2 +- src/allmydata/util/observer.py | 2 +- src/allmydata/util/pipeline.py | 2 +- src/allmydata/util/pollmixin.py | 2 +- src/allmydata/util/spans.py | 2 +- src/allmydata/util/statistics.py | 2 +- src/allmydata/util/time_format.py | 2 +- src/allmydata/util/verlib.py | 2 +- src/allmydata/version_checks.py | 2 +- 46 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index 4eb753b57..04b8f0cc3 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -16,4 +16,4 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 196d751bd..ad7cfcba4 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -16,7 +16,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 424dc23fa..098fa9758 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -24,7 +24,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index b105805a5..153e48d33 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class BadSignature(Exception): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95c39bcc6..b5d15ad4a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -19,7 +19,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 6c6ed1059..d377b6396 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 77798b3c2..4f8f9d6d7 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -54,7 +54,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.util import mathutil # from the pyutil library diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 97745e293..0dae05aa6 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 958c36742..3ef1e96a6 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import timedelta diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 47636a175..0b9a018b9 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import base64 diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index e26532e9d..8bbb6dfeb 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 498723cf4..0aefa757f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_bytes diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index fcff77b54..d8f386e5f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index 0868db124..9b7124114 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index b96f4abfb..d1d4cb252 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index cb0967b83..abcd4f0fb 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index ec6d3f666..94de8f6be 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index d3262a9ea..71374fec7 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os, socket import gc diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index b00c99bca..d5ff379cd 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index b37f0d3e1..0db13db58 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py index ab7059521..1295be363 100644 --- a/src/allmydata/test/test_pipeline.py +++ b/src/allmydata/test/test_pipeline.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import gc diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 1326c8282..7a6d0b282 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -10,7 +10,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.python.modules import ( getModule, diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index f62d6e684..02d8292f3 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -9,7 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 8d5837fc5..476f0a084 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import StringIO # native string StringIO diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index fa68ccc36..dc9c03b91 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -8,7 +8,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 1b447e346..7301399d9 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import pkg_resources diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 87fe265d6..b4b6b0695 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index e7bdd8410..f895c3727 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re from datetime import timedelta diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index af8817702..ed4b8599f 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -14,7 +14,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index ba5f80d67..287d214ea 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: def backwardscompat_bytes(b): diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 3d80671ba..964baff34 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: import string diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index e36892f96..9ce05ef3a 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index ade95341f..33f1f64f5 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "fileDescriptorResource", diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index d26f5a9b0..96d52c862 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 8c7079c3a..60ac57083 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from reprlib import Repr diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 8754fca53..bd5ea7e78 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, re, socket, subprocess, errno from sys import platform diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index be88f60b4..42863c30e 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -13,7 +13,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 7f2bef377..14e515619 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index d5003dfb3..4ebb598c1 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import weakref from twisted.internet import defer diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py index df80e2c6c..31f5d5d49 100644 --- a/src/allmydata/util/pipeline.py +++ b/src/allmydata/util/pipeline.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.internet import defer from twisted.python.failure import Failure diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 4eb19b8e3..5d1716853 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from twisted.internet import task diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 814904d68..b224f0950 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class Spans(object): diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index a690eb235..3da5da70a 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -18,7 +18,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from functools import reduce diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index 5807791e4..ff267485e 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str import calendar, datetime, re, time diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py index f69e34e3d..2dfc24a1b 100644 --- a/src/allmydata/util/verlib.py +++ b/src/allmydata/util/verlib.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 51a49d78a..d022055ea 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "PackagingError", From b8df17283b5956dcd1fdd2a15c0d4468cd4060a6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:54:09 -0400 Subject: [PATCH 0684/1054] News file. Not bothering with explanation since the regression was only in unreleased versions. --- newsfragments/3368.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3368.minor diff --git a/newsfragments/3368.minor b/newsfragments/3368.minor new file mode 100644 index 000000000..e69de29bb From 57fdd9f6b4c92d2e32d8bb908e68b38bbaff0d09 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Thu, 6 Aug 2020 15:11:41 +0200 Subject: [PATCH 0685/1054] s/unicode/str/ (from future.builtins import str kind of str) --- src/allmydata/util/fileutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 288875e0a..734fe05cd 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -265,7 +265,7 @@ def move_into_place(source, dest): def write_atomically(target, contents, mode="b"): assert ( isinstance(contents, bytes) and "b" in mode or - isinstance(contents, unicode) and "t" in mode or mode == ""), (type(contents), mode) + isinstance(contents, str) and "t" in mode or mode == ""), (type(contents), mode) with open(target+".tmp", "w"+mode) as f: f.write(contents) move_into_place(target+".tmp", target) From b71ae461a6857a788131ca1b5b9496f48cc32282 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Thu, 6 Aug 2020 16:06:01 +0200 Subject: [PATCH 0686/1054] use less hacks in yaml tests --- src/allmydata/test/test_util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index afcdda60c..eae8d15bc 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -554,7 +554,6 @@ class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) back = yamlutil.safe_load(data) - str_type = type("x") # workaround for future.builtins.str - self.failUnlessEqual(type(back[0]), str_type) - self.failUnlessEqual(type(back[1]), str_type) - self.failUnlessEqual(type(back[2]), str_type) + self.assertIsInstance(back[0], str) + self.assertIsInstance(back[1], str) + self.assertIsInstance(back[2], str) From 9b9d53015596dc2944d98cf8f79ccb747b4fc06a Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Thu, 6 Aug 2020 16:07:50 +0200 Subject: [PATCH 0687/1054] Fix some typos. Add docstring --- src/allmydata/test/test_util.py | 6 +++++- src/allmydata/util/fileutil.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index eae8d15bc..aa1c716a6 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,3 +1,7 @@ +""" +Ported to Python3. +""" + from __future__ import print_function from __future__ import absolute_import from __future__ import division @@ -5,7 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six import os, time, sys import yaml diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 734fe05cd..693cd1d63 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -1,5 +1,5 @@ """ -Ported to Python3 +Ported to Python3. Futz with files like a pro. """ @@ -12,7 +12,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: # open is not here because we want to use native strings on Py2 - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401from builtins import range + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys, os, stat, tempfile, time, binascii import six From 9669e4eaa7b121ac6672e6753d127124563ddbb0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:12:26 -0400 Subject: [PATCH 0688/1054] Drop int. --- src/allmydata/util/dictutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d5f0da29b..2a80e7e33 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -13,7 +13,7 @@ if PY2: # IMPORTANT: We deliberately don't import dict. The issue is that we're # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of # code that doesn't support it. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 class DictOfSets(dict): From dab690a757f0dab595012438821f3ab61aa5c3d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:13:10 -0400 Subject: [PATCH 0689/1054] Flake fixes. --- src/allmydata/test/test_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 330abebe5..0a542f444 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -5,7 +5,6 @@ import os, time, sys import yaml from twisted.trial import unittest -from twisted.internet import defer, reactor from twisted.python.failure import Failure from allmydata.util import idlib, mathutil From 32945b85f6d84820843465f39da732ad5170fdde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:14:39 -0400 Subject: [PATCH 0690/1054] Drop int. --- src/allmydata/test/test_log.py | 2 +- src/allmydata/util/log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 2c26e8ff4..dea198506 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index dcb5838a3..11c78a5a2 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from pyutil import nummedobj From ff5cfe0c71822ff13d39e0567320339dd6205687 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:15:31 -0400 Subject: [PATCH 0691/1054] Update ratchet with new test. --- misc/python3/ratchet-passing | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 2edba65c0..56d82d4e7 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -60,6 +60,7 @@ allmydata.test.test_log.Log.test_grandparent_id allmydata.test.test_log.Log.test_no_prefix allmydata.test.test_log.Log.test_numming allmydata.test.test_log.Log.test_parent_id +allmydata.test.test_log.Log.test_with_bytes_prefix allmydata.test.test_log.Log.test_with_prefix allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra From c6d4ec0295446071c6e1dd8036e876d1ed317137 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:33:04 -0400 Subject: [PATCH 0692/1054] Port the test module to Python 3. --- src/allmydata/test/test_happiness.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index a8ed06363..ffba402ef 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -1,5 +1,15 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 87fe265d6..be0a382e8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -52,6 +52,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", + "allmydata.test.test_happiness", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", From d8f74770ebd1a6f27f2a69d24ba22c5a0f83f010 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:38:19 -0400 Subject: [PATCH 0693/1054] Port to Python 3. --- misc/python3/ratchet-passing | 15 +++++++++ newsfragments/3370.minor | 0 src/allmydata/immutable/happiness_upload.py | 35 +++++++++++++-------- src/allmydata/util/_python3.py | 1 + 4 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3370.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 951603419..48db826f2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -57,6 +57,21 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls allmydata.test.test_dictutil.DictUtil.test_auxdict allmydata.test.test_dictutil.DictUtil.test_dict_of_sets +allmydata.test.test_happiness.Happiness.test_100 +allmydata.test.test_happiness.Happiness.test_calc_happy +allmydata.test.test_happiness.Happiness.test_everything_broken +allmydata.test.test_happiness.Happiness.test_hypothesis0 +allmydata.test.test_happiness.Happiness.test_hypothesis_0 +allmydata.test.test_happiness.Happiness.test_hypothesis_1 +allmydata.test.test_happiness.Happiness.test_placement_1 +allmydata.test.test_happiness.Happiness.test_placement_simple +allmydata.test.test_happiness.Happiness.test_redistribute +allmydata.test.test_happiness.Happiness.test_unhappy +allmydata.test.test_happiness.HappinessUtils.test_residual_0 +allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph +allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy +allmydata.test.test_happiness.PlacementTests.test_more_hypothesis allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes diff --git a/newsfragments/3370.minor b/newsfragments/3370.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 75edb74d9..a177660cc 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,5 +1,14 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from Queue import PriorityQueue +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things for external Python 2 code. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + +from queue import PriorityQueue def augmenting_path_for(graph): @@ -35,9 +44,9 @@ def bfs(graph, s): GRAY = 1 # BLACK vertices are those we have seen and explored BLACK = 2 - color = [WHITE for i in xrange(len(graph))] - predecessor = [None for i in xrange(len(graph))] - distance = [-1 for i in xrange(len(graph))] + color = [WHITE for i in range(len(graph))] + predecessor = [None for i in range(len(graph))] + distance = [-1 for i in range(len(graph))] queue = [s] # vertices that we haven't explored yet. color[s] = GRAY distance[s] = 0 @@ -58,9 +67,9 @@ def residual_network(graph, f): flow network represented by my graph and f arguments. graph is a flow network in adjacency-list form, and f is a flow in graph. """ - new_graph = [[] for i in xrange(len(graph))] - cf = [[0 for s in xrange(len(graph))] for sh in xrange(len(graph))] - for i in xrange(len(graph)): + new_graph = [[] for i in range(len(graph))] + cf = [[0 for s in range(len(graph))] for sh in range(len(graph))] + for i in range(len(graph)): for v in graph[i]: if f[i][v] == 1: # We add an edge (v, i) with cf[v,i] = 1. This means @@ -135,7 +144,7 @@ def _compute_maximum_graph(graph, shareIndices): return {} dim = len(graph) - flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)] + flow_function = [[0 for sh in range(dim)] for s in range(dim)] residual_graph, residual_function = residual_network(graph, flow_function) while augmenting_path_for(residual_graph): @@ -260,9 +269,9 @@ def _servermap_flow_graph(peers, shares, servermap): #print "share_to_index %s" % share_to_index #print "servermap %s" % servermap for peer in peers: - if servermap.has_key(peer): + if peer in servermap: for s in servermap[peer]: - if share_to_index.has_key(s): + if s in share_to_index: indexedShares.append(share_to_index[s]) graph.insert(peer_to_index[peer], indexedShares) for share in shares: @@ -373,7 +382,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): new_mappings = _calculate_mappings(new_peers, new_shares) #print "new_peers %s" % new_peers #print "new_mappings %s" % new_mappings - mappings = dict(readonly_mappings.items() + existing_mappings.items() + new_mappings.items()) + mappings = dict(list(readonly_mappings.items()) + list(existing_mappings.items()) + list(new_mappings.items())) homeless_shares = set() for share in mappings: if mappings[share] is None: @@ -384,7 +393,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): mappings, homeless_shares, { k: v - for k, v in peers_to_shares.items() + for k, v in list(peers_to_shares.items()) if k not in readonly_peers } ) @@ -401,5 +410,5 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): return { k: v.pop() if v else next(peer_iter) - for k, v in mappings.items() + for k, v in list(mappings.items()) } diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index be0a382e8..d0b2d4a21 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.immutable._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 3e11779976872bfbad20169b99f8c1a7d65d1f9e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:18:00 -0400 Subject: [PATCH 0694/1054] Do some error checking --- src/allmydata/web/check_results.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 27ca2fed7..d996058f7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -463,30 +463,34 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin): @renderer def root_storage_index(self, req, tag): + if not self.monitor.get_status(): + return "" return self.monitor.get_status().get_root_storage_index_string() def _get_monitor_counter(self, name): - return self.monitor.get_status().get_counters().get(name) + if not self.monitor.get_status(): + return "" + return str(self.monitor.get_status().get_counters().get(name)) @renderer def objects_checked(self, req, tag): - return str(self._get_monitor_counter("count-objects-checked")) + return self._get_monitor_counter("count-objects-checked") @renderer def objects_healthy(self, req, tag): - return str(self._get_monitor_counter("count-objects-healthy")) + return self._get_monitor_counter("count-objects-healthy") @renderer def objects_unhealthy(self, req, tag): - return str(self._get_monitor_counter("count-objects-unhealthy")) + return self._get_monitor_counter("count-objects-unhealthy") @renderer def objects_unrecoverable(self, req, tag): - return str(self._get_monitor_counter("count-objects-unrecoverable")) + return self._get_monitor_counter("count-objects-unrecoverable") @renderer def count_corrupt_shares(self, req, tag): - return str(self._get_monitor_counter("count-corrupt-shares")) + return self._get_monitor_counter("count-corrupt-shares") @renderer def problems_p(self, req, tag): From 76777cd4e128bc444949213c67f10a7df2f5ded3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:22:26 -0400 Subject: [PATCH 0695/1054] Inherit "deep check and repair" page from "deep check" page --- src/allmydata/web/check_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d996058f7..913c4d7e6 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -676,12 +676,12 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): return json.dumps(data, indent=1) + "\n" -class DeepCheckAndRepairResultsRendererElement(Element, ResultsBase, ReloadMixin): +class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml")) def __init__(self, monitor): - super(DeepCheckAndRepairResultsRendererElement, self).__init__() + super(DeepCheckAndRepairResultsRendererElement, self).__init__(monitor) self.monitor = monitor @renderer From c0109b70b645a95ad98d0a5fecd563ba2c6bb777 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:25:34 -0400 Subject: [PATCH 0696/1054] Remove methods common to base class --- src/allmydata/web/check_results.py | 56 ++++++++---------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 913c4d7e6..85e80a9f4 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -684,60 +684,45 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): super(DeepCheckAndRepairResultsRendererElement, self).__init__(monitor) self.monitor = monitor - @renderer - def root_storage_index(self, req, tag): - if not self.monitor.get_status(): - return "" - return self.monitor.get_status().get_root_storage_index_string() - - def _get_counter(self, name): - if not self.monitor.get_status(): - return "" - return str(self.monitor.get_status().get_counters().get(name)) - - @renderer - def objects_checked(self, req, tag): - return self._get_counter("count-objects-checked") - @renderer def objects_healthy(self, req, tag): - return self._get_counter("count-objects-healthy-pre-repair") + return self._get_monitor_counter("count-objects-healthy-pre-repair") @renderer def objects_unhealthy(self, req, tag): - return self._get_counter("count-objects-unhealthy-pre-repair") + return self._get_monitor_counter("count-objects-unhealthy-pre-repair") @renderer def corrupt_shares(self, req, tag): - return self._get_counter("count-corrupt-shares-pre-repair") + return self._get_monitor_counter("count-corrupt-shares-pre-repair") @renderer def repairs_attempted(self, req, tag): - return self._get_counter("count-repairs-attempted") + return self._get_monitor_counter("count-repairs-attempted") @renderer def repairs_successful(self, req, tag): - return self._get_counter("count-repairs-successful") + return self._get_monitor_counter("count-repairs-successful") @renderer def repairs_unsuccessful(self, req, tag): - return self._get_counter("count-repairs-unsuccessful") + return self._get_monitor_counter("count-repairs-unsuccessful") @renderer def objects_healthy_post(self, req, tag): - return self._get_counter("count-objects-healthy-post-repair") + return self._get_monitor_counter("count-objects-healthy-post-repair") @renderer def objects_unhealthy_post(self, req, tag): - return self._get_counter("count-objects-unhealthy-post-repair") + return self._get_monitor_counter("count-objects-unhealthy-post-repair") @renderer def corrupt_shares_post(self, req, tag): - return self._get_counter("count-corrupt-shares-post-repair") + return self._get_monitor_counter("count-corrupt-shares-post-repair") @renderer def pre_repair_problems_p(self, req, tag): - if self._get_counter("count-objects-unhealthy-pre-repair"): + if self._get_monitor_counter("count-objects-unhealthy-pre-repair"): return tag return "" @@ -758,8 +743,8 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): @renderer def post_repair_problems_p(self, req, tag): - if (self._get_counter("count-objects-unhealthy-post-repair") - or self._get_counter("count-corrupt-shares-post-repair")): + if (self._get_monitor_counter("count-objects-unhealthy-post-repair") + or self._get_monitor_counter("count-corrupt-shares-post-repair")): return tag return "" @@ -780,7 +765,7 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): @renderer def servers_with_corrupt_shares_p(self, req, tag): - if self._get_counter("count-corrupt-shares-pre-repair"): + if self._get_monitor_counter("count-corrupt-shares-pre-repair"): return tag return "" @@ -793,7 +778,7 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): @renderer def remaining_corrupt_shares_p(self, req, tag): - if self._get_counter("count-corrupt-shares-post-repair"): + if self._get_monitor_counter("count-corrupt-shares-post-repair"): return tag return "" @@ -804,13 +789,6 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): corrupt = [{"share":"unimplemented"}] return SlotsSequenceElement(tag, corrupt) - @renderer - def return_to(self, req, tag): - return_to = get_arg(req, "return_to", None) - if return_to: - return tags.div(tags.a("Return to file/directory.", href=return_to)) - return "" - @renderer def all_objects(self, req, tag): results = {} @@ -833,9 +811,3 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): return SlotsSequenceElement(tag, objects) - @renderer - def runtime(self, req, tag): - runtime = 'unknown' - if hasattr(req, 'processing_started_timestamp'): - runtime = time.time() - req.processing_started_timestamp - return tag("runtime: %s seconds" % runtime) From 36352ee1827370ef4b81ec31831d11aec5f4a273 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:28:32 -0400 Subject: [PATCH 0697/1054] Reuse "servers with corrupt shares" from base class --- src/allmydata/web/check_results.py | 13 ------------- .../web/deep-check-and-repair-results.xhtml | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 85e80a9f4..d7a19705a 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -763,19 +763,6 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): return SlotsSequenceElement(tag, problems) - @renderer - def servers_with_corrupt_shares_p(self, req, tag): - if self._get_monitor_counter("count-corrupt-shares-pre-repair"): - return tag - return "" - - @renderer - def servers_with_corrupt_shares(self, req, tag): - # TODO: this was originally unimplemented before porting to - # twisted.web.template; leaving it as such. - corrupt = [{"share":"unimplemented"}] - return SlotsSequenceElement(tag, corrupt) - @renderer def remaining_corrupt_shares_p(self, req, tag): if self._get_monitor_counter("count-corrupt-shares-post-repair"): diff --git a/src/allmydata/web/deep-check-and-repair-results.xhtml b/src/allmydata/web/deep-check-and-repair-results.xhtml index c8cf2deea..be7f35b1d 100644 --- a/src/allmydata/web/deep-check-and-repair-results.xhtml +++ b/src/allmydata/web/deep-check-and-repair-results.xhtml @@ -57,7 +57,7 @@

                Servers on which corrupt shares were found

                • - +
                • None
                From c52bc6a7818388453589b7c9e8bc5ae4b09dc801 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:32:00 -0400 Subject: [PATCH 0698/1054] Document "deep check and repair" renderer element --- src/allmydata/web/check_results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index d7a19705a..f43f93c5f 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -677,6 +677,10 @@ class DeepCheckAndRepairResultsRenderer(MultiFormatResource): class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): + """ + The page generated here has several elements common to "deep check + results" page; hence the code reuse. + """ loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml")) From f4cfc0c613df51cd64da9e440c7f1d0d24423f0d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 6 Aug 2020 22:40:44 -0400 Subject: [PATCH 0699/1054] Assign a ticket to unimplemented functionality --- src/allmydata/web/check_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index f43f93c5f..500ac15a7 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -777,6 +777,8 @@ class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement): def post_repair_corrupt_shares(self, req, tag): # TODO: this was not implemented before porting to # twisted.web.template; leaving it as such. + # + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3371 corrupt = [{"share":"unimplemented"}] return SlotsSequenceElement(tag, corrupt) From 49c631b4c8d20bda1bb0f7c6a0d4017e14be3355 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 7 Aug 2020 08:43:17 -0400 Subject: [PATCH 0700/1054] Sort the thing we said we'd sort --- newsfragments/3372.minor | 0 src/allmydata/util/_python3.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3372.minor diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b4b6b0695..ea852cc54 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,8 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.test.common_py3", + "allmydata.util._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", @@ -38,11 +40,9 @@ PORTED_MODULES = [ "allmydata.util.observer", "allmydata.util.pipeline", "allmydata.util.pollmixin", - "allmydata.util._python3", "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", - "allmydata.test.common_py3", ] PORTED_TEST_MODULES = [ From fb9bf5511ff18a407fdfc357fca94ec15d58450f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:09:41 -0400 Subject: [PATCH 0701/1054] Passing tests. --- src/allmydata/test/test_log.py | 49 +++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index dea198506..eecbda9e3 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -25,22 +25,6 @@ class SampleError(Exception): pass -class LoggingObject0(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject1(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject2(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject3(tahoe_log.PrefixingLogMixin): - pass - - class Log(unittest.TestCase): def setUp(self): self.messages = [] @@ -67,6 +51,9 @@ class Log(unittest.TestCase): If facility is passed to PrefixingLogMixin.__init__, it is used as default facility. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1(facility="defaultfac") obj.log("hello") obj.log("world", facility="override") @@ -78,28 +65,37 @@ class Log(unittest.TestCase): If prefix is passed to PrefixingLogMixin.__init__, it is used in message rendering. """ - obj = LoggingObject0("fac", prefix="pre1") + class LoggingObject4(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject4("fac", prefix="pre1") obj.log("hello") obj.log("world") - self.assertEqual(self.messages[-2][0], '(pre1): hello') - self.assertEqual(self.messages[-1][0], '(pre1): world') + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') def test_with_bytes_prefix(self): """ If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in message rendering. """ - obj = LoggingObject0("fac", prefix=b"pre1") + class LoggingObject5(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject5("fac", prefix=b"pre1") obj.log("hello") obj.log("world") - self.assertEqual(self.messages[-2][0], '(pre1): hello') - self.assertEqual(self.messages[-1][0], '(pre1): world') + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') def test_no_prefix(self): """ If no prefix is passed to PrefixingLogMixin.__init__, it is not used in message rendering. """ + class LoggingObject2(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject2() obj.log("hello") obj.log("world") @@ -111,6 +107,9 @@ class Log(unittest.TestCase): Objects inheriting from PrefixingLogMixin get a unique number from a class-specific counter. """ + class LoggingObject3(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject3() obj2 = LoggingObject3() obj.log("hello") @@ -125,6 +124,9 @@ class Log(unittest.TestCase): This logic is pretty bogus, but that's what the code does. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1() result = obj.log("zero") self.assertEqual(result, "msg1") @@ -140,6 +142,9 @@ class Log(unittest.TestCase): If grandparent message id is given, it's used as parent id of the first message. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1(grandparentmsgid="grand") result = obj.log("zero") self.assertEqual(result, "msg1") From 4d2193fe13bd12ff9d73025bb54dcaf25c7e0bad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:12:12 -0400 Subject: [PATCH 0702/1054] Fix the module name. --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa1ab2fb9..608d6ebea 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,7 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", - "allmydata.immutable._python3", + "allmydata.immutable.happiness_upload", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 80136da0a60688756be0029402a2fe7b0ff2087d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 11:14:58 -0400 Subject: [PATCH 0703/1054] This Fedora probably also has no Python version tag --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 203993f4e..f5b33b106 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,7 +314,7 @@ jobs: fedora-29: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:29-py2" + - image: "tahoelafsci/fedora:29-py" user: "nobody" From 44143d1b083acf23b93bbfb5eb37a23e9af22934 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:37:44 -0400 Subject: [PATCH 0704/1054] Move tests for happinessutil.py into test_happiness.py. --- src/allmydata/test/common_py3.py | 26 +++- src/allmydata/test/common_util.py | 25 +--- src/allmydata/test/test_happiness.py | 198 ++++++++++++++++++++++++++- src/allmydata/test/test_upload.py | 191 +------------------------- 4 files changed, 224 insertions(+), 216 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 0dae05aa6..b29cb9691 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -17,7 +17,8 @@ import os import time import signal -from twisted.internet import reactor +from twisted.internet import defer, reactor +from twisted.python import failure class TimezoneMixin(object): @@ -65,3 +66,26 @@ class SignalMixin(object): if self.sigchldHandler: signal.signal(signal.SIGCHLD, self.sigchldHandler) return super(SignalMixin, self).tearDown() + + +class ShouldFailMixin(object): + + def shouldFail(self, expected_failure, which, substring, + callable, *args, **kwargs): + assert substring is None or isinstance(substring, str) + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + res.trap(expected_failure) + if substring: + self.failUnless(substring in str(res), + "%s: substring '%s' not in '%s'" + % (which, substring, str(res))) + # return the Failure for further analysis, but in a form that + # doesn't make the Deferred chain think that we failed. + return [res] + else: + self.fail("%s was supposed to raise %s, not get '%s'" % + (which, expected_failure, res)) + d.addBoth(done) + return d diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 261d39af3..684aee5ba 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -5,14 +5,13 @@ from random import randrange from six.moves import StringIO from twisted.internet import reactor, defer -from twisted.python import failure from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) from ..scripts import runner -from .common_py3 import SignalMixin +from .common_py3 import SignalMixin, ShouldFailMixin # noqa def skip_if_cannot_represent_filename(u): @@ -96,28 +95,6 @@ class StallMixin(object): reactor.callLater(delay, d.callback, res) return d -class ShouldFailMixin(object): - - def shouldFail(self, expected_failure, which, substring, - callable, *args, **kwargs): - assert substring is None or isinstance(substring, str) - d = defer.maybeDeferred(callable, *args, **kwargs) - def done(res): - if isinstance(res, failure.Failure): - res.trap(expected_failure) - if substring: - self.failUnless(substring in str(res), - "%s: substring '%s' not in '%s'" - % (which, substring, str(res))) - # return the Failure for further analysis, but in a form that - # doesn't make the Deferred chain think that we failed. - return [res] - else: - self.fail("%s was supposed to raise %s, not get '%s'" % - (which, expected_failure, res)) - d.addBoth(done) - return d - class TestMixin(SignalMixin): def setUp(self): diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index ffba402ef..0f7ee13d5 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -13,12 +13,17 @@ if PY2: from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets + from allmydata.immutable import happiness_upload +from allmydata.util.happinessutil import servers_of_happiness, \ + shares_by_server, merge_servers +from allmydata.test.common_py3 import ShouldFailMixin -class HappinessUtils(unittest.TestCase): +class HappinessUploadUtils(unittest.TestCase): """ - test-cases for utility functions augmenting_path_for and residual_network + test-cases for happiness_upload utility functions augmenting_path_for and + residual_network. """ def test_residual_0(self): @@ -279,3 +284,192 @@ class PlacementTests(unittest.TestCase): # peers; if we have fewer shares than peers happiness is capped at # # of peers. assert happiness == min(len(peers), len(shares)) + + +class FakeServerTracker(object): + def __init__(self, serverid, buckets): + self._serverid = serverid + self.buckets = buckets + def get_serverid(self): + return self._serverid + + +class HappinessUtilTests(unittest.TestCase, ShouldFailMixin): + """Tests for happinesutil.py.""" + + def test_merge_servers(self): + # merge_servers merges a list of upload_servers and a dict of + # shareid -> serverid mappings. + shares = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4", "server5"]), + 5 : set(["server1", "server2"]), + } + # if not provided with a upload_servers argument, it should just + # return the first argument unchanged. + self.failUnlessEqual(shares, merge_servers(shares, set([]))) + trackers = [] + for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]: + t = FakeServerTracker(server, [i]) + trackers.append(t) + expected = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4", "server5"]), + 5 : set(["server1", "server2", "server5"]), + 6 : set(["server6"]), + 7 : set(["server7"]), + 8 : set(["server8"]), + } + self.failUnlessEqual(expected, merge_servers(shares, set(trackers))) + shares2 = {} + expected = { + 5 : set(["server5"]), + 6 : set(["server6"]), + 7 : set(["server7"]), + 8 : set(["server8"]), + } + self.failUnlessEqual(expected, merge_servers(shares2, set(trackers))) + shares3 = {} + trackers = [] + expected = {} + for (i, server) in [(i, "server%d" % i) for i in range(10)]: + shares3[i] = set([server]) + t = FakeServerTracker(server, [i]) + trackers.append(t) + expected[i] = set([server]) + self.failUnlessEqual(expected, merge_servers(shares3, set(trackers))) + + + def test_servers_of_happiness_utility_function(self): + # These tests are concerned with the servers_of_happiness() + # utility function, and its underlying matching algorithm. Other + # aspects of the servers_of_happiness behavior are tested + # elsehwere These tests exist to ensure that + # servers_of_happiness doesn't under or overcount the happiness + # value for given inputs. + + # servers_of_happiness expects a dict of + # shnum => set(serverids) as a preexisting shares argument. + test1 = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4"]) + } + happy = servers_of_happiness(test1) + self.failUnlessEqual(4, happy) + test1[4] = set(["server1"]) + # We've added a duplicate server, so now servers_of_happiness + # should be 3 instead of 4. + happy = servers_of_happiness(test1) + self.failUnlessEqual(3, happy) + # The second argument of merge_servers should be a set of objects with + # serverid and buckets as attributes. In actual use, these will be + # ServerTracker instances, but for testing it is fine to make a + # FakeServerTracker whose job is to hold those instance variables to + # test that part. + trackers = [] + for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]: + t = FakeServerTracker(server, [i]) + trackers.append(t) + # Recall that test1 is a server layout with servers_of_happiness + # = 3. Since there isn't any overlap between the shnum -> + # set([serverid]) correspondences in test1 and those in trackers, + # the result here should be 7. + test2 = merge_servers(test1, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(7, happy) + # Now add an overlapping server to trackers. This is redundant, + # so it should not cause the previously reported happiness value + # to change. + t = FakeServerTracker("server1", [1]) + trackers.append(t) + test2 = merge_servers(test1, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(7, happy) + test = {} + happy = servers_of_happiness(test) + self.failUnlessEqual(0, happy) + # Test a more substantial overlap between the trackers and the + # existing assignments. + test = { + 1 : set(['server1']), + 2 : set(['server2']), + 3 : set(['server3']), + 4 : set(['server4']), + } + trackers = [] + t = FakeServerTracker('server5', [4]) + trackers.append(t) + t = FakeServerTracker('server6', [3, 5]) + trackers.append(t) + # The value returned by servers_of_happiness is the size + # of a maximum matching in the bipartite graph that + # servers_of_happiness() makes between serverids and share + # numbers. It should find something like this: + # (server 1, share 1) + # (server 2, share 2) + # (server 3, share 3) + # (server 5, share 4) + # (server 6, share 5) + # + # and, since there are 5 edges in this matching, it should + # return 5. + test2 = merge_servers(test, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(5, happy) + # Zooko's first puzzle: + # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156) + # + # server 1: shares 0, 1 + # server 2: shares 1, 2 + # server 3: share 2 + # + # This should yield happiness of 3. + test = { + 0 : set(['server1']), + 1 : set(['server1', 'server2']), + 2 : set(['server2', 'server3']), + } + self.failUnlessEqual(3, servers_of_happiness(test)) + # Zooko's second puzzle: + # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158) + # + # server 1: shares 0, 1 + # server 2: share 1 + # + # This should yield happiness of 2. + test = { + 0 : set(['server1']), + 1 : set(['server1', 'server2']), + } + self.failUnlessEqual(2, servers_of_happiness(test)) + + + def test_shares_by_server(self): + test = dict([(i, set(["server%d" % i])) for i in range(1, 5)]) + sbs = shares_by_server(test) + self.failUnlessEqual(set([1]), sbs["server1"]) + self.failUnlessEqual(set([2]), sbs["server2"]) + self.failUnlessEqual(set([3]), sbs["server3"]) + self.failUnlessEqual(set([4]), sbs["server4"]) + test1 = { + 1 : set(["server1"]), + 2 : set(["server1"]), + 3 : set(["server1"]), + 4 : set(["server2"]), + 5 : set(["server2"]) + } + sbs = shares_by_server(test1) + self.failUnlessEqual(set([1, 2, 3]), sbs["server1"]) + self.failUnlessEqual(set([4, 5]), sbs["server2"]) + # This should fail unless the serverid part of the mapping is a set + test2 = {1: "server1"} + self.shouldFail(AssertionError, + "test_shares_by_server", + "", + shares_by_server, test2) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 165ca17d7..2ec426722 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -15,17 +15,15 @@ from allmydata.util import log, base32 from allmydata.util.assertutil import precondition from allmydata.util.deferredutil import DeferredListShouldSucceed from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_util import ShouldFailMixin -from allmydata.util.happinessutil import servers_of_happiness, \ - shares_by_server, merge_servers +from allmydata.test.common_py3 import ShouldFailMixin from allmydata.storage_client import StorageFarmBroker from allmydata.storage.server import storage_index_to_dir from allmydata.client import _Client - from .common import ( EMPTY_CLIENT_CONFIG, ) + MiB = 1024*1024 def extract_uri(results): @@ -864,12 +862,6 @@ def is_happy_enough(servertoshnums, h, k): return False return True -class FakeServerTracker(object): - def __init__(self, serverid, buckets): - self._serverid = serverid - self.buckets = buckets - def get_serverid(self): - return self._serverid class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, ShouldFailMixin): @@ -1499,185 +1491,6 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, self._do_upload_with_broken_servers, 2)) return d - - def test_merge_servers(self): - # merge_servers merges a list of upload_servers and a dict of - # shareid -> serverid mappings. - shares = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4", "server5"]), - 5 : set(["server1", "server2"]), - } - # if not provided with a upload_servers argument, it should just - # return the first argument unchanged. - self.failUnlessEqual(shares, merge_servers(shares, set([]))) - trackers = [] - for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]: - t = FakeServerTracker(server, [i]) - trackers.append(t) - expected = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4", "server5"]), - 5 : set(["server1", "server2", "server5"]), - 6 : set(["server6"]), - 7 : set(["server7"]), - 8 : set(["server8"]), - } - self.failUnlessEqual(expected, merge_servers(shares, set(trackers))) - shares2 = {} - expected = { - 5 : set(["server5"]), - 6 : set(["server6"]), - 7 : set(["server7"]), - 8 : set(["server8"]), - } - self.failUnlessEqual(expected, merge_servers(shares2, set(trackers))) - shares3 = {} - trackers = [] - expected = {} - for (i, server) in [(i, "server%d" % i) for i in xrange(10)]: - shares3[i] = set([server]) - t = FakeServerTracker(server, [i]) - trackers.append(t) - expected[i] = set([server]) - self.failUnlessEqual(expected, merge_servers(shares3, set(trackers))) - - - def test_servers_of_happiness_utility_function(self): - # These tests are concerned with the servers_of_happiness() - # utility function, and its underlying matching algorithm. Other - # aspects of the servers_of_happiness behavior are tested - # elsehwere These tests exist to ensure that - # servers_of_happiness doesn't under or overcount the happiness - # value for given inputs. - - # servers_of_happiness expects a dict of - # shnum => set(serverids) as a preexisting shares argument. - test1 = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4"]) - } - happy = servers_of_happiness(test1) - self.failUnlessEqual(4, happy) - test1[4] = set(["server1"]) - # We've added a duplicate server, so now servers_of_happiness - # should be 3 instead of 4. - happy = servers_of_happiness(test1) - self.failUnlessEqual(3, happy) - # The second argument of merge_servers should be a set of objects with - # serverid and buckets as attributes. In actual use, these will be - # ServerTracker instances, but for testing it is fine to make a - # FakeServerTracker whose job is to hold those instance variables to - # test that part. - trackers = [] - for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]: - t = FakeServerTracker(server, [i]) - trackers.append(t) - # Recall that test1 is a server layout with servers_of_happiness - # = 3. Since there isn't any overlap between the shnum -> - # set([serverid]) correspondences in test1 and those in trackers, - # the result here should be 7. - test2 = merge_servers(test1, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(7, happy) - # Now add an overlapping server to trackers. This is redundant, - # so it should not cause the previously reported happiness value - # to change. - t = FakeServerTracker("server1", [1]) - trackers.append(t) - test2 = merge_servers(test1, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(7, happy) - test = {} - happy = servers_of_happiness(test) - self.failUnlessEqual(0, happy) - # Test a more substantial overlap between the trackers and the - # existing assignments. - test = { - 1 : set(['server1']), - 2 : set(['server2']), - 3 : set(['server3']), - 4 : set(['server4']), - } - trackers = [] - t = FakeServerTracker('server5', [4]) - trackers.append(t) - t = FakeServerTracker('server6', [3, 5]) - trackers.append(t) - # The value returned by servers_of_happiness is the size - # of a maximum matching in the bipartite graph that - # servers_of_happiness() makes between serverids and share - # numbers. It should find something like this: - # (server 1, share 1) - # (server 2, share 2) - # (server 3, share 3) - # (server 5, share 4) - # (server 6, share 5) - # - # and, since there are 5 edges in this matching, it should - # return 5. - test2 = merge_servers(test, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(5, happy) - # Zooko's first puzzle: - # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156) - # - # server 1: shares 0, 1 - # server 2: shares 1, 2 - # server 3: share 2 - # - # This should yield happiness of 3. - test = { - 0 : set(['server1']), - 1 : set(['server1', 'server2']), - 2 : set(['server2', 'server3']), - } - self.failUnlessEqual(3, servers_of_happiness(test)) - # Zooko's second puzzle: - # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158) - # - # server 1: shares 0, 1 - # server 2: share 1 - # - # This should yield happiness of 2. - test = { - 0 : set(['server1']), - 1 : set(['server1', 'server2']), - } - self.failUnlessEqual(2, servers_of_happiness(test)) - - - def test_shares_by_server(self): - test = dict([(i, set(["server%d" % i])) for i in xrange(1, 5)]) - sbs = shares_by_server(test) - self.failUnlessEqual(set([1]), sbs["server1"]) - self.failUnlessEqual(set([2]), sbs["server2"]) - self.failUnlessEqual(set([3]), sbs["server3"]) - self.failUnlessEqual(set([4]), sbs["server4"]) - test1 = { - 1 : set(["server1"]), - 2 : set(["server1"]), - 3 : set(["server1"]), - 4 : set(["server2"]), - 5 : set(["server2"]) - } - sbs = shares_by_server(test1) - self.failUnlessEqual(set([1, 2, 3]), sbs["server1"]) - self.failUnlessEqual(set([4, 5]), sbs["server2"]) - # This should fail unless the serverid part of the mapping is a set - test2 = {1: "server1"} - self.shouldFail(AssertionError, - "test_shares_by_server", - "", - shares_by_server, test2) - - def test_existing_share_detection(self): self.basedir = self.mktemp() d = self._setup_and_upload() From 6c77a227ff5cb09e415cef754df0b03e2de19796 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:47:24 -0400 Subject: [PATCH 0705/1054] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/happinessutil.py | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 608d6ebea..691b53872 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -30,6 +30,7 @@ PORTED_MODULES = [ "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.gcutil", + "allmydata.util.happinessutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", diff --git a/src/allmydata/util/happinessutil.py b/src/allmydata/util/happinessutil.py index 75f7d4001..9f2617a5e 100644 --- a/src/allmydata/util/happinessutil.py +++ b/src/allmydata/util/happinessutil.py @@ -1,7 +1,18 @@ """ I contain utilities useful for calculating servers_of_happiness, and for -reporting it in messages +reporting it in messages. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 from copy import deepcopy from allmydata.immutable.happiness_upload import residual_network @@ -51,7 +62,7 @@ def shares_by_server(servermap): dictionary of sets of shares, indexed by peerids. """ ret = {} - for shareid, peers in servermap.iteritems(): + for shareid, peers in servermap.items(): assert isinstance(peers, set) for peerid in peers: ret.setdefault(peerid, set()).add(shareid) @@ -146,7 +157,7 @@ def servers_of_happiness(sharemap): # The implementation here is an adapation of an algorithm described in # "Introduction to Algorithms", Cormen et al, 2nd ed., pp 658-662. dim = len(graph) - flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)] + flow_function = [[0 for sh in range(dim)] for s in range(dim)] residual_graph, residual_function = residual_network(graph, flow_function) while augmenting_path_for(residual_graph): path = augmenting_path_for(residual_graph) @@ -169,7 +180,7 @@ def servers_of_happiness(sharemap): # our graph, so we can stop after summing flow across those. The # value of a flow computed in this way is the size of a maximum # matching on the bipartite graph described above. - return sum([flow_function[0][v] for v in xrange(1, num_servers+1)]) + return sum([flow_function[0][v] for v in range(1, num_servers+1)]) def _flow_network_for(servermap): """ @@ -198,14 +209,14 @@ def _flow_network_for(servermap): graph = [] # index -> [index], an adjacency list # Add an entry at the top (index 0) that has an edge to every server # in servermap - graph.append(servermap.keys()) + graph.append(list(servermap.keys())) # For each server, add an entry that has an edge to every share that it # contains (or will contain). for k in servermap: graph.append(servermap[k]) # For each share, add an entry that has an edge to the sink. sink_num = num_servers + num_shares + 1 - for i in xrange(num_shares): + for i in range(num_shares): graph.append([sink_num]) # Add an empty entry for the sink, which has no outbound edges. graph.append([]) @@ -231,8 +242,8 @@ def _reindex(servermap, base_index): # Number the shares for k in ret: for shnum in ret[k]: - if not shares.has_key(shnum): + if shnum not in shares: shares[shnum] = num num += 1 - ret[k] = map(lambda x: shares[x], ret[k]) + ret[k] = [shares[x] for x in ret[k]] return (ret, len(shares)) From 1308fc44c0ed7ea3bfd21cb9cee09bace895c451 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:51:43 -0400 Subject: [PATCH 0706/1054] Ratchet and news file. --- misc/python3/ratchet-passing | 9 ++++++--- newsfragments/3373.minor | 0 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3373.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 48db826f2..0bc4522b3 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -67,9 +67,12 @@ allmydata.test.test_happiness.Happiness.test_placement_1 allmydata.test.test_happiness.Happiness.test_placement_simple allmydata.test.test_happiness.Happiness.test_redistribute allmydata.test.test_happiness.Happiness.test_unhappy -allmydata.test.test_happiness.HappinessUtils.test_residual_0 -allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph -allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.HappinessUploadUtils.test_residual_0 +allmydata.test.test_happiness.HappinessUploadUtils.test_trivial_flow_graph +allmydata.test.test_happiness.HappinessUploadUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.HappinessUtilTests.test_merge_servers +allmydata.test.test_happiness.HappinessUtilTests.test_servers_of_happiness_utility_function +allmydata.test.test_happiness.HappinessUtilTests.test_shares_by_server allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy allmydata.test.test_happiness.PlacementTests.test_more_hypothesis allmydata.test.test_hashtree.Complete.test_create diff --git a/newsfragments/3373.minor b/newsfragments/3373.minor new file mode 100644 index 000000000..e69de29bb From b08a78e5b49799a5d3126355902074da0dc0f827 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:52:26 -0400 Subject: [PATCH 0707/1054] Docstring. --- src/allmydata/test/test_happiness.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index 0f7ee13d5..bf73ca2d8 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -1,4 +1,10 @@ # -*- coding: utf-8 -*- +""" +Tests for allmydata.immutable.happiness_upload and +allmydata.util.happinessutil. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division From 7516a5526e6561ca3c9f3306b7d46ed06f70047b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 12:57:46 -0400 Subject: [PATCH 0708/1054] Port to Python 3. --- src/allmydata/interfaces.py | 16 ++++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 17 insertions(+) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36eb55bfb..7b02ceea9 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,3 +1,19 @@ +""" +Interfaces for Tahoe-LAFS. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Don't import object/str/dict/etc. types, so we don't break any interfaces. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, max, min # noqa: F401 + +from past.builtins import long from zope.interface import Interface, Attribute from twisted.plugin import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 091c248af..c3de47608 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.interfaces", "allmydata.test.common_py3", "allmydata.util._python3", "allmydata.util.abbreviate", From 5384768f7608dd074a5c51c798e87b9f1cda6db2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:08:53 -0400 Subject: [PATCH 0709/1054] Port to Python 3. --- src/allmydata/codec.py | 21 +++++++++++++++++---- src/allmydata/test/test_codec.py | 25 +++++++++++++++++++------ src/allmydata/util/_python3.py | 2 ++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index 1ceb146e4..048355871 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -1,4 +1,17 @@ -# -*- test-case-name: allmydata.test.test_encode_share -*- +""" +CRS encoding and decoding. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from future.utils import native_str from zope.interface import implementer from twisted.internet import defer @@ -27,8 +40,8 @@ class CRSEncoder(object): return (self.data_size, self.required_shares, self.max_shares) def get_serialized_params(self): - return "%d-%d-%d" % (self.data_size, self.required_shares, - self.max_shares) + return native_str("%d-%d-%d" % (self.data_size, self.required_shares, + self.max_shares)) def get_block_size(self): return self.share_size @@ -37,7 +50,7 @@ class CRSEncoder(object): precondition(desired_share_ids is None or len(desired_share_ids) <= self.max_shares, desired_share_ids, self.max_shares) if desired_share_ids is None: - desired_share_ids = range(self.max_shares) + desired_share_ids = list(range(self.max_shares)) for inshare in inshares: assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares) diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py index 706e62dff..3076a322e 100644 --- a/src/allmydata/test/test_codec.py +++ b/src/allmydata/test/test_codec.py @@ -1,3 +1,16 @@ +""" +Tests for allmydata.codec. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from twisted.trial import unittest @@ -23,7 +36,7 @@ class T(unittest.TestCase): d.addCallback(_done_encoding_all) if fewer_shares is not None: # also validate that the desired_shareids= parameter works - desired_shareids = random.sample(range(max_shares), fewer_shares) + desired_shareids = random.sample(list(range(max_shares)), fewer_shares) d.addCallback(lambda res: enc.encode(data0s, desired_shareids)) def _check_fewer_shares(some_shares_and_their_shareids): (some_shares, their_shareids) = some_shares_and_their_shareids @@ -38,11 +51,11 @@ class T(unittest.TestCase): return d1 def _check_data(decoded_shares): - self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s))) + self.failUnlessEqual(len(b''.join(decoded_shares)), len(b''.join(data0s))) self.failUnlessEqual(len(decoded_shares), len(data0s)) for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)): self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],)) - self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",)) + self.failUnless(b''.join(decoded_shares) == b''.join(data0s), "%s" % ("???",)) # 0data0sclipped = tuple(data0s) # data0sclipped[-1] = # self.failUnless(tuple(decoded_shares) == tuple(data0s)) @@ -59,7 +72,7 @@ class T(unittest.TestCase): def _decode_some_random(res): log.msg("_decode_some_random") # use a randomly-selected minimal subset - l = random.sample(zip(self.shares, self.shareids), required_shares) + l = random.sample(list(zip(self.shares, self.shareids)), required_shares) some_shares = [ x[0] for x in l ] some_shareids = [ x[1] for x in l ] return _decode((some_shares, some_shareids)) @@ -70,10 +83,10 @@ class T(unittest.TestCase): log.msg("_decode_multiple") # make sure we can re-use the decoder object shares1 = random.sample(self.shares, required_shares) - sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares) + sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares) shares1 = [ x[0] for x in sharesl1 ] shareids1 = [ x[1] for x in sharesl1 ] - sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares) + sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares) shares2 = [ x[0] for x in sharesl2 ] shareids2 = [ x[1] for x in sharesl2 ] dec = CRSDecoder() diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c3de47608..2b18eaa1d 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.codec", "allmydata.crypto", "allmydata.crypto.aes", "allmydata.crypto.ed25519", @@ -51,6 +52,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_codec", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", From 43e36ebcb68a75161e807829f95de67310179c28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:09:25 -0400 Subject: [PATCH 0710/1054] Ignore Futurize-generated backup files. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8191c173b..1af859854 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ venv *~ *.DS_Store .*.kate-swp +*.bak /build/ /support/ From 9c0b5eac2b0a20fcad76cb715c21c8c5daba5694 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:10:55 -0400 Subject: [PATCH 0711/1054] Additional testing. --- src/allmydata/test/test_codec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py index 3076a322e..ee64e2bf2 100644 --- a/src/allmydata/test/test_codec.py +++ b/src/allmydata/test/test_codec.py @@ -15,7 +15,7 @@ if PY2: import os from twisted.trial import unittest from twisted.python import log -from allmydata.codec import CRSEncoder, CRSDecoder +from allmydata.codec import CRSEncoder, CRSDecoder, parse_params import random from allmydata.util import mathutil @@ -26,6 +26,8 @@ class T(unittest.TestCase): enc.set_params(size, required_shares, max_shares) params = enc.get_params() assert params == (size, required_shares, max_shares) + serialized_params = enc.get_serialized_params() + self.assertEqual(parse_params(serialized_params), params) log.msg("params: %s" % (params,)) d = enc.encode(data0s) def _done_encoding_all(shares_and_shareids): From 017104fec18339fe852e23e64f5b6503283c2fee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:11:24 -0400 Subject: [PATCH 0712/1054] News file. --- newsfragments/3374.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3374.minor diff --git a/newsfragments/3374.minor b/newsfragments/3374.minor new file mode 100644 index 000000000..e69de29bb From b82e2ad1c507c1b0e3dbf3fdfd53d2ee5e2ccddc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:26:44 -0400 Subject: [PATCH 0713/1054] Tests for allmydata.monitor. --- src/allmydata/test/test_monitor.py | 52 ++++++++++++++++++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 53 insertions(+) create mode 100644 src/allmydata/test/test_monitor.py diff --git a/src/allmydata/test/test_monitor.py b/src/allmydata/test/test_monitor.py new file mode 100644 index 000000000..7010da73a --- /dev/null +++ b/src/allmydata/test/test_monitor.py @@ -0,0 +1,52 @@ +""" +Tests for allmydata.monitor. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest + +from allmydata.monitor import Monitor, OperationCancelledError + + +class MonitorTests(unittest.TestCase): + """Tests for the Monitor class.""" + + def test_cancellation(self): + """The monitor can be cancelled.""" + m = Monitor() + self.assertFalse(m.is_cancelled()) + m.raise_if_cancelled() + m.cancel() + self.assertTrue(m.is_cancelled()) + with self.assertRaises(OperationCancelledError): + m.raise_if_cancelled() + + def test_status(self): + """The monitor can have its status set.""" + m = Monitor() + self.assertEqual(m.get_status(), None) + m.set_status("discombobulated") + self.assertEqual(m.get_status(), "discombobulated") + + def test_finish(self): + """The monitor can finish.""" + m = Monitor() + self.assertFalse(m.is_finished()) + d = m.when_done() + self.assertNoResult(d) + + result = m.finish(300) + self.assertEqual(result, 300) + self.assertEqual(m.get_status(), 300) + self.assertTrue(m.is_finished()) + + d.addBoth(self.assertEqual, 300) + return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2b18eaa1d..f8260078f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -61,6 +61,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", "allmydata.test.test_iputil", "allmydata.test.test_log", + "allmydata.test.test_monitor", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", From 0e034e06b771782b081986d12bd9dc066f3f3546 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:28:14 -0400 Subject: [PATCH 0714/1054] Port to Python 3. --- src/allmydata/monitor.py | 14 ++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+) diff --git a/src/allmydata/monitor.py b/src/allmydata/monitor.py index 98c463014..1559a30d9 100644 --- a/src/allmydata/monitor.py +++ b/src/allmydata/monitor.py @@ -1,7 +1,21 @@ +""" +Manage status of long-running operations. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import Interface, implementer from allmydata.util import observer + class IMonitor(Interface): """I manage status, progress, and cancellation for long-running operations. diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f8260078f..f05beed88 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -24,6 +24,7 @@ PORTED_MODULES = [ "allmydata.crypto.util", "allmydata.hashtree", "allmydata.interfaces", + "allmydata.monitor", "allmydata.test.common_py3", "allmydata.util._python3", "allmydata.util.abbreviate", From 6229bb4e04964fe5347eb79231cf43537367d460 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:29:30 -0400 Subject: [PATCH 0715/1054] Ratchet. --- misc/python3/ratchet-passing | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bbc6e6c7e..f5394e965 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -24,6 +24,9 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_codec.T.test_encode +allmydata.test.test_codec.T.test_encode1 +allmydata.test.test_codec.T.test_encode2 allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes allmydata.test.test_crypto.TestEd25519.test_key_serialization @@ -91,6 +94,9 @@ allmydata.test.test_log.Log.test_numming allmydata.test.test_log.Log.test_parent_id allmydata.test.test_log.Log.test_with_bytes_prefix allmydata.test.test_log.Log.test_with_prefix +allmydata.test.test_monitor.MonitorTests.test_cancellation +allmydata.test.test_monitor.MonitorTests.test_finish +allmydata.test.test_monitor.MonitorTests.test_status allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested From 0a2b797c4964d51a38cbc75718f80e7938868336 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 13:32:12 -0400 Subject: [PATCH 0716/1054] Lint fix. --- src/allmydata/interfaces.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 7b02ceea9..5126b8c90 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -10,8 +10,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - # Don't import object/str/dict/etc. types, so we don't break any interfaces. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, max, min # noqa: F401 + # Don't import object/str/dict/etc. types, so we don't break any + # interfaces. Not importing open() because it triggers bogus flake8 error. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401 from past.builtins import long From d6a80c891a8ec9796760dcf213eb8249f76dfb5d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 13:39:51 -0400 Subject: [PATCH 0717/1054] try to understand the CI failures --- misc/python3/ratchet.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index a6a2c53c3..ec67ea0a0 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -32,6 +32,11 @@ set -e if [ $TERM = 'dumb' ]; then export TERM=ansi fi -git diff "$tracking_filename" -exit $code +echo "The ${tracking_filename} diff is:" +echo "=================================" +git diff "${tracking_filename}" +echo "=================================" + +echo "Exiting with code ${code} from ratchet.py." +exit ${code} From 24772616c2125806608279b80145f287619941bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 14:13:20 -0400 Subject: [PATCH 0718/1054] Bytes not strings. --- src/allmydata/interfaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 5126b8c90..e9e5a5eac 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -87,7 +87,7 @@ class RIBucketReader(RemoteInterface): """ -TestVector = ListOf(TupleOf(Offset, ReadSize, str, str)) +TestVector = ListOf(TupleOf(Offset, ReadSize, bytes, bytes)) # elements are (offset, length, operator, specimen) # operator is one of "lt, le, eq, ne, ge, gt" # nop always passes and is used to fetch data while writing. @@ -111,7 +111,7 @@ class RIStorageServer(RemoteInterface): """ Return a dictionary of version information. """ - return DictOf(str, Any()) + return DictOf(bytes, Any()) def allocate_buckets(storage_index=StorageIndex, renew_secret=LeaseRenewSecret, From 5736f9cdebdbd0c4415c354276982c3fb74ab19b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:31:32 -0400 Subject: [PATCH 0719/1054] Let's just see, for fun --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index ec67ea0a0..e38999dd2 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,7 +35,7 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" -git diff "${tracking_filename}" +git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 1f3d24656750464c43d62e815c7f3a9944b6694d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:13:33 -0400 Subject: [PATCH 0720/1054] try to get more info --- misc/python3/ratchet.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index e38999dd2..da3c88e51 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,6 +35,12 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" +export GIT_TRACE=1 +export GIT_CURL_VERBOSE=2 +export GIT_TRACE_PACK_ACCESS=1 +export GIT_TRACE_PACKET=1 +export GIT_TRACE_PERFORMANCE=1 +export GIT_TRACE_SETUP=1 git diff -- "${tracking_filename}" echo "=================================" From c2b1fda4e7502c82b1b53fc5cbd8fba22e793d0c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:59:27 -0400 Subject: [PATCH 0721/1054] Add Ubuntu 20.04 --- .circleci/config.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..dae9e904e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,10 +11,13 @@ workflows: requires: - "debian-9" - - "ubuntu-18.04" + - "ubuntu-20.04" + - "ubuntu-18.04": + requires: + - "ubuntu-20.04" - "ubuntu-16.04": requires: - - "ubuntu-18.04" + - "ubuntu-20.04" - "fedora-29" - "fedora-28": @@ -65,6 +68,7 @@ workflows: - "build-image-debian-9" - "build-image-ubuntu-16.04" - "build-image-ubuntu-18.04" + - "build-image-ubuntu-20.04" - "build-image-fedora-28" - "build-image-fedora-29" - "build-image-centos-8" @@ -268,6 +272,13 @@ jobs: user: "nobody" + ubuntu-20.04: + <<: *DEBIAN + docker: + - image: "tahoelafsci/ubuntu:20.04" + user: "nobody" + + centos-8: &RHEL_DERIV docker: - image: "tahoelafsci/centos:8" @@ -480,6 +491,14 @@ jobs: TAG: "18.04" + build-image-ubuntu-20.04: + <<: *BUILD_IMAGE + + environment: + DISTRO: "ubuntu" + TAG: "20.04" + + build-image-centos-8: <<: *BUILD_IMAGE From 083be5f0c834e2ca16791d53e49c0beb2916af6c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:59:45 -0400 Subject: [PATCH 0722/1054] news fragment --- newsfragments/3328.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3328.installation diff --git a/newsfragments/3328.installation b/newsfragments/3328.installation new file mode 100644 index 000000000..7b08ffdc4 --- /dev/null +++ b/newsfragments/3328.installation @@ -0,0 +1 @@ +Tahoe-LAFS now supports Ubuntu 20.04. \ No newline at end of file From b1e67774b427fe57678a403192aed2fe450ef27d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:46:46 -0400 Subject: [PATCH 0723/1054] Try getting the _trial_temp out of the working tree --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index da3c88e51..65a270560 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,7 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata +SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" set -e From 7c922785245e93fd61398289e9328e3ae5cf47ac Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:52:57 -0400 Subject: [PATCH 0724/1054] getting ridiculous --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 65a270560..c92b22c5d 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,7 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -git diff -- "${tracking_filename}" +strace git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 8b44b89f7534efe02e386c2b883b33ed442da5a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 18:33:35 -0400 Subject: [PATCH 0725/1054] oi --- misc/python3/ratchet.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c92b22c5d..9d57a0e10 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,6 +33,7 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi +apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From 107861802bf46681a211195256a2bec3f24e4335 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:18:29 -0400 Subject: [PATCH 0726/1054] idk maybe this --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 9d57a0e10..3d824d199 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,7 +33,7 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi -apt-get install -y strace +sudo apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From a60d4f7df4c799e09c261015b5091f534e306f28 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:56:37 -0400 Subject: [PATCH 0727/1054] get strace --- .circleci/Dockerfile.ubuntu | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..98c3b5088 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -13,6 +13,7 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install git && \ apt-get --quiet --yes install \ sudo \ + strace \ build-essential \ python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev \ From a758b84e15126cf28ac0969a0f4c36db57d0713b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:56:51 -0400 Subject: [PATCH 0728/1054] handling this elsewhere --- misc/python3/ratchet.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 3d824d199..c92b22c5d 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,7 +33,6 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi -sudo apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From 9af1e741d8566a6f4bd8817531fafe1b510d022c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:19:38 -0400 Subject: [PATCH 0729/1054] that was kind of annoyingly non-illuminating --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c92b22c5d..6c428cdf9 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,7 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -strace git diff -- "${tracking_filename}" +strace -f git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 8d1798fbb6c8186ca346525ba0bf1fec98d2493a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:26:39 -0400 Subject: [PATCH 0730/1054] is the pager screwing up? --- misc/python3/ratchet.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 6c428cdf9..1d54a9cb0 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,8 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -strace -f git diff -- "${tracking_filename}" +strace -f git diff -- "${tracking_filename}" > tracking.diff +cat tracking.diff echo "=================================" echo "Exiting with code ${code} from ratchet.py." From fe850e724000970bff75a74797f99b33863f213e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:45:33 -0400 Subject: [PATCH 0731/1054] redirecting to a file seems to help --- misc/python3/ratchet.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 1d54a9cb0..c05e675f2 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,13 +35,9 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" -export GIT_TRACE=1 -export GIT_CURL_VERBOSE=2 -export GIT_TRACE_PACK_ACCESS=1 -export GIT_TRACE_PACKET=1 -export GIT_TRACE_PERFORMANCE=1 -export GIT_TRACE_SETUP=1 -strace -f git diff -- "${tracking_filename}" > tracking.diff +# "git diff" gets pretty confused in this execution context when trying to +# write to stdout. Somehow it fails with SIGTTOU. +git diff -- "${tracking_filename}" > tracking.diff cat tracking.diff echo "=================================" From ef3cbebe431b9978d211434dc6c141d3c26699eb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:57:59 -0400 Subject: [PATCH 0732/1054] Perhaps get this path right --- misc/python3/ratchet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c05e675f2..33183fce1 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,8 +11,8 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata -subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" +trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata +subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e # Okay, now we're clear. From 360a97a5794bd69a210d72f8a411412d47413a83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:08:48 -0400 Subject: [PATCH 0733/1054] Let the python36 job have the env vars --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab467abd0..27c9f36cc 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME +passenv = {[testenv]passenv} HOME commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 65520786cac9415af6ccfb71fa3e858a751b806e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:26:15 -0400 Subject: [PATCH 0734/1054] [drop] faster ci --- .circleci/config.yml | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5b33b106..1ae4c1326 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,50 +6,50 @@ workflows: ci: jobs: # Platforms - - "debian-9" - - "debian-8": - requires: - - "debian-9" + # - "debian-9" + # - "debian-8": + # requires: + # - "debian-9" - - "ubuntu-18.04" - - "ubuntu-16.04": - requires: - - "ubuntu-18.04" + # - "ubuntu-18.04" + # - "ubuntu-16.04": + # requires: + # - "ubuntu-18.04" - - "fedora-29" - - "fedora-28": - requires: - - "fedora-29" + # - "fedora-29" + # - "fedora-28": + # requires: + # - "fedora-29" - - "centos-8" + # - "centos-8" - - "slackware-14.2" + # - "slackware-14.2" - - "nixos-19.09" + # - "nixos-19.09" - # Test against PyPy 2.7 - - "pypy2.7-buster" + # # Test against PyPy 2.7 + # - "pypy2.7-buster" # Just one Python 3.6 configuration while the port is in-progress. - "python3.6" - # Other assorted tasks and configurations - - "lint" - - "pyinstaller" - - "deprecations" - - "c-locale" - # Any locale other than C or UTF-8. - - "another-locale" + # # Other assorted tasks and configurations + # - "lint" + # - "pyinstaller" + # - "deprecations" + # - "c-locale" + # # Any locale other than C or UTF-8. + # - "another-locale" - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-9" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-9" - # Generate the underlying data for a visualization to aid with Python 3 - # porting. - - "build-porting-depgraph" + # # Generate the underlying data for a visualization to aid with Python 3 + # # porting. + # - "build-porting-depgraph" images: # Build the Docker images used by the ci jobs. This makes the ci jobs From f8e44acdef3bb0aa06bc8d74a881bf7b69b6a77c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:27:36 -0400 Subject: [PATCH 0735/1054] did it get written --- misc/python3/ratchet.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 33183fce1..c9bf6bed1 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -12,6 +12,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata +find "${SUBUNITREPORTER_OUTPUT_PATH}" subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From c17337e732797d36bb46253b43f17b008ad86b07 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:47:11 -0400 Subject: [PATCH 0736/1054] We get passed an env var that tells us how to run trial Also temp-dir made no difference and don't need the find either. --- misc/python3/ratchet.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c9bf6bed1..e9f8a4c04 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,8 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata -find "${SUBUNITREPORTER_OUTPUT_PATH}" +trial ${TAHOE_LAFS_TRIAL_ARGS} allmydata subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From 971bca7402706f9b864f5d5e4746212775a629dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:51:15 -0400 Subject: [PATCH 0737/1054] --rterrors is broken with subunitreporter on python 3 --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index e9f8a4c04..aa768cd06 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,7 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial ${TAHOE_LAFS_TRIAL_ARGS} allmydata +trial --reporter=subunitv2-file allmydata subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From 23d7ffa369060286e6bcbe516288681dc980ed2c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:56:13 -0400 Subject: [PATCH 0738/1054] Check for subunit2 output here --- .circleci/run-tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index c26acdcbc..aab3271d6 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -81,6 +81,11 @@ ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ ${TAHOE_LAFS_TOX_ARGS} || "${alternative}" if [ -n "${ARTIFACTS}" ]; then + if [ ! -e "${SUBUNIT2}" ]; then + echo "subunitv2 output file does not exist: ${SUBUNIT2}" + exit 1 + fi + # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" From 7ba6daeed52aed906076c33ecb7946ee900de980 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:58:24 -0400 Subject: [PATCH 0739/1054] maybe get some more visibility into errors --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index aab3271d6..939f9b1e5 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" + cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml | cat > "${JUNITXML}" || "${alternative}" fi From a927ebbafaaf80fbb22b4cdd5bc0e57dca0958ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Aug 2020 00:04:13 -0400 Subject: [PATCH 0740/1054] what's up --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 939f9b1e5..197227442 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml | cat > "${JUNITXML}" || "${alternative}" + cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml fi From 0c2cf9f0c5fbc6b501af782f87bd741ad1f6d068 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:11:14 -0400 Subject: [PATCH 0741/1054] Don't fail if subunit2junitxml fails --- .circleci/run-tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 197227442..8894b9691 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,9 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml + # Always succeed even if subunit2junitxml fails. subunit2junitxml signals + # failure if the stream it is processing contains test failures. This is + # not what we care about. If we cared about it, the test command above + # would have signalled failure already and we wouldn't be here. + "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || true fi From 462e212a19f93c833ab308f7d8f5bb2ed848e035 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:16:19 -0400 Subject: [PATCH 0742/1054] Revert "[drop] faster ci" This reverts commit 65520786cac9415af6ccfb71fa3e858a751b806e. --- .circleci/config.yml | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ae4c1326..f5b33b106 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,50 +6,50 @@ workflows: ci: jobs: # Platforms - # - "debian-9" - # - "debian-8": - # requires: - # - "debian-9" + - "debian-9" + - "debian-8": + requires: + - "debian-9" - # - "ubuntu-18.04" - # - "ubuntu-16.04": - # requires: - # - "ubuntu-18.04" + - "ubuntu-18.04" + - "ubuntu-16.04": + requires: + - "ubuntu-18.04" - # - "fedora-29" - # - "fedora-28": - # requires: - # - "fedora-29" + - "fedora-29" + - "fedora-28": + requires: + - "fedora-29" - # - "centos-8" + - "centos-8" - # - "slackware-14.2" + - "slackware-14.2" - # - "nixos-19.09" + - "nixos-19.09" - # # Test against PyPy 2.7 - # - "pypy2.7-buster" + # Test against PyPy 2.7 + - "pypy2.7-buster" # Just one Python 3.6 configuration while the port is in-progress. - "python3.6" - # # Other assorted tasks and configurations - # - "lint" - # - "pyinstaller" - # - "deprecations" - # - "c-locale" - # # Any locale other than C or UTF-8. - # - "another-locale" + # Other assorted tasks and configurations + - "lint" + - "pyinstaller" + - "deprecations" + - "c-locale" + # Any locale other than C or UTF-8. + - "another-locale" - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-9" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-9" - # # Generate the underlying data for a visualization to aid with Python 3 - # # porting. - # - "build-porting-depgraph" + # Generate the underlying data for a visualization to aid with Python 3 + # porting. + - "build-porting-depgraph" images: # Build the Docker images used by the ci jobs. This makes the ci jobs From fdfa95e1e8b5bbc2aeadd9b999cc5fb57515beea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:59:28 -0400 Subject: [PATCH 0743/1054] news fragment --- newsfragments/3375.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3375.minor diff --git a/newsfragments/3375.minor b/newsfragments/3375.minor new file mode 100644 index 000000000..e69de29bb From 9853fa90904be82e71c7e9fd19d99d2177289bad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:59:36 -0400 Subject: [PATCH 0744/1054] Pin Python 2 compatible PyInstaller --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4356cfd24..908d24d54 100644 --- a/tox.ini +++ b/tox.ini @@ -210,7 +210,9 @@ extras = deps = {[testenv]deps} packaging - pyinstaller + # PyInstaller 4.0 drops Python 2 support. When we finish porting to + # Python 3 we can reconsider this constraint. + pyinstaller < 4.0 # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1 From 39fc75a3fdbea0a32835143c70a5b2de0c7a8baf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 Aug 2020 13:39:55 -0400 Subject: [PATCH 0745/1054] Tweaks in preparation for Python 3 support. --- src/allmydata/util/encodingutil.py | 40 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index b31fd1205..2881039e8 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -3,6 +3,8 @@ Functions used to convert inputs from whatever encoding used in the system to unicode and back. """ +from past.builtins import unicode + import sys, os, re, locale from allmydata.util.assertutil import precondition, _assert @@ -89,7 +91,7 @@ def argv_to_unicode(s): """ Decode given argv element to unicode. If this fails, raise a UsageError. """ - precondition(isinstance(s, str), s) + precondition(isinstance(s, bytes), s) try: return unicode(s, io_encoding) @@ -134,19 +136,19 @@ def unicode_to_url(s): #precondition(isinstance(s, unicode), s) #return s.encode('utf-8') -def to_str(s): - if s is None or isinstance(s, str): +def to_str(s): # TODO rename to to_bytes + if s is None or isinstance(s, bytes): return s return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, (NoneType, str)), s) + precondition(isinstance(s, str) or s is None, s) if s is None: return s return s.decode('utf-8') -PRINTABLE_ASCII = re.compile(r'^[\n\r\x20-\x7E]*$', re.DOTALL) -PRINTABLE_8BIT = re.compile(r'^[\n\r\x20-\x7E\x80-\xFF]*$', re.DOTALL) +PRINTABLE_ASCII = re.compile(br'^[\n\r\x20-\x7E]*$', re.DOTALL) +PRINTABLE_8BIT = re.compile(br'^[\n\r\x20-\x7E\x80-\xFF]*$', re.DOTALL) def is_printable_ascii(s): return PRINTABLE_ASCII.search(s) is not None @@ -188,14 +190,14 @@ def _unicode_escape(m, quote_newlines): else: return u'\\x%02x' % (codepoint,) -def _str_escape(m, quote_newlines): +def _str_escape(m, quote_newlines): # TODO rename to _bytes_escape c = m.group(0) - if c == '"' or c == '$' or c == '`' or c == '\\': - return '\\' + c - elif c == '\n' and not quote_newlines: + if c == b'"' or c == b'$' or c == b'`' or c == b'\\': + return b'\\' + c + elif c == b'\n' and not quote_newlines: return c else: - return '\\x%02x' % (ord(c),) + return b'\\x%02x' % (ord(c),) MUST_DOUBLE_QUOTE_NL = re.compile(u'[^\\x20-\\x26\\x28-\\x7E\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFC]', re.DOTALL) MUST_DOUBLE_QUOTE = re.compile(u'[^\\n\\x20-\\x26\\x28-\\x7E\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFC]', re.DOTALL) @@ -205,7 +207,7 @@ ESCAPABLE_UNICODE = re.compile(u'([\uD800-\uDBFF][\uDC00-\uDFFF])|' # valid sur u'[^ !#\\x25-\\x5B\\x5D-\\x5F\\x61-\\x7E\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFC]', re.DOTALL) -ESCAPABLE_8BIT = re.compile( r'[^ !#\x25-\x5B\x5D-\x5F\x61-\x7E]', re.DOTALL) +ESCAPABLE_8BIT = re.compile( br'[^ !#\x25-\x5B\x5D-\x5F\x61-\x7E]', re.DOTALL) def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): """ @@ -221,11 +223,11 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): If not explicitly given, quote_newlines is True when quotemarks is True. """ - precondition(isinstance(s, (str, unicode)), s) + precondition(isinstance(s, (bytes, unicode)), s) if quote_newlines is None: quote_newlines = quotemarks - if isinstance(s, str): + if isinstance(s, bytes): try: s = s.decode('utf-8') except UnicodeDecodeError: @@ -235,18 +237,18 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): if must_double_quote.search(s) is None: try: out = s.encode(encoding or io_encoding) - if quotemarks or out.startswith('"'): - return "'%s'" % (out,) + if quotemarks or out.startswith(b'"'): + return b"'%s'" % (out,) else: return out except (UnicodeDecodeError, UnicodeEncodeError): pass escaped = ESCAPABLE_UNICODE.sub(lambda m: _unicode_escape(m, quote_newlines), s) - return '"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),) + return b'"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),) def quote_path(path, quotemarks=True): - return quote_output("/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True) + return quote_output(b"/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True) def quote_local_unicode_path(path, quotemarks=True): precondition(isinstance(path, unicode), path) @@ -290,7 +292,7 @@ def to_filepath(path): return FilePath(path) def _decode(s): - precondition(isinstance(s, basestring), s=s) + precondition(isinstance(s, (bytes, unicode)), s=s) if isinstance(s, bytes): return s.decode(filesystem_encoding) From 0f65d10f47abf761d27bd4fb1e12e4d442553472 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 13:25:08 -0400 Subject: [PATCH 0746/1054] news fragment --- newsfragments/3323.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3323.removed diff --git a/newsfragments/3323.removed b/newsfragments/3323.removed new file mode 100644 index 000000000..356b4b2af --- /dev/null +++ b/newsfragments/3323.removed @@ -0,0 +1 @@ +Slackware 14.2 is no longer a Tahoe-LAFS supported platform. From a74ef86098c8a2e9676c226d6861903c2fac7a9d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 13:25:16 -0400 Subject: [PATCH 0747/1054] Remove the CI configuration --- .circleci/Dockerfile.slackware | 49 ---------------------------------- .circleci/config.yml | 31 --------------------- 2 files changed, 80 deletions(-) delete mode 100644 .circleci/Dockerfile.slackware diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware deleted file mode 100644 index 73ba6b32d..000000000 --- a/.circleci/Dockerfile.slackware +++ /dev/null @@ -1,49 +0,0 @@ -ARG TAG -FROM vbatts/slackware:${TAG} - -ENV WHEELHOUSE_PATH /tmp/wheelhouse -ENV VIRTUALENV_PATH /tmp/venv -# This will get updated by the CircleCI checkout step. -ENV BUILD_SRC_ROOT /tmp/project - -# Be careful with slackpkg. If the package name given doesn't match anything, -# slackpkg still claims to succeed but you're totally screwed. Slackware -# updates versions of packaged software so including too much version prefix -# is a good way to have your install commands suddenly begin not installing -# anything. -RUN slackpkg update && \ - slackpkg install \ - openssh-7 git-2 \ - ca-certificates \ - sudo-1 \ - make-4 \ - automake-1 \ - kernel-headers \ - glibc-2 \ - binutils-2 \ - gcc-5 \ - gcc-g++-5 \ - python-2 \ - libffi-3 \ - libyaml-0 \ - sqlite-3 \ - icu4c-56 \ - libmpc-1 Date: Tue, 11 Aug 2020 14:04:52 -0400 Subject: [PATCH 0748/1054] Move connection_status tests into their own module. --- src/allmydata/test/test_connection_status.py | 112 +++++++++++++++++++ src/allmydata/test/test_connections.py | 104 ----------------- 2 files changed, 112 insertions(+), 104 deletions(-) create mode 100644 src/allmydata/test/test_connection_status.py diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py new file mode 100644 index 000000000..763c14c57 --- /dev/null +++ b/src/allmydata/test/test_connection_status.py @@ -0,0 +1,112 @@ +""" +Tests for allmydata.util.connection_status. +""" + +import mock + +from twisted.trial import unittest + +from ..util import connection_status + +class Status(unittest.TestCase): + def test_hint_statuses(self): + ncs = connection_status._hint_statuses(["h2","h1"], + {"h1": "hand1", "h4": "hand4"}, + {"h1": "st1", "h2": "st2", + "h3": "st3"}) + self.assertEqual(ncs, {"h1 via hand1": "st1", + "h2": "st2"}) + + def test_reconnector_connected(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_others(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_listener(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.listenerStatus = ("listener1", "successful") + ci.winningHint = None + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected via listener (listener1)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connecting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "connecting" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, "Trying to connect") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_waiting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "waiting" + ri.lastAttempt = 10 + ri.nextAttempt = 20 + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + with mock.patch("time.time", return_value=12): + cs = connection_status.from_foolscap_reconnector(rc, 5) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, + "Reconnecting in 8 seconds (last attempt 2s ago)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 5) diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 3e2806dd0..9b5bd7f30 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -7,7 +7,6 @@ from foolscap.connections import tcp from ..node import PrivacyError, config_from_string from ..node import create_connection_handlers from ..node import create_main_tub, _tub_portlocation -from ..util import connection_status from ..util.i2p_provider import create as create_i2p_provider from ..util.tor_provider import create as create_tor_provider @@ -463,106 +462,3 @@ class Privacy(unittest.TestCase): str(ctx.exception), "tub.location includes tcp: hint", ) - -class Status(unittest.TestCase): - def test_hint_statuses(self): - ncs = connection_status._hint_statuses(["h2","h1"], - {"h1": "hand1", "h4": "hand4"}, - {"h1": "st1", "h2": "st2", - "h3": "st3"}) - self.assertEqual(ncs, {"h1 via hand1": "st1", - "h2": "st2"}) - - def test_reconnector_connected(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_others(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_listener(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.listenerStatus = ("listener1", "successful") - ci.winningHint = None - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected via listener (listener1)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connecting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "connecting" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, "Trying to connect") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_waiting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "waiting" - ri.lastAttempt = 10 - ri.nextAttempt = 20 - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - with mock.patch("time.time", return_value=12): - cs = connection_status.from_foolscap_reconnector(rc, 5) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, - "Reconnecting in 8 seconds (last attempt 2s ago)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 5) From 243d02ecb1a01216fc16331ec25a04186dfce063 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 14:08:51 -0400 Subject: [PATCH 0749/1054] Port to Python 3. --- src/allmydata/test/test_connection_status.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index 763c14c57..2bd8bf6ab 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.connection_status. + +Port to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import mock diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 091c248af..f074927b6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -50,6 +50,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_connection_status", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", From a08cde9a4da3bf636560fc72e2297a371953b38e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 14:30:16 -0400 Subject: [PATCH 0750/1054] Port to Python 3. --- src/allmydata/interfaces.py | 1 + src/allmydata/util/_python3.py | 1 + src/allmydata/util/connection_status.py | 22 ++++++++++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36eb55bfb..c93c1d81d 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,3 +1,4 @@ +from past.builtins import long from zope.interface import Interface, Attribute from twisted.plugin import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f074927b6..494d5e161 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -28,6 +28,7 @@ PORTED_MODULES = [ "allmydata.util.assertutil", "allmydata.util.base32", "allmydata.util.base62", + "allmydata.util.connection_status", "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.gcutil", diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 44c12f220..0e8595e81 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -1,3 +1,18 @@ +""" +Parse connection status from Foolscap. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from zope.interface import implementer from ..interfaces import IConnectionStatus @@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses): def from_foolscap_reconnector(rc, last_received): ri = rc.getReconnectionInfo() - # See foolscap/reconnector.py, ReconnectionInfo, for details about - # possible states. + # See foolscap/reconnector.py, ReconnectionInfo, for details about possible + # states. The returned result is a native string, it seems, so convert to + # unicode. state = ri.state + if isinstance(state, bytes): # Python 2 + state = str(state, "ascii") if state == "unstarted": return ConnectionStatus.unstarted() From 504258622c8e3cd87a2a4fdcc3b1ac1902618c78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 14:54:12 -0400 Subject: [PATCH 0751/1054] Docstring. --- src/allmydata/immutable/happiness_upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index a177660cc..9716aaef2 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,3 +1,9 @@ +""" +Algorithms for figuring out happiness, the number of unique nodes the data is +on. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function From e24c21bef779de6d1d419b2135231f628a0bcc20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 15:38:58 -0400 Subject: [PATCH 0752/1054] Make configutil tests more standalone, and less repetitive. --- src/allmydata/test/cli/test_create.py | 27 ++++++++ src/allmydata/test/test_configutil.py | 90 +++++++-------------------- 2 files changed, 49 insertions(+), 68 deletions(-) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 75162c39e..f013c0205 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -6,6 +6,8 @@ from twisted.python import usage from allmydata.util import configutil from ..common_util import run_cli, parse_cli from ...scripts import create_node +from ... import client + def read_config(basedir): tahoe_cfg = os.path.join(basedir, "tahoe.cfg") @@ -33,6 +35,31 @@ class Config(unittest.TestCase): e = self.assertRaises(usage.UsageError, parse_cli, verb, *args) self.assertIn("option %s not recognized" % (option,), str(e)) + def test_create_client_config(self): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + opts = {"nickname": "nick", + "webport": "tcp:3456", + "hide-ip": False, + "listen": "none", + "shares-needed": "1", + "shares-happy": "1", + "shares-total": "1", + } + create_node.write_node_config(f, opts) + create_node.write_client_config(f, opts) + + config = configutil.get_config(fname) + # should succeed, no exceptions + configutil.validate_config( + fname, + config, + client._valid_config(), + ) + @defer.inlineCallbacks def test_client(self): basedir = self.mktemp() diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index 45eb6ac25..ffb5f5320 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -3,12 +3,9 @@ import os.path from twisted.trial import unittest from allmydata.util import configutil -from allmydata.test.no_network import GridTestMixin -from ..scripts import create_node -from .. import client -class ConfigUtilTests(GridTestMixin, unittest.TestCase): +class ConfigUtilTests(unittest.TestCase): def setUp(self): super(ConfigUtilTests, self).setUp() self.static_valid_config = configutil.ValidConfiguration( @@ -20,10 +17,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): lambda section_name, item_name: (section_name, item_name) == ("node", "valid"), ) + def create_tahoe_cfg(self, cfg): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + with open(fname, "w") as f: + f.write(cfg) + return fname + def test_config_utils(self): - self.basedir = "cli/ConfigUtilTests/test-config-utils" - self.set_up_grid(oneshare=True) - tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg") + tahoe_cfg = self.create_tahoe_cfg("""\ +[node] +nickname = client-0 +web.port = adopt-socket:fd=5 +[storage] +enabled = false +""") # test that at least one option was read correctly config = configutil.get_config(tahoe_cfg) @@ -45,12 +54,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.failUnlessEqual(config.get("node", "descriptor"), descriptor) def test_config_validation_success(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -66,12 +70,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): validation but are matched by the dynamic validation is considered valid. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -82,12 +81,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): ) def test_config_validation_invalid_item(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -103,12 +97,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -124,12 +113,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -145,12 +129,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section, item pair that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -160,28 +139,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.dynamic_valid_config, ) self.assertIn("section [node] contains unknown option 'invalid'", str(e)) - - def test_create_client_config(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - opts = {"nickname": "nick", - "webport": "tcp:3456", - "hide-ip": False, - "listen": "none", - "shares-needed": "1", - "shares-happy": "1", - "shares-total": "1", - } - create_node.write_node_config(f, opts) - create_node.write_client_config(f, opts) - - config = configutil.get_config(fname) - # should succeed, no exceptions - configutil.validate_config( - fname, - config, - client._valid_config(), - ) From babe2dbc85d8c6d87bec5e1a919faf71f854b596 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 15:45:52 -0400 Subject: [PATCH 0753/1054] Port to Python 3. --- src/allmydata/test/test_configutil.py | 15 +++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 16 insertions(+) diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index ffb5f5320..c57381289 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -1,3 +1,18 @@ +""" +Tests for allmydata.util.configutil. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Omitted dict, cause worried about interactions. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 494d5e161..b6c4b6c87 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -51,6 +51,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_configutil", "allmydata.test.test_connection_status", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", From 11b934120c1b5f279e4c84c1d94de8bec9ea2bc8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 15:49:59 -0400 Subject: [PATCH 0754/1054] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/configutil.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b6c4b6c87..947fcf00b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -28,6 +28,7 @@ PORTED_MODULES = [ "allmydata.util.assertutil", "allmydata.util.base32", "allmydata.util.base62", + "allmydata.util.configutil", "allmydata.util.connection_status", "allmydata.util.deferredutil", "allmydata.util.dictutil", diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 3699db35d..295605b65 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -1,5 +1,13 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from ConfigParser import SafeConfigParser +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from configparser import SafeConfigParser import attr @@ -13,11 +21,7 @@ class UnknownConfigError(Exception): def get_config(tahoe_cfg): config = SafeConfigParser() - with open(tahoe_cfg, "rb") as f: - # Skip any initial Byte Order Mark. Since this is an ordinary file, we - # don't need to handle incomplete reads, and can assume seekability. - if f.read(3) != '\xEF\xBB\xBF': - f.seek(0) + with open(tahoe_cfg, "r") as f: config.readfp(f) return config @@ -28,7 +32,7 @@ def set_config(config, section, option, value): assert config.get(section, option) == value def write_config(tahoe_cfg, config): - with open(tahoe_cfg, "wb") as f: + with open(tahoe_cfg, "w") as f: config.write(f) def validate_config(fname, cfg, valid_config): From 277298050f16ab3794868f377b6a617820baefe7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 15:51:27 -0400 Subject: [PATCH 0755/1054] News file and ratchet. --- misc/python3/ratchet-passing | 13 +++++++++++++ newsfragments/3377.minor | 0 2 files changed, 13 insertions(+) create mode 100644 newsfragments/3377.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bbc6e6c7e..e6b0d60c2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -24,6 +24,19 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success +allmydata.test.test_configutil.ConfigUtilTests.test_config_utils +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success +allmydata.test.test_connection_status.Status.test_hint_statuses +allmydata.test.test_connection_status.Status.test_reconnector_connected +allmydata.test.test_connection_status.Status.test_reconnector_connected_listener +allmydata.test.test_connection_status.Status.test_reconnector_connected_others +allmydata.test.test_connection_status.Status.test_reconnector_connecting +allmydata.test.test_connection_status.Status.test_reconnector_waiting allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes allmydata.test.test_crypto.TestEd25519.test_key_serialization diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor new file mode 100644 index 000000000..e69de29bb From 10378541d72a6853b4cc1f3c86482979f14b3827 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 16:40:00 -0400 Subject: [PATCH 0756/1054] Use Python 2 ConfigParser, so correct exceptions get raised. --- src/allmydata/util/configutil.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 295605b65..79980b006 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -7,10 +7,17 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from configparser import SafeConfigParser +if PY2: + # In theory on Python 2 configparser also works, but then code gets the + # wrong exceptions and they don't get handled. So just use native parser + # for now. + from ConfigParser import SafeConfigParser +else: + from configparser import SafeConfigParser import attr + class UnknownConfigError(Exception): """ An unknown config item was found. From 9ad5602477a48b7da9b9e73f138d6fbe74de66d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Aug 2020 16:50:42 -0400 Subject: [PATCH 0757/1054] Fix test failures. --- src/allmydata/test/common_py3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index b29cb9691..157f22fdf 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -9,7 +9,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2 +from future.utils import PY2, native_str, native_bytes if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -72,7 +72,7 @@ class ShouldFailMixin(object): def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs): - assert substring is None or isinstance(substring, str) + assert substring is None or isinstance(substring, (native_str, native_bytes)) d = defer.maybeDeferred(callable, *args, **kwargs) def done(res): if isinstance(res, failure.Failure): From 513177c4b86469874aa93d3fdbe33a34f6a9ecc2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 18:33:01 -0400 Subject: [PATCH 0758/1054] strace not used by anything now --- .circleci/Dockerfile.ubuntu | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 98c3b5088..2fcc60f5a 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -13,7 +13,6 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install git && \ apt-get --quiet --yes install \ sudo \ - strace \ build-essential \ python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev \ From 9c4e68e0a7e65717e806e39fccb90181de17a34a Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 12 Aug 2020 15:33:54 +0200 Subject: [PATCH 0759/1054] maybe that's how you use subunit2pyunit --- misc/python3/ratchet.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index a6a2c53c3..aeffe2ff2 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -13,6 +13,7 @@ cd "../.." set +e SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" +subunit2pyunit < "$base/results.subunit2" set -e # Okay, now we're clear. From 14b273953acffb4d44dc9ca30d16f60b70864b14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 10:10:18 -0400 Subject: [PATCH 0760/1054] Better fix, maybe. --- src/allmydata/test/common_py3.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 157f22fdf..b4e9fb058 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -9,9 +9,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2, native_str, native_bytes +from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import unicode import os import time @@ -72,7 +73,7 @@ class ShouldFailMixin(object): def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs): - assert substring is None or isinstance(substring, (native_str, native_bytes)) + assert substring is None or isinstance(substring, (bytes, unicode)) d = defer.maybeDeferred(callable, *args, **kwargs) def done(res): if isinstance(res, failure.Failure): From e13c0cf4d34dacf00b51c0d788fd859197467ac2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 10:23:30 -0400 Subject: [PATCH 0761/1054] Reorganize imports to make Python 3 support easier. --- src/allmydata/dirnode.py | 10 ++-------- src/allmydata/test/common_py3.py | 23 +++++++++++++++++++++++ src/allmydata/test/common_util.py | 23 ++++------------------- src/allmydata/test/test_encodingutil.py | 10 +++++----- src/allmydata/util/encodingutil.py | 7 +++++++ 5 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index f1c95697b..38cb26caf 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -1,5 +1,5 @@ """Directory Node implementation.""" -import time, unicodedata +import time from zope.interface import implementer from twisted.internet import defer @@ -18,7 +18,7 @@ from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor from allmydata.util import hashutil, base32, log -from allmydata.util.encodingutil import quote_output +from allmydata.util.encodingutil import quote_output, normalize from allmydata.util.assertutil import precondition from allmydata.util.netstring import netstring, split_netstring from allmydata.util.consumer import download_to_data @@ -101,12 +101,6 @@ def update_metadata(metadata, new_metadata, now): return metadata -# 'x' at the end of a variable name indicates that it holds a Unicode string that may not -# be NFC-normalized. - -def normalize(namex): - return unicodedata.normalize('NFC', namex) - # TODO: {Deleter,MetadataSetter,Adder}.modify all start by unpacking the # contents and end by repacking them. It might be better to apply them to # the unpacked contents. diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 0dae05aa6..94f63b4da 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -13,11 +13,17 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import unicode + import os import time import signal from twisted.internet import reactor +from twisted.trial import unittest + +from ..util.assertutil import precondition +from ..util.encodingutil import unicode_platform, get_filesystem_encoding class TimezoneMixin(object): @@ -65,3 +71,20 @@ class SignalMixin(object): if self.sigchldHandler: signal.signal(signal.SIGCHLD, self.sigchldHandler) return super(SignalMixin, self).tearDown() + + +class ReallyEqualMixin(object): + def failUnlessReallyEqual(self, a, b, msg=None): + self.assertEqual(a, b, msg) + self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) + + +def skip_if_cannot_represent_filename(u): + precondition(isinstance(u, unicode)) + + enc = get_filesystem_encoding() + if not unicode_platform(): + try: + u.encode(enc) + except UnicodeEncodeError: + raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 4e25e897e..e16cca835 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -9,24 +9,15 @@ from twisted.python import failure from twisted.trial import unittest from ..util.assertutil import precondition -from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, - - get_io_encoding) +from allmydata.util.encodingutil import get_io_encoding from future.utils import PY2 if PY2: # XXX this is a hack that makes some tests pass on Python3, remove # in the future from ..scripts import runner -from .common_py3 import SignalMixin +from .common_py3 import ( + SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, +) -def skip_if_cannot_represent_filename(u): - precondition(isinstance(u, unicode)) - - enc = get_filesystem_encoding() - if not unicode_platform(): - try: - u.encode(enc) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") def skip_if_cannot_represent_argv(u): precondition(isinstance(u, unicode)) @@ -87,12 +78,6 @@ def flip_one_bit(s, offset=0, size=None): return result -class ReallyEqualMixin(object): - def failUnlessReallyEqual(self, a, b, msg=None): - self.assertEqual(a, b, msg) - self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) - - class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 771e9d879..d0b9793c4 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -64,15 +64,15 @@ from twisted.trial import unittest from twisted.python.filepath import FilePath -from allmydata.test.common_util import ReallyEqualMixin +from allmydata.test.common_py3 import ( + ReallyEqualMixin, skip_if_cannot_represent_filename, +) from allmydata.util import encodingutil, fileutil from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \ unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \ get_io_encoding, get_filesystem_encoding, to_str, from_utf8_or_none, _reload, \ to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from -from allmydata.dirnode import normalize -from .common_util import skip_if_cannot_represent_filename from twisted.python import usage @@ -258,7 +258,7 @@ class EncodingUtil(ReallyEqualMixin): _reload() filenames = listdir_unicode(u'/dummy') - self.failUnlessEqual(set([normalize(fname) for fname in filenames]), + self.failUnlessEqual(set([encodingutil.normalize(fname) for fname in filenames]), set(TEST_FILENAMES)) @@ -283,7 +283,7 @@ class StdlibUnicode(unittest.TestCase): # We only require that the listing includes a filename that is canonically equivalent # to lumiere_nfc (on Mac OS X, it will be the NFD equivalent). - self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames])) + self.failUnlessIn(lumiere_nfc + ".txt", set([encodingutil.normalize(fname) for fname in filenames])) expanded = fileutil.expanduser(u"~/" + lumiere_nfc) self.failIfIn(u"~", expanded) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 2881039e8..0ec6511bf 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -6,6 +6,7 @@ unicode and back. from past.builtins import unicode import sys, os, re, locale +import unicodedata from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -358,3 +359,9 @@ def listdir_unicode(path): def listdir_filepath(fp): return listdir_unicode(unicode_from_filepath(fp)) + + +# 'x' at the end of a variable name indicates that it holds a Unicode string that may not +# be NFC-normalized. +def normalize(namex): + return unicodedata.normalize('NFC', namex) From 32a17caddbc23205358b20853e06c398b37d5af4 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 12 Aug 2020 17:50:40 +0200 Subject: [PATCH 0762/1054] maybe that's how it should be? --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index aa768cd06..21fdb884f 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,7 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial --reporter=subunitv2-file allmydata +SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter=subunitv2-file allmydata subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From a4a44fe112c5d79aa02b4c8a4090031763cec139 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 12 Aug 2020 18:02:00 +0200 Subject: [PATCH 0763/1054] make some changes to an opaque binary file --- misc/python3/ratchet-passing | 3 --- 1 file changed, 3 deletions(-) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 200126859..1c2d1eda8 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -166,9 +166,6 @@ allmydata.test.test_util.FileUtil.test_windows_expanduser_win7 allmydata.test.test_util.FileUtil.test_windows_expanduser_xp allmydata.test.test_util.FileUtil.test_write_atomically allmydata.test.test_util.IDLib.test_nodeid_b2a -allmydata.test.test_util.Limiter.test_errors -allmydata.test.test_util.Limiter.test_limiter -allmydata.test.test_util.Log.test_err allmydata.test.test_util.Math.test_round_sigfigs allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True allmydata.test.test_util.PollMixinTests.test_PollMixin_True From c37a2b0a895702a07d4a3bc9a594c467ee722396 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 12 Aug 2020 18:15:14 +0200 Subject: [PATCH 0764/1054] maybe this is the magic incantation --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 21fdb884f..00f9853c0 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -12,7 +12,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter=subunitv2-file allmydata -subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" +subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" set -e # Okay, now we're clear. From 820586cadba69f644cd78ec0fb6865865bced861 Mon Sep 17 00:00:00 2001 From: Maciej Fijalkowski Date: Wed, 12 Aug 2020 18:27:50 +0200 Subject: [PATCH 0765/1054] try a bit harder to properly merge master --- misc/python3/ratchet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 00f9853c0..aa768cd06 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,8 +11,8 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter=subunitv2-file allmydata -subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" +trial --reporter=subunitv2-file allmydata +subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e # Okay, now we're clear. From 53bdc10e195228daf10a1af1cd244148edd42d35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 13:10:10 -0400 Subject: [PATCH 0766/1054] Some steps towards Python 3 support. --- src/allmydata/test/test_encodingutil.py | 76 ++++++++++++++----------- src/allmydata/util/encodingutil.py | 13 +++-- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index d0b9793c4..e0adc2dbf 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,5 +1,9 @@ from __future__ import print_function +from future.utils import PY2 + +from past.builtins import unicode + lumiere_nfc = u"lumi\u00E8re" Artonwall_nfc = u"\u00C4rtonwall.mp3" Artonwall_nfd = u"A\u0308rtonwall.mp3" @@ -44,7 +48,7 @@ if __name__ == "__main__": open(os.path.join(tmpdir, fname), 'w').close() # Use Unicode API under Windows or MacOS X - if sys.platform in ('win32', 'darwin'): + if PY2 and sys.platform in ('win32', 'darwin'): dirlist = os.listdir(unicode(tmpdir)) else: dirlist = os.listdir(tmpdir) @@ -278,7 +282,11 @@ class StdlibUnicode(unittest.TestCase): fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt' open(fn, 'wb').close() self.failUnless(os.path.exists(fn)) - self.failUnless(os.path.exists(os.path.join(os.getcwdu(), fn))) + if PY2: + getcwdu = os.getcwdu + else: + getcwdu = os.getcwd + self.failUnless(os.path.exists(os.path.join(getcwdu(), fn))) filenames = listdir_unicode(lumiere_nfc) # We only require that the listing includes a filename that is canonically equivalent @@ -314,7 +322,7 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(quote_output(inp, encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) if out[0:2] == 'b"': pass - elif isinstance(inp, str): + elif isinstance(inp, bytes): self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quote_newlines=quote_newlines), out) self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) else: @@ -326,47 +334,47 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): self._check(inp, out, enc, optional_quotes, quote_newlines) # optional single quotes - check("foo", "'foo'", True) - check("\\", "'\\'", True) - check("$\"`", "'$\"`'", True) - check("\n", "'\n'", True, quote_newlines=False) + check(b"foo", b"'foo'", True) + check(b"\\", b"'\\'", True) + check(b"$\"`", b"'$\"`'", True) + check(b"\n", b"'\n'", True, quote_newlines=False) # mandatory single quotes - check("\"", "'\"'") + check(b"\"", b"'\"'") # double quotes - check("'", "\"'\"") - check("\n", "\"\\x0a\"", quote_newlines=True) - check("\x00", "\"\\x00\"") + check(b"'", b"\"'\"") + check(b"\n", b"\"\\x0a\"", quote_newlines=True) + check(b"\x00", b"\"\\x00\"") # invalid Unicode and astral planes - check(u"\uFDD0\uFDEF", "\"\\ufdd0\\ufdef\"") - check(u"\uDC00\uD800", "\"\\udc00\\ud800\"") - check(u"\uDC00\uD800\uDC00", "\"\\udc00\\U00010000\"") - check(u"\uD800\uDC00", "\"\\U00010000\"") - check(u"\uD800\uDC01", "\"\\U00010001\"") - check(u"\uD801\uDC00", "\"\\U00010400\"") - check(u"\uDBFF\uDFFF", "\"\\U0010ffff\"") - check(u"'\uDBFF\uDFFF", "\"'\\U0010ffff\"") - check(u"\"\uDBFF\uDFFF", "\"\\\"\\U0010ffff\"") + check(u"\uFDD0\uFDEF", b"\"\\ufdd0\\ufdef\"") + check(u"\uDC00\uD800", b"\"\\udc00\\ud800\"") + check(u"\uDC00\uD800\uDC00", b"\"\\udc00\\U00010000\"") + check(u"\uD800\uDC00", b"\"\\U00010000\"") + check(u"\uD800\uDC01", b"\"\\U00010001\"") + check(u"\uD801\uDC00", b"\"\\U00010400\"") + check(u"\uDBFF\uDFFF", b"\"\\U0010ffff\"") + check(u"'\uDBFF\uDFFF", b"\"'\\U0010ffff\"") + check(u"\"\uDBFF\uDFFF", b"\"\\\"\\U0010ffff\"") # invalid UTF-8 - check("\xFF", "b\"\\xff\"") - check("\x00\"$\\`\x80\xFF", "b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"") + check("\xFF", b"b\"\\xff\"") + check("\x00\"$\\`\x80\xFF", b"b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"") def test_quote_output_ascii(self, enc='ascii'): def check(inp, out, optional_quotes=False, quote_newlines=None): self._check(inp, out, enc, optional_quotes, quote_newlines) self._test_quote_output_all(enc) - check(u"\u00D7", "\"\\xd7\"") - check(u"'\u00D7", "\"'\\xd7\"") - check(u"\"\u00D7", "\"\\\"\\xd7\"") - check(u"\u2621", "\"\\u2621\"") - check(u"'\u2621", "\"'\\u2621\"") - check(u"\"\u2621", "\"\\\"\\u2621\"") - check(u"\n", "'\n'", True, quote_newlines=False) - check(u"\n", "\"\\x0a\"", quote_newlines=True) + check(u"\u00D7", b"\"\\xd7\"") + check(u"'\u00D7", b"\"'\\xd7\"") + check(u"\"\u00D7", b"\"\\\"\\xd7\"") + check(u"\u2621", b"\"\\u2621\"") + check(u"'\u2621", b"\"'\\u2621\"") + check(u"\"\u2621", b"\"\\\"\\u2621\"") + check(u"\n", b"'\n'", True, quote_newlines=False) + check(u"\n", b"\"\\x0a\"", quote_newlines=True) def test_quote_output_latin1(self, enc='latin1'): def check(inp, out, optional_quotes=False, quote_newlines=None): @@ -548,14 +556,14 @@ class OpenBSD(EncodingUtil, unittest.TestCase): class TestToFromStr(ReallyEqualMixin, unittest.TestCase): def test_to_str(self): - self.failUnlessReallyEqual(to_str("foo"), "foo") - self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), "lumi\xc3\xa8re") + self.failUnlessReallyEqual(to_str("foo"), b"foo") + self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), b"lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str("\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? - self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), "lumi\xc3\xa8re") + self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), b"lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str(None), None) def test_from_utf8_or_none(self): self.failUnlessRaises(AssertionError, from_utf8_or_none, u"foo") - self.failUnlessReallyEqual(from_utf8_or_none("lumi\xc3\xa8re"), u"lumi\u00E8re") + self.failUnlessReallyEqual(from_utf8_or_none(b"lumi\xc3\xa8re"), u"lumi\u00E8re") self.failUnlessReallyEqual(from_utf8_or_none(None), None) self.failUnlessRaises(UnicodeDecodeError, from_utf8_or_none, "\xFF") diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 0ec6511bf..d795eb43c 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -3,6 +3,8 @@ Functions used to convert inputs from whatever encoding used in the system to unicode and back. """ +from future.utils import PY2 + from past.builtins import unicode import sys, os, re, locale @@ -65,13 +67,13 @@ def _reload(): check_encoding(io_encoding) - is_unicode_platform = sys.platform in ["win32", "darwin"] + is_unicode_platform = not PY2 or sys.platform in ["win32", "darwin"] # Despite the Unicode-mode FilePath support added to Twisted in # , we can't yet use # Unicode-mode FilePaths with INotify on non-Windows platforms # due to . - use_unicode_filepath = sys.platform == "win32" + use_unicode_filepath = not PY2 or sys.platform == "win32" _reload() @@ -92,6 +94,9 @@ def argv_to_unicode(s): """ Decode given argv element to unicode. If this fails, raise a UsageError. """ + if isinstance(s, unicode): + return s + precondition(isinstance(s, bytes), s) try: @@ -122,7 +127,7 @@ def unicode_to_argv(s, mangle=False): if mangle and sys.platform == "win32": # This must be the same as 'mangle' in bin/tahoe-script.template. - return str(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s)) + return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s)) else: return s.encode(io_encoding) @@ -143,7 +148,7 @@ def to_str(s): # TODO rename to to_bytes return s.encode('utf-8') def from_utf8_or_none(s): - precondition(isinstance(s, str) or s is None, s) + precondition(isinstance(s, bytes) or s is None, s) if s is None: return s return s.decode('utf-8') From f7e28ffb0b7ae8b3f2433a8fdc63cae87a832515 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 14:18:02 -0400 Subject: [PATCH 0767/1054] More progress towards Python 3. --- src/allmydata/test/test_encodingutil.py | 79 +++++++++++++++---------- src/allmydata/util/encodingutil.py | 2 +- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index e0adc2dbf..7fe497cde 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,6 +1,6 @@ from __future__ import print_function -from future.utils import PY2 +from future.utils import PY2, PY3 from past.builtins import unicode @@ -63,6 +63,7 @@ if __name__ == "__main__": import os, sys, locale +from unittest import skipIf from twisted.trial import unittest @@ -163,6 +164,7 @@ class EncodingUtilNonUnicodePlatform(unittest.TestCase): sys.platform = self.original_platform _reload() + @skipIf(PY3, "Python 3 has its own way of dealing with this.") def test_listdir_unicode(self): # What happens if latin1-encoded filenames are encountered on an UTF-8 # filesystem? @@ -228,7 +230,8 @@ class EncodingUtil(ReallyEqualMixin): _reload() self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv) - def test_unicode_platform(self): + @skipIf(PY3, "It's always Unicode on Python 2.") + def test_unicode_platform_py2(self): matrix = { 'linux2': False, 'linux3': False, @@ -240,6 +243,11 @@ class EncodingUtil(ReallyEqualMixin): _reload() self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform]) + @skipIf(PY2, "Python 3 isn't Python 2.") + def test_unicode_platform_py3(self): + _reload() + self.failUnlessReallyEqual(unicode_platform(), True) + def test_listdir_unicode(self): if 'dirlist' not in dir(self): return @@ -323,11 +331,22 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): if out[0:2] == 'b"': pass elif isinstance(inp, bytes): - self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quote_newlines=quote_newlines), out) - self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) + try: + unicode_inp = inp.decode("utf-8") + except UnicodeDecodeError: + # Some things decode on Python 2, but not Python 3... + return + self.failUnlessReallyEqual(quote_output(unicode_inp, encoding=enc, quote_newlines=quote_newlines), out) + self.failUnlessReallyEqual(quote_output(unicode_inp, encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) else: - self.failUnlessReallyEqual(quote_output(inp.encode('utf-8'), encoding=enc, quote_newlines=quote_newlines), out) - self.failUnlessReallyEqual(quote_output(inp.encode('utf-8'), encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) + try: + bytes_inp = inp.encode('utf-8') + except UnicodeEncodeError: + # Some things encode on Python 2, but not Python 3, e.g. + # surrogates like u"\uDC00\uD800"... + return + self.failUnlessReallyEqual(quote_output(bytes_inp, encoding=enc, quote_newlines=quote_newlines), out) + self.failUnlessReallyEqual(quote_output(bytes_inp, encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2) def _test_quote_output_all(self, enc): def check(inp, out, optional_quotes=False, quote_newlines=None): @@ -359,8 +378,8 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): check(u"\"\uDBFF\uDFFF", b"\"\\\"\\U0010ffff\"") # invalid UTF-8 - check("\xFF", b"b\"\\xff\"") - check("\x00\"$\\`\x80\xFF", b"b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"") + check(b"\xFF", b"b\"\\xff\"") + check(b"\x00\"$\\`\x80\xFF", b"b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"") def test_quote_output_ascii(self, enc='ascii'): def check(inp, out, optional_quotes=False, quote_newlines=None): @@ -419,43 +438,43 @@ def win32_other(win32, other): class QuotePaths(ReallyEqualMixin, unittest.TestCase): def test_quote_path(self): - self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'") - self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'") - self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar") - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), '"foo/\\x0abar"') - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"') - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"') + self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), b"'foo/bar'") + self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'") + self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar") + self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), b'"foo/\\x0abar"') + self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"') + self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), b'"foo/\\x0abar"') self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"), - win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")) + win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'")) self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True), - win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")) + win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'")) self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False), - win32_other("C:\\foo", "\\\\?\\C:\\foo")) + win32_other(b"C:\\foo", b"\\\\?\\C:\\foo")) self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), - win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'")) + win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'")) self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True), - win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'")) + win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'")) self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False), - win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar")) + win32_other(b"\\\\foo\\bar", b"\\\\?\\UNC\\foo\\bar")) def test_quote_filepath(self): foo_bar_fp = FilePath(win32_other(u'C:\\foo\\bar', u'/foo/bar')) self.failUnlessReallyEqual(quote_filepath(foo_bar_fp), - win32_other("'C:\\foo\\bar'", "'/foo/bar'")) + win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'")) self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=True), - win32_other("'C:\\foo\\bar'", "'/foo/bar'")) + win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'")) self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=False), - win32_other("C:\\foo\\bar", "/foo/bar")) + win32_other(b"C:\\foo\\bar", b"/foo/bar")) if sys.platform == "win32": foo_longfp = FilePath(u'\\\\?\\C:\\foo') self.failUnlessReallyEqual(quote_filepath(foo_longfp), - "'C:\\foo'") + b"'C:\\foo'") self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=True), - "'C:\\foo'") + b"'C:\\foo'") self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=False), - "C:\\foo") + b"C:\\foo") class FilePaths(ReallyEqualMixin, unittest.TestCase): @@ -556,9 +575,9 @@ class OpenBSD(EncodingUtil, unittest.TestCase): class TestToFromStr(ReallyEqualMixin, unittest.TestCase): def test_to_str(self): - self.failUnlessReallyEqual(to_str("foo"), b"foo") - self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), b"lumi\xc3\xa8re") - self.failUnlessReallyEqual(to_str("\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? + self.failUnlessReallyEqual(to_str(b"foo"), b"foo") + self.failUnlessReallyEqual(to_str(b"lumi\xc3\xa8re"), b"lumi\xc3\xa8re") + self.failUnlessReallyEqual(to_str(b"\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), b"lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str(None), None) @@ -566,4 +585,4 @@ class TestToFromStr(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(AssertionError, from_utf8_or_none, u"foo") self.failUnlessReallyEqual(from_utf8_or_none(b"lumi\xc3\xa8re"), u"lumi\u00E8re") self.failUnlessReallyEqual(from_utf8_or_none(None), None) - self.failUnlessRaises(UnicodeDecodeError, from_utf8_or_none, "\xFF") + self.failUnlessRaises(UnicodeDecodeError, from_utf8_or_none, b"\xFF") diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index d795eb43c..67c421ffe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -237,7 +237,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): try: s = s.decode('utf-8') except UnicodeDecodeError: - return 'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _str_escape(m, quote_newlines), s),) + return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _str_escape(m, quote_newlines), s),) must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE if must_double_quote.search(s) is None: From e9d67b8d0f0b6cb1034065e8c9fa256ea34bab7a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 14:32:22 -0400 Subject: [PATCH 0768/1054] Even more progress towards Python 3 support. --- src/allmydata/test/test_encodingutil.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 7fe497cde..1e85fd621 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -155,6 +155,7 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase): # The following tests apply only to platforms that don't store filenames as # Unicode entities on the filesystem. class EncodingUtilNonUnicodePlatform(unittest.TestCase): + @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.") def setUp(self): # Mock sys.platform because unicode_platform() uses it self.original_platform = sys.platform @@ -164,7 +165,6 @@ class EncodingUtilNonUnicodePlatform(unittest.TestCase): sys.platform = self.original_platform _reload() - @skipIf(PY3, "Python 3 has its own way of dealing with this.") def test_listdir_unicode(self): # What happens if latin1-encoded filenames are encountered on an UTF-8 # filesystem? @@ -260,7 +260,14 @@ class EncodingUtil(ReallyEqualMixin): % (self.filesystem_encoding,)) def call_os_listdir(path): - return self.dirlist + if PY2: + return self.dirlist + else: + # Python 3 always lists unicode filenames: + return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) + else d + for d in self.dirlist] + self.patch(os, 'listdir', call_os_listdir) def call_sys_getfilesystemencoding(): @@ -528,23 +535,23 @@ class FilePaths(ReallyEqualMixin, unittest.TestCase): class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' - argv = 'lumi\xc3\xa8re' + argv = b'lumi\xc3\xa8re' platform = 'linux2' filesystem_encoding = 'UTF-8' io_encoding = 'UTF-8' - dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt'] + dirlist = [b'test_file', b'\xc3\x84rtonwall.mp3', b'Blah blah.txt'] class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' - argv = 'lumi\xe8re' + argv = b'lumi\xe8re' platform = 'linux2' filesystem_encoding = 'ISO-8859-1' io_encoding = 'ISO-8859-1' - dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3'] + dirlist = [b'test_file', b'Blah blah.txt', b'\xc4rtonwall.mp3'] class Windows(EncodingUtil, unittest.TestCase): uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD' - argv = 'lumi\xc3\xa8re' + argv = b'lumi\xc3\xa8re' platform = 'win32' filesystem_encoding = 'mbcs' io_encoding = 'utf-8' @@ -552,7 +559,7 @@ class Windows(EncodingUtil, unittest.TestCase): class MacOSXLeopard(EncodingUtil, unittest.TestCase): uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc' - output = 'lumi\xc3\xa8re' + output = b'lumi\xc3\xa8re' platform = 'darwin' filesystem_encoding = 'utf-8' io_encoding = 'UTF-8' @@ -577,7 +584,7 @@ class TestToFromStr(ReallyEqualMixin, unittest.TestCase): def test_to_str(self): self.failUnlessReallyEqual(to_str(b"foo"), b"foo") self.failUnlessReallyEqual(to_str(b"lumi\xc3\xa8re"), b"lumi\xc3\xa8re") - self.failUnlessReallyEqual(to_str(b"\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? + self.failUnlessReallyEqual(to_str(b"\xFF"), b"\xFF") # passes through invalid UTF-8 -- is this what we want? self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), b"lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str(None), None) From b04a65fc165c5b18cfa2aaab520062cd7eae505b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 14:54:07 -0400 Subject: [PATCH 0769/1054] And more progress to Python 3. --- src/allmydata/test/test_encodingutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 1e85fd621..ff9e036d7 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -217,7 +217,7 @@ class EncodingUtil(ReallyEqualMixin): self.failUnlessReallyEqual(argv_to_unicode(argv), argu) def test_unicode_to_url(self): - self.failUnless(unicode_to_url(lumiere_nfc), "lumi\xc3\xa8re") + self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re") def test_unicode_to_output(self): if 'argv' not in dir(self): @@ -306,7 +306,7 @@ class StdlibUnicode(unittest.TestCase): # We only require that the listing includes a filename that is canonically equivalent # to lumiere_nfc (on Mac OS X, it will be the NFD equivalent). - self.failUnlessIn(lumiere_nfc + ".txt", set([encodingutil.normalize(fname) for fname in filenames])) + self.failUnlessIn(lumiere_nfc + u".txt", set([encodingutil.normalize(fname) for fname in filenames])) expanded = fileutil.expanduser(u"~/" + lumiere_nfc) self.failIfIn(u"~", expanded) From e03c2a759f21d21dd73c466ab05525efcf58fefd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:14:04 -0400 Subject: [PATCH 0770/1054] Port test_encodingutil.py to Python 3. --- src/allmydata/test/test_encodingutil.py | 10 ++++++++-- src/allmydata/util/_python3.py | 1 + src/allmydata/util/encodingutil.py | 14 +++++++------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index ff9e036d7..bb10bfd96 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,6 +1,12 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals from future.utils import PY2, PY3 +if PY2: + # We don't import str because omg way too ambiguous in this context. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from past.builtins import unicode @@ -95,7 +101,7 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase): mock_stdout.encoding = 'cp65001' _reload() - self.failUnlessReallyEqual(get_io_encoding(), 'utf-8') + self.assertEqual(get_io_encoding(), 'utf-8') mock_stdout.encoding = 'koi8-r' expected = sys.platform == "win32" and 'utf-8' or 'koi8-r' @@ -127,7 +133,7 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase): preferredencoding = None _reload() - self.failUnlessReallyEqual(get_io_encoding(), 'utf-8') + self.assertEqual(get_io_encoding(), 'utf-8') def test_argv_to_unicode(self): encodingutil.io_encoding = 'utf-8' diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a1750ca18..581b48005 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -55,6 +55,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", + "allmydata.test.test_encodingutil", "allmydata.test.test_happiness", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 67c421ffe..116e168a9 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -3,7 +3,7 @@ Functions used to convert inputs from whatever encoding used in the system to unicode and back. """ -from future.utils import PY2 +from future.utils import PY2, native_str from past.builtins import unicode @@ -168,14 +168,14 @@ def unicode_to_output(s): try: out = s.encode(io_encoding) except (UnicodeEncodeError, UnicodeDecodeError): - raise UnicodeEncodeError(io_encoding, s, 0, 0, - "A string could not be encoded as %s for output to the terminal:\n%r" % - (io_encoding, repr(s))) + raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, + native_str("A string could not be encoded as %s for output to the terminal:\n%r" % + (io_encoding, repr(s)))) if PRINTABLE_8BIT.search(out) is None: - raise UnicodeEncodeError(io_encoding, s, 0, 0, - "A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" % - (io_encoding, repr(s))) + raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, + native_str("A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" % + (io_encoding, repr(s)))) return out From b25f62d189539caea4a3ef03cfacbee7106914a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:40:45 -0400 Subject: [PATCH 0771/1054] Clarifications. --- src/allmydata/util/encodingutil.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 116e168a9..7cca3f0b2 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -3,7 +3,7 @@ Functions used to convert inputs from whatever encoding used in the system to unicode and back. """ -from future.utils import PY2, native_str +from future.utils import PY2, PY3, native_str from past.builtins import unicode @@ -67,13 +67,14 @@ def _reload(): check_encoding(io_encoding) - is_unicode_platform = not PY2 or sys.platform in ["win32", "darwin"] + is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"] # Despite the Unicode-mode FilePath support added to Twisted in # , we can't yet use # Unicode-mode FilePaths with INotify on non-Windows platforms - # due to . - use_unicode_filepath = not PY2 or sys.platform == "win32" + # due to . Supposedly + # 7928 is fixed, though... + use_unicode_filepath = PY3 or sys.platform == "win32" _reload() @@ -122,8 +123,12 @@ def unicode_to_argv(s, mangle=False): If the argument is to be passed to a different process, then the 'mangle' argument should be true; on Windows, this uses a mangled encoding that will be reversed by code in runner.py. + + On Python 3, just return the string unchanged, since argv is unicode. """ precondition(isinstance(s, unicode), s) + if PY3: + return s if mangle and sys.platform == "win32": # This must be the same as 'mangle' in bin/tahoe-script.template. @@ -133,7 +138,7 @@ def unicode_to_argv(s, mangle=False): def unicode_to_url(s): """ - Encode an unicode object used in an URL. + Encode an unicode object used in an URL to bytes. """ # According to RFC 2718, non-ascii characters in URLs must be UTF-8 encoded. @@ -283,7 +288,7 @@ def extend_filepath(fp, segments): return fp def to_filepath(path): - precondition(isinstance(path, unicode if use_unicode_filepath else basestring), + precondition(isinstance(path, unicode if use_unicode_filepath else (bytes, unicode)), path=path) if isinstance(path, unicode) and not use_unicode_filepath: From 5a0fa7ff73ddeb231f7d85bf97308c224779e03c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:40:52 -0400 Subject: [PATCH 0772/1054] Test unicode_to_argv(). --- src/allmydata/test/test_encodingutil.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index bb10bfd96..4b93b834b 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -83,7 +83,8 @@ from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \ unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \ get_io_encoding, get_filesystem_encoding, to_str, from_utf8_or_none, _reload, \ - to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from + to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \ + unicode_to_argv from twisted.python import usage @@ -236,7 +237,17 @@ class EncodingUtil(ReallyEqualMixin): _reload() self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv) - @skipIf(PY3, "It's always Unicode on Python 2.") + @skipIf(PY3, "Python 2 only.") + def test_unicode_to_argv_py2(self): + """unicode_to_argv() converts to bytes on Python 2.""" + self.assertEqual(unicode_to_argv("abc"), u"abc".encode(self.io_encoding)) + + @skipIf(PY2, "Python 3 only.") + def test_unicode_to_argv_py3(self): + """unicode_to_argv() is noop on Python 3.""" + self.assertEqual(unicode_to_argv("abc"), "abc") + + @skipIf(PY3, "Python 3 only.") def test_unicode_platform_py2(self): matrix = { 'linux2': False, From 5c9294ebf1fdbe574bcacaba29d6659cea69d2b1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:45:21 -0400 Subject: [PATCH 0773/1054] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/encodingutil.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 581b48005..3799cdeb7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -32,6 +32,7 @@ PORTED_MODULES = [ "allmydata.util.deferredutil", "allmydata.util.fileutil", "allmydata.util.dictutil", + "allmydata.util.encodingutil", "allmydata.util.gcutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 7cca3f0b2..3e8ac7e2d 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -1,9 +1,21 @@ """ Functions used to convert inputs from whatever encoding used in the system to unicode and back. + +Ported to Python 3. + +Once Python 2 support is dropped, most of this module will obsolete, since +Unicode is the default everywhere in Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals from future.utils import PY2, PY3, native_str +if PY2: + # We omit str() because that seems too tricky to get right. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from past.builtins import unicode From cdccb931790910d8f5628d39f200615b512631cd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:48:07 -0400 Subject: [PATCH 0774/1054] Flake fixes. --- src/allmydata/test/common_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e16cca835..fd9699e27 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -14,8 +14,9 @@ from future.utils import PY2 if PY2: # XXX this is a hack that makes some tests pass on Python3, remove # in the future from ..scripts import runner +# Imported for backwards compatibility: from .common_py3 import ( - SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, + SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, # noqa: F401 ) From 7505abe8265e37b45cf0f6b16243021a019e7f26 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Aug 2020 15:59:00 -0400 Subject: [PATCH 0775/1054] News file and ratchet (connection_status stuff is... iffy, but we have to live with it until the only-ported-modules bit is done). --- misc/python3/ratchet-passing | 13 +++++++++++++ newsfragments/3376.minor | 0 2 files changed, 13 insertions(+) create mode 100644 newsfragments/3376.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 200126859..4451cc36b 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -27,6 +27,19 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section +allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success +allmydata.test.test_configutil.ConfigUtilTests.test_config_utils +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section +allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success +allmydata.test.test_connection_status.Status.test_hint_statuses +allmydata.test.test_connection_status.Status.test_reconnector_connected +allmydata.test.test_connection_status.Status.test_reconnector_connected_listener +allmydata.test.test_connection_status.Status.test_reconnector_connected_others +allmydata.test.test_connection_status.Status.test_reconnector_connecting +allmydata.test.test_connection_status.Status.test_reconnector_waiting allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes allmydata.test.test_crypto.TestEd25519.test_key_serialization diff --git a/newsfragments/3376.minor b/newsfragments/3376.minor new file mode 100644 index 000000000..e69de29bb From 96e3d941728013e1c42241d5712e6e109a7909ed Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 12 Aug 2020 07:26:38 -0400 Subject: [PATCH 0776/1054] Fold py36 back into normal tox run This removes the individual-test-based ratchet. --- misc/python3/ratchet-passing | 183 ---------------- misc/python3/ratchet.py | 409 ----------------------------------- misc/python3/ratchet.sh | 45 ---- tox.ini | 8 +- 4 files changed, 2 insertions(+), 643 deletions(-) delete mode 100644 misc/python3/ratchet-passing delete mode 100755 misc/python3/ratchet.py delete mode 100755 misc/python3/ratchet.sh diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing deleted file mode 100644 index 1c2d1eda8..000000000 --- a/misc/python3/ratchet-passing +++ /dev/null @@ -1,183 +0,0 @@ -allmydata.test.mutable.test_exceptions.Exceptions.test_repr -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_1s -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_25s -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_day -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_future_5_minutes -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_hours -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_month -allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year -allmydata.test.test_abbreviate.Abbreviate.test_parse_space -allmydata.test.test_abbreviate.Abbreviate.test_space -allmydata.test.test_abbreviate.Abbreviate.test_time -allmydata.test.test_backupdb.BackupDB.test_basic -allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2 -allmydata.test.test_backupdb.BackupDB.test_wrong_version -allmydata.test.test_base32.Base32.test_a2b -allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons -allmydata.test.test_base32.Base32.test_b2a -allmydata.test.test_base32.Base32.test_b2a_or_none -allmydata.test.test_base62.Base62.test_ende_0x00 -allmydata.test.test_base62.Base62.test_ende_0x000000 -allmydata.test.test_base62.Base62.test_ende_0x01 -allmydata.test.test_base62.Base62.test_ende_0x0100 -allmydata.test.test_base62.Base62.test_ende_0x010000 -allmydata.test.test_base62.Base62.test_ende_longrandstr -allmydata.test.test_base62.Base62.test_ende_randstr -allmydata.test.test_base62.Base62.test_known_values -allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars -allmydata.test.test_base62.Base62.test_odd_sizes -allmydata.test.test_base62.Base62.test_roundtrip -allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes -allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes -allmydata.test.test_crypto.TestEd25519.test_key_serialization -allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey -allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes -allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes -allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes -allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey -allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input -allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input -allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input -allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input -allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair -allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair -allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes -allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size -allmydata.test.test_crypto.TestRegression.test_iv_not_bytes -allmydata.test.test_crypto.TestRegression.test_key_incorrect_size -allmydata.test.test_crypto.TestRegression.test_old_start_up_test -allmydata.test.test_crypto.TestRsa.test_keys -allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey -allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey -allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad -allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string -allmydata.test.test_crypto.TestUtil.test_remove_prefix_good -allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial -allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero -allmydata.test.test_deferredutil.DeferredUtilTests.test_failure -allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results -allmydata.test.test_deferredutil.DeferredUtilTests.test_success -allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls -allmydata.test.test_dictutil.DictUtil.test_auxdict -allmydata.test.test_dictutil.DictUtil.test_dict_of_sets -allmydata.test.test_happiness.Happiness.test_100 -allmydata.test.test_happiness.Happiness.test_calc_happy -allmydata.test.test_happiness.Happiness.test_everything_broken -allmydata.test.test_happiness.Happiness.test_hypothesis0 -allmydata.test.test_happiness.Happiness.test_hypothesis_0 -allmydata.test.test_happiness.Happiness.test_hypothesis_1 -allmydata.test.test_happiness.Happiness.test_placement_1 -allmydata.test.test_happiness.Happiness.test_placement_simple -allmydata.test.test_happiness.Happiness.test_redistribute -allmydata.test.test_happiness.Happiness.test_unhappy -allmydata.test.test_happiness.HappinessUtils.test_residual_0 -allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph -allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph -allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy -allmydata.test.test_happiness.PlacementTests.test_more_hypothesis -allmydata.test.test_hashtree.Complete.test_create -allmydata.test.test_hashtree.Complete.test_dump -allmydata.test.test_hashtree.Complete.test_needed_hashes -allmydata.test.test_hashtree.Incomplete.test_check -allmydata.test.test_hashtree.Incomplete.test_create -allmydata.test.test_hashtree.Incomplete.test_depth_of -allmydata.test.test_hashtree.Incomplete.test_large -allmydata.test.test_hashtree.Incomplete.test_needed_hashes -allmydata.test.test_hashutil.HashUtilTests.test_chk -allmydata.test.test_hashutil.HashUtilTests.test_hashers -allmydata.test.test_hashutil.HashUtilTests.test_known_answers -allmydata.test.test_hashutil.HashUtilTests.test_random_key -allmydata.test.test_hashutil.HashUtilTests.test_sha256d -allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated -allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare -allmydata.test.test_humanreadable.HumanReadable.test_repr -allmydata.test.test_iputil.GcUtil.test_gc_after_allocations -allmydata.test.test_iputil.GcUtil.test_release_delays_gc -allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for -allmydata.test.test_iputil.ListAddresses.test_list_async -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr -allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route -allmydata.test.test_iputil.ListenOnUsed.test_random_port -allmydata.test.test_iputil.ListenOnUsed.test_specific_port -allmydata.test.test_log.Log.test_default_facility -allmydata.test.test_log.Log.test_err -allmydata.test.test_log.Log.test_grandparent_id -allmydata.test.test_log.Log.test_no_prefix -allmydata.test.test_log.Log.test_numming -allmydata.test.test_log.Log.test_parent_id -allmydata.test.test_log.Log.test_with_bytes_prefix -allmydata.test.test_log.Log.test_with_prefix -allmydata.test.test_netstring.Netstring.test_encode -allmydata.test.test_netstring.Netstring.test_extra -allmydata.test.test_netstring.Netstring.test_nested -allmydata.test.test_netstring.Netstring.test_split -allmydata.test.test_observer.Observer.test_lazy_oneshot -allmydata.test.test_observer.Observer.test_observerlist -allmydata.test.test_observer.Observer.test_oneshot -allmydata.test.test_observer.Observer.test_oneshot_fireagain -allmydata.test.test_pipeline.Pipeline.test_basic -allmydata.test.test_pipeline.Pipeline.test_errors -allmydata.test.test_pipeline.Pipeline.test_errors2 -allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting -allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct -allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist -allmydata.test.test_spans.ByteSpans.test_basic -allmydata.test.test_spans.ByteSpans.test_large -allmydata.test.test_spans.ByteSpans.test_math -allmydata.test.test_spans.ByteSpans.test_overlap -allmydata.test.test_spans.ByteSpans.test_random -allmydata.test.test_spans.StringSpans.test_basic -allmydata.test.test_spans.StringSpans.test_random -allmydata.test.test_spans.StringSpans.test_test -allmydata.test.test_statistics.Statistics.test_binomial_coeff -allmydata.test.test_statistics.Statistics.test_binomial_distribution_pmf -allmydata.test.test_statistics.Statistics.test_convolve -allmydata.test.test_statistics.Statistics.test_find_k -allmydata.test.test_statistics.Statistics.test_pr_backup_file_loss -allmydata.test.test_statistics.Statistics.test_pr_file_loss -allmydata.test.test_statistics.Statistics.test_repair_cost -allmydata.test.test_statistics.Statistics.test_repair_count_pmf -allmydata.test.test_statistics.Statistics.test_survival_pmf -allmydata.test.test_time_format.TimeFormat.test_epoch -allmydata.test.test_time_format.TimeFormat.test_epoch_in_London -allmydata.test.test_time_format.TimeFormat.test_format_delta -allmydata.test.test_time_format.TimeFormat.test_format_time -allmydata.test.test_time_format.TimeFormat.test_format_time_y2038 -allmydata.test.test_time_format.TimeFormat.test_iso_utc -allmydata.test.test_time_format.TimeFormat.test_parse_date -allmydata.test.test_time_format.TimeFormat.test_parse_duration -allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode -allmydata.test.test_util.FileUtil.test_create_long_path -allmydata.test.test_util.FileUtil.test_disk_stats -allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative -allmydata.test.test_util.FileUtil.test_du -allmydata.test.test_util.FileUtil.test_encrypted_tempfile -allmydata.test.test_util.FileUtil.test_get_pathinfo -allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink -allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode -allmydata.test.test_util.FileUtil.test_remove_if_possible -allmydata.test.test_util.FileUtil.test_rename -allmydata.test.test_util.FileUtil.test_rename_no_overwrite -allmydata.test.test_util.FileUtil.test_replace_file -allmydata.test.test_util.FileUtil.test_rm_dir -allmydata.test.test_util.FileUtil.test_windows_expanduser_win7 -allmydata.test.test_util.FileUtil.test_windows_expanduser_xp -allmydata.test.test_util.FileUtil.test_write_atomically -allmydata.test.test_util.IDLib.test_nodeid_b2a -allmydata.test.test_util.Math.test_round_sigfigs -allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True -allmydata.test.test_util.PollMixinTests.test_PollMixin_True -allmydata.test.test_util.PollMixinTests.test_timeout -allmydata.test.test_util.YAML.test_convert -allmydata.test.test_version.CheckRequirement.test_cross_check -allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions -allmydata.test.test_version.CheckRequirement.test_extract_openssl_version -allmydata.test.test_version.CheckRequirement.test_packages_from_pkg_resources -allmydata.test.test_version.T.test_report_import_error -allmydata.test.test_version.VersionTestCase.test_basic_versions -allmydata.test.test_version.VersionTestCase.test_comparison -allmydata.test.test_version.VersionTestCase.test_from_parts -allmydata.test.test_version.VersionTestCase.test_irrational_versions -allmydata.test.test_version.VersionTestCase.test_suggest_normalized_version diff --git a/misc/python3/ratchet.py b/misc/python3/ratchet.py deleted file mode 100755 index cb672cf67..000000000 --- a/misc/python3/ratchet.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -'''Ratchet up passing tests, or ratchet down failing tests. - -Usage: - - ratchet.py <"up" or "down"> - -This script helps when you expect a large test suite to fail spectactularly in -some environment, and you want to gradually improve the situation with minimal -impact to forward development of the same codebase for other environments. The -initial and primary usecase is porting from Python 2 to Python 3. - -The idea is to emit JUnit XML from your test runner, and then invoke ratchet.py -to consume this XML output and operate on a so-called "tracking" file. When -ratcheting up passing tests, the tracking file will contain a list of tests, -one per line, that passed. When ratching down, the tracking file contains a -list of failing tests. On each subsequent run, ratchet.py will compare the -prior results in the tracking file with the new results in the XML, and will -report on both welcome and unwelcome changes. It will modify the tracking file -in the case of welcome changes, and therein lies the ratcheting. - -The exit codes are: - - 0 - no changes observed - 1 - changes observed, whether welcome or unwelcome - 2 - invocation error - -If does not exist, you'll get a FileNotFoundError: - - >>> _test('up', None, None) # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - FileNotFoundError: ... - -If does not exist, that's fine: - - >>> _test('up', '1', None) - Some tests not required to pass did: - c0.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 0 test(s) were required to pass, but instead 1 did. 🐭 - -Same if you're ratcheting down: - - >>> _test('down', '1', None) - All and only tests expected to fail did. 💃 - -If the test run has the same output as last time, it's all good: - - >>> _test('up', '01001110', '01001110') - All and only tests required to pass did. 💃 - - >>> _test('down', '01001110', '10110001') - All and only tests expected to fail did. 💃 - -If there's a welcome change, that's noted: - - >>> _test('up', '0101', '0100') - Some tests not required to pass did: - c3.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 1 test(s) were required to pass, but instead 2 did. 🐭 - - >>> _test('down', '0011', '1110') - Some tests expected to fail didn't: - c2.t - Conveniently, they have been removed from `` for you. Perhaps commit that? - Eep! 3 test(s) were expected to fail, but instead 2 did. 🐭 - -And if there is an unwelcome change, that is noted as well: - - >>> _test('up', '1101', '1111') - Some tests required to pass didn't: - c2.t - Eep! 4 test(s) were required to pass, but instead 3 did. 🐭 - - >>> _test('down', '0000', '1101') - Some tests not expected to fail did: - c2.t - Eep! 3 test(s) were expected to fail, but instead 4 did. 🐭 - -And if there are both welcome and unwelcome changes, they are both noted: - - >>> _test('up', '1101', '1011') - Some tests not required to pass did: - c1.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Some tests required to pass didn't: - c2.t - Eep! 3 test(s) were required to pass, but instead 3 did. 🐭 - - >>> _test('down', '0100', '1100') - Some tests not expected to fail did: - c2.t - c3.t - Some tests expected to fail didn't: - c1.t - Conveniently, they have been removed from `` for you. Perhaps commit that? - Eep! 2 test(s) were expected to fail, but instead 3 did. 🐭 - - -To test ratchet.py itself: - - python3 -m doctest ratchet.py - -''' -from __future__ import absolute_import, division, print_function, unicode_literals - -import io -import os -import re -import sys -import tempfile -import xml.etree.ElementTree as Etree - - -class JUnitXMLFile(object): - '''Represent a file containing test results in JUnit XML format. - - >>> eg = _mktemp_junitxml('0100111') - >>> results = JUnitXMLFile(eg.name).parse() - >>> results.failed - ['c0.t', 'c2.t', 'c3.t'] - >>> results.passed - ['c1.t', 'c4.t', 'c5.t', 'c6.t'] - - ''' - - def __init__(self, filepath): - self.filepath = filepath - self.failed = [] - self.failed_aggregates = {} - self.stderr_output = [] - self.passed = [] - self._tree = None - - def parse(self): - if self._tree: - raise RuntimeError('already parsed') - self._tree = Etree.parse(self.filepath) - for testcase in self._tree.findall('testcase'): - self.process_testcase(testcase) - return self - - def process_testcase(self, case): - key = self.case_key(case) - - # look at children but throw away stderr output - nonpassing = [c for c in case if not c.tag == 'system-err'] - n = len(nonpassing) - if n > 1: - raise RuntimeError(f'multiple results for {key}: {nonpassing}') - elif n == 1: - result = nonpassing.pop() - self.failed.append(key) - message = result.get('message') - self.failed_aggregates.setdefault(message, []).append(key) - else: - self.passed.append(key) - - @staticmethod - def case_key(case): - return f'{case.get("classname")}.{case.get("name")}' - - def report(self, details=False): - for k, v in sorted( - self.failed_aggregates.items(), - key = lambda i: len(i[1]), - reverse=True): - print(f'# {k}') - for t in v: - print(f' - {t}') - - -def load_previous_results(txt): - try: - previous_results = open(txt).read() - except FileNotFoundError: - previous_results = '' - parsed = set() - for line in previous_results.splitlines(): - if not line or line.startswith('#'): - continue - parsed.add(line) - return parsed - - -def print_tests(tests): - for test in sorted(tests): - print(' ', test) - - -def ratchet_up_passing(tracking_path, tests): - try: - old = set(open(tracking_path, 'r')) - except FileNotFoundError: - old = set() - new = set(t + '\n' for t in tests) - merged = sorted(old | new) - open(tracking_path, 'w+').writelines(merged) - - -def ratchet_down_failing(tracking_path, tests): - new = set(t + '\n' for t in tests) - open(tracking_path, 'w+').writelines(sorted(new)) - - -def main(direction, junitxml_path, tracking_path): - '''Takes a string indicating which direction to ratchet, "up" or "down," - and two paths, one to test-runner output in JUnit XML format, the other to - a file tracking test results (one test case dotted name per line). Walk the - former looking for the latter, and react appropriately. - - >>> inp = _mktemp_junitxml('0100111') - >>> out = _mktemp_tracking('0000000') - >>> _test_main('up', inp.name, out.name) - Some tests not required to pass did: - c1.t - c4.t - c5.t - c6.t - Conveniently, they have been added to `` for you. Perhaps commit that? - Eep! 0 test(s) were required to pass, but instead 4 did. 🐭 - - ''' - - results = JUnitXMLFile(junitxml_path).parse() - - if tracking_path == '...': - # Shortcut to aid in debugging XML parsing issues. - results.report() - return - - previous = load_previous_results(tracking_path) - current = set(results.passed if direction == 'up' else results.failed) - - subjunctive = {'up': 'required to pass', 'down': 'expected to fail'}[direction] - ratchet = None - - too_many = current - previous - if too_many: - print(f'Some tests not {subjunctive} did:') - print_tests(too_many) - if direction == 'up': - # Too many passing tests is good -- let's do more of those! - ratchet_up_passing(tracking_path, current) - print(f'Conveniently, they have been added to `{tracking_path}` for you. Perhaps commit that?') - - not_enough = previous - current - if not_enough: - print(f'Some tests {subjunctive} didn\'t:') - print_tests(not_enough) - if direction == 'down': - # Not enough failing tests is good -- let's do more of those! - ratchet_down_failing(tracking_path, current) - print(f'Conveniently, they have been removed from `{tracking_path}` for you. Perhaps commit that?') - - if too_many or not_enough: - print(f'Eep! {len(previous)} test(s) were {subjunctive}, but instead {len(current)} did. 🐭') - return 1 - - print(f'All and only tests {subjunctive} did. 💃') - return 0 - - -# When called as an executable ... - -if __name__ == '__main__': - try: - direction, junitxml_path, tracking_path = sys.argv[1:4] - if direction not in ('up', 'down'): - raise ValueError - except ValueError: - doc = '\n'.join(__doc__.splitlines()[:6]) - doc = re.sub(' ratchet.py', f' {sys.argv[0]}', doc) - print(doc, file=sys.stderr) - exit_code = 2 - else: - exit_code = main(direction, junitxml_path, tracking_path) - sys.exit(exit_code) - - -# Helpers for when called under doctest ... - -def _test(*a): - return _test_main(*_mk(*a)) - - -def _test_main(direction, junitxml, tracking): - '''Takes a string 'up' or 'down' and paths to (or open file objects for) - the JUnit XML and tracking files to use for this test run. Captures and - emits stdout (slightly modified) for inspection via doctest.''' - junitxml_path = junitxml.name if hasattr(junitxml, 'name') else junitxml - tracking_path = tracking.name if hasattr(tracking, 'name') else tracking - - old_stdout = sys.stdout - sys.stdout = io.StringIO() - try: - main(direction, junitxml_path, tracking_path) - finally: - sys.stdout.seek(0) - out = sys.stdout.read() - out = re.sub('`.*?`', '``', out).strip() - sys.stdout = old_stdout - print(out) - - -class _PotentialFile(object): - '''Represent a file that we are able to create but which doesn't exist yet, - and which, if we create it, will be automatically torn down when the test - run is over.''' - - def __init__(self, filename): - self.d = tempfile.TemporaryDirectory() - self.name = os.path.join(self.d.name, filename) - - -def _mk(direction, spec_junitxml, spec_tracking): - '''Takes a string 'up' or 'down' and two bit strings specifying the state - of the JUnit XML results file and the tracking file to set up for this test - case. Returns the direction (unharmed) and two file-ish objects. - - If a spec string is None the corresponding return value will be a - _PotentialFile object, which has a .name attribute (like a true file - object) that points to a file that does not exist, but could. - - The reason not to simply return the path in all cases is that the file - objects are actually temporary file objects that destroy the underlying - file when they go out of scope, and we want to keep the underlying file - around until the end of the test run.''' - - if None not in(spec_junitxml, spec_tracking): - if len(spec_junitxml) != len(spec_tracking): - raise ValueError('if both given, must be the same length: `{spec_junitxml}` and `{spec_tracking}`') - if spec_junitxml is None: - junitxml_fp = _PotentialFile('results.xml') - else: - junitxml_fp = _mktemp_junitxml(spec_junitxml) - if spec_tracking is None: - tracking_fp = _PotentialFile('tracking') - else: - tracking_fp = _mktemp_tracking(spec_tracking) - return direction, junitxml_fp, tracking_fp - - -def _mktemp_junitxml(spec): - '''Test helper to generate a raw JUnit XML file. - - >>> fp = _mktemp_junitxml('00101') - >>> open(fp.name).read()[:11] - '' - - ''' - fp = tempfile.NamedTemporaryFile() - fp.write(b'') - - passed = '''\ - -''' - failed = '''\ - -Traceback (most recent call last): - File "/foo/bar/baz/buz.py", line 1, in <module> -NameError: name 'heck' is not defined - - -''' - - i = 0 - for c in spec: - if c == '0': - out = failed - elif c == '1': - out = passed - else: - raise ValueError(f'bad c: `{c}`') - fp.write(out.format(i=i).encode('utf8')) - i += 1 - - fp.write(b'') - fp.flush() - return fp - - -def _mktemp_tracking(spec): - '''Test helper to prefabricate a tracking file. - - >>> fp = _mktemp_tracking('01101') - >>> print(open(fp.name).read()[:-1]) - c1.t - c2.t - c4.t - - ''' - fp = tempfile.NamedTemporaryFile() - - i = 0 - for c in spec: - if c == '0': - pass - elif c == '1': - fp.write(f'c{i}.t\n'.encode('utf8')) - else: - raise ValueError(f'bad c: `{c}`') - i += 1 - - fp.flush() - return fp diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh deleted file mode 100755 index aa768cd06..000000000 --- a/misc/python3/ratchet.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -tracking_filename="ratchet-passing" - -# Start somewhere predictable. -cd "$(dirname $0)" -base=$(pwd) - -# Actually, though, trial outputs some things that are only gitignored in the project root. -cd "../.." - -# Since both of the next calls are expected to exit non-0, relax our guard. -set +e -trial --reporter=subunitv2-file allmydata -subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" -set -e - -# Okay, now we're clear. -cd "$base" - -# Make sure ratchet.py itself is clean. -python3 -m doctest ratchet.py - -# Now see about Tahoe-LAFS (also expected to fail) ... -set +e -python3 ratchet.py up results.xml "$tracking_filename" -code=$? -set -e - -# Emit a diff of the tracking file, to aid in the situation where changes are -# not discovered until CI (where TERM might `dumb`). -if [ $TERM = 'dumb' ]; then - export TERM=ansi -fi - -echo "The ${tracking_filename} diff is:" -echo "=================================" -# "git diff" gets pretty confused in this execution context when trying to -# write to stdout. Somehow it fails with SIGTTOU. -git diff -- "${tracking_filename}" > tracking.diff -cat tracking.diff -echo "=================================" - -echo "Exiting with code ${code} from ratchet.py." -exit ${code} diff --git a/tox.ini b/tox.ini index 0376ab28e..92f387d68 100644 --- a/tox.ini +++ b/tox.ini @@ -45,13 +45,9 @@ usedevelop = False # tests. extras = test commands = +# `tahoe --version` is not ready for prime time yet under py36. May it be some day! + !py36: tahoe --version trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} - tahoe --version - -[testenv:py36] -# On macOS, git inside of ratchet.sh needs $HOME. -passenv = {[testenv]passenv} HOME -commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] setenv = From 306df5301269a39b38715734c7cb4436afe8aa01 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 13 Aug 2020 05:53:17 -0400 Subject: [PATCH 0777/1054] Expose TAHOE_LAFS_TRIAL_ARGS to config This allows for working around a bug in trial under Python 3 where --reporter and --rterror collide. --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 8894b9691..bf1b6ee8e 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -65,7 +65,7 @@ TIMEOUT="timeout --kill-after 1m 15m" # Send the output directly to a file because transporting the binary subunit2 # via tox and then scraping it out is hideous and failure prone. export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" -export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" +export TAHOE_LAFS_TRIAL_ARGS="${TAHOE_LAFS_TRIAL_ARGS:---reporter=subunitv2-file --rterrors}" export PIP_NO_INDEX="1" if [ "${ALLOWED_FAILURE}" = "yes" ]; then From 60078ce47f0527af75e97b6e0f309c7c4aa996d9 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 13 Aug 2020 06:23:21 -0400 Subject: [PATCH 0778/1054] Satisfy towncrier --- newsfragments/3372.minor | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/3372.minor +++ b/newsfragments/3372.minor @@ -0,0 +1 @@ + From ae87037e737267b1b49ee9a270927228fbae1b36 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 13 Aug 2020 06:17:14 -0400 Subject: [PATCH 0779/1054] Filter tests we care about in CI for Python 3 --- .circleci/config.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e75a6ba0f..9c0808e17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,6 +163,7 @@ jobs: "${TAHOE_LAFS_TOX_ARGS}" - run: &RUN_TESTS + # Please keep python3.6 in sync with this if you can. Thanks. name: "Run test suite" command: | /tmp/project/.circleci/run-tests.sh \ @@ -282,6 +283,45 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + # Work around a conflict with trial under Python 3 between --reporter and + # --rterrors (by unspecifying --rterrors). + TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" + + steps: + - "checkout" + - run: *SETUP_VIRTUALENV + - run: + name: "Run test suite (python3.6)" + command: | + + # Constrain the modules exercised under Python 3 based on the list + # we keep in _python3. Exporting inside a command seems to be the + # preferred way of setting an environment variable (or otherwise + # constraining the test run) dynamically: + # + # https://ideas.circleci.com/ideas/CCI-I-67 + # + # This seems to force a choice between DRY code with an overloaded + # RUN_TESTS, or an uncluttered RUN_TESTS and code duplication here. + # Que sera, sera. + + export TAHOE_LAFS_TOX_ARGS="$(/tmp/tahoe-lafs.tox/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/python -c 'from allmydata.util._python3 import PORTED_TEST_MODULES; print(" ".join(PORTED_TEST_MODULES))')" \ + + /tmp/project/.circleci/run-tests.sh \ + "/tmp/venv" \ + "/tmp/project" \ + "${ALLOWED_FAILURE}" \ + "${ARTIFACTS_OUTPUT_PATH}" \ + "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ + "${TAHOE_LAFS_TOX_ARGS}" + # trial output gets directed straight to a log. avoid the circleci + # timeout while the test suite runs. + no_output_timeout: "20m" + + - store_test_results: *STORE_TEST_RESULTS + - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE ubuntu-20.04: From e113cba8d0e73cd0f5c79cc12e6bd90c2854cd89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:02:49 -0400 Subject: [PATCH 0780/1054] Address review comments. --- src/allmydata/test/test_encodingutil.py | 4 +++- src/allmydata/util/encodingutil.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 4b93b834b..9d2c1cf5f 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -53,7 +53,9 @@ if __name__ == "__main__": for fname in TEST_FILENAMES: open(os.path.join(tmpdir, fname), 'w').close() - # Use Unicode API under Windows or MacOS X + # On Python 2, listing directories returns unicode under Windows or + # MacOS X if the input is unicode. On Python 3, it always returns + # Unicode. if PY2 and sys.platform in ('win32', 'darwin'): dirlist = os.listdir(unicode(tmpdir)) else: diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 3e8ac7e2d..c29933620 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -214,6 +214,9 @@ def _unicode_escape(m, quote_newlines): return u'\\x%02x' % (codepoint,) def _str_escape(m, quote_newlines): # TODO rename to _bytes_escape + """ + Takes a re match on bytes, the result is escaped bytes of group(0). + """ c = m.group(0) if c == b'"' or c == b'$' or c == b'`' or c == b'\\': return b'\\' + c From 2157da524d0f3a8c71661e702ae2b04561b0f8b1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:04:44 -0400 Subject: [PATCH 0781/1054] This needs an encoding too. --- src/allmydata/util/encodingutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index c29933620..07416cc93 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -144,7 +144,7 @@ def unicode_to_argv(s, mangle=False): if mangle and sys.platform == "win32": # This must be the same as 'mangle' in bin/tahoe-script.template. - return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s)) + return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s), io_encoding) else: return s.encode(io_encoding) From 8167e85f7b96136d65199cf56bc028c20da6e4ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:11:14 -0400 Subject: [PATCH 0782/1054] Fix flake8 with __all__. --- src/allmydata/test/common_util.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index fd9699e27..110122b25 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -16,7 +16,7 @@ if PY2: # XXX this is a hack that makes some tests pass on Python3, remove from ..scripts import runner # Imported for backwards compatibility: from .common_py3 import ( - SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, # noqa: F401 + SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, ) @@ -172,3 +172,11 @@ except ImportError: os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) make_readonly = _make_readonly make_accessible = _make_accessible + + +__all__ = [ + "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin", + "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", + "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", + "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" +] From a2e707225b7913219ae7307a5b6a40d2be0f9fa7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:48:21 -0400 Subject: [PATCH 0783/1054] Fix the ratchet.sh script. --- misc/python3/ratchet.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index aa768cd06..b383ce199 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -9,11 +9,10 @@ base=$(pwd) # Actually, though, trial outputs some things that are only gitignored in the project root. cd "../.." -# Since both of the next calls are expected to exit non-0, relax our guard. -set +e -trial --reporter=subunitv2-file allmydata -subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" -set -e +export SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" +# Since the next two calls are expected to exit non-0, relax our guard. +trial --reporter=subunitv2-file allmydata || true +subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" || true # Okay, now we're clear. cd "$base" From 942c5d52cf49e1ba8259edc4cddf616b0423bbfe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:49:47 -0400 Subject: [PATCH 0784/1054] New passing tests. --- misc/python3/ratchet-passing | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 26f05de18..bf56e0de2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -73,6 +73,59 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls allmydata.test.test_dictutil.DictUtil.test_auxdict allmydata.test.test_dictutil.DictUtil.test_dict_of_sets +allmydata.test.test_encodingutil.EncodingUtilErrors.test_argv_to_unicode +allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding +allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding_not_from_stdout +allmydata.test.test_encodingutil.EncodingUtilErrors.test_no_unicode_normalization +allmydata.test.test_encodingutil.EncodingUtilErrors.test_unicode_to_output +allmydata.test.test_encodingutil.FilePaths.test_extend_filepath +allmydata.test.test_encodingutil.FilePaths.test_to_filepath +allmydata.test.test_encodingutil.FilePaths.test_unicode_from_filepath +allmydata.test.test_encodingutil.FilePaths.test_unicode_segments_from +allmydata.test.test_encodingutil.MacOSXLeopard.test_argv_to_unicode +allmydata.test.test_encodingutil.MacOSXLeopard.test_listdir_unicode +allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_platform_py3 +allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_output +allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_url +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_argv_to_unicode +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_listdir_unicode +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_platform_py3 +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_output +allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_url +allmydata.test.test_encodingutil.OpenBSD.test_argv_to_unicode +allmydata.test.test_encodingutil.OpenBSD.test_listdir_unicode +allmydata.test.test_encodingutil.OpenBSD.test_unicode_platform_py3 +allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_output +allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_url +allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_ascii +allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_default +allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_latin1 +allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_utf8 +allmydata.test.test_encodingutil.QuotePaths.test_quote_filepath +allmydata.test.test_encodingutil.QuotePaths.test_quote_path +allmydata.test.test_encodingutil.StdlibUnicode.test_mkdir_open_exists_abspath_listdir_expanduser +allmydata.test.test_encodingutil.TestToFromStr.test_from_utf8_or_none +allmydata.test.test_encodingutil.TestToFromStr.test_to_str +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_argv_to_unicode +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_listdir_unicode +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_platform_py3 +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_output +allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_url +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_argv_to_unicode +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_listdir_unicode +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_platform_py3 +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_output +allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_url +allmydata.test.test_encodingutil.Windows.test_argv_to_unicode +allmydata.test.test_encodingutil.Windows.test_unicode_platform_py3 +allmydata.test.test_encodingutil.Windows.test_unicode_to_argv_py3 +allmydata.test.test_encodingutil.Windows.test_unicode_to_output +allmydata.test.test_encodingutil.Windows.test_unicode_to_url allmydata.test.test_happiness.Happiness.test_100 allmydata.test.test_happiness.Happiness.test_calc_happy allmydata.test.test_happiness.Happiness.test_everything_broken From af61571fa61616e4bdd43326919075d46eeaf6cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 15:56:08 -0400 Subject: [PATCH 0785/1054] Accidentally passing test :( --- misc/python3/ratchet-passing | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bf56e0de2..01e58a2fa 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -206,6 +206,7 @@ allmydata.test.test_statistics.Statistics.test_pr_file_loss allmydata.test.test_statistics.Statistics.test_repair_cost allmydata.test.test_statistics.Statistics.test_repair_count_pmf allmydata.test.test_statistics.Statistics.test_survival_pmf +allmydata.test.test_stats.CPUUsage.test_monitor allmydata.test.test_time_format.TimeFormat.test_epoch allmydata.test.test_time_format.TimeFormat.test_epoch_in_London allmydata.test.test_time_format.TimeFormat.test_format_delta From b0c4f6d2abaea2d2a2ca5408072cb8ba68fb827a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 16:30:27 -0400 Subject: [PATCH 0786/1054] Fix Python 2 tests. --- src/allmydata/util/configutil.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 79980b006..1a1a93f18 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -1,3 +1,10 @@ +""" +Read/write config files. + +Configuration is returned as native strings. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -5,7 +12,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # We don't do open(), because we want files to read/write native strs when + # we do "r" or "w". + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: # In theory on Python 2 configparser also works, but then code gets the @@ -27,8 +36,17 @@ class UnknownConfigError(Exception): def get_config(tahoe_cfg): + """Load the config, returning a SafeConfigParser. + + Configuration is returned as native strings. + """ config = SafeConfigParser() with open(tahoe_cfg, "r") as f: + # On Python 2, where we read in bytes, skip any initial Byte Order + # Mark. Since this is an ordinary file, we don't need to handle + # incomplete reads, and can assume seekability. + if PY2 and f.read(3) != b'\xEF\xBB\xBF': + f.seek(0) config.readfp(f) return config From a3714a9970bc84ae43efaca6c21f868910cca305 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 16:32:32 -0400 Subject: [PATCH 0787/1054] Fix whitespace. --- src/allmydata/test/test_encodingutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 9d2c1cf5f..cf72fc9ac 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -286,7 +286,7 @@ class EncodingUtil(ReallyEqualMixin): return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) else d for d in self.dirlist] - + self.patch(os, 'listdir', call_os_listdir) def call_sys_getfilesystemencoding(): From a79c58133299e76a06471f72d5af02cb05317cf6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Aug 2020 16:37:24 -0400 Subject: [PATCH 0788/1054] Wrong branch. --- misc/python3/ratchet-passing | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bf56e0de2..78e9e29ae 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -27,19 +27,6 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip -allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item -allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section -allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success -allmydata.test.test_configutil.ConfigUtilTests.test_config_utils -allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item -allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section -allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success -allmydata.test.test_connection_status.Status.test_hint_statuses -allmydata.test.test_connection_status.Status.test_reconnector_connected -allmydata.test.test_connection_status.Status.test_reconnector_connected_listener -allmydata.test.test_connection_status.Status.test_reconnector_connected_others -allmydata.test.test_connection_status.Status.test_reconnector_connecting -allmydata.test.test_connection_status.Status.test_reconnector_waiting allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes allmydata.test.test_crypto.TestEd25519.test_key_serialization From 647ed5d6e1ccdb118112e64b3101e58352dcd997 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 14 Aug 2020 07:43:25 -0400 Subject: [PATCH 0789/1054] Use BASH_ENV to avoid duplication of job steps --- .circleci/config.yml | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c0808e17..dc5c5e553 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,33 +291,18 @@ jobs: - "checkout" - run: *SETUP_VIRTUALENV - run: - name: "Run test suite (python3.6)" + name: "Limit Modules" command: | - # Constrain the modules exercised under Python 3 based on the list - # we keep in _python3. Exporting inside a command seems to be the - # preferred way of setting an environment variable (or otherwise - # constraining the test run) dynamically: + # Limit the modules exercised under Python 3 dynamically, based + # on the list we keep in allmydata.util._python3. Use BASH_ENV to + # communicate this to the RUN_TESTS step: # - # https://ideas.circleci.com/ideas/CCI-I-67 - # - # This seems to force a choice between DRY code with an overloaded - # RUN_TESTS, or an uncluttered RUN_TESTS and code duplication here. - # Que sera, sera. + # https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment - export TAHOE_LAFS_TOX_ARGS="$(/tmp/tahoe-lafs.tox/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/python -c 'from allmydata.util._python3 import PORTED_TEST_MODULES; print(" ".join(PORTED_TEST_MODULES))')" \ - - /tmp/project/.circleci/run-tests.sh \ - "/tmp/venv" \ - "/tmp/project" \ - "${ALLOWED_FAILURE}" \ - "${ARTIFACTS_OUTPUT_PATH}" \ - "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ - "${TAHOE_LAFS_TOX_ARGS}" - # trial output gets directed straight to a log. avoid the circleci - # timeout while the test suite runs. - no_output_timeout: "20m" + echo "export TAHOE_LAFS_TOX_ARGS=\"$(/tmp/tahoe-lafs.tox/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/python -c 'from allmydata.util._python3 import PORTED_TEST_MODULES; print(" ".join(PORTED_TEST_MODULES))')\"" >> ${BASH_ENV} + - run: *RUN_TESTS - store_test_results: *STORE_TEST_RESULTS - store_artifacts: *STORE_TEST_LOG - store_artifacts: *STORE_OTHER_ARTIFACTS From da9913be45630ba4ee1602102c9e7f6f580aa138 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Aug 2020 10:47:55 -0400 Subject: [PATCH 0790/1054] Make sure it ends up in artifacts. --- misc/python3/ratchet.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index b383ce199..72a6ac4ed 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -40,5 +40,9 @@ git diff -- "${tracking_filename}" > tracking.diff cat tracking.diff echo "=================================" +if [ -n "${ARTIFACTS}" ]; then + cp "${SUBUNITREPORTER_OUTPUT_PATH}" "$ARTIFACTS" +fi + echo "Exiting with code ${code} from ratchet.py." exit ${code} From 9695e87fbd610da2c6d10d37c90add3e3cb852ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 11:16:45 -0400 Subject: [PATCH 0791/1054] Revert to master version of these configs --- .circleci/config.yml | 25 ------------------------- tox.ini | 8 ++++++-- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc5c5e553..e75a6ba0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -163,7 +163,6 @@ jobs: "${TAHOE_LAFS_TOX_ARGS}" - run: &RUN_TESTS - # Please keep python3.6 in sync with this if you can. Thanks. name: "Run test suite" command: | /tmp/project/.circleci/run-tests.sh \ @@ -283,30 +282,6 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" - # Work around a conflict with trial under Python 3 between --reporter and - # --rterrors (by unspecifying --rterrors). - TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - - steps: - - "checkout" - - run: *SETUP_VIRTUALENV - - run: - name: "Limit Modules" - command: | - - # Limit the modules exercised under Python 3 dynamically, based - # on the list we keep in allmydata.util._python3. Use BASH_ENV to - # communicate this to the RUN_TESTS step: - # - # https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment - - echo "export TAHOE_LAFS_TOX_ARGS=\"$(/tmp/tahoe-lafs.tox/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/python -c 'from allmydata.util._python3 import PORTED_TEST_MODULES; print(" ".join(PORTED_TEST_MODULES))')\"" >> ${BASH_ENV} - - - run: *RUN_TESTS - - store_test_results: *STORE_TEST_RESULTS - - store_artifacts: *STORE_TEST_LOG - - store_artifacts: *STORE_OTHER_ARTIFACTS - - run: *SUBMIT_COVERAGE ubuntu-20.04: diff --git a/tox.ini b/tox.ini index 92f387d68..0376ab28e 100644 --- a/tox.ini +++ b/tox.ini @@ -45,9 +45,13 @@ usedevelop = False # tests. extras = test commands = -# `tahoe --version` is not ready for prime time yet under py36. May it be some day! - !py36: tahoe --version trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} + tahoe --version + +[testenv:py36] +# On macOS, git inside of ratchet.sh needs $HOME. +passenv = {[testenv]passenv} HOME +commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] setenv = From f7c3c5320621844e804de7c33748a034dc54e624 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 11:19:01 -0400 Subject: [PATCH 0792/1054] Go directly to trial instead of the ratchet helper --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 0376ab28e..98ca90c39 100644 --- a/tox.ini +++ b/tox.ini @@ -49,9 +49,8 @@ commands = tahoe --version [testenv:py36] -# On macOS, git inside of ratchet.sh needs $HOME. -passenv = {[testenv]passenv} HOME -commands = {toxinidir}/misc/python3/ratchet.sh +commands = + trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests} [testenv:integration] setenv = From 76e5c40fc6c1851b3931575eaf6ddf566afc0a79 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 11:21:32 -0400 Subject: [PATCH 0793/1054] Add a module to the test suite which contains all of the other ported modules --- src/allmydata/test/python3_tests.py | 37 +++++++++++++++++++++++++++++ src/allmydata/util/_python3.py | 13 ++++++---- 2 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 src/allmydata/test/python3_tests.py diff --git a/src/allmydata/test/python3_tests.py b/src/allmydata/test/python3_tests.py new file mode 100644 index 000000000..9326caa51 --- /dev/null +++ b/src/allmydata/test/python3_tests.py @@ -0,0 +1,37 @@ +""" +This module defines the subset of the full test suite which is expected to +pass on Python 3 in a way which makes that suite discoverable by trial. + +This module has been ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from twisted.python.reflect import ( + namedModule, +) +from twisted.trial.runner import ( + TestLoader, +) +from twisted.trial.unittest import ( + TestSuite, +) + +from allmydata.util._python3 import ( + PORTED_TEST_MODULES, +) + +def testSuite(): + loader = TestLoader() + return TestSuite(list( + loader.loadModule(namedModule(module)) + for module + in PORTED_TEST_MODULES + )) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a1750ca18..52d7ddd77 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,6 +1,15 @@ """ Track the port to Python 3. +The two easiest ways to run the part of the test suite which is expected to +pass on Python 3 are:: + + $ tox -e py36 + +and:: + + $ trial allmydata.test.python3_tests + This module has been ported to Python 3. """ @@ -71,7 +80,3 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.test_version", ] - -if __name__ == '__main__': - from subprocess import check_call - check_call(["trial"] + PORTED_TEST_MODULES) From 0a7589f0c2eead6e1f2dda69b100732938a49d11 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 11:24:42 -0400 Subject: [PATCH 0794/1054] Update the CircleCI configuration to get the reporter working for py36 --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e75a6ba0f..9f7381f33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,10 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT + # The default trial args include --rterrors which is incompatible with + # this reporter on Python 3. So drop that and just specify the + # reporter. + TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" TAHOE_LAFS_TOX_ENVIRONMENT: "py36" From 97dd2fab72a265ec0f4cba78450661be6f7f0fe2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Aug 2020 13:08:24 -0400 Subject: [PATCH 0795/1054] Delete. --- misc/python3/ratchet.sh | 48 ----------------------------------------- 1 file changed, 48 deletions(-) delete mode 100755 misc/python3/ratchet.sh diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh deleted file mode 100755 index 72a6ac4ed..000000000 --- a/misc/python3/ratchet.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -tracking_filename="ratchet-passing" - -# Start somewhere predictable. -cd "$(dirname $0)" -base=$(pwd) - -# Actually, though, trial outputs some things that are only gitignored in the project root. -cd "../.." - -export SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" -# Since the next two calls are expected to exit non-0, relax our guard. -trial --reporter=subunitv2-file allmydata || true -subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" || true - -# Okay, now we're clear. -cd "$base" - -# Make sure ratchet.py itself is clean. -python3 -m doctest ratchet.py - -# Now see about Tahoe-LAFS (also expected to fail) ... -set +e -python3 ratchet.py up results.xml "$tracking_filename" -code=$? -set -e - -# Emit a diff of the tracking file, to aid in the situation where changes are -# not discovered until CI (where TERM might `dumb`). -if [ $TERM = 'dumb' ]; then - export TERM=ansi -fi - -echo "The ${tracking_filename} diff is:" -echo "=================================" -# "git diff" gets pretty confused in this execution context when trying to -# write to stdout. Somehow it fails with SIGTTOU. -git diff -- "${tracking_filename}" > tracking.diff -cat tracking.diff -echo "=================================" - -if [ -n "${ARTIFACTS}" ]; then - cp "${SUBUNITREPORTER_OUTPUT_PATH}" "$ARTIFACTS" -fi - -echo "Exiting with code ${code} from ratchet.py." -exit ${code} From 41fe40506845ba7de38a7ddf3995c781794838fc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 13:09:56 -0400 Subject: [PATCH 0796/1054] news fragment --- newsfragments/3380.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3380.minor diff --git a/newsfragments/3380.minor b/newsfragments/3380.minor new file mode 100644 index 000000000..e69de29bb From 6eba1469f3b7c707e2f5678b4960c6c0cfb8299e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 14 Aug 2020 13:10:54 -0400 Subject: [PATCH 0797/1054] Back out subunit2junitxml exit status change from ticket:3336 I don't know why I included this change. It was certainly based on a faulty assumption (that the tox command would exit with error if a test failed - it won't). Apart from that, the motivation was the Python 3.6 ratchet job. The test run was expected to fail but then the *ratchet* tool was in charge of deciding if this indicated real failure or not. Failing at this point prevented that logic from running. Now the ratchet is gone and we want CI to turn red if anything fails. --- .circleci/run-tests.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index bf1b6ee8e..48f500280 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,9 +88,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - # Always succeed even if subunit2junitxml fails. subunit2junitxml signals - # failure if the stream it is processing contains test failures. This is - # not what we care about. If we cared about it, the test command above - # would have signalled failure already and we wouldn't be here. - "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || true + "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" fi From 2cc2cb6a7f655bd62ab8973e86a6c7847fa5c19f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Aug 2020 13:49:48 -0400 Subject: [PATCH 0798/1054] Rename to_str() to the more accurate to_bytes(). --- src/allmydata/scripts/backupdb.py | 6 ++-- src/allmydata/scripts/debug.py | 12 ++++---- src/allmydata/scripts/tahoe_backup.py | 4 +-- src/allmydata/scripts/tahoe_cp.py | 34 ++++++++++----------- src/allmydata/scripts/tahoe_ls.py | 6 ++-- src/allmydata/scripts/tahoe_mv.py | 4 +-- src/allmydata/test/cli/test_check.py | 4 +-- src/allmydata/test/cli/test_cp.py | 24 +++++++-------- src/allmydata/test/test_encodingutil.py | 14 ++++----- src/allmydata/test/web/test_grid.py | 30 +++++++++---------- src/allmydata/test/web/test_web.py | 40 ++++++++++++------------- src/allmydata/util/encodingutil.py | 10 +++++-- src/allmydata/web/common.py | 6 ++-- src/allmydata/web/directory.py | 4 +-- 14 files changed, 101 insertions(+), 97 deletions(-) diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 234f8524a..d188eec3c 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -6,7 +6,7 @@ from allmydata.util.netstring import netstring from allmydata.util.hashutil import backupdb_dirhash from allmydata.util import base32 from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import to_str +from allmydata.util.encodingutil import to_bytes from allmydata.util.dbutil import get_db, DBError @@ -218,7 +218,7 @@ class BackupDB_v2(object): probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) - return FileResult(self, to_str(filecap), should_check, + return FileResult(self, to_bytes(filecap), should_check, path, mtime, ctime, size) def get_or_allocate_fileid_for_cap(self, filecap): @@ -321,7 +321,7 @@ class BackupDB_v2(object): probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) - return DirectoryResult(self, dirhash_s, to_str(dircap), should_check) + return DirectoryResult(self, dirhash_s, to_bytes(dircap), should_check) def did_create_directory(self, dircap, dirhash): now = time.time() diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fff58caff..789218f8b 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -63,7 +63,7 @@ def dump_immutable_chk_share(f, out, options): from allmydata import uri from allmydata.util import base32 from allmydata.immutable.layout import ReadBucketProxy - from allmydata.util.encodingutil import quote_output, to_str + from allmydata.util.encodingutil import quote_output, to_bytes # use a ReadBucketProxy to parse the bucket and find the uri extension bp = ReadBucketProxy(None, None, '') @@ -109,7 +109,7 @@ def dump_immutable_chk_share(f, out, options): # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: - piece = to_str(pieces[-2]) + piece = to_bytes(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) uri_extension_hash = base32.a2b(unpacked["UEB_hash"]) @@ -221,7 +221,7 @@ def dump_SDMF_share(m, length, options): from allmydata.mutable.common import NeedMoreDataError from allmydata.util import base32, hashutil from allmydata.uri import SSKVerifierURI - from allmydata.util.encodingutil import quote_output, to_str + from allmydata.util.encodingutil import quote_output, to_bytes offset = m.DATA_OFFSET @@ -269,7 +269,7 @@ def dump_SDMF_share(m, length, options): # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: - piece = to_str(pieces[-2]) + piece = to_bytes(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey) @@ -307,7 +307,7 @@ def dump_MDMF_share(m, length, options): from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.util import base32, hashutil from allmydata.uri import MDMFVerifierURI - from allmydata.util.encodingutil import quote_output, to_str + from allmydata.util.encodingutil import quote_output, to_bytes offset = m.DATA_OFFSET out = options.stdout @@ -363,7 +363,7 @@ def dump_MDMF_share(m, length, options): # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: - piece = to_str(pieces[-2]) + piece = to_bytes(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 558c3d6d3..c63558eb1 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -11,7 +11,7 @@ from allmydata.scripts.common_http import do_http, HTTPError, format_http_error from allmydata.util import time_format from allmydata.scripts import backupdb from allmydata.util.encodingutil import listdir_unicode, quote_output, \ - quote_local_unicode_path, to_str, FilenameEncodingError, unicode_to_url + quote_local_unicode_path, to_bytes, FilenameEncodingError, unicode_to_url from allmydata.util.assertutil import precondition from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath @@ -47,7 +47,7 @@ def mkdir(contents, options): if resp.status < 200 or resp.status >= 300: raise HTTPError("Error during mkdir", resp) - dircap = to_str(resp.read().strip()) + dircap = to_bytes(resp.read().strip()) return dircap def put_child(dirurl, childname, childcap): diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 5d0849c56..c90dca072 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -13,7 +13,7 @@ from allmydata import uri from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \ - quote_local_unicode_path, to_str + quote_local_unicode_path, to_bytes from allmydata.util.assertutil import precondition, _assert @@ -254,8 +254,8 @@ class TahoeDirectorySource(object): def init_from_parsed(self, parsed): nodetype, d = parsed - self.writecap = to_str(d.get("rw_uri")) - self.readcap = to_str(d.get("ro_uri")) + self.writecap = to_bytes(d.get("rw_uri")) + self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) @@ -270,13 +270,13 @@ class TahoeDirectorySource(object): self.progressfunc("examining %d of %d" % (i+1, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) - writecap = to_str(data[1].get("rw_uri")) - readcap = to_str(data[1].get("ro_uri")) + writecap = to_bytes(data[1].get("rw_uri")) + readcap = to_bytes(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) elif data[0] == "dirnode": - writecap = to_str(data[1].get("rw_uri")) - readcap = to_str(data[1].get("ro_uri")) + writecap = to_bytes(data[1].get("rw_uri")) + readcap = to_bytes(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: @@ -324,8 +324,8 @@ class TahoeDirectoryTarget(object): def init_from_parsed(self, parsed): nodetype, d = parsed - self.writecap = to_str(d.get("rw_uri")) - self.readcap = to_str(d.get("ro_uri")) + self.writecap = to_bytes(d.get("rw_uri")) + self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) @@ -365,8 +365,8 @@ class TahoeDirectoryTarget(object): self.progressfunc("examining %d of %d" % (i+1, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) - writecap = to_str(data[1].get("rw_uri")) - readcap = to_str(data[1].get("ro_uri")) + writecap = to_bytes(data[1].get("rw_uri")) + readcap = to_bytes(data[1].get("ro_uri")) url = None if self.writecap: url = self.nodeurl + "/".join(["uri", @@ -375,8 +375,8 @@ class TahoeDirectoryTarget(object): self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) elif data[0] == "dirnode": - writecap = to_str(data[1].get("rw_uri")) - readcap = to_str(data[1].get("ro_uri")) + writecap = to_bytes(data[1].get("rw_uri")) + readcap = to_bytes(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: @@ -619,8 +619,8 @@ class Copier(object): self.progress) t.init_from_parsed(parsed) else: - writecap = to_str(d.get("rw_uri")) - readcap = to_str(d.get("ro_uri")) + writecap = to_bytes(d.get("rw_uri")) + readcap = to_bytes(d.get("ro_uri")) mutable = d.get("mutable", False) t = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) @@ -682,8 +682,8 @@ class Copier(object): else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec) - writecap = to_str(d.get("rw_uri")) - readcap = to_str(d.get("ro_uri")) + writecap = to_bytes(d.get("rw_uri")) + readcap = to_bytes(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 1332da32a..2bfe16d27 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -5,7 +5,7 @@ import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_str +from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_bytes def list(options): nodeurl = options['node-url'] @@ -94,8 +94,8 @@ def list(options): mtime = child[1].get("metadata", {}).get('tahoe', {}).get("linkmotime") if not mtime: mtime = child[1]["metadata"].get("mtime") - rw_uri = to_str(child[1].get("rw_uri")) - ro_uri = to_str(child[1].get("ro_uri")) + rw_uri = to_bytes(child[1].get("rw_uri")) + ro_uri = to_bytes(child[1].get("ro_uri")) if ctime: # match for formatting that GNU 'ls' does if (now - ctime) > 6*30*24*60*60: diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 4caaca928..7d13ea72a 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -6,7 +6,7 @@ import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.encodingutil import to_str +from allmydata.util.encodingutil import to_bytes # this script is used for both 'mv' and 'ln' @@ -35,7 +35,7 @@ def mv(options, mode="move"): return 1 data = resp.read() nodetype, attrs = json.loads(data) - cap = to_str(attrs.get("rw_uri") or attrs["ro_uri"]) + cap = to_bytes(attrs.get("rw_uri") or attrs["ro_uri"]) # now get the target try: diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index a2476e6c7..85649e262 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -5,7 +5,7 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import quote_output, to_str +from allmydata.util.encodingutil import quote_output, to_bytes from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug @@ -41,7 +41,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) - self.failUnlessReallyEqual(to_str(data["summary"]), "Healthy") + self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 7b076f327..59331029b 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -8,7 +8,7 @@ from twisted.internet import defer from allmydata.scripts import cli from allmydata.util import fileutil from allmydata.util.encodingutil import (quote_output, get_io_encoding, - unicode_to_output, to_str) + unicode_to_output, to_bytes) from allmydata.util.assertutil import _assert from ..no_network import GridTestMixin from .common import CLITestMixin @@ -272,9 +272,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) - self.rw_uri = to_str(data["rw_uri"]) + self.rw_uri = to_bytes(data["rw_uri"]) self.failUnlessIn("ro_uri", data) - self.ro_uri = to_str(data["ro_uri"]) + self.ro_uri = to_bytes(data["ro_uri"]) d.addCallback(_get_test_txt_uris) # Now make a new file to copy in place of test.txt. @@ -306,9 +306,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) - self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri) + self.failUnlessEqual(to_bytes(data["ro_uri"]), self.ro_uri) self.failUnlessIn("rw_uri", data) - self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri) + self.failUnlessEqual(to_bytes(data["rw_uri"]), self.rw_uri) d.addCallback(_check_json) # and, finally, doing a GET directly on one of the old uris @@ -381,7 +381,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIf(childdata['mutable']) self.failUnlessIn("ro_uri", childdata) uri_key = "ro_uri" - self.childuris[k] = to_str(childdata[uri_key]) + self.childuris[k] = to_bytes(childdata[uri_key]) d.addCallback(_process_directory_json) # Now build a local directory to copy into place, like the following: # test2/ @@ -410,11 +410,11 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): if "mutable" in fn: self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) - self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn]) + self.failUnlessEqual(to_bytes(data["rw_uri"]), self.childuris[fn]) else: self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) - self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn]) + self.failIfEqual(to_bytes(data["ro_uri"]), self.childuris[fn]) for fn in ("mutable1", "mutable2"): d.addCallback(lambda ignored, fn=fn: @@ -456,7 +456,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(filetype, "filenode") self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) - self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"]) + self.failUnlessEqual(to_bytes(data["ro_uri"]), self.childuris["imm2"]) d.addCallback(_process_imm2_json) return d @@ -497,7 +497,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) - self._test_read_uri = to_str(data["ro_uri"]) + self._test_read_uri = to_bytes(data["ro_uri"]) d.addCallback(_process_test_json) # Now we'll link the readonly URI into the tahoe: alias. d.addCallback(lambda ignored: @@ -521,7 +521,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(testtype, "filenode") self.failUnless(testdata['mutable']) self.failUnlessIn("ro_uri", testdata) - self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri) + self.failUnlessEqual(to_bytes(testdata["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", testdata) d.addCallback(_process_tahoe_json) # Okay, now we're going to try uploading another mutable file in @@ -589,7 +589,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(file2type, "filenode") self.failUnless(file2data['mutable']) self.failUnlessIn("ro_uri", file2data) - self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri) + self.failUnlessEqual(to_bytes(file2data["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", file2data) d.addCallback(_got_testdir_json) return d diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index cf72fc9ac..208b0f98d 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -84,7 +84,7 @@ from allmydata.util import encodingutil, fileutil from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \ unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \ - get_io_encoding, get_filesystem_encoding, to_str, from_utf8_or_none, _reload, \ + get_io_encoding, get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \ to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \ unicode_to_argv from twisted.python import usage @@ -600,12 +600,12 @@ class OpenBSD(EncodingUtil, unittest.TestCase): class TestToFromStr(ReallyEqualMixin, unittest.TestCase): - def test_to_str(self): - self.failUnlessReallyEqual(to_str(b"foo"), b"foo") - self.failUnlessReallyEqual(to_str(b"lumi\xc3\xa8re"), b"lumi\xc3\xa8re") - self.failUnlessReallyEqual(to_str(b"\xFF"), b"\xFF") # passes through invalid UTF-8 -- is this what we want? - self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), b"lumi\xc3\xa8re") - self.failUnlessReallyEqual(to_str(None), None) + def test_to_bytes(self): + self.failUnlessReallyEqual(to_bytes(b"foo"), b"foo") + self.failUnlessReallyEqual(to_bytes(b"lumi\xc3\xa8re"), b"lumi\xc3\xa8re") + self.failUnlessReallyEqual(to_bytes(b"\xFF"), b"\xFF") # passes through invalid UTF-8 -- is this what we want? + self.failUnlessReallyEqual(to_bytes(u"lumi\u00E8re"), b"lumi\xc3\xa8re") + self.failUnlessReallyEqual(to_bytes(None), None) def test_from_utf8_or_none(self): self.failUnlessRaises(AssertionError, from_utf8_or_none, u"foo") diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 73c354567..2a52d98e5 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -10,7 +10,7 @@ from twisted.web import resource from twisted.trial import unittest from allmydata import uri, dirnode from allmydata.util import base32 -from allmydata.util.encodingutil import to_str +from allmydata.util.encodingutil import to_bytes from allmydata.util.consumer import download_to_data from allmydata.util.netstring import split_netstring from allmydata.unknown import UnknownNode @@ -367,13 +367,13 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi f = data[1]["children"][name] self.failUnlessEqual(f[0], "unknown") if expect_rw_uri: - self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data) + self.failUnlessReallyEqual(to_bytes(f[1]["rw_uri"]), unknown_rwcap, data) else: self.failIfIn("rw_uri", f[1]) if immutable: - self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data) + self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_immcap, data) else: - self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data) + self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_rocap, data) self.failUnlessIn("metadata", f[1]) d.addCallback(_check_directory_json, expect_rw_uri=not immutable) @@ -406,18 +406,18 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi data = json.loads(res) self.failUnlessEqual(data[0], "unknown") if expect_rw_uri: - self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data) + self.failUnlessReallyEqual(to_bytes(data[1]["rw_uri"]), unknown_rwcap, data) else: self.failIfIn("rw_uri", data[1]) if immutable: - self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data) + self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_immcap, data) self.failUnlessReallyEqual(data[1]["mutable"], False) elif expect_rw_uri: - self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data) + self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_rocap, data) self.failUnlessReallyEqual(data[1]["mutable"], True) else: - self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data) + self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_rocap, data) self.failIfIn("mutable", data[1]) # TODO: check metadata contents @@ -581,7 +581,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi ll_type, ll_data = listed_children[u"lonely"] self.failUnlessEqual(ll_type, "filenode") self.failIfIn("rw_uri", ll_data) - self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri) + self.failUnlessReallyEqual(to_bytes(ll_data["ro_uri"]), lonely_uri) d.addCallback(_check_json) return d @@ -643,14 +643,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi u0 = units[0] self.failUnlessEqual(u0["path"], []) self.failUnlessEqual(u0["type"], "directory") - self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri()) + self.failUnlessReallyEqual(to_bytes(u0["cap"]), self.rootnode.get_uri()) u0cr = u0["check-results"] self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10) self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10) ugood = [u for u in units if u["type"] == "file" and u["path"] == [u"good"]][0] - self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"]) + self.failUnlessReallyEqual(to_bytes(ugood["cap"]), self.uris["good"]) ugoodcr = ugood["check-results"] self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10) self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10) @@ -672,7 +672,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessEqual(units[-1]["type"], "stats") first = units[0] self.failUnlessEqual(first["path"], []) - self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri()) + self.failUnlessEqual(to_bytes(first["cap"]), self.rootnode.get_uri()) self.failUnlessEqual(first["type"], "directory") stats = units[-1]["stats"] self.failUnlessReallyEqual(stats["count-immutable-files"], 2) @@ -826,7 +826,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi u0 = units[0] self.failUnlessEqual(u0["path"], []) self.failUnlessEqual(u0["type"], "directory") - self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri()) + self.failUnlessReallyEqual(to_bytes(u0["cap"]), self.rootnode.get_uri()) u0crr = u0["check-and-repair-results"] self.failUnlessReallyEqual(u0crr["repair-attempted"], False) self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10) @@ -834,7 +834,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi ugood = [u for u in units if u["type"] == "file" and u["path"] == [u"good"]][0] - self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"]) + self.failUnlessEqual(to_bytes(ugood["cap"]), self.uris["good"]) ugoodcrr = ugood["check-and-repair-results"] self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False) self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10) @@ -842,7 +842,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi usick = [u for u in units if u["type"] == "file" and u["path"] == [u"sick"]][0] - self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"]) + self.failUnlessReallyEqual(to_bytes(usick["cap"]), self.uris["sick"]) usickcrr = usick["check-and-repair-results"] self.failUnlessReallyEqual(usickcrr["repair-attempted"], True) self.failUnlessReallyEqual(usickcrr["repair-successful"], True) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b4d604ed4..0dc12a4c5 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -36,7 +36,7 @@ from allmydata.nodemaker import NodeMaker from allmydata.web.common import WebError, MultiFormatPage from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data -from allmydata.util.encodingutil import to_str +from allmydata.util.encodingutil import to_bytes from ...util.connection_status import ConnectionStatus from ..common import ( EMPTY_CLIENT_CONFIG, @@ -457,8 +457,8 @@ class WebMixin(TimezoneMixin): self.failUnless(isinstance(data[1], dict)) self.failIf(data[1]["mutable"]) self.failIfIn("rw_uri", data[1]) # immutable - self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri) - self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap) + self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), self._bar_txt_uri) + self.failUnlessReallyEqual(to_bytes(data[1]["verify_uri"]), self._bar_txt_verifycap) self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS)) def failUnlessIsQuuxJSON(self, res, readonly=False): @@ -487,9 +487,9 @@ class WebMixin(TimezoneMixin): self.failUnless(isinstance(data[1], dict)) self.failUnless(data[1]["mutable"]) self.failUnlessIn("rw_uri", data[1]) # mutable - self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri) - self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri) - self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap) + self.failUnlessReallyEqual(to_bytes(data[1]["rw_uri"]), self._foo_uri) + self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), self._foo_readonly_uri) + self.failUnlessReallyEqual(to_bytes(data[1]["verify_uri"]), self._foo_verifycap) kidnames = sorted([unicode(n) for n in data[1]["children"]]) self.failUnlessEqual(kidnames, @@ -506,19 +506,19 @@ class WebMixin(TimezoneMixin): self.failUnlessIn("linkmotime", tahoe_md) self.failUnlessEqual(kids[u"bar.txt"][0], "filenode") self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS)) - self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri) - self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]), + self.failUnlessReallyEqual(to_bytes(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri) + self.failUnlessReallyEqual(to_bytes(kids[u"bar.txt"][1]["verify_uri"]), self._bar_txt_verifycap) self.failUnlessIn("metadata", kids[u"bar.txt"][1]) self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"]) self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"], self._bar_txt_metadata["tahoe"]["linkcrtime"]) - self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]), + self.failUnlessReallyEqual(to_bytes(kids[u"n\u00fc.txt"][1]["ro_uri"]), self._bar_txt_uri) self.failUnlessIn("quux.txt", kids) - self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]), + self.failUnlessReallyEqual(to_bytes(kids[u"quux.txt"][1]["rw_uri"]), self._quux_txt_uri) - self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]), + self.failUnlessReallyEqual(to_bytes(kids[u"quux.txt"][1]["ro_uri"]), self._quux_txt_readonly_uri) @inlineCallbacks @@ -2181,7 +2181,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi got = {} for (path_list, cap) in data: got[tuple(path_list)] = cap - self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri) + self.failUnlessReallyEqual(to_bytes(got[(u"sub",)]), self._sub_uri) self.failUnlessIn((u"sub", u"baz.txt"), got) self.failUnlessIn("finished", res) self.failUnlessIn("origin", res) @@ -2266,9 +2266,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(units[-1]["type"], "stats") first = units[0] self.failUnlessEqual(first["path"], []) - self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri) + self.failUnlessReallyEqual(to_bytes(first["cap"]), self._foo_uri) self.failUnlessEqual(first["type"], "directory") - baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0] + baz = [u for u in units[:-1] if to_bytes(u["cap"]) == self._baz_file_uri][0] self.failUnlessEqual(baz["path"], ["sub", "baz.txt"]) self.failIfEqual(baz["storage-index"], None) self.failIfEqual(baz["verifycap"], None) @@ -2281,14 +2281,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_GET_DIRURL_uri(self): d = self.GET(self.public_url + "/foo?t=uri") def _check(res): - self.failUnlessReallyEqual(to_str(res), self._foo_uri) + self.failUnlessReallyEqual(to_bytes(res), self._foo_uri) d.addCallback(_check) return d def test_GET_DIRURL_readonly_uri(self): d = self.GET(self.public_url + "/foo?t=readonly-uri") def _check(res): - self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri) + self.failUnlessReallyEqual(to_bytes(res), self._foo_readonly_uri) d.addCallback(_check) return d @@ -2950,9 +2950,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi new_json = children[u"new.txt"] self.failUnlessEqual(new_json[0], "filenode") self.failUnless(new_json[1]["mutable"]) - self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri) + self.failUnlessReallyEqual(to_bytes(new_json[1]["rw_uri"]), self._mutable_uri) ro_uri = self._mutable_node.get_readonly().to_string() - self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri) + self.failUnlessReallyEqual(to_bytes(new_json[1]["ro_uri"]), ro_uri) d.addCallback(_check_page_json) # and the JSON form of the file @@ -2962,9 +2962,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi parsed = json.loads(res) self.failUnlessEqual(parsed[0], "filenode") self.failUnless(parsed[1]["mutable"]) - self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri) + self.failUnlessReallyEqual(to_bytes(parsed[1]["rw_uri"]), self._mutable_uri) ro_uri = self._mutable_node.get_readonly().to_string() - self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri) + self.failUnlessReallyEqual(to_bytes(parsed[1]["ro_uri"]), ro_uri) d.addCallback(_check_file_json) # and look at t=uri and t=readonly-uri diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 07416cc93..24e420496 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -155,11 +155,15 @@ def unicode_to_url(s): # According to RFC 2718, non-ascii characters in URLs must be UTF-8 encoded. # FIXME - return to_str(s) + return to_bytes(s) #precondition(isinstance(s, unicode), s) #return s.encode('utf-8') -def to_str(s): # TODO rename to to_bytes +def to_bytes(s): + """Convert unicode to bytes. + + None and bytes are passed through unchanged. + """ if s is None or isinstance(s, bytes): return s return s.encode('utf-8') @@ -274,7 +278,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): return b'"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),) def quote_path(path, quotemarks=True): - return quote_output(b"/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True) + return quote_output(b"/".join(map(to_bytes, path)), quotemarks=quotemarks, quote_newlines=True) def quote_local_unicode_path(path, quotemarks=True): precondition(isinstance(path, unicode), path) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index a930fd2b1..788557480 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -18,7 +18,7 @@ from allmydata.mutable.common import UnrecoverableFileError from allmydata.util import abbreviate from allmydata.util.hashutil import timing_safe_compare from allmydata.util.time_format import format_time, format_delta -from allmydata.util.encodingutil import to_str, quote_output +from allmydata.util.encodingutil import to_bytes, quote_output def get_filenode_metadata(filenode): @@ -133,8 +133,8 @@ def convert_children_json(nodemaker, children_json): data = json.loads(children_json) for (namex, (ctype, propdict)) in data.iteritems(): namex = unicode(namex) - writecap = to_str(propdict.get("rw_uri")) - readcap = to_str(propdict.get("ro_uri")) + writecap = to_bytes(propdict.get("rw_uri")) + readcap = to_bytes(propdict.get("ro_uri")) metadata = propdict.get("metadata", {}) # name= argument is just for error reporting childnode = nodemaker.create_from_cap(writecap, readcap, name=namex) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index a5ba3bed5..9fdecbcb4 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -22,7 +22,7 @@ from twisted.python.filepath import FilePath from allmydata.util import base32 from allmydata.util.encodingutil import ( - to_str, + to_bytes, quote_output, ) from allmydata.uri import ( @@ -484,7 +484,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): to_dir = to_dir.decode(charset) assert isinstance(to_dir, unicode) to_path = to_dir.split(u"/") - to_root = self.client.nodemaker.create_from_cap(to_str(to_path[0])) + to_root = self.client.nodemaker.create_from_cap(to_bytes(to_path[0])) if not IDirectoryNode.providedBy(to_root): raise WebError("to_dir is not a directory", http.BAD_REQUEST) d = to_root.get_child_at_path(to_path[1:]) From 03ed0fd66fba26d622167000dc3f41717e8d7e00 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Aug 2020 11:29:09 -0400 Subject: [PATCH 0799/1054] Another function that should be a no-op on Python 3. --- src/allmydata/test/test_encodingutil.py | 8 +++++++- src/allmydata/util/encodingutil.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 208b0f98d..61e90c068 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -144,6 +144,7 @@ class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase): argv_to_unicode, lumiere_nfc.encode('latin1')) + @skipIf(PY3, "Python 2 only.") def test_unicode_to_output(self): encodingutil.io_encoding = 'koi8-r' self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc) @@ -228,7 +229,8 @@ class EncodingUtil(ReallyEqualMixin): def test_unicode_to_url(self): self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re") - def test_unicode_to_output(self): + @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.") + def test_unicode_to_output_py2(self): if 'argv' not in dir(self): return @@ -239,6 +241,10 @@ class EncodingUtil(ReallyEqualMixin): _reload() self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv) + @skipIf(PY2, "Python 3 only.") + def test_unicode_to_output_py3(self): + self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), lumiere_nfc) + @skipIf(PY3, "Python 2 only.") def test_unicode_to_argv_py2(self): """unicode_to_argv() converts to bytes on Python 2.""" diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 24e420496..bfcbb13d9 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -21,6 +21,7 @@ from past.builtins import unicode import sys, os, re, locale import unicodedata +import warnings from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -140,6 +141,8 @@ def unicode_to_argv(s, mangle=False): """ precondition(isinstance(s, unicode), s) if PY3: + warnings.warn("This will be unnecessary once Python 2 is dropped.", + DeprecationWarning) return s if mangle and sys.platform == "win32": @@ -185,6 +188,10 @@ def unicode_to_output(s): Encode an unicode object for representation on stdout or stderr. """ precondition(isinstance(s, unicode), s) + if PY3: + warnings.warn("This will be unnecessary once Python 2 is dropped.", + DeprecationWarning) + return s try: out = s.encode(io_encoding) From af5e9eaf67f7af6959b63b519b8197dfc06618ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Aug 2020 11:29:27 -0400 Subject: [PATCH 0800/1054] A better name. --- src/allmydata/util/encodingutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index bfcbb13d9..1d41f337a 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -224,7 +224,7 @@ def _unicode_escape(m, quote_newlines): else: return u'\\x%02x' % (codepoint,) -def _str_escape(m, quote_newlines): # TODO rename to _bytes_escape +def _bytes_escape(m, quote_newlines): """ Takes a re match on bytes, the result is escaped bytes of group(0). """ @@ -268,7 +268,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): try: s = s.decode('utf-8') except UnicodeDecodeError: - return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _str_escape(m, quote_newlines), s),) + return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE if must_double_quote.search(s) is None: From f95f9c481eb9a9d3a728e92de7df32e47710e7a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Aug 2020 12:04:52 -0400 Subject: [PATCH 0801/1054] Explanation. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 1d41f337a..56c6deb88 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -186,6 +186,9 @@ def is_printable_ascii(s): def unicode_to_output(s): """ Encode an unicode object for representation on stdout or stderr. + + On Python 3 just returns the string unchanged, since it is not necessary to + encode in any way. """ precondition(isinstance(s, unicode), s) if PY3: From 97c3be05093f6e62f8ac8f5b8efa7c9505d103d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Aug 2020 13:29:49 -0400 Subject: [PATCH 0802/1054] Make quote_* and friends return unicode. --- src/allmydata/test/test_encodingutil.py | 55 ++++++++++++++++--------- src/allmydata/util/encodingutil.py | 53 ++++++++++++++++-------- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 61e90c068..376bd6ec6 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -355,6 +355,8 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): _reload() def _check(self, inp, out, enc, optional_quotes, quote_newlines): + if PY3 and isinstance(out, bytes): + out = out.decode(enc or encodingutil.io_encoding) out2 = out if optional_quotes: out2 = out2[1:-1] @@ -382,6 +384,9 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): def _test_quote_output_all(self, enc): def check(inp, out, optional_quotes=False, quote_newlines=None): + if PY3: + # Result is always Unicode on Python 3 + out = out.decode("ascii") self._check(inp, out, enc, optional_quotes, quote_newlines) # optional single quotes @@ -444,7 +449,10 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): def test_quote_output_utf8(self, enc='utf-8'): def check(inp, out, optional_quotes=False, quote_newlines=None): - self._check(inp, out.encode('utf-8'), enc, optional_quotes, quote_newlines) + if PY2: + # On Python 3 output is always Unicode: + out = out.encode('utf-8') + self._check(inp, out, enc, optional_quotes, quote_newlines) self._test_quote_output_all(enc) check(u"\u2621", u"'\u2621'", True) @@ -469,43 +477,50 @@ def win32_other(win32, other): return win32 if sys.platform == "win32" else other class QuotePaths(ReallyEqualMixin, unittest.TestCase): - def test_quote_path(self): - self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), b"'foo/bar'") - self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'") - self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar") - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), b'"foo/\\x0abar"') - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"') - self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), b'"foo/\\x0abar"') - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"), + def assertPathsEqual(self, actual, expected): + if PY3: + # On Python 3, results should be unicode: + expected = expected.decode("ascii") + self.failUnlessReallyEqual(actual, expected) + + def test_quote_path(self): + self.assertPathsEqual(quote_path([u'foo', u'bar']), b"'foo/bar'") + self.assertPathsEqual(quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'") + self.assertPathsEqual(quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar") + self.assertPathsEqual(quote_path([u'foo', u'\nbar']), b'"foo/\\x0abar"') + self.assertPathsEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"') + self.assertPathsEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), b'"foo/\\x0abar"') + + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"), win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'")) - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True), + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True), win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'")) - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False), + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False), win32_other(b"C:\\foo", b"\\\\?\\C:\\foo")) - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'")) - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True), + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True), win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'")) - self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False), + self.assertPathsEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False), win32_other(b"\\\\foo\\bar", b"\\\\?\\UNC\\foo\\bar")) def test_quote_filepath(self): foo_bar_fp = FilePath(win32_other(u'C:\\foo\\bar', u'/foo/bar')) - self.failUnlessReallyEqual(quote_filepath(foo_bar_fp), + self.assertPathsEqual(quote_filepath(foo_bar_fp), win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'")) - self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=True), + self.assertPathsEqual(quote_filepath(foo_bar_fp, quotemarks=True), win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'")) - self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=False), + self.assertPathsEqual(quote_filepath(foo_bar_fp, quotemarks=False), win32_other(b"C:\\foo\\bar", b"/foo/bar")) if sys.platform == "win32": foo_longfp = FilePath(u'\\\\?\\C:\\foo') - self.failUnlessReallyEqual(quote_filepath(foo_longfp), + self.assertPathsEqual(quote_filepath(foo_longfp), b"'C:\\foo'") - self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=True), + self.assertPathsEqual(quote_filepath(foo_longfp, quotemarks=True), b"'C:\\foo'") - self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=False), + self.assertPathsEqual(quote_filepath(foo_longfp, quotemarks=False), b"C:\\foo") diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 56c6deb88..35ee7f64c 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -262,30 +262,49 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): Python-compatible backslash escaping is used. If not explicitly given, quote_newlines is True when quotemarks is True. + + On Python 3, returns Unicode strings. """ precondition(isinstance(s, (bytes, unicode)), s) + encoding = encoding or io_encoding + if quote_newlines is None: quote_newlines = quotemarks - if isinstance(s, bytes): - try: - s = s.decode('utf-8') - except UnicodeDecodeError: - return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) + def _encode(s): + if isinstance(s, bytes): + try: + s = s.decode('utf-8') + except UnicodeDecodeError: + return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) - must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE - if must_double_quote.search(s) is None: - try: - out = s.encode(encoding or io_encoding) - if quotemarks or out.startswith(b'"'): - return b"'%s'" % (out,) - else: - return out - except (UnicodeDecodeError, UnicodeEncodeError): - pass + must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE + if must_double_quote.search(s) is None: + try: + out = s.encode(encoding) + if quotemarks or out.startswith(b'"'): + return b"'%s'" % (out,) + else: + return out + except (UnicodeDecodeError, UnicodeEncodeError): + pass + + escaped = ESCAPABLE_UNICODE.sub(lambda m: _unicode_escape(m, quote_newlines), s) + return b'"%s"' % (escaped.encode(encoding, 'backslashreplace'),) + + result = _encode(s) + if PY3: + # On Python half of what this function does is unnecessary, since + # output is always Unicode. To ensure no encode errors, one can do: + # + # sys.stdout.reconfigure(encoding=sys.stdout.encoding, errors="backslashreplace") + # + # Although the problem is that doesn't work in Python 3.6, only 3.7 or + # later... For now not thinking about it, just returning unicode since + # that is the right thing to do on Python 3. + result = result.decode(encoding) + return result - escaped = ESCAPABLE_UNICODE.sub(lambda m: _unicode_escape(m, quote_newlines), s) - return b'"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),) def quote_path(path, quotemarks=True): return quote_output(b"/".join(map(to_bytes, path)), quotemarks=quotemarks, quote_newlines=True) From 01ba0d0316860f18a7cf4b3994c971b5893aaf6f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Aug 2020 13:30:12 -0400 Subject: [PATCH 0803/1054] News file. --- newsfragments/3378.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3378.minor diff --git a/newsfragments/3378.minor b/newsfragments/3378.minor new file mode 100644 index 000000000..e69de29bb From c66d38e1912063a437191a39c0e0b63188e22275 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Aug 2020 08:37:59 -0400 Subject: [PATCH 0804/1054] Make web.directory.DeepSizeResults a MultiFormatResource --- src/allmydata/web/directory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index a5ba3bed5..949dbae46 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -53,7 +53,6 @@ from allmydata.web.common import ( get_mutable_type, get_filenode_metadata, render_time, - MultiFormatPage, MultiFormatResource, SlotsSequenceElement, ) @@ -1213,7 +1212,7 @@ class ManifestElement(ReloadableMonitorElement): class ManifestResults(MultiFormatResource, ReloadMixin): - # Control MultiFormatPage + # Control MultiFormatResource formatArgument = "output" formatDefault = "html" @@ -1268,8 +1267,9 @@ class ManifestResults(MultiFormatResource, ReloadMixin): return json.dumps(status, indent=1) -class DeepSizeResults(MultiFormatPage): - # Control MultiFormatPage +class DeepSizeResults(MultiFormatResource): + + # Control MultiFormatResource formatArgument = "output" formatDefault = "html" From 0de92a22656a62355180225f8a5fb8661f845545 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Aug 2020 08:38:55 -0400 Subject: [PATCH 0805/1054] Add newsfragment --- newsfragments/3381.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3381.minor diff --git a/newsfragments/3381.minor b/newsfragments/3381.minor new file mode 100644 index 000000000..e69de29bb From d5853ccb6050c5f361f25250dffa453f562293e1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 18 Aug 2020 08:48:02 -0400 Subject: [PATCH 0806/1054] Remove web.common.MultiFormatPage --- newsfragments/3382.minor | 0 src/allmydata/test/web/test_web.py | 112 +---------------------------- src/allmydata/web/common.py | 58 --------------- 3 files changed, 1 insertion(+), 169 deletions(-) create mode 100644 newsfragments/3382.minor diff --git a/newsfragments/3382.minor b/newsfragments/3382.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b4d604ed4..202d7216e 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -33,7 +33,7 @@ from allmydata.immutable import upload from allmydata.immutable.downloader.status import DownloadStatus from allmydata.dirnode import DirectoryNode from allmydata.nodemaker import NodeMaker -from allmydata.web.common import WebError, MultiFormatPage +from allmydata.web.common import WebError from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_str @@ -657,116 +657,6 @@ class WebMixin(TimezoneMixin): (which, res)) - -class MultiFormatPageTests(TrialTestCase): - """ - Tests for ``MultiFormatPage``. - """ - def resource(self): - """ - Create and return an instance of a ``MultiFormatPage`` subclass with two - formats: ``a`` and ``b``. - """ - class Content(MultiFormatPage): - docFactory = stan("doc factory") - - def render_A(self, req): - return "a" - - def render_B(self, req): - return "b" - return Content() - - - def render(self, resource, **query_args): - """ - Render a Nevow ``Page`` against a request with the given query arguments. - - :param resource: The Nevow resource to render. - - :param query_args: The query arguments to put into the request being - rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. - - :return: The rendered response body as ``bytes``. - """ - ctx = WebContext(tag=resource) - req = FakeRequest(args=query_args) - ctx.remember(DefaultExceptionHandler(), ICanHandleException) - ctx.remember(req, IRequest) - ctx.remember(None, IData) - - d = maybeDeferred(resource.renderHTTP, ctx) - d.addErrback(processingFailed, req, ctx) - res = self.successResultOf(d) - if isinstance(res, bytes): - return req.v + res - return req.v - - - def test_select_format(self): - """ - The ``formatArgument`` attribute of a ``MultiFormatPage`` subclass - identifies the query argument which selects the result format. - """ - resource = self.resource() - resource.formatArgument = "foo" - self.assertEqual("a", self.render(resource, foo=["a"])) - - - def test_default_format_argument(self): - """ - If a ``MultiFormatPage`` subclass does not set ``formatArgument`` then the - ``t`` argument is used. - """ - resource = self.resource() - self.assertEqual("a", self.render(resource, t=["a"])) - - - def test_no_format(self): - """ - If no value is given for the format argument and no default format has - been defined, the base Nevow rendering behavior is used - (``renderHTTP``). - """ - resource = self.resource() - self.assertEqual("doc factory", self.render(resource)) - - - def test_default_format(self): - """ - If no value is given for the format argument and the ``MultiFormatPage`` - subclass defines a ``formatDefault``, that value is used as the format - to render. - """ - resource = self.resource() - resource.formatDefault = "b" - self.assertEqual("b", self.render(resource)) - - - def test_explicit_none_format_renderer(self): - """ - If a format is selected which has a renderer set to ``None``, the base - Nevow rendering behavior is used (``renderHTTP``). - """ - resource = self.resource() - resource.render_FOO = None - self.assertEqual("doc factory", self.render(resource, t=["foo"])) - - - def test_unknown_format(self): - """ - If a format is selected for which there is no renderer, an error is - returned. - """ - resource = self.resource() - self.assertIn( - "Exception", - self.render(resource, t=["foo"]), - ) - self.flushLoggedErrors(WebError) - - - class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, TrialTestCase): maxDiff = None diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index a930fd2b1..e057621fd 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,6 @@ from twisted.web import http, server, resource, template from twisted.python import log from twisted.python.failure import Failure from nevow import loaders, appserver -from nevow.rend import Page from nevow.inevow import IRequest from nevow.util import resource_filename from allmydata import blacklist @@ -364,63 +363,6 @@ class MyExceptionHandler(appserver.DefaultExceptionHandler, object): class NeedOperationHandleError(WebError): pass - -class MultiFormatPage(Page): - """ - ```MultiFormatPage`` is a ``rend.Page`` that can be rendered in a number - of different formats. - - Rendered format is controlled by a query argument (given by - ``self.formatArgument``). Different resources may support different - formats but ``json`` is a pretty common one. - """ - formatArgument = "t" - formatDefault = None - - def renderHTTP(self, ctx): - """ - Dispatch to a renderer for a particular format, as selected by a query - argument. - - A renderer for the format given by the query argument matching - ``formatArgument`` will be selected and invoked. The default ``Page`` - rendering behavior will be used if no format is selected (either by - query arguments or by ``formatDefault``). - - :return: The result of the selected renderer. - """ - req = IRequest(ctx) - t = get_arg(req, self.formatArgument, self.formatDefault) - renderer = self._get_renderer(t) - result = renderer(ctx) - return result - - - def _get_renderer(self, fmt): - """ - Get the renderer for the indicated format. - - :param bytes fmt: The format. If a method with a prefix of - ``render_`` and a suffix of this format (upper-cased) is found, it - will be used. - - :return: A callable which takes a Nevow context and renders a - response. - """ - if fmt is None: - return super(MultiFormatPage, self).renderHTTP - try: - renderer = getattr(self, "render_{}".format(fmt.upper())) - except AttributeError: - raise WebError( - "Unknown {} value: {!r}".format(self.formatArgument, fmt), - ) - else: - if renderer is None: - return super(MultiFormatPage, self).renderHTTP - return lambda ctx: renderer(IRequest(ctx)) - - class MultiFormatResource(resource.Resource, object): """ ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in From 80b9a9b3ce2b3d832ff5b00f6944ee1fa2c53d64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 18 Aug 2020 13:03:36 -0400 Subject: [PATCH 0807/1054] Split the test module in two. --- src/allmydata/test/test_storage.py | 1293 +---------------------- src/allmydata/test/test_storage_web.py | 1309 ++++++++++++++++++++++++ 2 files changed, 1310 insertions(+), 1292 deletions(-) create mode 100644 src/allmydata/test/test_storage_web.py diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index ccf9fc5cb..ca7f2b0d0 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2,8 +2,6 @@ import time import os.path import platform import stat -import re -import json import struct import shutil import gc @@ -11,30 +9,16 @@ import gc from twisted.trial import unittest from twisted.internet import defer -from twisted.application import service -from twisted.web.template import flattenString -# We need to use `nevow.inevow.IRequest` for now for compatibility -# with the code in web/common.py. Once nevow bits are gone from -# web/common.py, we can use `twisted.web.iweb.IRequest` here. -from nevow.inevow import IRequest - -from twisted.web.server import Request -from twisted.web.test.requesthelper import DummyChannel -from zope.interface import implementer - -from foolscap.api import fireEventually import itertools from allmydata import interfaces -from allmydata.util import fileutil, hashutil, base32, pollmixin +from allmydata.util import fileutil, hashutil, base32 from allmydata.storage.server import StorageServer from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.storage.lease import LeaseInfo -from allmydata.storage.crawler import BucketCountingCrawler -from allmydata.storage.expirer import LeaseCheckingCrawler from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \ ReadBucketProxy from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ @@ -48,11 +32,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import ( - StorageStatus, - StorageStatusElement, - remove_prefix -) from allmydata.storage_client import ( _StorageServer, ) @@ -2980,1273 +2959,3 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["95_0_percentile"] is None, output) self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) - -def remove_tags(s): - s = re.sub(r'<[^>]*>', ' ', s) - s = re.sub(r'\s+', ' ', s) - return s - -def renderSynchronously(ss): - """ - Return fully rendered HTML document. - - :param _StorageStatus ss: a StorageStatus instance. - """ - return unittest.TestCase().successResultOf(renderDeferred(ss)) - -def renderDeferred(ss): - """ - Return a `Deferred` HTML renderer. - - :param _StorageStatus ss: a StorageStatus instance. - """ - elem = StorageStatusElement(ss._storage, ss._nickname) - return flattenString(None, elem) - -def renderJSON(resource): - """Render a JSON from the given resource.""" - - @implementer(IRequest) - class JSONRequest(Request): - """ - A Request with t=json argument added to it. This is useful to - invoke a Resouce.render_JSON() method. - """ - def __init__(self): - Request.__init__(self, DummyChannel()) - self.args = {"t": ["json"]} - self.fields = {} - - return resource.render(JSONRequest()) - -class MyBucketCountingCrawler(BucketCountingCrawler): - def finished_prefix(self, cycle, prefix): - BucketCountingCrawler.finished_prefix(self, cycle, prefix) - if self.hook_ds: - d = self.hook_ds.pop(0) - d.callback(None) - -class MyStorageServer(StorageServer): - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = MyBucketCountingCrawler(self, statefile) - self.bucket_counter.setServiceParent(self) - -class BucketCounter(unittest.TestCase, pollmixin.PollMixin): - - def setUp(self): - self.s = service.MultiService() - self.s.startService() - def tearDown(self): - return self.s.stopService() - - def test_bucket_counter(self): - basedir = "storage/BucketCounter/bucket_counter" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - # to make sure we capture the bucket-counting-crawler in the middle - # of a cycle, we reach in and reduce its maximum slice time to 0. We - # also make it start sooner than usual. - ss.bucket_counter.slow_start = 0 - orig_cpu_slice = ss.bucket_counter.cpu_slice - ss.bucket_counter.cpu_slice = 0 - ss.setServiceParent(self.s) - - w = StorageStatus(ss) - - # this sample is before the crawler has started doing anything - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Reserved space: - 0 B (0)", s) - self.failUnlessIn("Total buckets: Not computed yet", s) - self.failUnlessIn("Next crawl in", s) - - # give the bucket-counting-crawler one tick to get started. The - # cpu_slice=0 will force it to yield right after it processes the - # first prefix - - d = fireEventually() - def _check(ignored): - # are we really right after the first prefix? - state = ss.bucket_counter.get_state() - if state["last-complete-prefix"] is None: - d2 = fireEventually() - d2.addCallback(_check) - return d2 - self.failUnlessEqual(state["last-complete-prefix"], - ss.bucket_counter.prefixes[0]) - ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible - html = renderSynchronously(w) - s = remove_tags(html) - self.failUnlessIn(" Current crawl ", s) - self.failUnlessIn(" (next work in ", s) - d.addCallback(_check) - - # now give it enough time to complete a full cycle - def _watch(): - return not ss.bucket_counter.get_progress()["cycle-in-progress"] - d.addCallback(lambda ignored: self.poll(_watch)) - def _check2(ignored): - ss.bucket_counter.cpu_slice = orig_cpu_slice - html = renderSynchronously(w) - s = remove_tags(html) - self.failUnlessIn("Total buckets: 0 (the number of", s) - self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) - d.addCallback(_check2) - return d - - def test_bucket_counter_cleanup(self): - basedir = "storage/BucketCounter/bucket_counter_cleanup" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - # to make sure we capture the bucket-counting-crawler in the middle - # of a cycle, we reach in and reduce its maximum slice time to 0. - ss.bucket_counter.slow_start = 0 - orig_cpu_slice = ss.bucket_counter.cpu_slice - ss.bucket_counter.cpu_slice = 0 - ss.setServiceParent(self.s) - - d = fireEventually() - - def _after_first_prefix(ignored): - state = ss.bucket_counter.state - if state["last-complete-prefix"] is None: - d2 = fireEventually() - d2.addCallback(_after_first_prefix) - return d2 - ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible - # now sneak in and mess with its state, to make sure it cleans up - # properly at the end of the cycle - self.failUnlessEqual(state["last-complete-prefix"], - ss.bucket_counter.prefixes[0]) - state["bucket-counts"][-12] = {} - state["storage-index-samples"]["bogusprefix!"] = (-12, []) - ss.bucket_counter.save_state() - d.addCallback(_after_first_prefix) - - # now give it enough time to complete a cycle - def _watch(): - return not ss.bucket_counter.get_progress()["cycle-in-progress"] - d.addCallback(lambda ignored: self.poll(_watch)) - def _check2(ignored): - ss.bucket_counter.cpu_slice = orig_cpu_slice - s = ss.bucket_counter.get_state() - self.failIf(-12 in s["bucket-counts"], s["bucket-counts"].keys()) - self.failIf("bogusprefix!" in s["storage-index-samples"], - s["storage-index-samples"].keys()) - d.addCallback(_check2) - return d - - def test_bucket_counter_eta(self): - basedir = "storage/BucketCounter/bucket_counter_eta" - fileutil.make_dirs(basedir) - ss = MyStorageServer(basedir, "\x00" * 20) - ss.bucket_counter.slow_start = 0 - # these will be fired inside finished_prefix() - hooks = ss.bucket_counter.hook_ds = [defer.Deferred() for i in range(3)] - w = StorageStatus(ss) - - d = defer.Deferred() - - def _check_1(ignored): - # no ETA is available yet - html = renderSynchronously(w) - s = remove_tags(html) - self.failUnlessIn("complete (next work", s) - - def _check_2(ignored): - # one prefix has finished, so an ETA based upon that elapsed time - # should be available. - html = renderSynchronously(w) - s = remove_tags(html) - self.failUnlessIn("complete (ETA ", s) - - def _check_3(ignored): - # two prefixes have finished - html = renderSynchronously(w) - s = remove_tags(html) - self.failUnlessIn("complete (ETA ", s) - d.callback("done") - - hooks[0].addCallback(_check_1).addErrback(d.errback) - hooks[1].addCallback(_check_2).addErrback(d.errback) - hooks[2].addCallback(_check_3).addErrback(d.errback) - - ss.setServiceParent(self.s) - return d - -class InstrumentedLeaseCheckingCrawler(LeaseCheckingCrawler): - stop_after_first_bucket = False - def process_bucket(self, *args, **kwargs): - LeaseCheckingCrawler.process_bucket(self, *args, **kwargs) - if self.stop_after_first_bucket: - self.stop_after_first_bucket = False - self.cpu_slice = -1.0 - def yielding(self, sleep_time): - if not self.stop_after_first_bucket: - self.cpu_slice = 500 - -class BrokenStatResults(object): - pass - -class No_ST_BLOCKS_LeaseCheckingCrawler(LeaseCheckingCrawler): - def stat(self, fn): - s = os.stat(fn) - bsr = BrokenStatResults() - for attrname in dir(s): - if attrname.startswith("_"): - continue - if attrname == "st_blocks": - continue - setattr(bsr, attrname, getattr(s, attrname)) - return bsr - -class InstrumentedStorageServer(StorageServer): - LeaseCheckerClass = InstrumentedLeaseCheckingCrawler -class No_ST_BLOCKS_StorageServer(StorageServer): - LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler - -class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): - - def setUp(self): - self.s = service.MultiService() - self.s.startService() - def tearDown(self): - return self.s.stopService() - - def make_shares(self, ss): - def make(si): - return (si, hashutil.tagged_hash("renew", si), - hashutil.tagged_hash("cancel", si)) - def make_mutable(si): - return (si, hashutil.tagged_hash("renew", si), - hashutil.tagged_hash("cancel", si), - hashutil.tagged_hash("write-enabler", si)) - def make_extra_lease(si, num): - return (hashutil.tagged_hash("renew-%d" % num, si), - hashutil.tagged_hash("cancel-%d" % num, si)) - - immutable_si_0, rs0, cs0 = make("\x00" * 16) - immutable_si_1, rs1, cs1 = make("\x01" * 16) - rs1a, cs1a = make_extra_lease(immutable_si_1, 1) - mutable_si_2, rs2, cs2, we2 = make_mutable("\x02" * 16) - mutable_si_3, rs3, cs3, we3 = make_mutable("\x03" * 16) - rs3a, cs3a = make_extra_lease(mutable_si_3, 1) - sharenums = [0] - canary = FakeCanary() - # note: 'tahoe debug dump-share' will not handle this file, since the - # inner contents are not a valid CHK share - data = "\xff" * 1000 - - a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - - a,w = ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - ss.remote_add_lease(immutable_si_1, rs1a, cs1a) - - writev = ss.remote_slot_testv_and_readv_and_writev - writev(mutable_si_2, (we2, rs2, cs2), - {0: ([], [(0,data)], len(data))}, []) - writev(mutable_si_3, (we3, rs3, cs3), - {0: ([], [(0,data)], len(data))}, []) - ss.remote_add_lease(mutable_si_3, rs3a, cs3a) - - self.sis = [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] - self.renew_secrets = [rs0, rs1, rs1a, rs2, rs3, rs3a] - self.cancel_secrets = [cs0, cs1, cs1a, cs2, cs3, cs3a] - - def test_basic(self): - basedir = "storage/LeaseCrawler/basic" - fileutil.make_dirs(basedir) - ss = InstrumentedStorageServer(basedir, "\x00" * 20) - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - lc.cpu_slice = 500 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) - - # create a few shares, with some leases on them - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - - # add a non-sharefile to exercise another code path - fn = os.path.join(ss.sharedir, - storage_index_to_dir(immutable_si_0), - "not-a-share") - f = open(fn, "wb") - f.write("I am not a share.\n") - f.close() - - # this is before the crawl has started, so we're not in a cycle yet - initial_state = lc.get_state() - self.failIf(lc.get_progress()["cycle-in-progress"]) - self.failIfIn("cycle-to-date", initial_state) - self.failIfIn("estimated-remaining-cycle", initial_state) - self.failIfIn("estimated-current-cycle", initial_state) - self.failUnlessIn("history", initial_state) - self.failUnlessEqual(initial_state["history"], {}) - - ss.setServiceParent(self.s) - - DAY = 24*60*60 - - d = fireEventually() - - # now examine the state right after the first bucket has been - # processed. - def _after_first_bucket(ignored): - initial_state = lc.get_state() - if "cycle-to-date" not in initial_state: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - self.failUnlessIn("cycle-to-date", initial_state) - self.failUnlessIn("estimated-remaining-cycle", initial_state) - self.failUnlessIn("estimated-current-cycle", initial_state) - self.failUnlessIn("history", initial_state) - self.failUnlessEqual(initial_state["history"], {}) - - so_far = initial_state["cycle-to-date"] - self.failUnlessEqual(so_far["expiration-enabled"], False) - self.failUnlessIn("configured-expiration-mode", so_far) - self.failUnlessIn("lease-age-histogram", so_far) - lah = so_far["lease-age-histogram"] - self.failUnlessEqual(type(lah), list) - self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) - self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) - self.failUnlessEqual(so_far["corrupt-shares"], []) - sr1 = so_far["space-recovered"] - self.failUnlessEqual(sr1["examined-buckets"], 1) - self.failUnlessEqual(sr1["examined-shares"], 1) - self.failUnlessEqual(sr1["actual-shares"], 0) - self.failUnlessEqual(sr1["configured-diskbytes"], 0) - self.failUnlessEqual(sr1["original-sharebytes"], 0) - left = initial_state["estimated-remaining-cycle"] - sr2 = left["space-recovered"] - self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"]) - self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"]) - self.failIfEqual(sr2["actual-shares"], None) - self.failIfEqual(sr2["configured-diskbytes"], None) - self.failIfEqual(sr2["original-sharebytes"], None) - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - self.failUnlessIn("So far, this cycle has examined " - "1 shares in 1 buckets (0 mutable / 1 immutable) ", s) - self.failUnlessIn("and has recovered: " - "0 shares, 0 buckets (0 mutable / 0 immutable), " - "0 B (0 B / 0 B)", s) - self.failUnlessIn("If expiration were enabled, " - "we would have recovered: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B) by now", s) - self.failUnlessIn("and the remainder of this cycle " - "would probably recover: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B)", s) - self.failUnlessIn("and the whole cycle would probably recover: " - "0 shares, 0 buckets (0 mutable / 0 immutable)," - " 0 B (0 B / 0 B)", s) - self.failUnlessIn("if we were strictly using each lease's default " - "31-day lease lifetime", s) - self.failUnlessIn("this cycle would be expected to recover: ", s) - d.addCallback(_check_html_in_cycle) - - # wait for the crawler to finish the first cycle. Nothing should have - # been removed. - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) - - def _after_first_cycle(ignored): - s = lc.get_state() - self.failIf("cycle-to-date" in s) - self.failIf("estimated-remaining-cycle" in s) - self.failIf("estimated-current-cycle" in s) - last = s["history"][0] - self.failUnlessIn("cycle-start-finish-times", last) - self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) - self.failUnlessEqual(last["expiration-enabled"], False) - self.failUnlessIn("configured-expiration-mode", last) - - self.failUnlessIn("lease-age-histogram", last) - lah = last["lease-age-histogram"] - self.failUnlessEqual(type(lah), list) - self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) - - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) - self.failUnlessEqual(last["corrupt-shares"], []) - - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 0) - self.failUnlessEqual(rec["original-buckets"], 0) - self.failUnlessEqual(rec["configured-buckets"], 0) - self.failUnlessEqual(rec["actual-shares"], 0) - self.failUnlessEqual(rec["original-shares"], 0) - self.failUnlessEqual(rec["configured-shares"], 0) - self.failUnlessEqual(rec["actual-diskbytes"], 0) - self.failUnlessEqual(rec["original-diskbytes"], 0) - self.failUnlessEqual(rec["configured-diskbytes"], 0) - self.failUnlessEqual(rec["actual-sharebytes"], 0) - self.failUnlessEqual(rec["original-sharebytes"], 0) - self.failUnlessEqual(rec["configured-sharebytes"], 0) - - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("recovered: 0 shares, 0 buckets " - "(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s) - self.failUnlessIn("and saw a total of 4 shares, 4 buckets " - "(2 mutable / 2 immutable),", s) - self.failUnlessIn("but expiration was not enabled", s) - d.addCallback(_check_html) - d.addCallback(lambda ign: renderJSON(webstatus)) - def _check_json(raw): - data = json.loads(raw) - self.failUnlessIn("lease-checker", data) - self.failUnlessIn("lease-checker-progress", data) - d.addCallback(_check_json) - return d - - def backdate_lease(self, sf, renew_secret, new_expire_time): - # ShareFile.renew_lease ignores attempts to back-date a lease (i.e. - # "renew" a lease with a new_expire_time that is older than what the - # current lease has), so we have to reach inside it. - for i,lease in enumerate(sf.get_leases()): - if lease.renew_secret == renew_secret: - lease.expiration_time = new_expire_time - f = open(sf.home, 'rb+') - sf._write_lease_record(f, i, lease) - f.close() - return - raise IndexError("unable to renew non-existent lease") - - def test_expire_age(self): - basedir = "storage/LeaseCrawler/expire_age" - fileutil.make_dirs(basedir) - # setting expiration_time to 2000 means that any lease which is more - # than 2000s old will be expired. - ss = InstrumentedStorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="age", - expiration_override_lease_duration=2000) - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) - - # create a few shares, with some leases on them - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - - # artificially crank back the expiration time on the first lease of - # each share, to make it look like it expired already (age=1000s). - # Some shares have an extra lease which is set to expire at the - # default time in 31 days from now (age=31days). We then run the - # crawler, which will expire the first lease, making some shares get - # deleted and others stay alive (with one remaining lease) - now = time.time() - - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], now - 1000) - sf0_size = os.stat(sf0.home).st_size - - # immutable_si_1 gets an extra lease - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], now - 1000) - - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], now - 1000) - sf2_size = os.stat(sf2.home).st_size - - # mutable_si_3 gets an extra lease - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], now - 1000) - - ss.setServiceParent(self.s) - - d = fireEventually() - # examine the state right after the first bucket has been processed - def _after_first_bucket(ignored): - p = lc.get_progress() - if not p["cycle-in-progress"]: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - # the first bucket encountered gets deleted, and its prefix - # happens to be about 1/5th of the way through the ring, so the - # predictor thinks we'll have 5 shares and that we'll delete them - # all. This part of the test depends upon the SIs landing right - # where they do now. - self.failUnlessIn("The remainder of this cycle is expected to " - "recover: 4 shares, 4 buckets", s) - self.failUnlessIn("The whole cycle is expected to examine " - "5 shares in 5 buckets and to recover: " - "5 shares, 5 buckets", s) - d.addCallback(_check_html_in_cycle) - - # wait for the crawler to finish the first cycle. Two shares should - # have been removed - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) - - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 1) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 1) - - s = lc.get_state() - last = s["history"][0] - - self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], - ("age", 2000, None, ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) - - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 2) - self.failUnlessEqual(rec["original-buckets"], 2) - self.failUnlessEqual(rec["configured-buckets"], 2) - self.failUnlessEqual(rec["actual-shares"], 2) - self.failUnlessEqual(rec["original-shares"], 2) - self.failUnlessEqual(rec["configured-shares"], 2) - size = sf0_size + sf2_size - self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-sharebytes"], size) - self.failUnlessEqual(rec["configured-sharebytes"], size) - # different platforms have different notions of "blocks used by - # this file", so merely assert that it's a number - self.failUnless(rec["actual-diskbytes"] >= 0, - rec["actual-diskbytes"]) - self.failUnless(rec["original-diskbytes"] >= 0, - rec["original-diskbytes"]) - self.failUnless(rec["configured-diskbytes"] >= 0, - rec["configured-diskbytes"]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) - self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) - d.addCallback(_check_html) - return d - - def test_expire_cutoff_date(self): - basedir = "storage/LeaseCrawler/expire_cutoff_date" - fileutil.make_dirs(basedir) - # setting cutoff-date to 2000 seconds ago means that any lease which - # is more than 2000s old will be expired. - now = time.time() - then = int(now - 2000) - ss = InstrumentedStorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then) - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - lc.stop_after_first_bucket = True - webstatus = StorageStatus(ss) - - # create a few shares, with some leases on them - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - - # artificially crank back the expiration time on the first lease of - # each share, to make it look like was renewed 3000s ago. To achieve - # this, we need to set the expiration time to now-3000+31days. This - # will change when the lease format is improved to contain both - # create/renew time and duration. - new_expiration_time = now - 3000 + 31*24*60*60 - - # Some shares have an extra lease which is set to expire at the - # default time in 31 days from now (age=31days). We then run the - # crawler, which will expire the first lease, making some shares get - # deleted and others stay alive (with one remaining lease) - - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf0_size = os.stat(sf0.home).st_size - - # immutable_si_1 gets an extra lease - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) - - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf2_size = os.stat(sf2.home).st_size - - # mutable_si_3 gets an extra lease - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) - - ss.setServiceParent(self.s) - - d = fireEventually() - # examine the state right after the first bucket has been processed - def _after_first_bucket(ignored): - p = lc.get_progress() - if not p["cycle-in-progress"]: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html_in_cycle(html): - s = remove_tags(html) - # the first bucket encountered gets deleted, and its prefix - # happens to be about 1/5th of the way through the ring, so the - # predictor thinks we'll have 5 shares and that we'll delete them - # all. This part of the test depends upon the SIs landing right - # where they do now. - self.failUnlessIn("The remainder of this cycle is expected to " - "recover: 4 shares, 4 buckets", s) - self.failUnlessIn("The whole cycle is expected to examine " - "5 shares in 5 buckets and to recover: " - "5 shares, 5 buckets", s) - d.addCallback(_check_html_in_cycle) - - # wait for the crawler to finish the first cycle. Two shares should - # have been removed - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) - - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 1) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 1) - - s = lc.get_state() - last = s["history"][0] - - self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], - ("cutoff-date", None, then, - ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) - - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 4) - self.failUnlessEqual(rec["examined-shares"], 4) - self.failUnlessEqual(rec["actual-buckets"], 2) - self.failUnlessEqual(rec["original-buckets"], 0) - self.failUnlessEqual(rec["configured-buckets"], 2) - self.failUnlessEqual(rec["actual-shares"], 2) - self.failUnlessEqual(rec["original-shares"], 0) - self.failUnlessEqual(rec["configured-shares"], 2) - size = sf0_size + sf2_size - self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-sharebytes"], 0) - self.failUnlessEqual(rec["configured-sharebytes"], size) - # different platforms have different notions of "blocks used by - # this file", so merely assert that it's a number - self.failUnless(rec["actual-diskbytes"] >= 0, - rec["actual-diskbytes"]) - self.failUnless(rec["original-diskbytes"] >= 0, - rec["original-diskbytes"]) - self.failUnless(rec["configured-diskbytes"] >= 0, - rec["configured-diskbytes"]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Expiration Enabled:" - " expired leases will be removed", s) - date = time.strftime("%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then)) - substr = "Leases created or last renewed before %s will be considered expired." % date - self.failUnlessIn(substr, s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) - d.addCallback(_check_html) - return d - - def test_only_immutable(self): - basedir = "storage/LeaseCrawler/only_immutable" - fileutil.make_dirs(basedir) - now = time.time() - then = int(now - 2000) - ss = StorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then, - expiration_sharetypes=("immutable",)) - lc = ss.lease_checker - lc.slow_start = 0 - webstatus = StorageStatus(ss) - - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - # set all leases to be expirable - new_expiration_time = now - 3000 + 31*24*60*60 - - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) - self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) - self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) - - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) - - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 0) - self.failUnlessEqual(count_shares(immutable_si_1), 0) - self.failUnlessEqual(count_shares(mutable_si_2), 1) - self.failUnlessEqual(count_leases(mutable_si_2), 1) - self.failUnlessEqual(count_shares(mutable_si_3), 1) - self.failUnlessEqual(count_leases(mutable_si_3), 2) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("The following sharetypes will be expired: immutable.", s) - d.addCallback(_check_html) - return d - - def test_only_mutable(self): - basedir = "storage/LeaseCrawler/only_mutable" - fileutil.make_dirs(basedir) - now = time.time() - then = int(now - 2000) - ss = StorageServer(basedir, "\x00" * 20, - expiration_enabled=True, - expiration_mode="cutoff-date", - expiration_cutoff_date=then, - expiration_sharetypes=("mutable",)) - lc = ss.lease_checker - lc.slow_start = 0 - webstatus = StorageStatus(ss) - - self.make_shares(ss) - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - # set all leases to be expirable - new_expiration_time = now - 3000 + 31*24*60*60 - - def count_shares(si): - return len(list(ss._iter_share_files(si))) - def _get_sharefile(si): - return list(ss._iter_share_files(si))[0] - def count_leases(si): - return len(list(_get_sharefile(si).get_leases())) - - sf0 = _get_sharefile(immutable_si_0) - self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) - sf1 = _get_sharefile(immutable_si_1) - self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) - self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) - sf2 = _get_sharefile(mutable_si_2) - self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) - sf3 = _get_sharefile(mutable_si_3) - self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) - self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) - - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) - - def _after_first_cycle(ignored): - self.failUnlessEqual(count_shares(immutable_si_0), 1) - self.failUnlessEqual(count_leases(immutable_si_0), 1) - self.failUnlessEqual(count_shares(immutable_si_1), 1) - self.failUnlessEqual(count_leases(immutable_si_1), 2) - self.failUnlessEqual(count_shares(mutable_si_2), 0) - self.failUnlessEqual(count_shares(mutable_si_3), 0) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderDeferred(webstatus)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("The following sharetypes will be expired: mutable.", s) - d.addCallback(_check_html) - return d - - def test_bad_mode(self): - basedir = "storage/LeaseCrawler/bad_mode" - fileutil.make_dirs(basedir) - e = self.failUnlessRaises(ValueError, - StorageServer, basedir, "\x00" * 20, - expiration_mode="bogus") - self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e)) - - def test_limited_history(self): - basedir = "storage/LeaseCrawler/limited_history" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - lc.cpu_slice = 500 - - # create a few shares, with some leases on them - self.make_shares(ss) - - ss.setServiceParent(self.s) - - def _wait_until_15_cycles_done(): - last = lc.state["last-cycle-finished"] - if last is not None and last >= 15: - return True - if lc.timer: - lc.timer.reset(0) - return False - d = self.poll(_wait_until_15_cycles_done) - - def _check(ignored): - s = lc.get_state() - h = s["history"] - self.failUnlessEqual(len(h), 10) - self.failUnlessEqual(max(h.keys()), 15) - self.failUnlessEqual(min(h.keys()), 6) - d.addCallback(_check) - return d - - def test_unpredictable_future(self): - basedir = "storage/LeaseCrawler/unpredictable_future" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - lc.cpu_slice = -1.0 # stop quickly - - self.make_shares(ss) - - ss.setServiceParent(self.s) - - d = fireEventually() - def _check(ignored): - # this should fire after the first bucket is complete, but before - # the first prefix is complete, so the progress-measurer won't - # think we've gotten far enough to raise our percent-complete - # above 0%, triggering the cannot-predict-the-future code in - # expirer.py . This will have to change if/when the - # progress-measurer gets smart enough to count buckets (we'll - # have to interrupt it even earlier, before it's finished the - # first bucket). - s = lc.get_state() - if "cycle-to-date" not in s: - d2 = fireEventually() - d2.addCallback(_check) - return d2 - self.failUnlessIn("cycle-to-date", s) - self.failUnlessIn("estimated-remaining-cycle", s) - self.failUnlessIn("estimated-current-cycle", s) - - left = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(left["actual-buckets"], None) - self.failUnlessEqual(left["original-buckets"], None) - self.failUnlessEqual(left["configured-buckets"], None) - self.failUnlessEqual(left["actual-shares"], None) - self.failUnlessEqual(left["original-shares"], None) - self.failUnlessEqual(left["configured-shares"], None) - self.failUnlessEqual(left["actual-diskbytes"], None) - self.failUnlessEqual(left["original-diskbytes"], None) - self.failUnlessEqual(left["configured-diskbytes"], None) - self.failUnlessEqual(left["actual-sharebytes"], None) - self.failUnlessEqual(left["original-sharebytes"], None) - self.failUnlessEqual(left["configured-sharebytes"], None) - - full = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(full["actual-buckets"], None) - self.failUnlessEqual(full["original-buckets"], None) - self.failUnlessEqual(full["configured-buckets"], None) - self.failUnlessEqual(full["actual-shares"], None) - self.failUnlessEqual(full["original-shares"], None) - self.failUnlessEqual(full["configured-shares"], None) - self.failUnlessEqual(full["actual-diskbytes"], None) - self.failUnlessEqual(full["original-diskbytes"], None) - self.failUnlessEqual(full["configured-diskbytes"], None) - self.failUnlessEqual(full["actual-sharebytes"], None) - self.failUnlessEqual(full["original-sharebytes"], None) - self.failUnlessEqual(full["configured-sharebytes"], None) - - d.addCallback(_check) - return d - - def test_no_st_blocks(self): - basedir = "storage/LeaseCrawler/no_st_blocks" - fileutil.make_dirs(basedir) - ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, - expiration_mode="age", - expiration_override_lease_duration=-1000) - # a negative expiration_time= means the "configured-" - # space-recovered counts will be non-zero, since all shares will have - # expired by then - - # make it start sooner than usual. - lc = ss.lease_checker - lc.slow_start = 0 - - self.make_shares(ss) - ss.setServiceParent(self.s) - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d = self.poll(_wait) - - def _check(ignored): - s = lc.get_state() - last = s["history"][0] - rec = last["space-recovered"] - self.failUnlessEqual(rec["configured-buckets"], 4) - self.failUnlessEqual(rec["configured-shares"], 4) - self.failUnless(rec["configured-sharebytes"] > 0, - rec["configured-sharebytes"]) - # without the .st_blocks field in os.stat() results, we should be - # reporting diskbytes==sharebytes - self.failUnlessEqual(rec["configured-sharebytes"], - rec["configured-diskbytes"]) - d.addCallback(_check) - return d - - def test_share_corruption(self): - self._poll_should_ignore_these_errors = [ - UnknownMutableContainerVersionError, - UnknownImmutableContainerVersionError, - ] - basedir = "storage/LeaseCrawler/share_corruption" - fileutil.make_dirs(basedir) - ss = InstrumentedStorageServer(basedir, "\x00" * 20) - w = StorageStatus(ss) - # make it start sooner than usual. - lc = ss.lease_checker - lc.stop_after_first_bucket = True - lc.slow_start = 0 - lc.cpu_slice = 500 - - # create a few shares, with some leases on them - self.make_shares(ss) - - # now corrupt one, and make sure the lease-checker keeps going - [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis - first = min(self.sis) - first_b32 = base32.b2a(first) - fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0") - f = open(fn, "rb+") - f.seek(0) - f.write("BAD MAGIC") - f.close() - # if get_share_file() doesn't see the correct mutable magic, it - # assumes the file is an immutable share, and then - # immutable.ShareFile sees a bad version. So regardless of which kind - # of share we corrupted, this will trigger an - # UnknownImmutableContainerVersionError. - - # also create an empty bucket - empty_si = base32.b2a("\x04"*16) - empty_bucket_dir = os.path.join(ss.sharedir, - storage_index_to_dir(empty_si)) - fileutil.make_dirs(empty_bucket_dir) - - ss.setServiceParent(self.s) - - d = fireEventually() - - # now examine the state right after the first bucket has been - # processed. - def _after_first_bucket(ignored): - s = lc.get_state() - if "cycle-to-date" not in s: - d2 = fireEventually() - d2.addCallback(_after_first_bucket) - return d2 - so_far = s["cycle-to-date"] - rec = so_far["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 1) - self.failUnlessEqual(rec["examined-shares"], 0) - self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) - d.addCallback(_after_first_bucket) - - d.addCallback(lambda ign: renderJSON(w)) - def _check_json(raw): - data = json.loads(raw) - # grr. json turns all dict keys into strings. - so_far = data["lease-checker"]["cycle-to-date"] - corrupt_shares = so_far["corrupt-shares"] - # it also turns all tuples into lists - self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) - d.addCallback(_check_json) - d.addCallback(lambda ign: renderDeferred(w)) - def _check_html(html): - s = remove_tags(html) - self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) - d.addCallback(_check_html) - - def _wait(): - return bool(lc.get_state()["last-cycle-finished"] is not None) - d.addCallback(lambda ign: self.poll(_wait)) - - def _after_first_cycle(ignored): - s = lc.get_state() - last = s["history"][0] - rec = last["space-recovered"] - self.failUnlessEqual(rec["examined-buckets"], 5) - self.failUnlessEqual(rec["examined-shares"], 3) - self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) - d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: renderJSON(w)) - def _check_json_history(raw): - data = json.loads(raw) - last = data["lease-checker"]["history"]["0"] - corrupt_shares = last["corrupt-shares"] - self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) - d.addCallback(_check_json_history) - d.addCallback(lambda ign: renderDeferred(w)) - def _check_html_history(html): - s = remove_tags(html) - self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) - d.addCallback(_check_html_history) - - def _cleanup(res): - self.flushLoggedErrors(UnknownMutableContainerVersionError, - UnknownImmutableContainerVersionError) - return res - d.addBoth(_cleanup) - return d - - -class WebStatus(unittest.TestCase, pollmixin.PollMixin): - - def setUp(self): - self.s = service.MultiService() - self.s.startService() - def tearDown(self): - return self.s.stopService() - - def test_no_server(self): - w = StorageStatus(None) - html = renderSynchronously(w) - self.failUnlessIn("

                No Storage Server Running

                ", html) - - def test_status(self): - basedir = "storage/WebStatus/status" - fileutil.make_dirs(basedir) - nodeid = "\x00" * 20 - ss = StorageServer(basedir, nodeid) - ss.setServiceParent(self.s) - w = StorageStatus(ss, "nickname") - d = renderDeferred(w) - def _check_html(html): - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Server Nickname: nickname", s) - self.failUnlessIn("Server Nodeid: %s" % base32.b2a(nodeid), s) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Reserved space: - 0 B (0)", s) - d.addCallback(_check_html) - d.addCallback(lambda ign: renderJSON(w)) - def _check_json(raw): - data = json.loads(raw) - s = data["stats"] - self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1) - self.failUnlessEqual(s["storage_server.reserved_space"], 0) - self.failUnlessIn("bucket-counter", data) - self.failUnlessIn("lease-checker", data) - d.addCallback(_check_json) - return d - - - def test_status_no_disk_stats(self): - def call_get_disk_stats(whichdir, reserved_space=0): - raise AttributeError() - self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) - - # Some platforms may have no disk stats API. Make sure the code can handle that - # (test runs on all platforms). - basedir = "storage/WebStatus/status_no_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Accepting new shares: Yes", s) - self.failUnlessIn("Total disk space: ?", s) - self.failUnlessIn("Space Available to Tahoe: ?", s) - self.failUnless(ss.get_available_space() is None) - - def test_status_bad_disk_stats(self): - def call_get_disk_stats(whichdir, reserved_space=0): - raise OSError() - self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) - - # If the API to get disk stats exists but a call to it fails, then the status should - # show that no shares will be accepted, and get_available_space() should be 0. - basedir = "storage/WebStatus/status_bad_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Accepting new shares: No", s) - self.failUnlessIn("Total disk space: ?", s) - self.failUnlessIn("Space Available to Tahoe: ?", s) - self.failUnlessEqual(ss.get_available_space(), 0) - - def test_status_right_disk_stats(self): - GB = 1000000000 - total = 5*GB - free_for_root = 4*GB - free_for_nonroot = 3*GB - reserved = 1*GB - - basedir = "storage/WebStatus/status_right_disk_stats" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=reserved) - expecteddir = ss.sharedir - - def call_get_disk_stats(whichdir, reserved_space=0): - self.failUnlessEqual(whichdir, expecteddir) - self.failUnlessEqual(reserved_space, reserved) - used = total - free_for_root - avail = max(free_for_nonroot - reserved_space, 0) - return { - 'total': total, - 'free_for_root': free_for_root, - 'free_for_nonroot': free_for_nonroot, - 'used': used, - 'avail': avail, - } - self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) - - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Total disk space: 5.00 GB", s) - self.failUnlessIn("Disk space used: - 1.00 GB", s) - self.failUnlessIn("Disk space free (root): 4.00 GB", s) - self.failUnlessIn("Disk space free (non-root): 3.00 GB", s) - self.failUnlessIn("Reserved space: - 1.00 GB", s) - self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s) - self.failUnlessEqual(ss.get_available_space(), 2*GB) - - def test_readonly(self): - basedir = "storage/WebStatus/readonly" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Accepting new shares: No", s) - - def test_reserved(self): - basedir = "storage/WebStatus/reserved" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) - - def test_huge_reserved(self): - basedir = "storage/WebStatus/reserved" - fileutil.make_dirs(basedir) - ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) - ss.setServiceParent(self.s) - w = StorageStatus(ss) - html = renderSynchronously(w) - self.failUnlessIn("

                Storage Server Status

                ", html) - s = remove_tags(html) - self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) - - def test_util(self): - w = StorageStatusElement(None, None) - self.failUnlessEqual(w.render_space(None), "?") - self.failUnlessEqual(w.render_space(10e6), "10000000") - self.failUnlessEqual(w.render_abbrev_space(None), "?") - self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB") - self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") - self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py new file mode 100644 index 000000000..ee6d7a393 --- /dev/null +++ b/src/allmydata/test/test_storage_web.py @@ -0,0 +1,1309 @@ +""" +Tests for twisted.storage that uses Web APIs. +""" + +from __future__ import absolute_import + +import time +import os.path +import re +import json + +from twisted.trial import unittest + +from twisted.internet import defer +from twisted.application import service +from twisted.web.template import flattenString + +# We need to use `nevow.inevow.IRequest` for now for compatibility +# with the code in web/common.py. Once nevow bits are gone from +# web/common.py, we can use `twisted.web.iweb.IRequest` here. +from nevow.inevow import IRequest + +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel +from zope.interface import implementer + +from foolscap.api import fireEventually +from allmydata.util import fileutil, hashutil, base32, pollmixin +from allmydata.storage.common import storage_index_to_dir, \ + UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError +from allmydata.storage.server import StorageServer +from allmydata.storage.crawler import BucketCountingCrawler +from allmydata.storage.expirer import LeaseCheckingCrawler +from allmydata.web.storage import ( + StorageStatus, + StorageStatusElement, + remove_prefix +) +from .test_storage import FakeCanary + +def remove_tags(s): + s = re.sub(r'<[^>]*>', ' ', s) + s = re.sub(r'\s+', ' ', s) + return s + +def renderSynchronously(ss): + """ + Return fully rendered HTML document. + + :param _StorageStatus ss: a StorageStatus instance. + """ + return unittest.TestCase().successResultOf(renderDeferred(ss)) + +def renderDeferred(ss): + """ + Return a `Deferred` HTML renderer. + + :param _StorageStatus ss: a StorageStatus instance. + """ + elem = StorageStatusElement(ss._storage, ss._nickname) + return flattenString(None, elem) + +def renderJSON(resource): + """Render a JSON from the given resource.""" + + @implementer(IRequest) + class JSONRequest(Request): + """ + A Request with t=json argument added to it. This is useful to + invoke a Resouce.render_JSON() method. + """ + def __init__(self): + Request.__init__(self, DummyChannel()) + self.args = {"t": ["json"]} + self.fields = {} + + return resource.render(JSONRequest()) + +class MyBucketCountingCrawler(BucketCountingCrawler): + def finished_prefix(self, cycle, prefix): + BucketCountingCrawler.finished_prefix(self, cycle, prefix) + if self.hook_ds: + d = self.hook_ds.pop(0) + d.callback(None) + +class MyStorageServer(StorageServer): + def add_bucket_counter(self): + statefile = os.path.join(self.storedir, "bucket_counter.state") + self.bucket_counter = MyBucketCountingCrawler(self, statefile) + self.bucket_counter.setServiceParent(self) + +class BucketCounter(unittest.TestCase, pollmixin.PollMixin): + + def setUp(self): + self.s = service.MultiService() + self.s.startService() + def tearDown(self): + return self.s.stopService() + + def test_bucket_counter(self): + basedir = "storage/BucketCounter/bucket_counter" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + # to make sure we capture the bucket-counting-crawler in the middle + # of a cycle, we reach in and reduce its maximum slice time to 0. We + # also make it start sooner than usual. + ss.bucket_counter.slow_start = 0 + orig_cpu_slice = ss.bucket_counter.cpu_slice + ss.bucket_counter.cpu_slice = 0 + ss.setServiceParent(self.s) + + w = StorageStatus(ss) + + # this sample is before the crawler has started doing anything + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Accepting new shares: Yes", s) + self.failUnlessIn("Reserved space: - 0 B (0)", s) + self.failUnlessIn("Total buckets: Not computed yet", s) + self.failUnlessIn("Next crawl in", s) + + # give the bucket-counting-crawler one tick to get started. The + # cpu_slice=0 will force it to yield right after it processes the + # first prefix + + d = fireEventually() + def _check(ignored): + # are we really right after the first prefix? + state = ss.bucket_counter.get_state() + if state["last-complete-prefix"] is None: + d2 = fireEventually() + d2.addCallback(_check) + return d2 + self.failUnlessEqual(state["last-complete-prefix"], + ss.bucket_counter.prefixes[0]) + ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible + html = renderSynchronously(w) + s = remove_tags(html) + self.failUnlessIn(" Current crawl ", s) + self.failUnlessIn(" (next work in ", s) + d.addCallback(_check) + + # now give it enough time to complete a full cycle + def _watch(): + return not ss.bucket_counter.get_progress()["cycle-in-progress"] + d.addCallback(lambda ignored: self.poll(_watch)) + def _check2(ignored): + ss.bucket_counter.cpu_slice = orig_cpu_slice + html = renderSynchronously(w) + s = remove_tags(html) + self.failUnlessIn("Total buckets: 0 (the number of", s) + self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) + d.addCallback(_check2) + return d + + def test_bucket_counter_cleanup(self): + basedir = "storage/BucketCounter/bucket_counter_cleanup" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + # to make sure we capture the bucket-counting-crawler in the middle + # of a cycle, we reach in and reduce its maximum slice time to 0. + ss.bucket_counter.slow_start = 0 + orig_cpu_slice = ss.bucket_counter.cpu_slice + ss.bucket_counter.cpu_slice = 0 + ss.setServiceParent(self.s) + + d = fireEventually() + + def _after_first_prefix(ignored): + state = ss.bucket_counter.state + if state["last-complete-prefix"] is None: + d2 = fireEventually() + d2.addCallback(_after_first_prefix) + return d2 + ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible + # now sneak in and mess with its state, to make sure it cleans up + # properly at the end of the cycle + self.failUnlessEqual(state["last-complete-prefix"], + ss.bucket_counter.prefixes[0]) + state["bucket-counts"][-12] = {} + state["storage-index-samples"]["bogusprefix!"] = (-12, []) + ss.bucket_counter.save_state() + d.addCallback(_after_first_prefix) + + # now give it enough time to complete a cycle + def _watch(): + return not ss.bucket_counter.get_progress()["cycle-in-progress"] + d.addCallback(lambda ignored: self.poll(_watch)) + def _check2(ignored): + ss.bucket_counter.cpu_slice = orig_cpu_slice + s = ss.bucket_counter.get_state() + self.failIf(-12 in s["bucket-counts"], s["bucket-counts"].keys()) + self.failIf("bogusprefix!" in s["storage-index-samples"], + s["storage-index-samples"].keys()) + d.addCallback(_check2) + return d + + def test_bucket_counter_eta(self): + basedir = "storage/BucketCounter/bucket_counter_eta" + fileutil.make_dirs(basedir) + ss = MyStorageServer(basedir, "\x00" * 20) + ss.bucket_counter.slow_start = 0 + # these will be fired inside finished_prefix() + hooks = ss.bucket_counter.hook_ds = [defer.Deferred() for i in range(3)] + w = StorageStatus(ss) + + d = defer.Deferred() + + def _check_1(ignored): + # no ETA is available yet + html = renderSynchronously(w) + s = remove_tags(html) + self.failUnlessIn("complete (next work", s) + + def _check_2(ignored): + # one prefix has finished, so an ETA based upon that elapsed time + # should be available. + html = renderSynchronously(w) + s = remove_tags(html) + self.failUnlessIn("complete (ETA ", s) + + def _check_3(ignored): + # two prefixes have finished + html = renderSynchronously(w) + s = remove_tags(html) + self.failUnlessIn("complete (ETA ", s) + d.callback("done") + + hooks[0].addCallback(_check_1).addErrback(d.errback) + hooks[1].addCallback(_check_2).addErrback(d.errback) + hooks[2].addCallback(_check_3).addErrback(d.errback) + + ss.setServiceParent(self.s) + return d + +class InstrumentedLeaseCheckingCrawler(LeaseCheckingCrawler): + stop_after_first_bucket = False + def process_bucket(self, *args, **kwargs): + LeaseCheckingCrawler.process_bucket(self, *args, **kwargs) + if self.stop_after_first_bucket: + self.stop_after_first_bucket = False + self.cpu_slice = -1.0 + def yielding(self, sleep_time): + if not self.stop_after_first_bucket: + self.cpu_slice = 500 + +class BrokenStatResults(object): + pass + +class No_ST_BLOCKS_LeaseCheckingCrawler(LeaseCheckingCrawler): + def stat(self, fn): + s = os.stat(fn) + bsr = BrokenStatResults() + for attrname in dir(s): + if attrname.startswith("_"): + continue + if attrname == "st_blocks": + continue + setattr(bsr, attrname, getattr(s, attrname)) + return bsr + +class InstrumentedStorageServer(StorageServer): + LeaseCheckerClass = InstrumentedLeaseCheckingCrawler +class No_ST_BLOCKS_StorageServer(StorageServer): + LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler + +class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): + + def setUp(self): + self.s = service.MultiService() + self.s.startService() + def tearDown(self): + return self.s.stopService() + + def make_shares(self, ss): + def make(si): + return (si, hashutil.tagged_hash("renew", si), + hashutil.tagged_hash("cancel", si)) + def make_mutable(si): + return (si, hashutil.tagged_hash("renew", si), + hashutil.tagged_hash("cancel", si), + hashutil.tagged_hash("write-enabler", si)) + def make_extra_lease(si, num): + return (hashutil.tagged_hash("renew-%d" % num, si), + hashutil.tagged_hash("cancel-%d" % num, si)) + + immutable_si_0, rs0, cs0 = make("\x00" * 16) + immutable_si_1, rs1, cs1 = make("\x01" * 16) + rs1a, cs1a = make_extra_lease(immutable_si_1, 1) + mutable_si_2, rs2, cs2, we2 = make_mutable("\x02" * 16) + mutable_si_3, rs3, cs3, we3 = make_mutable("\x03" * 16) + rs3a, cs3a = make_extra_lease(mutable_si_3, 1) + sharenums = [0] + canary = FakeCanary() + # note: 'tahoe debug dump-share' will not handle this file, since the + # inner contents are not a valid CHK share + data = "\xff" * 1000 + + a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, + 1000, canary) + w[0].remote_write(0, data) + w[0].remote_close() + + a,w = ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, sharenums, + 1000, canary) + w[0].remote_write(0, data) + w[0].remote_close() + ss.remote_add_lease(immutable_si_1, rs1a, cs1a) + + writev = ss.remote_slot_testv_and_readv_and_writev + writev(mutable_si_2, (we2, rs2, cs2), + {0: ([], [(0,data)], len(data))}, []) + writev(mutable_si_3, (we3, rs3, cs3), + {0: ([], [(0,data)], len(data))}, []) + ss.remote_add_lease(mutable_si_3, rs3a, cs3a) + + self.sis = [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] + self.renew_secrets = [rs0, rs1, rs1a, rs2, rs3, rs3a] + self.cancel_secrets = [cs0, cs1, cs1a, cs2, cs3, cs3a] + + def test_basic(self): + basedir = "storage/LeaseCrawler/basic" + fileutil.make_dirs(basedir) + ss = InstrumentedStorageServer(basedir, "\x00" * 20) + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + lc.cpu_slice = 500 + lc.stop_after_first_bucket = True + webstatus = StorageStatus(ss) + + # create a few shares, with some leases on them + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + + # add a non-sharefile to exercise another code path + fn = os.path.join(ss.sharedir, + storage_index_to_dir(immutable_si_0), + "not-a-share") + f = open(fn, "wb") + f.write("I am not a share.\n") + f.close() + + # this is before the crawl has started, so we're not in a cycle yet + initial_state = lc.get_state() + self.failIf(lc.get_progress()["cycle-in-progress"]) + self.failIfIn("cycle-to-date", initial_state) + self.failIfIn("estimated-remaining-cycle", initial_state) + self.failIfIn("estimated-current-cycle", initial_state) + self.failUnlessIn("history", initial_state) + self.failUnlessEqual(initial_state["history"], {}) + + ss.setServiceParent(self.s) + + DAY = 24*60*60 + + d = fireEventually() + + # now examine the state right after the first bucket has been + # processed. + def _after_first_bucket(ignored): + initial_state = lc.get_state() + if "cycle-to-date" not in initial_state: + d2 = fireEventually() + d2.addCallback(_after_first_bucket) + return d2 + self.failUnlessIn("cycle-to-date", initial_state) + self.failUnlessIn("estimated-remaining-cycle", initial_state) + self.failUnlessIn("estimated-current-cycle", initial_state) + self.failUnlessIn("history", initial_state) + self.failUnlessEqual(initial_state["history"], {}) + + so_far = initial_state["cycle-to-date"] + self.failUnlessEqual(so_far["expiration-enabled"], False) + self.failUnlessIn("configured-expiration-mode", so_far) + self.failUnlessIn("lease-age-histogram", so_far) + lah = so_far["lease-age-histogram"] + self.failUnlessEqual(type(lah), list) + self.failUnlessEqual(len(lah), 1) + self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) + self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) + self.failUnlessEqual(so_far["corrupt-shares"], []) + sr1 = so_far["space-recovered"] + self.failUnlessEqual(sr1["examined-buckets"], 1) + self.failUnlessEqual(sr1["examined-shares"], 1) + self.failUnlessEqual(sr1["actual-shares"], 0) + self.failUnlessEqual(sr1["configured-diskbytes"], 0) + self.failUnlessEqual(sr1["original-sharebytes"], 0) + left = initial_state["estimated-remaining-cycle"] + sr2 = left["space-recovered"] + self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"]) + self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"]) + self.failIfEqual(sr2["actual-shares"], None) + self.failIfEqual(sr2["configured-diskbytes"], None) + self.failIfEqual(sr2["original-sharebytes"], None) + d.addCallback(_after_first_bucket) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html_in_cycle(html): + s = remove_tags(html) + self.failUnlessIn("So far, this cycle has examined " + "1 shares in 1 buckets (0 mutable / 1 immutable) ", s) + self.failUnlessIn("and has recovered: " + "0 shares, 0 buckets (0 mutable / 0 immutable), " + "0 B (0 B / 0 B)", s) + self.failUnlessIn("If expiration were enabled, " + "we would have recovered: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B) by now", s) + self.failUnlessIn("and the remainder of this cycle " + "would probably recover: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) + self.failUnlessIn("and the whole cycle would probably recover: " + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) + self.failUnlessIn("if we were strictly using each lease's default " + "31-day lease lifetime", s) + self.failUnlessIn("this cycle would be expected to recover: ", s) + d.addCallback(_check_html_in_cycle) + + # wait for the crawler to finish the first cycle. Nothing should have + # been removed. + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d.addCallback(lambda ign: self.poll(_wait)) + + def _after_first_cycle(ignored): + s = lc.get_state() + self.failIf("cycle-to-date" in s) + self.failIf("estimated-remaining-cycle" in s) + self.failIf("estimated-current-cycle" in s) + last = s["history"][0] + self.failUnlessIn("cycle-start-finish-times", last) + self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) + self.failUnlessEqual(last["expiration-enabled"], False) + self.failUnlessIn("configured-expiration-mode", last) + + self.failUnlessIn("lease-age-histogram", last) + lah = last["lease-age-histogram"] + self.failUnlessEqual(type(lah), list) + self.failUnlessEqual(len(lah), 1) + self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) + + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + self.failUnlessEqual(last["corrupt-shares"], []) + + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 0) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 0) + self.failUnlessEqual(rec["actual-shares"], 0) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 0) + self.failUnlessEqual(rec["actual-diskbytes"], 0) + self.failUnlessEqual(rec["original-diskbytes"], 0) + self.failUnlessEqual(rec["configured-diskbytes"], 0) + self.failUnlessEqual(rec["actual-sharebytes"], 0) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], 0) + + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + self.failUnlessEqual(count_leases(immutable_si_0), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 2) + self.failUnlessEqual(count_leases(mutable_si_2), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 2) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("recovered: 0 shares, 0 buckets " + "(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s) + self.failUnlessIn("and saw a total of 4 shares, 4 buckets " + "(2 mutable / 2 immutable),", s) + self.failUnlessIn("but expiration was not enabled", s) + d.addCallback(_check_html) + d.addCallback(lambda ign: renderJSON(webstatus)) + def _check_json(raw): + data = json.loads(raw) + self.failUnlessIn("lease-checker", data) + self.failUnlessIn("lease-checker-progress", data) + d.addCallback(_check_json) + return d + + def backdate_lease(self, sf, renew_secret, new_expire_time): + # ShareFile.renew_lease ignores attempts to back-date a lease (i.e. + # "renew" a lease with a new_expire_time that is older than what the + # current lease has), so we have to reach inside it. + for i,lease in enumerate(sf.get_leases()): + if lease.renew_secret == renew_secret: + lease.expiration_time = new_expire_time + f = open(sf.home, 'rb+') + sf._write_lease_record(f, i, lease) + f.close() + return + raise IndexError("unable to renew non-existent lease") + + def test_expire_age(self): + basedir = "storage/LeaseCrawler/expire_age" + fileutil.make_dirs(basedir) + # setting expiration_time to 2000 means that any lease which is more + # than 2000s old will be expired. + ss = InstrumentedStorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode="age", + expiration_override_lease_duration=2000) + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + lc.stop_after_first_bucket = True + webstatus = StorageStatus(ss) + + # create a few shares, with some leases on them + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + self.failUnlessEqual(count_shares(immutable_si_0), 1) + self.failUnlessEqual(count_leases(immutable_si_0), 1) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 2) + self.failUnlessEqual(count_shares(mutable_si_2), 1) + self.failUnlessEqual(count_leases(mutable_si_2), 1) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 2) + + # artificially crank back the expiration time on the first lease of + # each share, to make it look like it expired already (age=1000s). + # Some shares have an extra lease which is set to expire at the + # default time in 31 days from now (age=31days). We then run the + # crawler, which will expire the first lease, making some shares get + # deleted and others stay alive (with one remaining lease) + now = time.time() + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], now - 1000) + sf0_size = os.stat(sf0.home).st_size + + # immutable_si_1 gets an extra lease + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], now - 1000) + + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], now - 1000) + sf2_size = os.stat(sf2.home).st_size + + # mutable_si_3 gets an extra lease + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], now - 1000) + + ss.setServiceParent(self.s) + + d = fireEventually() + # examine the state right after the first bucket has been processed + def _after_first_bucket(ignored): + p = lc.get_progress() + if not p["cycle-in-progress"]: + d2 = fireEventually() + d2.addCallback(_after_first_bucket) + return d2 + d.addCallback(_after_first_bucket) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html_in_cycle(html): + s = remove_tags(html) + # the first bucket encountered gets deleted, and its prefix + # happens to be about 1/5th of the way through the ring, so the + # predictor thinks we'll have 5 shares and that we'll delete them + # all. This part of the test depends upon the SIs landing right + # where they do now. + self.failUnlessIn("The remainder of this cycle is expected to " + "recover: 4 shares, 4 buckets", s) + self.failUnlessIn("The whole cycle is expected to examine " + "5 shares in 5 buckets and to recover: " + "5 shares, 5 buckets", s) + d.addCallback(_check_html_in_cycle) + + # wait for the crawler to finish the first cycle. Two shares should + # have been removed + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d.addCallback(lambda ign: self.poll(_wait)) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 0) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 1) + self.failUnlessEqual(count_shares(mutable_si_2), 0) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 1) + + s = lc.get_state() + last = s["history"][0] + + self.failUnlessEqual(last["expiration-enabled"], True) + self.failUnlessEqual(last["configured-expiration-mode"], + ("age", 2000, None, ("mutable", "immutable"))) + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 2) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 2) + self.failUnlessEqual(rec["configured-shares"], 2) + size = sf0_size + sf2_size + self.failUnlessEqual(rec["actual-sharebytes"], size) + self.failUnlessEqual(rec["original-sharebytes"], size) + self.failUnlessEqual(rec["configured-sharebytes"], size) + # different platforms have different notions of "blocks used by + # this file", so merely assert that it's a number + self.failUnless(rec["actual-diskbytes"] >= 0, + rec["actual-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) + self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) + d.addCallback(_check_html) + return d + + def test_expire_cutoff_date(self): + basedir = "storage/LeaseCrawler/expire_cutoff_date" + fileutil.make_dirs(basedir) + # setting cutoff-date to 2000 seconds ago means that any lease which + # is more than 2000s old will be expired. + now = time.time() + then = int(now - 2000) + ss = InstrumentedStorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode="cutoff-date", + expiration_cutoff_date=then) + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + lc.stop_after_first_bucket = True + webstatus = StorageStatus(ss) + + # create a few shares, with some leases on them + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + self.failUnlessEqual(count_shares(immutable_si_0), 1) + self.failUnlessEqual(count_leases(immutable_si_0), 1) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 2) + self.failUnlessEqual(count_shares(mutable_si_2), 1) + self.failUnlessEqual(count_leases(mutable_si_2), 1) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 2) + + # artificially crank back the expiration time on the first lease of + # each share, to make it look like was renewed 3000s ago. To achieve + # this, we need to set the expiration time to now-3000+31days. This + # will change when the lease format is improved to contain both + # create/renew time and duration. + new_expiration_time = now - 3000 + 31*24*60*60 + + # Some shares have an extra lease which is set to expire at the + # default time in 31 days from now (age=31days). We then run the + # crawler, which will expire the first lease, making some shares get + # deleted and others stay alive (with one remaining lease) + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) + sf0_size = os.stat(sf0.home).st_size + + # immutable_si_1 gets an extra lease + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) + sf2_size = os.stat(sf2.home).st_size + + # mutable_si_3 gets an extra lease + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + + ss.setServiceParent(self.s) + + d = fireEventually() + # examine the state right after the first bucket has been processed + def _after_first_bucket(ignored): + p = lc.get_progress() + if not p["cycle-in-progress"]: + d2 = fireEventually() + d2.addCallback(_after_first_bucket) + return d2 + d.addCallback(_after_first_bucket) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html_in_cycle(html): + s = remove_tags(html) + # the first bucket encountered gets deleted, and its prefix + # happens to be about 1/5th of the way through the ring, so the + # predictor thinks we'll have 5 shares and that we'll delete them + # all. This part of the test depends upon the SIs landing right + # where they do now. + self.failUnlessIn("The remainder of this cycle is expected to " + "recover: 4 shares, 4 buckets", s) + self.failUnlessIn("The whole cycle is expected to examine " + "5 shares in 5 buckets and to recover: " + "5 shares, 5 buckets", s) + d.addCallback(_check_html_in_cycle) + + # wait for the crawler to finish the first cycle. Two shares should + # have been removed + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d.addCallback(lambda ign: self.poll(_wait)) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 0) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 1) + self.failUnlessEqual(count_shares(mutable_si_2), 0) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 1) + + s = lc.get_state() + last = s["history"][0] + + self.failUnlessEqual(last["expiration-enabled"], True) + self.failUnlessEqual(last["configured-expiration-mode"], + ("cutoff-date", None, then, + ("mutable", "immutable"))) + self.failUnlessEqual(last["leases-per-share-histogram"], + {1: 2, 2: 2}) + + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 2) + size = sf0_size + sf2_size + self.failUnlessEqual(rec["actual-sharebytes"], size) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], size) + # different platforms have different notions of "blocks used by + # this file", so merely assert that it's a number + self.failUnless(rec["actual-diskbytes"] >= 0, + rec["actual-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Expiration Enabled:" + " expired leases will be removed", s) + date = time.strftime("%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then)) + substr = "Leases created or last renewed before %s will be considered expired." % date + self.failUnlessIn(substr, s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) + d.addCallback(_check_html) + return d + + def test_only_immutable(self): + basedir = "storage/LeaseCrawler/only_immutable" + fileutil.make_dirs(basedir) + now = time.time() + then = int(now - 2000) + ss = StorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode="cutoff-date", + expiration_cutoff_date=then, + expiration_sharetypes=("immutable",)) + lc = ss.lease_checker + lc.slow_start = 0 + webstatus = StorageStatus(ss) + + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + + ss.setServiceParent(self.s) + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d = self.poll(_wait) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 0) + self.failUnlessEqual(count_shares(immutable_si_1), 0) + self.failUnlessEqual(count_shares(mutable_si_2), 1) + self.failUnlessEqual(count_leases(mutable_si_2), 1) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 2) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("The following sharetypes will be expired: immutable.", s) + d.addCallback(_check_html) + return d + + def test_only_mutable(self): + basedir = "storage/LeaseCrawler/only_mutable" + fileutil.make_dirs(basedir) + now = time.time() + then = int(now - 2000) + ss = StorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode="cutoff-date", + expiration_cutoff_date=then, + expiration_sharetypes=("mutable",)) + lc = ss.lease_checker + lc.slow_start = 0 + webstatus = StorageStatus(ss) + + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + + ss.setServiceParent(self.s) + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d = self.poll(_wait) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 1) + self.failUnlessEqual(count_leases(immutable_si_0), 1) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 2) + self.failUnlessEqual(count_shares(mutable_si_2), 0) + self.failUnlessEqual(count_shares(mutable_si_3), 0) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderDeferred(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("The following sharetypes will be expired: mutable.", s) + d.addCallback(_check_html) + return d + + def test_bad_mode(self): + basedir = "storage/LeaseCrawler/bad_mode" + fileutil.make_dirs(basedir) + e = self.failUnlessRaises(ValueError, + StorageServer, basedir, "\x00" * 20, + expiration_mode="bogus") + self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e)) + + def test_limited_history(self): + basedir = "storage/LeaseCrawler/limited_history" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + lc.cpu_slice = 500 + + # create a few shares, with some leases on them + self.make_shares(ss) + + ss.setServiceParent(self.s) + + def _wait_until_15_cycles_done(): + last = lc.state["last-cycle-finished"] + if last is not None and last >= 15: + return True + if lc.timer: + lc.timer.reset(0) + return False + d = self.poll(_wait_until_15_cycles_done) + + def _check(ignored): + s = lc.get_state() + h = s["history"] + self.failUnlessEqual(len(h), 10) + self.failUnlessEqual(max(h.keys()), 15) + self.failUnlessEqual(min(h.keys()), 6) + d.addCallback(_check) + return d + + def test_unpredictable_future(self): + basedir = "storage/LeaseCrawler/unpredictable_future" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + lc.cpu_slice = -1.0 # stop quickly + + self.make_shares(ss) + + ss.setServiceParent(self.s) + + d = fireEventually() + def _check(ignored): + # this should fire after the first bucket is complete, but before + # the first prefix is complete, so the progress-measurer won't + # think we've gotten far enough to raise our percent-complete + # above 0%, triggering the cannot-predict-the-future code in + # expirer.py . This will have to change if/when the + # progress-measurer gets smart enough to count buckets (we'll + # have to interrupt it even earlier, before it's finished the + # first bucket). + s = lc.get_state() + if "cycle-to-date" not in s: + d2 = fireEventually() + d2.addCallback(_check) + return d2 + self.failUnlessIn("cycle-to-date", s) + self.failUnlessIn("estimated-remaining-cycle", s) + self.failUnlessIn("estimated-current-cycle", s) + + left = s["estimated-remaining-cycle"]["space-recovered"] + self.failUnlessEqual(left["actual-buckets"], None) + self.failUnlessEqual(left["original-buckets"], None) + self.failUnlessEqual(left["configured-buckets"], None) + self.failUnlessEqual(left["actual-shares"], None) + self.failUnlessEqual(left["original-shares"], None) + self.failUnlessEqual(left["configured-shares"], None) + self.failUnlessEqual(left["actual-diskbytes"], None) + self.failUnlessEqual(left["original-diskbytes"], None) + self.failUnlessEqual(left["configured-diskbytes"], None) + self.failUnlessEqual(left["actual-sharebytes"], None) + self.failUnlessEqual(left["original-sharebytes"], None) + self.failUnlessEqual(left["configured-sharebytes"], None) + + full = s["estimated-remaining-cycle"]["space-recovered"] + self.failUnlessEqual(full["actual-buckets"], None) + self.failUnlessEqual(full["original-buckets"], None) + self.failUnlessEqual(full["configured-buckets"], None) + self.failUnlessEqual(full["actual-shares"], None) + self.failUnlessEqual(full["original-shares"], None) + self.failUnlessEqual(full["configured-shares"], None) + self.failUnlessEqual(full["actual-diskbytes"], None) + self.failUnlessEqual(full["original-diskbytes"], None) + self.failUnlessEqual(full["configured-diskbytes"], None) + self.failUnlessEqual(full["actual-sharebytes"], None) + self.failUnlessEqual(full["original-sharebytes"], None) + self.failUnlessEqual(full["configured-sharebytes"], None) + + d.addCallback(_check) + return d + + def test_no_st_blocks(self): + basedir = "storage/LeaseCrawler/no_st_blocks" + fileutil.make_dirs(basedir) + ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, + expiration_mode="age", + expiration_override_lease_duration=-1000) + # a negative expiration_time= means the "configured-" + # space-recovered counts will be non-zero, since all shares will have + # expired by then + + # make it start sooner than usual. + lc = ss.lease_checker + lc.slow_start = 0 + + self.make_shares(ss) + ss.setServiceParent(self.s) + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d = self.poll(_wait) + + def _check(ignored): + s = lc.get_state() + last = s["history"][0] + rec = last["space-recovered"] + self.failUnlessEqual(rec["configured-buckets"], 4) + self.failUnlessEqual(rec["configured-shares"], 4) + self.failUnless(rec["configured-sharebytes"] > 0, + rec["configured-sharebytes"]) + # without the .st_blocks field in os.stat() results, we should be + # reporting diskbytes==sharebytes + self.failUnlessEqual(rec["configured-sharebytes"], + rec["configured-diskbytes"]) + d.addCallback(_check) + return d + + def test_share_corruption(self): + self._poll_should_ignore_these_errors = [ + UnknownMutableContainerVersionError, + UnknownImmutableContainerVersionError, + ] + basedir = "storage/LeaseCrawler/share_corruption" + fileutil.make_dirs(basedir) + ss = InstrumentedStorageServer(basedir, "\x00" * 20) + w = StorageStatus(ss) + # make it start sooner than usual. + lc = ss.lease_checker + lc.stop_after_first_bucket = True + lc.slow_start = 0 + lc.cpu_slice = 500 + + # create a few shares, with some leases on them + self.make_shares(ss) + + # now corrupt one, and make sure the lease-checker keeps going + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + first = min(self.sis) + first_b32 = base32.b2a(first) + fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0") + f = open(fn, "rb+") + f.seek(0) + f.write("BAD MAGIC") + f.close() + # if get_share_file() doesn't see the correct mutable magic, it + # assumes the file is an immutable share, and then + # immutable.ShareFile sees a bad version. So regardless of which kind + # of share we corrupted, this will trigger an + # UnknownImmutableContainerVersionError. + + # also create an empty bucket + empty_si = base32.b2a("\x04"*16) + empty_bucket_dir = os.path.join(ss.sharedir, + storage_index_to_dir(empty_si)) + fileutil.make_dirs(empty_bucket_dir) + + ss.setServiceParent(self.s) + + d = fireEventually() + + # now examine the state right after the first bucket has been + # processed. + def _after_first_bucket(ignored): + s = lc.get_state() + if "cycle-to-date" not in s: + d2 = fireEventually() + d2.addCallback(_after_first_bucket) + return d2 + so_far = s["cycle-to-date"] + rec = so_far["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 1) + self.failUnlessEqual(rec["examined-shares"], 0) + self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) + d.addCallback(_after_first_bucket) + + d.addCallback(lambda ign: renderJSON(w)) + def _check_json(raw): + data = json.loads(raw) + # grr. json turns all dict keys into strings. + so_far = data["lease-checker"]["cycle-to-date"] + corrupt_shares = so_far["corrupt-shares"] + # it also turns all tuples into lists + self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) + d.addCallback(_check_json) + d.addCallback(lambda ign: renderDeferred(w)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) + d.addCallback(_check_html) + + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d.addCallback(lambda ign: self.poll(_wait)) + + def _after_first_cycle(ignored): + s = lc.get_state() + last = s["history"][0] + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 5) + self.failUnlessEqual(rec["examined-shares"], 3) + self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: renderJSON(w)) + def _check_json_history(raw): + data = json.loads(raw) + last = data["lease-checker"]["history"]["0"] + corrupt_shares = last["corrupt-shares"] + self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) + d.addCallback(_check_json_history) + d.addCallback(lambda ign: renderDeferred(w)) + def _check_html_history(html): + s = remove_tags(html) + self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) + d.addCallback(_check_html_history) + + def _cleanup(res): + self.flushLoggedErrors(UnknownMutableContainerVersionError, + UnknownImmutableContainerVersionError) + return res + d.addBoth(_cleanup) + return d + + +class WebStatus(unittest.TestCase, pollmixin.PollMixin): + + def setUp(self): + self.s = service.MultiService() + self.s.startService() + def tearDown(self): + return self.s.stopService() + + def test_no_server(self): + w = StorageStatus(None) + html = renderSynchronously(w) + self.failUnlessIn("

                No Storage Server Running

                ", html) + + def test_status(self): + basedir = "storage/WebStatus/status" + fileutil.make_dirs(basedir) + nodeid = "\x00" * 20 + ss = StorageServer(basedir, nodeid) + ss.setServiceParent(self.s) + w = StorageStatus(ss, "nickname") + d = renderDeferred(w) + def _check_html(html): + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Server Nickname: nickname", s) + self.failUnlessIn("Server Nodeid: %s" % base32.b2a(nodeid), s) + self.failUnlessIn("Accepting new shares: Yes", s) + self.failUnlessIn("Reserved space: - 0 B (0)", s) + d.addCallback(_check_html) + d.addCallback(lambda ign: renderJSON(w)) + def _check_json(raw): + data = json.loads(raw) + s = data["stats"] + self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1) + self.failUnlessEqual(s["storage_server.reserved_space"], 0) + self.failUnlessIn("bucket-counter", data) + self.failUnlessIn("lease-checker", data) + d.addCallback(_check_json) + return d + + + def test_status_no_disk_stats(self): + def call_get_disk_stats(whichdir, reserved_space=0): + raise AttributeError() + self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) + + # Some platforms may have no disk stats API. Make sure the code can handle that + # (test runs on all platforms). + basedir = "storage/WebStatus/status_no_disk_stats" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Accepting new shares: Yes", s) + self.failUnlessIn("Total disk space: ?", s) + self.failUnlessIn("Space Available to Tahoe: ?", s) + self.failUnless(ss.get_available_space() is None) + + def test_status_bad_disk_stats(self): + def call_get_disk_stats(whichdir, reserved_space=0): + raise OSError() + self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) + + # If the API to get disk stats exists but a call to it fails, then the status should + # show that no shares will be accepted, and get_available_space() should be 0. + basedir = "storage/WebStatus/status_bad_disk_stats" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20) + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Accepting new shares: No", s) + self.failUnlessIn("Total disk space: ?", s) + self.failUnlessIn("Space Available to Tahoe: ?", s) + self.failUnlessEqual(ss.get_available_space(), 0) + + def test_status_right_disk_stats(self): + GB = 1000000000 + total = 5*GB + free_for_root = 4*GB + free_for_nonroot = 3*GB + reserved = 1*GB + + basedir = "storage/WebStatus/status_right_disk_stats" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20, reserved_space=reserved) + expecteddir = ss.sharedir + + def call_get_disk_stats(whichdir, reserved_space=0): + self.failUnlessEqual(whichdir, expecteddir) + self.failUnlessEqual(reserved_space, reserved) + used = total - free_for_root + avail = max(free_for_nonroot - reserved_space, 0) + return { + 'total': total, + 'free_for_root': free_for_root, + 'free_for_nonroot': free_for_nonroot, + 'used': used, + 'avail': avail, + } + self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) + + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Total disk space: 5.00 GB", s) + self.failUnlessIn("Disk space used: - 1.00 GB", s) + self.failUnlessIn("Disk space free (root): 4.00 GB", s) + self.failUnlessIn("Disk space free (non-root): 3.00 GB", s) + self.failUnlessIn("Reserved space: - 1.00 GB", s) + self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s) + self.failUnlessEqual(ss.get_available_space(), 2*GB) + + def test_readonly(self): + basedir = "storage/WebStatus/readonly" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Accepting new shares: No", s) + + def test_reserved(self): + basedir = "storage/WebStatus/reserved" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) + + def test_huge_reserved(self): + basedir = "storage/WebStatus/reserved" + fileutil.make_dirs(basedir) + ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) + ss.setServiceParent(self.s) + w = StorageStatus(ss) + html = renderSynchronously(w) + self.failUnlessIn("

                Storage Server Status

                ", html) + s = remove_tags(html) + self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) + + def test_util(self): + w = StorageStatusElement(None, None) + self.failUnlessEqual(w.render_space(None), "?") + self.failUnlessEqual(w.render_space(10e6), "10000000") + self.failUnlessEqual(w.render_abbrev_space(None), "?") + self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB") + self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") + self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) From 41ad4aea915480cb6f150760b9050eab590b87e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 18 Aug 2020 13:10:57 -0400 Subject: [PATCH 0808/1054] Make imports work. --- src/allmydata/interfaces.py | 2 ++ src/allmydata/storage/crawler.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36eb55bfb..081d9a33d 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,4 +1,6 @@ +from past.builtins import long + from zope.interface import Interface, Attribute from twisted.plugin import ( IPlugin, diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 008619d64..14139d81e 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -1,6 +1,9 @@ import os, time, struct -import cPickle as pickle +try: + import cPickle as pickle +except ImportError: + import pickle from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a From ed2f6bf3dfdb76b573bc6c0afc30cc336ffcbec9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 18 Aug 2020 13:12:05 -0400 Subject: [PATCH 0809/1054] News file. --- newsfragments/3383.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3383.minor diff --git a/newsfragments/3383.minor b/newsfragments/3383.minor new file mode 100644 index 000000000..e69de29bb From 416ab643359c10e80f9ef0b18e8e4d173757bde5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 10:50:44 -0400 Subject: [PATCH 0810/1054] Fix an import. --- src/allmydata/test/common_py3.py | 23 +++++++++++++++++++++++ src/allmydata/test/test_crawler.py | 2 +- src/allmydata/test/test_storage.py | 18 +----------------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 88f52ce8d..0daf66e62 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -112,3 +112,26 @@ def skip_if_cannot_represent_filename(u): except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") + +class Marker(object): + pass + +class FakeCanary(object): + """For use in storage tests. + + Can be moved back to test_storage.py once enough Python 3 porting has been + done. + """ + def __init__(self, ignore_disconnectors=False): + self.ignore = ignore_disconnectors + self.disconnectors = {} + def notifyOnDisconnect(self, f, *args, **kwargs): + if self.ignore: + return + m = Marker() + self.disconnectors[m] = (f, args, kwargs) + return m + def dontNotifyOnDisconnect(self, marker): + if self.ignore: + return + del self.disconnectors[marker] diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 48d1ba26e..95cb2ce75 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -11,7 +11,7 @@ from allmydata.util import fileutil, hashutil, pollmixin from allmydata.storage.server import StorageServer, si_b2a from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded -from allmydata.test.test_storage import FakeCanary +from allmydata.test.common_py3 import FakeCanary from allmydata.test.common_util import StallMixin class BucketEnumeratingCrawler(ShareCrawler): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index ca7f2b0d0..14c342c41 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -35,24 +35,8 @@ from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, ) +from .common_py3 import FakeCanary -class Marker(object): - pass - -class FakeCanary(object): - def __init__(self, ignore_disconnectors=False): - self.ignore = ignore_disconnectors - self.disconnectors = {} - def notifyOnDisconnect(self, f, *args, **kwargs): - if self.ignore: - return - m = Marker() - self.disconnectors[m] = (f, args, kwargs) - return m - def dontNotifyOnDisconnect(self, marker): - if self.ignore: - return - del self.disconnectors[marker] class FakeStatsProvider(object): def count(self, name, delta=1): From 14f349e846492215c59d397c423a1aa303d1accd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 11:02:26 -0400 Subject: [PATCH 0811/1054] Manual porting to Python 3. --- src/allmydata/test/test_crawler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 95cb2ce75..12f456f7c 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -92,27 +92,27 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): return self.s.stopService() def si(self, i): - return hashutil.storage_index_hash(str(i)) + return hashutil.storage_index_hash(bytes(i)) def rs(self, i, serverid): - return hashutil.bucket_renewal_secret_hash(str(i), serverid) + return hashutil.bucket_renewal_secret_hash(bytes(i), serverid) def cs(self, i, serverid): - return hashutil.bucket_cancel_secret_hash(str(i), serverid) + return hashutil.bucket_cancel_secret_hash(bytes(i), serverid) def write(self, i, ss, serverid, tail=0): si = self.si(i) - si = si[:-1] + chr(tail) + si = si[:-1] + bytearray((tail,)) had,made = ss.remote_allocate_buckets(si, self.rs(i, serverid), self.cs(i, serverid), set([0]), 99, FakeCanary()) - made[0].remote_write(0, "data") + made[0].remote_write(0, b"data") made[0].remote_close() return si_b2a(si) def test_immediate(self): self.basedir = "crawler/Basic/immediate" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -141,7 +141,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def test_service(self): self.basedir = "crawler/Basic/service" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -169,7 +169,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def test_paced(self): self.basedir = "crawler/Basic/paced" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -271,7 +271,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def test_paced_service(self): self.basedir = "crawler/Basic/paced_service" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -338,7 +338,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): self.basedir = "crawler/Basic/cpu_usage" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -383,7 +383,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def test_empty_subclass(self): self.basedir = "crawler/Basic/empty_subclass" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) @@ -411,7 +411,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def test_oneshot(self): self.basedir = "crawler/Basic/oneshot" fileutil.make_dirs(self.basedir) - serverid = "\x00" * 20 + serverid = b"\x00" * 20 ss = StorageServer(self.basedir, serverid) ss.setServiceParent(self.s) From e044309bd37418432f1b1a7cbaa79003b02b9646 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 11:03:54 -0400 Subject: [PATCH 0812/1054] Finish port to Python 3. --- src/allmydata/test/test_crawler.py | 7 +++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 8 insertions(+) diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 12f456f7c..8be9a3d92 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -1,4 +1,11 @@ from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time import os.path diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6f85ce214..ad71909f9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -63,6 +63,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_crawler", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", From e971ccf58e5de3835c4e47499f177358c8cf1e96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 11:12:29 -0400 Subject: [PATCH 0813/1054] Unbreak so tests pass on Python 2 again. --- src/allmydata/test/test_crawler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 8be9a3d92..1b9d85458 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -5,7 +5,8 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # Don't use future bytes, since it breaks tests. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import time import os.path @@ -107,7 +108,7 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def write(self, i, ss, serverid, tail=0): si = self.si(i) - si = si[:-1] + bytearray((tail,)) + si = si[:-1] + bytes(bytearray((tail,))) had,made = ss.remote_allocate_buckets(si, self.rs(i, serverid), self.cs(i, serverid), From ff582c5129dd9f575057f3713a06ce5f8bc0ec52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 11:38:59 -0400 Subject: [PATCH 0814/1054] Some progress towards running crawler on Python 3. --- src/allmydata/storage/common.py | 8 ++++++++ src/allmydata/storage/crawler.py | 7 ++++++- src/allmydata/storage/lease.py | 2 +- src/allmydata/storage/server.py | 2 +- src/allmydata/test/test_crawler.py | 6 +++--- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 865275bc1..55036eea7 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -1,3 +1,4 @@ +from future.utils import PY3 import os.path from allmydata.util import base32 @@ -17,5 +18,12 @@ def si_a2b(ascii_storageindex): return base32.a2b(ascii_storageindex) def storage_index_to_dir(storageindex): + """Convert storage index to directory path. + + Returns native string. + """ sia = si_b2a(storageindex) + if PY3: + # On Python 3 we expect paths to be unicode. + sia = sia.decode("ascii") return os.path.join(sia[:2], sia) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 14139d81e..3f48a127d 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -1,3 +1,4 @@ +from future.utils import native_str, PY3 import os, time, struct try: @@ -77,6 +78,9 @@ class ShareCrawler(service.MultiService): self.statefile = statefile self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] + if PY3: + # On Python 3 we expect the paths to be unicode, not bytes. + self.prefixes = [p.decode("ascii") for p in self.prefixes] self.prefixes.sort() self.timer = None self.bucket_cache = (None, []) @@ -314,7 +318,8 @@ class ShareCrawler(service.MultiService): try: buckets = os.listdir(prefixdir) buckets.sort() - except EnvironmentError: + except EnvironmentError as e: + print(e) buckets = [] self.bucket_cache = (i, buckets) self.process_prefixdir(cycle, prefix, prefixdir, diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 5e3e01716..4d2dce8c4 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -8,7 +8,7 @@ class LeaseInfo(object): self.cancel_secret = cancel_secret self.expiration_time = expiration_time if nodeid is not None: - assert isinstance(nodeid, str) + assert isinstance(nodeid, bytes) assert len(nodeid) == 20 self.nodeid = nodeid diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 26823957e..3ffb58b68 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -49,7 +49,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_cutoff_date=None, expiration_sharetypes=("mutable", "immutable")): service.MultiService.__init__(self) - assert isinstance(nodeid, str) + assert isinstance(nodeid, bytes) assert len(nodeid) == 20 self.my_nodeid = nodeid self.storedir = storedir diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 1b9d85458..1e1e79adb 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -100,11 +100,11 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): return self.s.stopService() def si(self, i): - return hashutil.storage_index_hash(bytes(i)) + return hashutil.storage_index_hash(b"%d" % (i,)) def rs(self, i, serverid): - return hashutil.bucket_renewal_secret_hash(bytes(i), serverid) + return hashutil.bucket_renewal_secret_hash(b"%d" % (i,), serverid) def cs(self, i, serverid): - return hashutil.bucket_cancel_secret_hash(bytes(i), serverid) + return hashutil.bucket_cancel_secret_hash(b"%d" % (i,), serverid) def write(self, i, ss, serverid, tail=0): si = self.si(i) From 35ac5a62e721a0ae913d8f817f16387dad37136b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 12:15:39 -0400 Subject: [PATCH 0815/1054] Tests now pass on Python 3 too. --- src/allmydata/storage/crawler.py | 6 +++--- src/allmydata/test/test_crawler.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 3f48a127d..75dd723ce 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -318,8 +318,7 @@ class ShareCrawler(service.MultiService): try: buckets = os.listdir(prefixdir) buckets.sort() - except EnvironmentError as e: - print(e) + except EnvironmentError: buckets = [] self.bucket_cache = (i, buckets) self.process_prefixdir(cycle, prefix, prefixdir, @@ -361,7 +360,8 @@ class ShareCrawler(service.MultiService): """ for bucket in buckets: - if bucket <= self.state["last-complete-bucket"]: + last_complete = self.state["last-complete-bucket"] + if last_complete is not None and bucket <= last_complete: continue self.process_bucket(cycle, prefix, prefixdir, bucket) self.state["last-complete-bucket"] = bucket diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 1e1e79adb..cfab04c31 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -3,7 +3,7 @@ from __future__ import division from __future__ import absolute_import from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, PY3, native_str if PY2: # Don't use future bytes, since it breaks tests. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 @@ -30,6 +30,10 @@ class BucketEnumeratingCrawler(ShareCrawler): self.all_buckets = [] self.finished_d = defer.Deferred() def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + if PY3: + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) @@ -44,6 +48,10 @@ class PacedCrawler(ShareCrawler): self.finished_d = defer.Deferred() self.yield_cb = None def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + if PY3: + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) self.countdown -= 1 if self.countdown == 0: From 8279be38c110ee52538d814cf66f0ab56644eb40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 13:25:11 -0400 Subject: [PATCH 0816/1054] Finish porting to Python 3. --- newsfragments/3386.minor | 0 src/allmydata/storage/crawler.py | 17 ++++++++++++++++- src/allmydata/test/test_crawler.py | 6 ++++++ src/allmydata/util/_python3.py | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3386.minor diff --git a/newsfragments/3386.minor b/newsfragments/3386.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 75dd723ce..24042c38b 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -1,4 +1,19 @@ -from future.utils import native_str, PY3 +""" +Crawl the storage server shares. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2, PY3 +if PY2: + # We don't import bytes, object, dict, and list just in case they're used, + # so as not to create brittle pickles with random magic objects. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min # noqa: F401 import os, time, struct try: diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index cfab04c31..04128769e 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -1,3 +1,9 @@ +""" +Tests for allmydata.storage.crawler. + +Ported to Python 3. +""" + from __future__ import print_function from __future__ import division from __future__ import absolute_import diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ad71909f9..794edef40 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -32,6 +32,7 @@ PORTED_MODULES = [ "allmydata.crypto.util", "allmydata.hashtree", "allmydata.immutable.happiness_upload", + "allmydata.storage.crawler", "allmydata.test.common_py3", "allmydata.util._python3", "allmydata.util.abbreviate", From 733b2cab361879aaf5ce0cffa73b5ef275645a29 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Aug 2020 13:26:57 -0400 Subject: [PATCH 0817/1054] Remove unnecessary import. --- src/allmydata/test/test_crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 04128769e..38bc8dcc6 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -9,7 +9,7 @@ from __future__ import division from __future__ import absolute_import from __future__ import unicode_literals -from future.utils import PY2, PY3, native_str +from future.utils import PY2, PY3 if PY2: # Don't use future bytes, since it breaks tests. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 From 320830cf907aefcdfea97c0e8c4e15feb2037609 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 19 Aug 2020 12:20:51 -0400 Subject: [PATCH 0818/1054] Reuse MultiFormatPage tests for MultiFormatResource --- src/allmydata/test/web/test_web.py | 116 ++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 202d7216e..70fea4f54 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -33,7 +33,7 @@ from allmydata.immutable import upload from allmydata.immutable.downloader.status import DownloadStatus from allmydata.dirnode import DirectoryNode from allmydata.nodemaker import NodeMaker -from allmydata.web.common import WebError +from allmydata.web.common import WebError, MultiFormatResource from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_str @@ -657,6 +657,120 @@ class WebMixin(TimezoneMixin): (which, res)) +class MultiFormatResourceTests(TrialTestCase): + """ + Tests for ``MultiFormatResource``. + """ + + def resource(self): + """ + Create and return an instance of a ``MultiFormatResource`` + subclass with a default HTML format, and two custom formats: + ``a`` and ``b``. + """ + class Content(MultiFormatResource): + + def render_HTML(self, req): + return "html" + + def render_A(self, req): + return "a" + + def render_B(self, req): + return "b" + + return Content() + + + def render(self, resource, **query_args): + """ + Render a ``Resource`` against a request with the given query arguments. + + :param resource: The Nevow resource to render. + + :param query_args: The query arguments to put into the request being + rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. + + :return: The rendered response body as ``bytes``. + """ + ctx = WebContext(tag=resource) + req = FakeRequest(args=query_args) + ctx.remember(DefaultExceptionHandler(), ICanHandleException) + ctx.remember(req, IRequest) + ctx.remember(None, IData) + + d = maybeDeferred(resource.render, ctx) + d.addErrback(processingFailed, req, ctx) + res = self.successResultOf(d) + if isinstance(res, bytes): + return req.v + res + return req.v + + + def test_select_format(self): + """ + The ``formatArgument`` attribute of a ``MultiFormatResource`` + subclass identifies the query argument which selects the + result format. + """ + resource = self.resource() + resource.formatArgument = "foo" + self.assertEqual("a", self.render(resource, foo=["a"])) + + + def test_default_format_argument(self): + """ + If a ``MultiFormatResource`` subclass does not set + ``formatArgument`` then the ``t`` argument is used. + """ + resource = self.resource() + self.assertEqual("a", self.render(resource, t=["a"])) + + + def test_no_format(self): + """ + If no value is given for the format argument and no default + format has been defined, the base rendering behavior is used + (``render_HTML``). + """ + resource = self.resource() + self.assertEqual("html", self.render(resource)) + + + def test_default_format(self): + """ + If no value is given for the format argument and the + ``MultiFormatResource`` subclass defines a ``formatDefault``, + that value is used as the format to render. + """ + resource = self.resource() + resource.formatDefault = "b" + self.assertEqual("b", self.render(resource)) + + + def test_explicit_none_format_renderer(self): + """ + If a format is selected which has a renderer set to ``None``, + the base rendering behavior is used (``render_HTML``). + """ + resource = self.resource() + resource.render_FOO = None + self.assertEqual("html", self.render(resource, t=["foo"])) + + + def test_unknown_format(self): + """ + If a format is selected for which there is no renderer, an error is + returned. + """ + resource = self.resource() + self.assertIn( + "Exception", + self.render(resource, t=["foo"]), + ) + self.flushLoggedErrors(WebError) + + class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, TrialTestCase): maxDiff = None From d2bcebecaa96d0d634c921214f46b628e9e11bba Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 19 Aug 2020 14:52:20 -0400 Subject: [PATCH 0819/1054] Render resource in a simpler manner --- src/allmydata/test/web/test_web.py | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 70fea4f54..b34a50c49 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -691,20 +691,27 @@ class MultiFormatResourceTests(TrialTestCase): :param query_args: The query arguments to put into the request being rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. - :return: The rendered response body as ``bytes``. + :return: The rendered response body as ``str``. """ - ctx = WebContext(tag=resource) - req = FakeRequest(args=query_args) - ctx.remember(DefaultExceptionHandler(), ICanHandleException) - ctx.remember(req, IRequest) - ctx.remember(None, IData) - d = maybeDeferred(resource.render, ctx) - d.addErrback(processingFailed, req, ctx) - res = self.successResultOf(d) - if isinstance(res, bytes): - return req.v + res - return req.v + # TODO: probably should: (1) refactor this out of here to a + # common module (test.common_web maybe?), and (2) replace + # nevow.inevow.IRequest with twisted.web.iweb.IRequest. For + # (2) to happen, we will have to update web.common.get_arg() + # etc first. + from zope.interface import implementer + from nevow.inevow import IRequest + from twisted.web.server import Request + from twisted.web.test.requesthelper import DummyChannel + + @implementer(IRequest) + class FakeRequest(Request): + def __init__(self, args): + Request.__init__(self, DummyChannel()) + self.args = args + self.fields = dict() + + return resource.render(FakeRequest(args=query_args)) def test_select_format(self): From d87ffb5ad0aebdc84038c9fa7e680260c3278ccb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 19 Aug 2020 15:17:49 -0400 Subject: [PATCH 0820/1054] Begone, (most) nevow imports! --- src/allmydata/test/web/test_web.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b34a50c49..bf8c7976b 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -8,24 +8,12 @@ from bs4 import BeautifulSoup from twisted.application import service from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from twisted.web import client, error, http from twisted.python import failure, log -from nevow.context import WebContext -from nevow.inevow import ( - ICanHandleException, - IRequest, - IData, -) from nevow.util import escapeToXML -from nevow.loaders import stan -from nevow.testutil import FakeRequest -from nevow.appserver import ( - processingFailed, - DefaultExceptionHandler, -) from allmydata import interfaces, uri, webish from allmydata.storage_client import StorageFarmBroker, StubServer From 9882d22101e32bfc613d9a0b56e53dea99b18b51 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 19 Aug 2020 15:19:59 -0400 Subject: [PATCH 0821/1054] Check for WebError when rendering resource in unknown format --- src/allmydata/test/web/test_web.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index bf8c7976b..3c20d8891 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -759,11 +759,8 @@ class MultiFormatResourceTests(TrialTestCase): returned. """ resource = self.resource() - self.assertIn( - "Exception", + with self.assertRaises(WebError): self.render(resource, t=["foo"]), - ) - self.flushLoggedErrors(WebError) class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, TrialTestCase): From ce994ff24636078497b70f5098c269f504ea650c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 19 Aug 2020 15:42:01 -0400 Subject: [PATCH 0822/1054] Use twisted's escapeToXml --- src/allmydata/test/web/test_web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 3c20d8891..f18a2efb5 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -13,7 +13,7 @@ from twisted.internet.task import Clock from twisted.web import client, error, http from twisted.python import failure, log -from nevow.util import escapeToXML +from twisted.words.xish.domish import escapeToXml from allmydata import interfaces, uri, webish from allmydata.storage_client import StorageFarmBroker, StubServer @@ -365,9 +365,9 @@ class WebMixin(TimezoneMixin): self._htmlname_unicode = u"<&weirdly'named\"file>>>_
                Relative Path Healthy Pre-Repair Recoverable Pre-Repair