From 0cf1117165bc022c2b519a49a592189afa1da45c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:20:12 -0400 Subject: [PATCH 1/3] Move the tests for deferredutil into their own module. --- src/allmydata/test/test_deferredutil.py | 76 +++++++++++++++++++++++++ src/allmydata/test/test_util.py | 56 +----------------- src/allmydata/util/_python3.py | 1 + 3 files changed, 78 insertions(+), 55 deletions(-) create mode 100644 src/allmydata/test/test_deferredutil.py diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py new file mode 100644 index 000000000..fcff77b54 --- /dev/null +++ b/src/allmydata/test/test_deferredutil.py @@ -0,0 +1,76 @@ +""" +Tests for allmydata.util.deferredutil. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.python.failure import Failure + +from allmydata.util import deferredutil + + +class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin): + def test_gather_results(self): + d1 = defer.Deferred() + d2 = defer.Deferred() + res = deferredutil.gatherResults([d1, d2]) + d1.errback(ValueError("BAD")) + def _callb(res): + self.fail("Should have errbacked, not resulted in %s" % (res,)) + def _errb(thef): + thef.trap(ValueError) + res.addCallbacks(_callb, _errb) + return res + + def test_success(self): + d1, d2 = defer.Deferred(), defer.Deferred() + good = [] + bad = [] + dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) + dlss.addCallbacks(good.append, bad.append) + d1.callback(1) + d2.callback(2) + self.failUnlessEqual(good, [[1,2]]) + self.failUnlessEqual(bad, []) + + def test_failure(self): + d1, d2 = defer.Deferred(), defer.Deferred() + good = [] + bad = [] + dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) + dlss.addCallbacks(good.append, bad.append) + d1.addErrback(lambda _ignore: None) + d2.addErrback(lambda _ignore: None) + d1.callback(1) + d2.errback(ValueError()) + self.failUnlessEqual(good, []) + self.failUnlessEqual(len(bad), 1) + f = bad[0] + self.failUnless(isinstance(f, Failure)) + self.failUnless(f.check(ValueError)) + + def test_wait_for_delayed_calls(self): + """ + This tests that 'wait_for_delayed_calls' does in fact wait for a + delayed call that is active when the test returns. If it didn't, + Trial would report an unclean reactor error for this test. + """ + def _trigger(): + #print "trigger" + pass + reactor.callLater(0.1, _trigger) + + d = defer.succeed(None) + d.addBoth(self.wait_for_delayed_calls) + return d diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 65ba308d9..b6f2074aa 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,7 +15,7 @@ from twisted.python.failure import Failure from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil -from allmydata.util import assertutil, fileutil, deferredutil, abbreviate +from allmydata.util import assertutil, fileutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log @@ -750,60 +750,6 @@ class PollMixinTests(unittest.TestCase): d.addCallbacks(_suc, _err) return d -class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin): - def test_gather_results(self): - d1 = defer.Deferred() - d2 = defer.Deferred() - res = deferredutil.gatherResults([d1, d2]) - d1.errback(ValueError("BAD")) - def _callb(res): - self.fail("Should have errbacked, not resulted in %s" % (res,)) - def _errb(thef): - thef.trap(ValueError) - res.addCallbacks(_callb, _errb) - return res - - def test_success(self): - d1, d2 = defer.Deferred(), defer.Deferred() - good = [] - bad = [] - dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) - dlss.addCallbacks(good.append, bad.append) - d1.callback(1) - d2.callback(2) - self.failUnlessEqual(good, [[1,2]]) - self.failUnlessEqual(bad, []) - - def test_failure(self): - d1, d2 = defer.Deferred(), defer.Deferred() - good = [] - bad = [] - dlss = deferredutil.DeferredListShouldSucceed([d1,d2]) - dlss.addCallbacks(good.append, bad.append) - d1.addErrback(lambda _ignore: None) - d2.addErrback(lambda _ignore: None) - d1.callback(1) - d2.errback(ValueError()) - self.failUnlessEqual(good, []) - self.failUnlessEqual(len(bad), 1) - f = bad[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(ValueError)) - - def test_wait_for_delayed_calls(self): - """ - This tests that 'wait_for_delayed_calls' does in fact wait for a - delayed call that is active when the test returns. If it didn't, - Trial would report an unclean reactor error for this test. - """ - def _trigger(): - #print "trigger" - pass - reactor.callLater(0.1, _trigger) - - d = defer.succeed(None) - d.addBoth(self.wait_for_delayed_calls) - return d class HashUtilTests(unittest.TestCase): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index fd679e014..3196a8f90 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.test_deferredutil", "allmydata.test.test_humanreadable", "allmydata.test.test_python3", ] From 7d1fdd8284c6901e85985927ef4f6424834cb46c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:32:16 -0400 Subject: [PATCH 2/3] Port modules to Python 3. --- newsfragments/3341.other | 0 src/allmydata/util/deferredutil.py | 59 ++++++++++-------------------- src/allmydata/util/pollmixin.py | 13 +++++++ 3 files changed, 33 insertions(+), 39 deletions(-) create mode 100644 newsfragments/3341.other diff --git a/newsfragments/3341.other b/newsfragments/3341.other new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 671d8bfd9..e36892f96 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -1,7 +1,21 @@ +""" +Utilities for working with Twisted Deferreds. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import time -from foolscap.api import eventually, fireEventually +from foolscap.api import eventually from twisted.internet import defer, reactor, error from twisted.python.failure import Failure @@ -130,7 +144,7 @@ class HookMixin(object): self._hooks[name] = (d, ignore_count) return d - def _call_hook(self, res, name, async=False): + def _call_hook(self, res, name, **kwargs): """ Called to trigger the hook, with argument 'res'. This is a no-op if the hook is unset. If the hook's ignore_count is positive, it will be @@ -142,7 +156,10 @@ class HookMixin(object): which will typically cause the test to also fail. 'res' is returned so that the current result or failure will be passed through. + + Accepts a single keyword argument, async, defaulting to False. """ + async_ = kwargs.get("async", False) hook = self._hooks[name] if hook is None: return res # pass on error/result @@ -153,7 +170,7 @@ class HookMixin(object): self._hooks[name] = (d, ignore_count - 1) else: self._hooks[name] = None - if async: + if async_: _with_log(eventually_callback(d), res) else: _with_log(d.callback, res) @@ -163,42 +180,6 @@ class HookMixin(object): log.msg(msg, level=log.NOISY) -def async_iterate(process, iterable, *extra_args, **kwargs): - """ - I iterate over the elements of 'iterable' (which may be deferred), eventually - applying 'process' to each one, optionally with 'extra_args' and 'kwargs'. - 'process' should return a (possibly deferred) boolean: True to continue the - iteration, False to stop. - - I return a Deferred that fires with True if all elements of the iterable - were processed (i.e. 'process' only returned True values); with False if - the iteration was stopped by 'process' returning False; or that fails with - the first failure of either 'process' or the iterator. - """ - iterator = iter(iterable) - - d = defer.succeed(None) - def _iterate(ign): - d2 = defer.maybeDeferred(iterator.next) - def _cb(item): - d3 = defer.maybeDeferred(process, item, *extra_args, **kwargs) - def _maybe_iterate(res): - if res: - d4 = fireEventually() - d4.addCallback(_iterate) - return d4 - return False - d3.addCallback(_maybe_iterate) - return d3 - def _eb(f): - f.trap(StopIteration) - return True - d2.addCallbacks(_cb, _eb) - return d2 - d.addCallback(_iterate) - return d - - def for_items(cb, mapping): """ For each (key, value) pair in a mapping, I add a callback to cb(None, key, value) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 14f5cd390..4eb19b8e3 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -1,4 +1,17 @@ +""" +Polling utility that returns Deferred. + +Ported to Python 3. +""" + from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import time from twisted.internet import task From 1e41df9f48337b7f4411f21618d120972174128e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jul 2020 14:38:07 -0400 Subject: [PATCH 3/3] Add to list of ported modules. --- src/allmydata/util/_python3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 3196a8f90..74f5d0385 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,8 +15,10 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.util.deferredutil", "allmydata.util.humanreadable", "allmydata.util.namespace", + "allmydata.util.pollmixin", "allmydata.util._python3", ]