Merge pull request #741 from tahoe-lafs/3341-more-python-3-utils

Port deferredutil to Python 3

src/allmydata/test/test_deferredutil.py

@@ -0,0 +1,76 @@
+"""
+Tests for allmydata.util.deferredutil.
+
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
+from twisted.trial import unittest
+from twisted.internet import defer, reactor
+from twisted.python.failure import Failure
+
+from allmydata.util import deferredutil
+
+
+class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin):
+    def test_gather_results(self):
+        d1 = defer.Deferred()
+        d2 = defer.Deferred()
+        res = deferredutil.gatherResults([d1, d2])
+        d1.errback(ValueError("BAD"))
+        def _callb(res):
+            self.fail("Should have errbacked, not resulted in %s" % (res,))
+        def _errb(thef):
+            thef.trap(ValueError)
+        res.addCallbacks(_callb, _errb)
+        return res
+
+    def test_success(self):
+        d1, d2 = defer.Deferred(), defer.Deferred()
+        good = []
+        bad = []
+        dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
+        dlss.addCallbacks(good.append, bad.append)
+        d1.callback(1)
+        d2.callback(2)
+        self.failUnlessEqual(good, [[1,2]])
+        self.failUnlessEqual(bad, [])
+
+    def test_failure(self):
+        d1, d2 = defer.Deferred(), defer.Deferred()
+        good = []
+        bad = []
+        dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
+        dlss.addCallbacks(good.append, bad.append)
+        d1.addErrback(lambda _ignore: None)
+        d2.addErrback(lambda _ignore: None)
+        d1.callback(1)
+        d2.errback(ValueError())
+        self.failUnlessEqual(good, [])
+        self.failUnlessEqual(len(bad), 1)
+        f = bad[0]
+        self.failUnless(isinstance(f, Failure))
+        self.failUnless(f.check(ValueError))
+
+    def test_wait_for_delayed_calls(self):
+        """
+        This tests that 'wait_for_delayed_calls' does in fact wait for a
+        delayed call that is active when the test returns. If it didn't,
+        Trial would report an unclean reactor error for this test.
+        """
+        def _trigger():
+            #print "trigger"
+            pass
+        reactor.callLater(0.1, _trigger)
+
+        d = defer.succeed(None)
+        d.addBoth(self.wait_for_delayed_calls)
+        return d

src/allmydata/test/test_util.py

@@ -15,7 +15,7 @@ from twisted.python.failure import Failure
 from twisted.python import log
 
 from allmydata.util import base32, idlib, mathutil, hashutil
-from allmydata.util import fileutil, deferredutil, abbreviate
+from allmydata.util import fileutil, abbreviate
 from allmydata.util import limiter, time_format, pollmixin
 from allmydata.util import statistics, dictutil, pipeline, yamlutil
 from allmydata.util import log as tahoe_log
@@ -600,60 +600,6 @@ class PollMixinTests(unittest.TestCase):
         d.addCallbacks(_suc, _err)
         return d
 
-class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin):
-    def test_gather_results(self):
-        d1 = defer.Deferred()
-        d2 = defer.Deferred()
-        res = deferredutil.gatherResults([d1, d2])
-        d1.errback(ValueError("BAD"))
-        def _callb(res):
-            self.fail("Should have errbacked, not resulted in %s" % (res,))
-        def _errb(thef):
-            thef.trap(ValueError)
-        res.addCallbacks(_callb, _errb)
-        return res
-
-    def test_success(self):
-        d1, d2 = defer.Deferred(), defer.Deferred()
-        good = []
-        bad = []
-        dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
-        dlss.addCallbacks(good.append, bad.append)
-        d1.callback(1)
-        d2.callback(2)
-        self.failUnlessEqual(good, [[1,2]])
-        self.failUnlessEqual(bad, [])
-
-    def test_failure(self):
-        d1, d2 = defer.Deferred(), defer.Deferred()
-        good = []
-        bad = []
-        dlss = deferredutil.DeferredListShouldSucceed([d1,d2])
-        dlss.addCallbacks(good.append, bad.append)
-        d1.addErrback(lambda _ignore: None)
-        d2.addErrback(lambda _ignore: None)
-        d1.callback(1)
-        d2.errback(ValueError())
-        self.failUnlessEqual(good, [])
-        self.failUnlessEqual(len(bad), 1)
-        f = bad[0]
-        self.failUnless(isinstance(f, Failure))
-        self.failUnless(f.check(ValueError))
-
-    def test_wait_for_delayed_calls(self):
-        """
-        This tests that 'wait_for_delayed_calls' does in fact wait for a
-        delayed call that is active when the test returns. If it didn't,
-        Trial would report an unclean reactor error for this test.
-        """
-        def _trigger():
-            #print "trigger"
-            pass
-        reactor.callLater(0.1, _trigger)
-
-        d = defer.succeed(None)
-        d.addBoth(self.wait_for_delayed_calls)
-        return d
 
 class HashUtilTests(unittest.TestCase):
 

src/allmydata/util/_python3.py

@@ -16,13 +16,16 @@ if PY2:
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
     "allmydata.util.assertutil",
+    "allmydata.util.deferredutil",
     "allmydata.util.humanreadable",
     "allmydata.util.mathutil",
     "allmydata.util.namespace",
+    "allmydata.util.pollmixin",
     "allmydata.util._python3",
 ]
 
 PORTED_TEST_MODULES = [
+    "allmydata.test.test_deferredutil",
     "allmydata.test.test_humanreadable",
     "allmydata.test.test_python3",
 ]

src/allmydata/util/deferredutil.py

@@ -1,7 +1,21 @@
+"""
+Utilities for working with Twisted Deferreds.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
 
 import time
 
-from foolscap.api import eventually, fireEventually
+from foolscap.api import eventually
 from twisted.internet import defer, reactor, error
 from twisted.python.failure import Failure
 
@@ -130,7 +144,7 @@ class HookMixin(object):
         self._hooks[name] = (d, ignore_count)
         return d
 
-    def _call_hook(self, res, name, async=False):
+    def _call_hook(self, res, name, **kwargs):
         """
         Called to trigger the hook, with argument 'res'. This is a no-op if
         the hook is unset. If the hook's ignore_count is positive, it will be
@@ -142,7 +156,10 @@ class HookMixin(object):
         which will typically cause the test to also fail.
         'res' is returned so that the current result or failure will be passed
         through.
+
+        Accepts a single keyword argument, async, defaulting to False.
         """
+        async_ = kwargs.get("async", False)
         hook = self._hooks[name]
         if hook is None:
             return res  # pass on error/result
@@ -153,7 +170,7 @@ class HookMixin(object):
             self._hooks[name] = (d, ignore_count - 1)
         else:
             self._hooks[name] = None
-            if async:
+            if async_:
                 _with_log(eventually_callback(d), res)
             else:
                 _with_log(d.callback, res)
@@ -163,42 +180,6 @@ class HookMixin(object):
         log.msg(msg, level=log.NOISY)
 
 
-def async_iterate(process, iterable, *extra_args, **kwargs):
-    """
-    I iterate over the elements of 'iterable' (which may be deferred), eventually
-    applying 'process' to each one, optionally with 'extra_args' and 'kwargs'.
-    'process' should return a (possibly deferred) boolean: True to continue the
-    iteration, False to stop.
-
-    I return a Deferred that fires with True if all elements of the iterable
-    were processed (i.e. 'process' only returned True values); with False if
-    the iteration was stopped by 'process' returning False; or that fails with
-    the first failure of either 'process' or the iterator.
-    """
-    iterator = iter(iterable)
-
-    d = defer.succeed(None)
-    def _iterate(ign):
-        d2 = defer.maybeDeferred(iterator.next)
-        def _cb(item):
-            d3 = defer.maybeDeferred(process, item, *extra_args, **kwargs)
-            def _maybe_iterate(res):
-                if res:
-                    d4 = fireEventually()
-                    d4.addCallback(_iterate)
-                    return d4
-                return False
-            d3.addCallback(_maybe_iterate)
-            return d3
-        def _eb(f):
-            f.trap(StopIteration)
-            return True
-        d2.addCallbacks(_cb, _eb)
-        return d2
-    d.addCallback(_iterate)
-    return d
-
-
 def for_items(cb, mapping):
     """
     For each (key, value) pair in a mapping, I add a callback to cb(None, key, value)

src/allmydata/util/pollmixin.py

@@ -1,4 +1,17 @@
+"""
+Polling utility that returns Deferred.
+
+Ported to Python 3.
+"""
+
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
 
 import time
 from twisted.internet import task