Merge pull request #835 from tahoe-lafs/3417.audit-for-loops-mutable-dict-views-python-3

Audit for loops for mutable dict views bugs on Python 3

Fixes ticket:3417

misc/python3/audit-dict-for-loops.py

@@ -0,0 +1,43 @@
+"""
+The following code is valid in Python 2:
+
+for x in my_dict.keys():
+    if something(x):
+        del my_dict[x]
+
+But broken in Python 3.
+
+One solution is:
+
+for x in list(my_dict.keys()):
+    if something(x):
+        del my_dict[x]
+
+Some but not all code in Tahoe has been changed to that. In other cases, the code was left unchanged since there was no `del`.
+
+However, some mistakes may have slept through.
+
+To help catch cases that were incorrectly ported, this script runs futurize on all ported modules, which should convert it into the `list()` form.
+You can then look at git diffs to see if any of the impacted would be buggy without the newly added `list()`.
+"""
+
+import os
+from subprocess import check_call
+
+from allmydata.util import _python3
+
+
+def fix_potential_issue():
+    for module in _python3.PORTED_MODULES + _python3.PORTED_TEST_MODULES:
+        filename = "src/" + module.replace(".", "/") + ".py"
+        if not os.path.exists(filename):
+            # Package, probably
+            filename = "src/" + module.replace(".", "/") + "/__init__.py"
+        check_call(["futurize", "-f", "lib2to3.fixes.fix_dict", "-w", filename])
+    print(
+        "All loops converted. Check diff to see if there are any that need to be commitedd."
+    )
+
+
+if __name__ == "__main__":
+    fix_potential_issue()

src/allmydata/test/test_storage.py

@@ -547,7 +547,7 @@ class Server(unittest.TestCase):
         already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75, canary)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
-        for (f,args,kwargs) in canary.disconnectors.values():
+        for (f,args,kwargs) in list(canary.disconnectors.values()):
             f(*args, **kwargs)
         del already
         del writers

src/allmydata/util/deferredutil.py

@@ -180,17 +180,6 @@ class HookMixin(object):
         log.msg(msg, level=log.NOISY)
 
 
-def for_items(cb, mapping):
-    """
-    For each (key, value) pair in a mapping, I add a callback to cb(None, key, value)
-    to a Deferred that fires immediately. I return that Deferred.
-    """
-    d = defer.succeed(None)
-    for k, v in mapping.items():
-        d.addCallback(lambda ign, k=k, v=v: cb(None, k, v))
-    return d
-
-
 class WaitForDelayedCallsMixin(PollMixin):
     def _delayed_calls_done(self):
         # We're done when the only remaining DelayedCalls fire after threshold.

src/allmydata/util/dictutil.py

@@ -24,7 +24,7 @@ class DictOfSets(dict):
             self[key] = set([value])
 
     def update(self, otherdictofsets):
-        for key, values in otherdictofsets.items():
+        for key, values in list(otherdictofsets.items()):
             if key in self:
                 self[key].update(values)
             else: