Merge remote-tracking branch 'origin/master' into 3365.log-python-3

misc/python3/ratchet-passing

@@ -24,6 +24,33 @@ allmydata.test.test_base62.Base62.test_known_values
 allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
 allmydata.test.test_base62.Base62.test_odd_sizes
 allmydata.test.test_base62.Base62.test_roundtrip
+allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
+allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
+allmydata.test.test_crypto.TestEd25519.test_key_serialization
+allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey
+allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes
+allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes
+allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes
+allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey
+allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input
+allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input
+allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input
+allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input
+allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair
+allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair
+allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes
+allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size
+allmydata.test.test_crypto.TestRegression.test_iv_not_bytes
+allmydata.test.test_crypto.TestRegression.test_key_incorrect_size
+allmydata.test.test_crypto.TestRegression.test_old_start_up_test
+allmydata.test.test_crypto.TestRsa.test_keys
+allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey
+allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey
+allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad
+allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string
+allmydata.test.test_crypto.TestUtil.test_remove_prefix_good
+allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial
+allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero
 allmydata.test.test_deferredutil.DeferredUtilTests.test_failure
 allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results
 allmydata.test.test_deferredutil.DeferredUtilTests.test_success
@@ -46,6 +73,8 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d
 allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated
 allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare
 allmydata.test.test_humanreadable.HumanReadable.test_repr
+allmydata.test.test_iputil.GcUtil.test_gc_after_allocations
+allmydata.test.test_iputil.GcUtil.test_release_delays_gc
 allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for
 allmydata.test.test_iputil.ListAddresses.test_list_async
 allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin

setup.cfg

@@ -7,6 +7,8 @@ bdist_egg = update_version bdist_egg
 bdist_wheel = update_version bdist_wheel
 
 [flake8]
-# For now, only use pyflakes errors; flake8 is still helpful because it allows
+# Enforce all pyflakes constraints, and also prohibit tabs for indentation.
-# ignoring specific errors/warnings when needed.
+# Reference:
-select = F
+#   https://flake8.pycqa.org/en/latest/user/error-codes.html
+#   https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
+select = F, W191

src/allmydata/crypto/__init__.py

@@ -5,4 +5,15 @@ For the most part, these functions use and return objects that are
 documented in the `cryptography` library -- however, code inside Tahoe
 should only use these functions and not rely on features of any
 objects that `cryptography` documents.
+
+Ported to Python 3.
 """
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

src/allmydata/crypto/aes.py

@@ -6,7 +6,17 @@ These functions use and return objects that are documented in the
 `cryptography` library -- however, code inside Tahoe should only use
 functions from allmydata.crypto.aes and not rely on features of any
 objects that `cryptography` documents.
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import six
 

src/allmydata/crypto/ed25519.py

@@ -13,7 +13,18 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form,
 but the minor usability improvement is not worth the documentation and
 specification confusion of using a non-standard encoding. So we stick with
 base32.
+
+Ported to Python 3.
 '''
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import six
 

src/allmydata/crypto/error.py

@@ -1,6 +1,16 @@
 """
 Exceptions raise by allmydata.crypto.* modules
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
 class BadSignature(Exception):

src/allmydata/crypto/rsa.py

@@ -9,8 +9,17 @@ features of any objects that `cryptography` documents.
 
 That is, the public and private keys are opaque objects; DO NOT depend
 on any of their methods.
-"""
 
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend

src/allmydata/crypto/util.py

@@ -1,6 +1,16 @@
 """
 Utilities used by allmydata.crypto modules
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from allmydata.crypto.error import BadPrefixError
 

src/allmydata/hashtree.py

@@ -56,7 +56,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from allmydata.util import mathutil # from the pyutil library
 

src/allmydata/test/common_py3.py

@@ -11,7 +11,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 import time

src/allmydata/test/test_abbreviate.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from datetime import timedelta
 

src/allmydata/test/test_base32.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import base64
 

src/allmydata/test/test_base62.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import chr as byteschr
 

src/allmydata/test/test_crypto.py

@@ -1,4 +1,14 @@
-import six
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from future.utils import native_bytes
+
 import unittest
 
 from base64 import b64decode
@@ -37,17 +47,18 @@ class TestRegression(unittest.TestCase):
         #     priv = rsa.generate(2048)
         #     priv_str = b64encode(priv.serialize())
         #     pub_str = b64encode(priv.get_verifying_key().serialize())
-        RSA_2048_PRIV_KEY = six.b(b64decode(f.read().strip()))
+        RSA_2048_PRIV_KEY = b64decode(f.read().strip())
+        assert isinstance(RSA_2048_PRIV_KEY, native_bytes)
 
     with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f:
         # Signature created using `RSA_2048_PRIV_KEY` via:
         #
         #     sig = priv.sign(b'test')
-        RSA_2048_SIG = six.b(b64decode(f.read().strip()))
+        RSA_2048_SIG = b64decode(f.read().strip())
 
     with RESOURCE_DIR.child('pycryptopp-rsa-2048-pub.txt').open('r') as f:
         # The public key corresponding to `RSA_2048_PRIV_KEY`.
-        RSA_2048_PUB_KEY = six.b(b64decode(f.read().strip()))
+        RSA_2048_PUB_KEY = b64decode(f.read().strip())
 
     def test_old_start_up_test(self):
         """
@@ -283,7 +294,7 @@ class TestEd25519(unittest.TestCase):
         private_key, public_key = ed25519.create_signing_keypair()
         private_key_str = ed25519.string_from_signing_key(private_key)
 
-        self.assertIsInstance(private_key_str, six.string_types)
+        self.assertIsInstance(private_key_str, native_bytes)
 
         private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str)
 
@@ -299,7 +310,7 @@ class TestEd25519(unittest.TestCase):
 
         # ditto, but for the verifying keys
         public_key_str = ed25519.string_from_verifying_key(public_key)
-        self.assertIsInstance(public_key_str, six.string_types)
+        self.assertIsInstance(public_key_str, native_bytes)
 
         public_key2 = ed25519.verifying_key_from_string(public_key_str)
         self.assertEqual(
@@ -403,7 +414,7 @@ class TestRsa(unittest.TestCase):
         priv_key, pub_key = rsa.create_signing_keypair(2048)
         priv_key_str = rsa.der_string_from_signing_key(priv_key)
 
-        self.assertIsInstance(priv_key_str, six.string_types)
+        self.assertIsInstance(priv_key_str, native_bytes)
 
         priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str)
 

src/allmydata/test/test_deferredutil.py

@@ -11,7 +11,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest
 from twisted.internet import defer, reactor

src/allmydata/test/test_dictutil.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest
 

src/allmydata/test/test_hashtree.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
 from twisted.trial import unittest

src/allmydata/test/test_hashutil.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest
 

src/allmydata/test/test_humanreadable.py

@@ -11,7 +11,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import long
 

src/allmydata/test/test_iputil.py

@@ -11,9 +11,10 @@ from __future__ import unicode_literals
 
 from future.utils import PY2, native_str
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import re, errno, subprocess, os, socket
+import gc
 
 from twisted.trial import unittest
 
@@ -21,7 +22,7 @@ from tenacity import retry, stop_after_attempt
 
 from foolscap.api import Tub
 
-from allmydata.util import iputil
+from allmydata.util import iputil, gcutil
 import allmydata.test.common_py3 as testutil
 from allmydata.util.namespace import Namespace
 
@@ -228,3 +229,35 @@ class ListenOnUsed(unittest.TestCase):
         s.close()
         port2 = iputil.listenOnUnused(tub, port)
         self.assertEqual(port, port2)
+
+
+class GcUtil(unittest.TestCase):
+    """Tests for allmydata.util.gcutil, which is used only by listenOnUnused."""
+
+    def test_gc_after_allocations(self):
+        """The resource tracker triggers allocations every 26 allocations."""
+        tracker = gcutil._ResourceTracker()
+        collections = []
+        self.patch(gc, "collect", lambda: collections.append(1))
+        for _ in range(2):
+            for _ in range(25):
+                tracker.allocate()
+                self.assertEqual(len(collections), 0)
+            tracker.allocate()
+            self.assertEqual(len(collections), 1)
+            del collections[:]
+
+    def test_release_delays_gc(self):
+        """Releasing a file descriptor resource delays GC collection."""
+        tracker = gcutil._ResourceTracker()
+        collections = []
+        self.patch(gc, "collect", lambda: collections.append(1))
+        for _ in range(2):
+            tracker.allocate()
+        for _ in range(3):
+            tracker.release()
+        for _ in range(25):
+            tracker.allocate()
+            self.assertEqual(len(collections), 0)
+        tracker.allocate()
+        self.assertEqual(len(collections), 1)

src/allmydata/test/test_netstring.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest
 

src/allmydata/test/test_observer.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest
 from twisted.internet import defer, reactor

src/allmydata/test/test_pipeline.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import gc
 

src/allmydata/test/test_python3.py

@@ -10,7 +10,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.python.modules import (
     getModule,

src/allmydata/test/test_spans.py

@@ -9,7 +9,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import long
 

src/allmydata/test/test_statistics.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from six.moves import StringIO  # native string StringIO
 

src/allmydata/test/test_time_format.py

@@ -8,7 +8,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import long
 

src/allmydata/test/test_util.py

@@ -5,11 +5,10 @@ import os, time, sys
 import yaml
 
 from twisted.trial import unittest
-from twisted.internet import defer, reactor
 
 from allmydata.util import idlib, mathutil
 from allmydata.util import fileutil
-from allmydata.util import limiter, pollmixin
+from allmydata.util import pollmixin
 from allmydata.util import yamlutil
 from allmydata.util.fileutil import EncryptedTemporaryFile
 from allmydata.test.common_util import ReallyEqualMixin
@@ -427,81 +426,6 @@ class PollMixinTests(unittest.TestCase):
         return d
 
 
-class Limiter(unittest.TestCase):
-
-    def job(self, i, foo):
-        self.calls.append( (i, foo) )
-        self.simultaneous += 1
-        self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous)
-        d = defer.Deferred()
-        def _done():
-            self.simultaneous -= 1
-            d.callback("done %d" % i)
-        reactor.callLater(1.0, _done)
-        return d
-
-    def bad_job(self, i, foo):
-        raise ValueError("bad_job %d" % i)
-
-    def test_limiter(self):
-        self.calls = []
-        self.simultaneous = 0
-        self.peak_simultaneous = 0
-        l = limiter.ConcurrencyLimiter()
-        dl = []
-        for i in range(20):
-            dl.append(l.add(self.job, i, foo=str(i)))
-        d = defer.DeferredList(dl, fireOnOneErrback=True)
-        def _done(res):
-            self.failUnlessEqual(self.simultaneous, 0)
-            self.failUnless(self.peak_simultaneous <= 10)
-            self.failUnlessEqual(len(self.calls), 20)
-            for i in range(20):
-                self.failUnless( (i, str(i)) in self.calls)
-        d.addCallback(_done)
-        return d
-
-    def test_errors(self):
-        self.calls = []
-        self.simultaneous = 0
-        self.peak_simultaneous = 0
-        l = limiter.ConcurrencyLimiter()
-        dl = []
-        for i in range(20):
-            dl.append(l.add(self.job, i, foo=str(i)))
-        d2 = l.add(self.bad_job, 21, "21")
-        d = defer.DeferredList(dl, fireOnOneErrback=True)
-        def _most_done(res):
-            results = []
-            for (success, result) in res:
-                self.failUnlessEqual(success, True)
-                results.append(result)
-            results.sort()
-            expected_results = ["done %d" % i for i in range(20)]
-            expected_results.sort()
-            self.failUnlessEqual(results, expected_results)
-            self.failUnless(self.peak_simultaneous <= 10)
-            self.failUnlessEqual(len(self.calls), 20)
-            for i in range(20):
-                self.failUnless( (i, str(i)) in self.calls)
-            def _good(res):
-                self.fail("should have failed, not got %s" % (res,))
-            def _err(f):
-                f.trap(ValueError)
-                self.failUnless("bad_job 21" in str(f))
-            d2.addCallbacks(_good, _err)
-            return d2
-        d.addCallback(_most_done)
-        def _all_done(res):
-            self.failUnlessEqual(self.simultaneous, 0)
-            self.failUnless(self.peak_simultaneous <= 10)
-            self.failUnlessEqual(len(self.calls), 20)
-            for i in range(20):
-                self.failUnless( (i, str(i)) in self.calls)
-        d.addCallback(_all_done)
-        return d
-
-
 ctr = [0]
 class EqButNotIs(object):
     def __init__(self, x):

src/allmydata/test/test_version.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import sys
 import pkg_resources

src/allmydata/util/_python3.py

@@ -11,10 +11,16 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
+    "allmydata.crypto",
+    "allmydata.crypto.aes",
+    "allmydata.crypto.ed25519",
+    "allmydata.crypto.error",
+    "allmydata.crypto.rsa",
+    "allmydata.crypto.util",
     "allmydata.hashtree",
     "allmydata.util.abbreviate",
     "allmydata.util.assertutil",
@@ -22,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.base62",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
+    "allmydata.util.gcutil",
     "allmydata.util.hashutil",
     "allmydata.util.humanreadable",
     "allmydata.util.iputil",
@@ -43,6 +50,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",
     "allmydata.test.test_hashtree",

src/allmydata/util/abbreviate.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import re
 from datetime import timedelta

src/allmydata/util/assertutil.py

@@ -14,7 +14,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
 # The API importers expect:

src/allmydata/util/base32.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 if PY2:
     def backwardscompat_bytes(b):

src/allmydata/util/base62.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 if PY2:
     import string

src/allmydata/util/deferredutil.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 

src/allmydata/util/dictutil.py

@@ -13,7 +13,7 @@ if PY2:
     # IMPORTANT: We deliberately don't import dict. The issue is that we're
     # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of
     # code that doesn't support it.
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
 
 
 class DictOfSets(dict):

src/allmydata/util/gcutil.py

@@ -7,7 +7,17 @@ Helpers for managing garbage collection.
     a result.  Register allocation and release of *bare* file descriptors with
     this object (file objects, socket objects, etc, have their own integration
     with the garbage collector and don't need to bother with this).
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 __all__ = [
     "fileDescriptorResource",

src/allmydata/util/hashutil.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import chr as byteschr
 

src/allmydata/util/humanreadable.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 from reprlib import Repr

src/allmydata/util/iputil.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2, native_str
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, re, socket, subprocess, errno
 from sys import platform

src/allmydata/util/limiter.py

@@ -1,40 +0,0 @@
-
-from twisted.internet import defer
-from foolscap.api import eventually
-
-class ConcurrencyLimiter(object):
-    """I implement a basic concurrency limiter. Add work to it in the form of
-    (callable, args, kwargs) tuples. No more than LIMIT callables will be
-    outstanding at any one time.
-    """
-
-    def __init__(self, limit=10):
-        self.limit = limit
-        self.pending = []
-        self.active = 0
-
-    def __repr__(self):
-        return "<Limiter with %d/%d/%d>" % (self.active, len(self.pending),
-                                            self.limit)
-
-    def add(self, cb, *args, **kwargs):
-        d = defer.Deferred()
-        task = (cb, args, kwargs, d)
-        self.pending.append(task)
-        self.maybe_start_task()
-        return d
-
-    def maybe_start_task(self):
-        if self.active >= self.limit:
-            return
-        if not self.pending:
-            return
-        (cb, args, kwargs, done_d) = self.pending.pop(0)
-        self.active += 1
-        d = defer.maybeDeferred(cb, *args, **kwargs)
-        d.addBoth(self._done, done_d)
-
-    def _done(self, res, done_d):
-        self.active -= 1
-        eventually(done_d.callback, res)
-        eventually(self.maybe_start_task)

src/allmydata/util/mathutil.py

@@ -13,7 +13,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
 # The API importers expect:

src/allmydata/util/netstring.py

@@ -10,7 +10,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from past.builtins import long
 

src/allmydata/util/observer.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import weakref
 from twisted.internet import defer

src/allmydata/util/pipeline.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.internet import defer
 from twisted.python.failure import Failure

src/allmydata/util/pollmixin.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 from twisted.internet import task

src/allmydata/util/spans.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
 class Spans(object):

src/allmydata/util/statistics.py

@@ -18,7 +18,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from functools import reduce
 

src/allmydata/util/time_format.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from future.utils import native_str
 
 import calendar, datetime, re, time

src/allmydata/util/verlib.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import re
 

src/allmydata/version_checks.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 __all__ = [
     "PackagingError",

tox.ini

@@ -49,7 +49,7 @@ commands =
          tahoe --version
 
 [testenv:py36]
-# git inside of ratchet.sh needs $HOME.
+# On macOS, git inside of ratchet.sh needs $HOME.
 passenv = HOME
 commands = {toxinidir}/misc/python3/ratchet.sh
 
@@ -77,6 +77,8 @@ commands =
          coverage xml
 
 [testenv:codechecks]
+# On macOS, git inside of towncrier needs $HOME.
+passenv = HOME
 whitelist_externals =
          /bin/mv
 commands =