Merge branch 'master' into 3254.status-status-nevow-to-twisted-web

misc/python3/tahoe-depgraph.py

@@ -60,7 +60,8 @@ class mymf(modulefinder.ModuleFinder):
                 self._depgraph[last_caller.__name__].add(fqname)
         return r
 
-    def load_module(self, fqname, fp, pathname, (suffix, mode, type)):
+    def load_module(self, fqname, fp, pathname, additional_info):
+        (suffix, mode, type) = additional_info
         r = modulefinder.ModuleFinder.load_module(
             self, fqname, fp, pathname, (suffix, mode, type))
         if r is not None:
@@ -71,7 +72,7 @@ class mymf(modulefinder.ModuleFinder):
         return {
             'depgraph': {
                 name: dict.fromkeys(deps, 1)
-                for name, deps in self._depgraph.iteritems()},
+                for name, deps in self._depgraph.items()},
             'types': self._types,
         }
 
@@ -101,20 +102,25 @@ def main(target):
                 filepath = path
             moduleNames.append(reflect.filenameToModuleName(filepath))
 
-    with tempfile.NamedTemporaryFile() as tmpfile:
+    with tempfile.NamedTemporaryFile("w") as tmpfile:
         for moduleName in moduleNames:
             tmpfile.write('import %s\n' % moduleName)
         tmpfile.flush()
         mf.run_script(tmpfile.name)
 
-    with open('tahoe-deps.json', 'wb') as outfile:
+    with open('tahoe-deps.json', 'w') as outfile:
         json_dump(mf.as_json(), outfile)
         outfile.write('\n')
 
-    ported_modules_path = os.path.join(target, "src", "allmydata", "ported-modules.txt")
-    with open(ported_modules_path) as ported_modules:
-        port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported")
-    with open('tahoe-ported.json', 'wb') as outfile:
+    ported_modules_path = os.path.join(target, "src", "allmydata", "util", "_python3.py")
+    with open(ported_modules_path) as f:
+        ported_modules = {}
+        exec(f.read(), ported_modules, ported_modules)
+        port_status = dict.fromkeys(
+            ported_modules["PORTED_MODULES"] + ported_modules["PORTED_TEST_MODULES"],
+            "ported"
+        )
+    with open('tahoe-ported.json', 'w') as outfile:
         json_dump(port_status, outfile)
         outfile.write('\n')
 

pyinstaller.spec

@@ -27,15 +27,29 @@ added_files = [
     ('src/allmydata/web/static/img/*.png', 'allmydata/web/static/img')]
 
 hidden_imports = [
+    '__builtin__',
     'allmydata.client',
     'allmydata.introducer',
     'allmydata.stats',
+    'base64',
     'cffi',
+    'collections',
+    'commands',
     'Crypto',
+    'functools',
+    'future.backports.misc',
+    'itertools',
+    'math',
     'packaging.specifiers',
+    're',
+    'reprlib',
     'six.moves.html_parser',
+    'subprocess',
+    'UserDict',
+    'UserList',
+    'UserString',
     'yaml',
-    'zfec'
+    'zfec',
 ]
 
 a = Analysis(

setup.py

@@ -54,7 +54,9 @@ install_requires = [
     # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo
     # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs
     # * foolscap 0.13.2 drops i2p support completely
-    "foolscap == 0.13.1",
+    # * foolscap >= 20.4 is necessary for Python 3
+    "foolscap == 0.13.1 ; python_version < '3.0'",
+    "foolscap >= 20.4.0 ; python_version > '3.0'",
 
     # * cryptography 2.6 introduced some ed25519 APIs we rely on.  Note that
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
@@ -119,6 +121,9 @@ install_requires = [
 
     # WebSocket library for twisted and asyncio
     "autobahn >= 19.5.2",
+
+    # Support for Python 3 transition
+    "future >= 0.18.2",
 ]
 
 setup_requires = [

src/allmydata/__init__.py

@@ -37,3 +37,8 @@ __appname__ = "tahoe-lafs"
 # in the "application" part of the Tahoe versioning scheme:
 # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning
 __full_version__ = __appname__ + '/' + str(__version__)
+
+
+# Install Python 3 module locations in Python 2:
+from future import standard_library
+standard_library.install_aliases()

src/allmydata/ported-modules.txt

@@ -1 +0,0 @@
-allmydata.util.namespace

src/allmydata/test/test_humanreadable.py

@@ -0,0 +1,64 @@
+"""
+Tests for allmydata.util.humanreadable.
+
+This module has been ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
+from past.builtins import long
+
+from twisted.trial import unittest
+
+from allmydata.util import humanreadable
+
+
+
+def foo(): pass # FYI foo()'s line number is used in the test below
+
+
+class NoArgumentException(Exception):
+    def __init__(self):
+        pass
+
+class HumanReadable(unittest.TestCase):
+    def test_repr(self):
+        hr = humanreadable.hr
+        self.failUnlessEqual(hr(foo), "<foo() at test_humanreadable.py:24>")
+        self.failUnlessEqual(hr(self.test_repr),
+                             "<bound method HumanReadable.test_repr of <allmydata.test.test_humanreadable.HumanReadable testMethod=test_repr>>")
+        self.failUnlessEqual(hr(long(1)), "1")
+        self.assertIn(hr(10**40),
+                      ["100000000000000000...000000000000000000",
+                       "100000000000000000...0000000000000000000"])
+        self.failUnlessEqual(hr(self), "<allmydata.test.test_humanreadable.HumanReadable testMethod=test_repr>")
+        self.failUnlessEqual(hr([1,2]), "[1, 2]")
+        self.failUnlessEqual(hr({1:2}), "{1:2}")
+        try:
+            raise ValueError
+        except Exception as e:
+            self.failUnless(
+                hr(e) == "<ValueError: ()>" # python-2.4
+                or hr(e) == "ValueError()") # python-2.5
+        try:
+            raise ValueError("oops")
+        except Exception as e:
+            self.failUnless(
+                hr(e) == "<ValueError: 'oops'>" # python-2.4
+                or hr(e) == "ValueError('oops',)" # python-2.5
+                or hr(e) == "ValueError(u'oops',)" # python 2 during py3 transition
+            )
+        try:
+            raise NoArgumentException
+        except Exception as e:
+            self.failUnless(
+                hr(e) == "<NoArgumentException>" # python-2.4
+                or hr(e) == "NoArgumentException()" # python-2.5
+                or hr(e) == "<NoArgumentException: ()>", hr(e)) # python-3

src/allmydata/test/test_python3.py

@@ -1,10 +1,16 @@
 """
 Tests related to the Python 3 porting effort itself.
-"""
 
-from pkg_resources import (
-    resource_stream,
-)
+This module has been ported to Python 3.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
 
 from twisted.python.modules import (
     getModule,
@@ -13,11 +19,18 @@ from twisted.trial.unittest import (
     SynchronousTestCase,
 )
 
+from allmydata.util._python3 import PORTED_MODULES, PORTED_TEST_MODULES
+
 
 class Python3PortingEffortTests(SynchronousTestCase):
+
     def test_finished_porting(self):
         """
         Tahoe-LAFS has been ported to Python 3.
+
+        Once
+        https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203
+        is completed this test should pass (and can be deleted!).
         """
         tahoe_lafs_module_names = set(all_module_names("allmydata"))
         ported_names = set(ported_module_names())
@@ -31,7 +44,10 @@ class Python3PortingEffortTests(SynchronousTestCase):
                 ),
             ),
         )
-    test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed"
+    if PY2:
+        test_finished_porting.skip = "For some reason todo isn't working on Python 2 now"
+    else:
+        test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed"
 
     def test_ported_modules_exist(self):
         """
@@ -70,18 +86,18 @@ def all_module_names(toplevel):
     """
     allmydata = getModule(toplevel)
     for module in allmydata.walkModules():
-        yield module.name.decode("utf-8")
+        name = module.name
+        if PY2:
+            name = name.decode("utf-8")
+        yield name
 
 
 def ported_module_names():
     """
-    :return list[unicode]: A ``set`` of ``unicode`` giving the names of
+    :return list[unicode]: A ``list`` of ``unicode`` giving the names of
         Tahoe-LAFS modules which have been ported to Python 3.
     """
-    return resource_stream(
-        "allmydata",
-        u"ported-modules.txt",
-    ).read().splitlines()
+    return PORTED_MODULES + PORTED_TEST_MODULES
 
 
 def unported_report(tahoe_lafs_module_names, ported_names):
@@ -100,8 +116,8 @@ def count_lines(module_name):
     try:
         source = module.filePath.getContent()
     except Exception as e:
-        print(module_name, e)
+        print((module_name, e))
         return 0
     lines = source.splitlines()
-    nonblank = filter(None, lines)
+    nonblank = [_f for _f in lines if _f]
     return len(nonblank)

src/allmydata/test/test_util.py

@@ -1,8 +1,5 @@
 from __future__ import print_function
 
-
-def foo(): pass # keep the line number constant
-
 import binascii
 import six
 import hashlib
@@ -17,7 +14,7 @@ from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 from twisted.python import log
 
-from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil
+from allmydata.util import base32, idlib, mathutil, hashutil
 from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
 from allmydata.util import limiter, time_format, pollmixin
 from allmydata.util import statistics, dictutil, pipeline, yamlutil
@@ -57,81 +54,11 @@ class Base32(unittest.TestCase):
         self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34")
         self.failUnlessRaises(AssertionError, base32.a2b, "b0gus")
 
+
 class IDLib(unittest.TestCase):
     def test_nodeid_b2a(self):
         self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
 
-class NoArgumentException(Exception):
-    def __init__(self):
-        pass
-
-class HumanReadable(unittest.TestCase):
-    def test_repr(self):
-        hr = humanreadable.hr
-        self.failUnlessEqual(hr(foo), "<foo() at test_util.py:4>")
-        self.failUnlessEqual(hr(self.test_repr),
-                             "<bound method HumanReadable.test_repr of <allmydata.test.test_util.HumanReadable testMethod=test_repr>>")
-        self.failUnlessEqual(hr(long(1)), "1")
-        self.failUnlessEqual(hr(10**40),
-                             "100000000000000000...000000000000000000")
-        self.failUnlessEqual(hr(self), "<allmydata.test.test_util.HumanReadable testMethod=test_repr>")
-        self.failUnlessEqual(hr([1,2]), "[1, 2]")
-        self.failUnlessEqual(hr({1:2}), "{1:2}")
-        try:
-            raise ValueError
-        except Exception as e:
-            self.failUnless(
-                hr(e) == "<ValueError: ()>" # python-2.4
-                or hr(e) == "ValueError()") # python-2.5
-        try:
-            raise ValueError("oops")
-        except Exception as e:
-            self.failUnless(
-                hr(e) == "<ValueError: 'oops'>" # python-2.4
-                or hr(e) == "ValueError('oops',)") # python-2.5
-        try:
-            raise NoArgumentException
-        except Exception as e:
-            self.failUnless(
-                hr(e) == "<NoArgumentException>" # python-2.4
-                or hr(e) == "NoArgumentException()") # python-2.5
-
-    def test_abbrev_time_1s(self):
-        diff = timedelta(seconds=1)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('1 second ago', s)
-
-    def test_abbrev_time_25s(self):
-        diff = timedelta(seconds=25)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('25 seconds ago', s)
-
-    def test_abbrev_time_future_5_minutes(self):
-        diff = timedelta(minutes=-5)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('5 minutes in the future', s)
-
-    def test_abbrev_time_hours(self):
-        diff = timedelta(hours=4)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('4 hours ago', s)
-
-    def test_abbrev_time_day(self):
-        diff = timedelta(hours=49)  # must be more than 2 days
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('2 days ago', s)
-
-    def test_abbrev_time_month(self):
-        diff = timedelta(days=91)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('3 months ago', s)
-
-    def test_abbrev_time_year(self):
-        diff = timedelta(weeks=(5 * 52) + 1)
-        s = abbreviate.abbreviate_time(diff)
-        self.assertEqual('5 years ago', s)
-
-
 
 class MyList(list):
     pass
@@ -984,6 +911,41 @@ class HashUtilTests(unittest.TestCase):
                         )
 
 class Abbreviate(unittest.TestCase):
+    def test_abbrev_time_1s(self):
+        diff = timedelta(seconds=1)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('1 second ago', s)
+
+    def test_abbrev_time_25s(self):
+        diff = timedelta(seconds=25)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('25 seconds ago', s)
+
+    def test_abbrev_time_future_5_minutes(self):
+        diff = timedelta(minutes=-5)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('5 minutes in the future', s)
+
+    def test_abbrev_time_hours(self):
+        diff = timedelta(hours=4)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('4 hours ago', s)
+
+    def test_abbrev_time_day(self):
+        diff = timedelta(hours=49)  # must be more than 2 days
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('2 days ago', s)
+
+    def test_abbrev_time_month(self):
+        diff = timedelta(days=91)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('3 months ago', s)
+
+    def test_abbrev_time_year(self):
+        diff = timedelta(weeks=(5 * 52) + 1)
+        s = abbreviate.abbreviate_time(diff)
+        self.assertEqual('5 years ago', s)
+
     def test_time(self):
         a = abbreviate.abbreviate_time
         self.failUnlessEqual(a(None), "unknown")

src/allmydata/util/_python3.py

@@ -0,0 +1,26 @@
+"""
+Track the port to Python 3.
+
+This module has been ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
+# Keep these sorted alphabetically, to reduce merge conflicts:
+PORTED_MODULES = [
+    "allmydata.util.humanreadable",
+    "allmydata.util.namespace",
+    "allmydata.util._python3",
+]
+
+PORTED_TEST_MODULES = [
+    "allmydata.test.test_humanreadable",
+    "allmydata.test.test_python3",
+]

src/allmydata/util/humanreadable.py

@@ -1,5 +1,20 @@
-import exceptions, os
-from repr import Repr
+"""
+Utilities for turning objects into human-readable strings.
+
+This module has been ported to Python 3.
+"""
+
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
+import os
+from reprlib import Repr
 
 class BetterRepr(Repr, object):
     def __init__(self):
@@ -14,21 +29,21 @@ class BetterRepr(Repr, object):
         self.maxother = 300
 
     def repr_function(self, obj, level):
-        if hasattr(obj, 'func_code'):
-            return '<' + obj.func_name + '() at ' + os.path.basename(obj.func_code.co_filename) + ':' + str(obj.func_code.co_firstlineno) + '>'
+        if hasattr(obj, '__code__'):
+            return '<' + obj.__name__ + '() at ' + os.path.basename(obj.__code__.co_filename) + ':' + str(obj.__code__.co_firstlineno) + '>'
         else:
-            return '<' + obj.func_name + '() at (builtin)'
+            return '<' + obj.__name__ + '() at (builtin)'
 
     def repr_instance_method(self, obj, level):
-        if hasattr(obj, 'func_code'):
-            return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at ' + os.path.basename(obj.im_func.func_code.co_filename) + ':' + str(obj.im_func.func_code.co_firstlineno) + '>'
+        if hasattr(obj, '__code__'):
+            return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at ' + os.path.basename(obj.__func__.__code__.co_filename) + ':' + str(obj.__func__.__code__.co_firstlineno) + '>'
         else:
-            return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at (builtin)'
+            return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at (builtin)'
 
     def repr_long(self, obj, level):
         s = repr(obj) # XXX Hope this isn't too slow...
         if len(s) > self.maxlong:
-            i = max(0, (self.maxlong-3)/2)
+            i = max(0, (self.maxlong-3) // 2)
             j = max(0, self.maxlong-3-i)
             s = s[:i] + '...' + s[len(s)-j:]
         if s[-1] == 'L':
@@ -43,7 +58,7 @@ class BetterRepr(Repr, object):
         on it.  If it is an instance of list call self.repr_list() on it. Else
         call Repr.repr_instance().
         """
-        if isinstance(obj, exceptions.Exception):
+        if isinstance(obj, Exception):
             # Don't cut down exception strings so much.
             tms = self.maxstring
             self.maxstring = max(512, tms * 4)
@@ -91,7 +106,7 @@ class BetterRepr(Repr, object):
         if level <= 0: return '{...}'
         s = ''
         n = len(obj)
-        items = obj.items()[:min(n, self.maxdict)]
+        items = list(obj.items())[:min(n, self.maxdict)]
         items.sort()
         for key, val in items:
             entry = self.repr1(key, level-1) + ':' + self.repr1(val, level-1)

src/allmydata/util/namespace.py

@@ -1,3 +1,6 @@
+"""
+This module has been ported to Python 3.
+"""
 
 class Namespace(object):
     pass