Merge pull request #739 from tahoe-lafs/3339-assertutil-python-3

Port assertutil and mathutil to Python 3 (by switching to already-ported pyutil versions)

nix/overlays.nix

@@ -15,6 +15,9 @@ self: super: {
       # slightly newer version than appears in nixos 19.09 is helpful.
       future = python-super.callPackage ./future.nix { };
 
+      # Need version of pyutil that supports Python 3. The version in 19.09
+      # is too old.
+      pyutil = python-super.callPackage ./pyutil.nix { };
     };
   };
 }

nix/pyutil.nix

@@ -0,0 +1,48 @@
+{ stdenv
+, buildPythonPackage
+, fetchPypi
+, setuptoolsDarcs
+, setuptoolsTrial
+, simplejson
+, twisted
+, isPyPy
+}:
+
+buildPythonPackage rec {
+  pname = "pyutil";
+  version = "3.3.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848";
+  };
+
+  buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []);
+  propagatedBuildInputs = [ twisted ];
+
+  # Tests fail because they try to write new code into the twisted
+  # package, apparently some kind of plugin.
+  doCheck = false;
+
+  prePatch = stdenv.lib.optionalString isPyPy ''
+    grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Pyutil, a collection of mature utilities for Python programmers";
+
+    longDescription = ''
+      These are a few data structures, classes and functions which
+      we've needed over many years of Python programming and which
+      seem to be of general use to other Python programmers. Many of
+      the modules that have existed in pyutil over the years have
+      subsequently been obsoleted by new features added to the
+      Python language or its standard library, thus showing that
+      we're not alone in wanting tools like these.
+    '';
+
+    homepage = "http://allmydata.org/trac/pyutil";
+    license = licenses.gpl2Plus;
+  };
+
+}

nix/tahoe-lafs.nix

@@ -4,7 +4,7 @@
 , setuptools, setuptoolsTrial, pyasn1, zope_interface
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
 , beautifulsoup4, eliot, autobahn, cryptography
-, html5lib
+, html5lib, pyutil
 }:
 python.pkgs.buildPythonPackage rec {
   version = "1.14.0.dev";
@@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec {
     setuptoolsTrial pyasn1 zope_interface
     service-identity pyyaml magic-wormhole treq
     eliot autobahn cryptography setuptools
-    future
+    future pyutil
   ];
 
   checkInputs = with python.pkgs; [

setup.py

@@ -124,6 +124,9 @@ install_requires = [
 
     # Support for Python 3 transition
     "future >= 0.18.2",
+
+    # Utility code:
+    "pyutil >= 3.3.0",
 ]
 
 setup_requires = [

src/allmydata/test/test_util.py

@@ -15,7 +15,7 @@ from twisted.python.failure import Failure
 from twisted.python import log
 
 from allmydata.util import base32, idlib, mathutil, hashutil
-from allmydata.util import assertutil, fileutil, deferredutil, abbreviate
+from allmydata.util import fileutil, deferredutil, abbreviate
 from allmydata.util import limiter, time_format, pollmixin
 from allmydata.util import statistics, dictutil, pipeline, yamlutil
 from allmydata.util import log as tahoe_log
@@ -64,97 +64,6 @@ class MyList(list):
     pass
 
 class Math(unittest.TestCase):
-    def test_div_ceil(self):
-        f = mathutil.div_ceil
-        self.failUnlessEqual(f(0, 1), 0)
-        self.failUnlessEqual(f(0, 2), 0)
-        self.failUnlessEqual(f(0, 3), 0)
-        self.failUnlessEqual(f(1, 3), 1)
-        self.failUnlessEqual(f(2, 3), 1)
-        self.failUnlessEqual(f(3, 3), 1)
-        self.failUnlessEqual(f(4, 3), 2)
-        self.failUnlessEqual(f(5, 3), 2)
-        self.failUnlessEqual(f(6, 3), 2)
-        self.failUnlessEqual(f(7, 3), 3)
-
-    def test_next_multiple(self):
-        f = mathutil.next_multiple
-        self.failUnlessEqual(f(5, 1), 5)
-        self.failUnlessEqual(f(5, 2), 6)
-        self.failUnlessEqual(f(5, 3), 6)
-        self.failUnlessEqual(f(5, 4), 8)
-        self.failUnlessEqual(f(5, 5), 5)
-        self.failUnlessEqual(f(5, 6), 6)
-        self.failUnlessEqual(f(32, 1), 32)
-        self.failUnlessEqual(f(32, 2), 32)
-        self.failUnlessEqual(f(32, 3), 33)
-        self.failUnlessEqual(f(32, 4), 32)
-        self.failUnlessEqual(f(32, 5), 35)
-        self.failUnlessEqual(f(32, 6), 36)
-        self.failUnlessEqual(f(32, 7), 35)
-        self.failUnlessEqual(f(32, 8), 32)
-        self.failUnlessEqual(f(32, 9), 36)
-        self.failUnlessEqual(f(32, 10), 40)
-        self.failUnlessEqual(f(32, 11), 33)
-        self.failUnlessEqual(f(32, 12), 36)
-        self.failUnlessEqual(f(32, 13), 39)
-        self.failUnlessEqual(f(32, 14), 42)
-        self.failUnlessEqual(f(32, 15), 45)
-        self.failUnlessEqual(f(32, 16), 32)
-        self.failUnlessEqual(f(32, 17), 34)
-        self.failUnlessEqual(f(32, 18), 36)
-        self.failUnlessEqual(f(32, 589), 589)
-
-    def test_pad_size(self):
-        f = mathutil.pad_size
-        self.failUnlessEqual(f(0, 4), 0)
-        self.failUnlessEqual(f(1, 4), 3)
-        self.failUnlessEqual(f(2, 4), 2)
-        self.failUnlessEqual(f(3, 4), 1)
-        self.failUnlessEqual(f(4, 4), 0)
-        self.failUnlessEqual(f(5, 4), 3)
-
-    def test_is_power_of_k(self):
-        f = mathutil.is_power_of_k
-        for i in range(1, 100):
-            if i in (1, 2, 4, 8, 16, 32, 64):
-                self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
-            else:
-                self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
-        for i in range(1, 100):
-            if i in (1, 3, 9, 27, 81):
-                self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
-            else:
-                self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
-
-    def test_next_power_of_k(self):
-        f = mathutil.next_power_of_k
-        self.failUnlessEqual(f(0,2), 1)
-        self.failUnlessEqual(f(1,2), 1)
-        self.failUnlessEqual(f(2,2), 2)
-        self.failUnlessEqual(f(3,2), 4)
-        self.failUnlessEqual(f(4,2), 4)
-        for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
-        for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
-        for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
-        for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
-        for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
-
-        self.failUnlessEqual(f(0,3), 1)
-        self.failUnlessEqual(f(1,3), 1)
-        self.failUnlessEqual(f(2,3), 3)
-        self.failUnlessEqual(f(3,3), 3)
-        for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
-        for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
-        for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
-        for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
-
-    def test_ave(self):
-        f = mathutil.ave
-        self.failUnlessEqual(f([1,2,3]), 2)
-        self.failUnlessEqual(f([0,0,0,4]), 1)
-        self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
-
     def test_round_sigfigs(self):
         f = mathutil.round_sigfigs
         self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002)
@@ -297,65 +206,6 @@ class Statistics(unittest.TestCase):
         self.failUnlessEqual(f(plist, .5, 3), .02734375)
 
 
-class Asserts(unittest.TestCase):
-    def should_assert(self, func, *args, **kwargs):
-        try:
-            func(*args, **kwargs)
-        except AssertionError as e:
-            return str(e)
-        except Exception as e:
-            self.fail("assert failed with non-AssertionError: %s" % e)
-        self.fail("assert was not caught")
-
-    def should_not_assert(self, func, *args, **kwargs):
-        try:
-            func(*args, **kwargs)
-        except AssertionError as e:
-            self.fail("assertion fired when it should not have: %s" % e)
-        except Exception as e:
-            self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e)
-        return # we're happy
-
-
-    def test_assert(self):
-        f = assertutil._assert
-        self.should_assert(f)
-        self.should_assert(f, False)
-        self.should_not_assert(f, True)
-
-        m = self.should_assert(f, False, "message")
-        self.failUnlessEqual(m, "'message' <type 'str'>", m)
-        m = self.should_assert(f, False, "message1", othermsg=12)
-        self.failUnlessEqual("'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
-        m = self.should_assert(f, False, othermsg="message2")
-        self.failUnlessEqual("othermsg: 'message2' <type 'str'>", m)
-
-    def test_precondition(self):
-        f = assertutil.precondition
-        self.should_assert(f)
-        self.should_assert(f, False)
-        self.should_not_assert(f, True)
-
-        m = self.should_assert(f, False, "message")
-        self.failUnlessEqual("precondition: 'message' <type 'str'>", m)
-        m = self.should_assert(f, False, "message1", othermsg=12)
-        self.failUnlessEqual("precondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
-        m = self.should_assert(f, False, othermsg="message2")
-        self.failUnlessEqual("precondition: othermsg: 'message2' <type 'str'>", m)
-
-    def test_postcondition(self):
-        f = assertutil.postcondition
-        self.should_assert(f)
-        self.should_assert(f, False)
-        self.should_not_assert(f, True)
-
-        m = self.should_assert(f, False, "message")
-        self.failUnlessEqual("postcondition: 'message' <type 'str'>", m)
-        m = self.should_assert(f, False, "message1", othermsg=12)
-        self.failUnlessEqual("postcondition: 'message1' <type 'str'>, othermsg: 12 <type 'int'>", m)
-        m = self.should_assert(f, False, othermsg="message2")
-        self.failUnlessEqual("postcondition: othermsg: 'message2' <type 'str'>", m)
-
 class FileUtil(ReallyEqualMixin, unittest.TestCase):
     def mkdir(self, basedir, path, mode=0o777):
         fn = os.path.join(basedir, path)

src/allmydata/util/_python3.py

@@ -15,7 +15,9 @@ if PY2:
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
+    "allmydata.util.assertutil",
     "allmydata.util.humanreadable",
+    "allmydata.util.mathutil",
     "allmydata.util.namespace",
     "allmydata.util._python3",
 ]

src/allmydata/util/assertutil.py

@@ -1,57 +1,23 @@
 """
 Tests useful in assertion checking, prints out nicely formated messages too.
+
+Backwards compatibility layer, the versions in pyutil are better maintained and
+have tests.
+
+Ported to Python 3.
 """
 
-from allmydata.util.humanreadable import hr
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
 
-def _assert(___cond=False, *___args, **___kwargs):
+from future.utils import PY2
-    if ___cond:
+if PY2:
-        return True
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
-    msgbuf=[]
-    if ___args:
-        msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),))))
-        msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]])
-        if ___kwargs:
-            msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    else:
-        if ___kwargs:
-            msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]])
 
-    raise AssertionError("".join(msgbuf))
 
-def precondition(___cond=False, *___args, **___kwargs):
+# The API importers expect:
-    if ___cond:
+from pyutil.assertutil import _assert, precondition, postcondition
-        return True
-    msgbuf=["precondition", ]
-    if ___args or ___kwargs:
-        msgbuf.append(": ")
-    if ___args:
-        msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),))))
-        msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]])
-        if ___kwargs:
-            msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    else:
-        if ___kwargs:
-            msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]])
 
-    raise AssertionError("".join(msgbuf))
+__all__ = ["_assert", "precondition", "postcondition"]
-
-def postcondition(___cond=False, *___args, **___kwargs):
-    if ___cond:
-        return True
-    msgbuf=["postcondition", ]
-    if ___args or ___kwargs:
-        msgbuf.append(": ")
-    if ___args:
-        msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),))))
-        msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]])
-        if ___kwargs:
-            msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    else:
-        if ___kwargs:
-            msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),)))))
-    msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]])
-
-    raise AssertionError("".join(msgbuf))

src/allmydata/util/mathutil.py

@@ -1,71 +1,28 @@
 """
 A few commonly needed functions.
+
+Backwards compatibility for direct imports.
+
+Ported to Python 3.
 """
 
-import math
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
 
-def div_ceil(n, d):
+from future.utils import PY2
-    """
+if PY2:
-    The smallest integer k such that k*d >= n.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
-    """
-    return (n/d) + (n%d != 0)
 
-def next_multiple(n, k):
-    """
-    The smallest multiple of k which is >= n.
-    """
-    return div_ceil(n, k) * k
 
-def pad_size(n, k):
+# The API importers expect:
-    """
+from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor
-    The smallest number that has to be added to n so that n is a multiple of k.
-    """
-    if n%k:
-        return k - n%k
-    else:
-        return 0
 
-def is_power_of_k(n, k):
-    return k**int(math.log(n, k) + 0.5) == n
-
-def next_power_of_k(n, k):
-    if n == 0:
-        x = 0
-    else:
-        x = int(math.log(n, k) + 0.5)
-    if k**x < n:
-        return k**(x+1)
-    else:
-        return k**x
-
-def ave(l):
-    return sum(l) / len(l)
-
-def log_ceil(n, b):
-    """
-    The smallest integer k such that b^k >= n.
-
-    log_ceil(n, 2) is the number of bits needed to store any of n values, e.g.
-    the number of bits needed to store any of 128 possible values is 7.
-    """
-    p = 1
-    k = 0
-    while p < n:
-        p *= b
-        k += 1
-    return k
-
-def log_floor(n, b):
-    """
-    The largest integer k such that b^k <= n.
-    """
-    p = 1
-    k = 0
-    while p <= n:
-        p *= b
-        k += 1
-    return k - 1
 
+# This function is not present in pyutil.mathutil:
 def round_sigfigs(f, n):
     fmt = "%." + str(n-1) + "e"
     return float(fmt % f)
+
+__all__ = ["div_ceil", "next_multiple", "pad_size", "is_power_of_k", "next_power_of_k", "ave", "log_ceil", "log_floor", "round_sigfigs"]