diff --git a/newsfragments/3339.other b/newsfragments/3339.other new file mode 100644 index 000000000..e69de29bb diff --git a/nix/overlays.nix b/nix/overlays.nix index 7f19a8f1a..ba3c9c885 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -15,6 +15,9 @@ self: super: { # slightly newer version than appears in nixos 19.09 is helpful. future = python-super.callPackage ./future.nix { }; + # Need version of pyutil that supports Python 3. The version in 19.09 + # is too old. + pyutil = python-super.callPackage ./pyutil.nix { }; }; }; } diff --git a/nix/pyutil.nix b/nix/pyutil.nix new file mode 100644 index 000000000..6852c2acc --- /dev/null +++ b/nix/pyutil.nix @@ -0,0 +1,48 @@ +{ stdenv +, buildPythonPackage +, fetchPypi +, setuptoolsDarcs +, setuptoolsTrial +, simplejson +, twisted +, isPyPy +}: + +buildPythonPackage rec { + pname = "pyutil"; + version = "3.3.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848"; + }; + + buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []); + propagatedBuildInputs = [ twisted ]; + + # Tests fail because they try to write new code into the twisted + # package, apparently some kind of plugin. + doCheck = false; + + prePatch = stdenv.lib.optionalString isPyPy '' + grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g" + ''; + + meta = with stdenv.lib; { + description = "Pyutil, a collection of mature utilities for Python programmers"; + + longDescription = '' + These are a few data structures, classes and functions which + we've needed over many years of Python programming and which + seem to be of general use to other Python programmers. Many of + the modules that have existed in pyutil over the years have + subsequently been obsoleted by new features added to the + Python language or its standard library, thus showing that + we're not alone in wanting tools like these. + ''; + + homepage = "http://allmydata.org/trac/pyutil"; + license = licenses.gpl2Plus; + }; + +} \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index ae5c05f70..097a463d1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib +, html5lib, pyutil }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future + future pyutil ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index 919c84818..047d08030 100644 --- a/setup.py +++ b/setup.py @@ -124,6 +124,9 @@ install_requires = [ # Support for Python 3 transition "future >= 0.18.2", + + # Utility code: + "pyutil >= 3.3.0", ] setup_requires = [ diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 65ba308d9..14d9f5dd7 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,7 +15,7 @@ from twisted.python.failure import Failure from twisted.python import log from allmydata.util import base32, idlib, mathutil, hashutil -from allmydata.util import assertutil, fileutil, deferredutil, abbreviate +from allmydata.util import fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil from allmydata.util import log as tahoe_log @@ -64,97 +64,6 @@ class MyList(list): pass class Math(unittest.TestCase): - def test_div_ceil(self): - f = mathutil.div_ceil - self.failUnlessEqual(f(0, 1), 0) - self.failUnlessEqual(f(0, 2), 0) - self.failUnlessEqual(f(0, 3), 0) - self.failUnlessEqual(f(1, 3), 1) - self.failUnlessEqual(f(2, 3), 1) - self.failUnlessEqual(f(3, 3), 1) - self.failUnlessEqual(f(4, 3), 2) - self.failUnlessEqual(f(5, 3), 2) - self.failUnlessEqual(f(6, 3), 2) - self.failUnlessEqual(f(7, 3), 3) - - def test_next_multiple(self): - f = mathutil.next_multiple - self.failUnlessEqual(f(5, 1), 5) - self.failUnlessEqual(f(5, 2), 6) - self.failUnlessEqual(f(5, 3), 6) - self.failUnlessEqual(f(5, 4), 8) - self.failUnlessEqual(f(5, 5), 5) - self.failUnlessEqual(f(5, 6), 6) - self.failUnlessEqual(f(32, 1), 32) - self.failUnlessEqual(f(32, 2), 32) - self.failUnlessEqual(f(32, 3), 33) - self.failUnlessEqual(f(32, 4), 32) - self.failUnlessEqual(f(32, 5), 35) - self.failUnlessEqual(f(32, 6), 36) - self.failUnlessEqual(f(32, 7), 35) - self.failUnlessEqual(f(32, 8), 32) - self.failUnlessEqual(f(32, 9), 36) - self.failUnlessEqual(f(32, 10), 40) - self.failUnlessEqual(f(32, 11), 33) - self.failUnlessEqual(f(32, 12), 36) - self.failUnlessEqual(f(32, 13), 39) - self.failUnlessEqual(f(32, 14), 42) - self.failUnlessEqual(f(32, 15), 45) - self.failUnlessEqual(f(32, 16), 32) - self.failUnlessEqual(f(32, 17), 34) - self.failUnlessEqual(f(32, 18), 36) - self.failUnlessEqual(f(32, 589), 589) - - def test_pad_size(self): - f = mathutil.pad_size - self.failUnlessEqual(f(0, 4), 0) - self.failUnlessEqual(f(1, 4), 3) - self.failUnlessEqual(f(2, 4), 2) - self.failUnlessEqual(f(3, 4), 1) - self.failUnlessEqual(f(4, 4), 0) - self.failUnlessEqual(f(5, 4), 3) - - def test_is_power_of_k(self): - f = mathutil.is_power_of_k - for i in range(1, 100): - if i in (1, 2, 4, 8, 16, 32, 64): - self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) - else: - self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) - for i in range(1, 100): - if i in (1, 3, 9, 27, 81): - self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) - else: - self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) - - def test_next_power_of_k(self): - f = mathutil.next_power_of_k - self.failUnlessEqual(f(0,2), 1) - self.failUnlessEqual(f(1,2), 1) - self.failUnlessEqual(f(2,2), 2) - self.failUnlessEqual(f(3,2), 4) - self.failUnlessEqual(f(4,2), 4) - for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) - for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) - for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) - for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) - for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) - - self.failUnlessEqual(f(0,3), 1) - self.failUnlessEqual(f(1,3), 1) - self.failUnlessEqual(f(2,3), 3) - self.failUnlessEqual(f(3,3), 3) - for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) - for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) - for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) - for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) - - def test_ave(self): - f = mathutil.ave - self.failUnlessEqual(f([1,2,3]), 2) - self.failUnlessEqual(f([0,0,0,4]), 1) - self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) - def test_round_sigfigs(self): f = mathutil.round_sigfigs self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002) @@ -297,65 +206,6 @@ class Statistics(unittest.TestCase): self.failUnlessEqual(f(plist, .5, 3), .02734375) -class Asserts(unittest.TestCase): - def should_assert(self, func, *args, **kwargs): - try: - func(*args, **kwargs) - except AssertionError as e: - return str(e) - except Exception as e: - self.fail("assert failed with non-AssertionError: %s" % e) - self.fail("assert was not caught") - - def should_not_assert(self, func, *args, **kwargs): - try: - func(*args, **kwargs) - except AssertionError as e: - self.fail("assertion fired when it should not have: %s" % e) - except Exception as e: - self.fail("assertion (which shouldn't have failed) failed with non-AssertionError: %s" % e) - return # we're happy - - - def test_assert(self): - f = assertutil._assert - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual(m, "'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("othermsg: 'message2' ", m) - - def test_precondition(self): - f = assertutil.precondition - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual("precondition: 'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("precondition: 'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("precondition: othermsg: 'message2' ", m) - - def test_postcondition(self): - f = assertutil.postcondition - self.should_assert(f) - self.should_assert(f, False) - self.should_not_assert(f, True) - - m = self.should_assert(f, False, "message") - self.failUnlessEqual("postcondition: 'message' ", m) - m = self.should_assert(f, False, "message1", othermsg=12) - self.failUnlessEqual("postcondition: 'message1' , othermsg: 12 ", m) - m = self.should_assert(f, False, othermsg="message2") - self.failUnlessEqual("postcondition: othermsg: 'message2' ", m) - class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0o777): fn = os.path.join(basedir, path) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index fd679e014..c8fe8fddd 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,7 +15,9 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.util.assertutil", "allmydata.util.humanreadable", + "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util._python3", ] diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index 5c48500c3..af8817702 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -1,57 +1,23 @@ """ Tests useful in assertion checking, prints out nicely formated messages too. + +Backwards compatibility layer, the versions in pyutil are better maintained and +have tests. + +Ported to Python 3. """ -from allmydata.util.humanreadable import hr +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals -def _assert(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=[] - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 - raise AssertionError("".join(msgbuf)) -def precondition(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=["precondition", ] - if ___args or ___kwargs: - msgbuf.append(": ") - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) +# The API importers expect: +from pyutil.assertutil import _assert, precondition, postcondition - raise AssertionError("".join(msgbuf)) - -def postcondition(___cond=False, *___args, **___kwargs): - if ___cond: - return True - msgbuf=["postcondition", ] - if ___args or ___kwargs: - msgbuf.append(": ") - if ___args: - msgbuf.append("%s %s" % tuple(map(hr, (___args[0], type(___args[0]),)))) - msgbuf.extend([", %s %s" % tuple(map(hr, (arg, type(arg),))) for arg in ___args[1:]]) - if ___kwargs: - msgbuf.append(", %s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - else: - if ___kwargs: - msgbuf.append("%s: %s %s" % ((___kwargs.items()[0][0],) + tuple(map(hr, (___kwargs.items()[0][1], type(___kwargs.items()[0][1]),))))) - msgbuf.extend([", %s: %s %s" % tuple(map(hr, (k, v, type(v),))) for k, v in ___kwargs.items()[1:]]) - - raise AssertionError("".join(msgbuf)) +__all__ = ["_assert", "precondition", "postcondition"] diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index bbdb40b08..be88f60b4 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -1,71 +1,28 @@ """ A few commonly needed functions. + +Backwards compatibility for direct imports. + +Ported to Python 3. """ -import math +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals -def div_ceil(n, d): - """ - The smallest integer k such that k*d >= n. - """ - return (n/d) + (n%d != 0) +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -def next_multiple(n, k): - """ - The smallest multiple of k which is >= n. - """ - return div_ceil(n, k) * k -def pad_size(n, k): - """ - The smallest number that has to be added to n so that n is a multiple of k. - """ - if n%k: - return k - n%k - else: - return 0 +# The API importers expect: +from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor -def is_power_of_k(n, k): - return k**int(math.log(n, k) + 0.5) == n - -def next_power_of_k(n, k): - if n == 0: - x = 0 - else: - x = int(math.log(n, k) + 0.5) - if k**x < n: - return k**(x+1) - else: - return k**x - -def ave(l): - return sum(l) / len(l) - -def log_ceil(n, b): - """ - The smallest integer k such that b^k >= n. - - log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. - the number of bits needed to store any of 128 possible values is 7. - """ - p = 1 - k = 0 - while p < n: - p *= b - k += 1 - return k - -def log_floor(n, b): - """ - The largest integer k such that b^k <= n. - """ - p = 1 - k = 0 - while p <= n: - p *= b - k += 1 - return k - 1 +# This function is not present in pyutil.mathutil: def round_sigfigs(f, n): fmt = "%." + str(n-1) + "e" return float(fmt % f) + +__all__ = ["div_ceil", "next_multiple", "pad_size", "is_power_of_k", "next_power_of_k", "ave", "log_ceil", "log_floor", "round_sigfigs"]