From d6f24af424ac5975d0d52a4a987cd555da61b53b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 15:10:43 -0400 Subject: [PATCH 01/40] Move span tests into their own module. --- src/allmydata/test/test_spans.py | 605 +++++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 595 ------------------------------ 2 files changed, 605 insertions(+), 595 deletions(-) create mode 100644 src/allmydata/test/test_spans.py diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py new file mode 100644 index 000000000..9901a24be --- /dev/null +++ b/src/allmydata/test/test_spans.py @@ -0,0 +1,605 @@ +""" +Tests for allmydata.util.spans. +""" + +from __future__ import print_function + +import binascii +import hashlib + +from twisted.trial import unittest + +from allmydata.util.spans import Spans, overlap, DataSpans + + +def sha256(data): + """ + :param bytes data: data to hash + + :returns: a hex-encoded SHA256 hash of the data + """ + return binascii.hexlify(hashlib.sha256(data).digest()) + + +class SimpleSpans(object): + # this is a simple+inefficient form of util.spans.Spans . We compare the + # behavior of this reference model against the real (efficient) form. + + def __init__(self, _span_or_start=None, length=None): + self._have = set() + if length is not None: + for i in range(_span_or_start, _span_or_start+length): + self._have.add(i) + elif _span_or_start: + for (start,length) in _span_or_start: + self.add(start, length) + + def add(self, start, length): + for i in range(start, start+length): + self._have.add(i) + return self + + def remove(self, start, length): + for i in range(start, start+length): + self._have.discard(i) + return self + + def each(self): + return sorted(self._have) + + def __iter__(self): + items = sorted(self._have) + prevstart = None + prevend = None + for i in items: + if prevstart is None: + prevstart = prevend = i + continue + if i == prevend+1: + prevend = i + continue + yield (prevstart, prevend-prevstart+1) + prevstart = prevend = i + if prevstart is not None: + yield (prevstart, prevend-prevstart+1) + + def __nonzero__(self): # this gets us bool() + return self.len() + + def len(self): + return len(self._have) + + def __add__(self, other): + s = self.__class__(self) + for (start, length) in other: + s.add(start, length) + return s + + def __sub__(self, other): + s = self.__class__(self) + for (start, length) in other: + s.remove(start, length) + return s + + def __iadd__(self, other): + for (start, length) in other: + self.add(start, length) + return self + + def __isub__(self, other): + for (start, length) in other: + self.remove(start, length) + return self + + def __and__(self, other): + s = self.__class__() + for i in other.each(): + if i in self._have: + s.add(i, 1) + return s + + def __contains__(self, start_and_length): + (start, length) = start_and_length + for i in range(start, start+length): + if i not in self._have: + return False + return True + +class ByteSpans(unittest.TestCase): + def test_basic(self): + s = Spans() + self.failUnlessEqual(list(s), []) + self.failIf(s) + self.failIf((0,1) in s) + self.failUnlessEqual(s.len(), 0) + + s1 = Spans(3, 4) # 3,4,5,6 + self._check1(s1) + + s1 = Spans(long(3), long(4)) # 3,4,5,6 + self._check1(s1) + + s2 = Spans(s1) + self._check1(s2) + + s2.add(10,2) # 10,11 + self._check1(s1) + self.failUnless((10,1) in s2) + self.failIf((10,1) in s1) + self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11]) + self.failUnlessEqual(s2.len(), 6) + + s2.add(15,2).add(20,2) + self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11,15,16,20,21]) + self.failUnlessEqual(s2.len(), 10) + + s2.remove(4,3).remove(15,1) + self.failUnlessEqual(list(s2.each()), [3,10,11,16,20,21]) + self.failUnlessEqual(s2.len(), 6) + + s1 = SimpleSpans(3, 4) # 3 4 5 6 + s2 = SimpleSpans(5, 4) # 5 6 7 8 + i = s1 & s2 + self.failUnlessEqual(list(i.each()), [5, 6]) + + def _check1(self, s): + self.failUnlessEqual(list(s), [(3,4)]) + self.failUnless(s) + self.failUnlessEqual(s.len(), 4) + self.failIf((0,1) in s) + self.failUnless((3,4) in s) + self.failUnless((3,1) in s) + self.failUnless((5,2) in s) + self.failUnless((6,1) in s) + self.failIf((6,2) in s) + self.failIf((7,1) in s) + self.failUnlessEqual(list(s.each()), [3,4,5,6]) + + def test_large(self): + s = Spans(4, 2**65) # don't do this with a SimpleSpans + self.failUnlessEqual(list(s), [(4, 2**65)]) + self.failUnless(s) + self.failUnlessEqual(s.len(), 2**65) + self.failIf((0,1) in s) + self.failUnless((4,2) in s) + self.failUnless((2**65,2) in s) + + def test_math(self): + s1 = Spans(0, 10) # 0,1,2,3,4,5,6,7,8,9 + s2 = Spans(5, 3) # 5,6,7 + s3 = Spans(8, 4) # 8,9,10,11 + + s = s1 - s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,8,9]) + s = s1 - s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7]) + s = s2 - s3 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s1 & s2 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s2 & s1 + self.failUnlessEqual(list(s.each()), [5,6,7]) + s = s1 & s3 + self.failUnlessEqual(list(s.each()), [8,9]) + s = s3 & s1 + self.failUnlessEqual(list(s.each()), [8,9]) + s = s2 & s3 + self.failUnlessEqual(list(s.each()), []) + s = s3 & s2 + self.failUnlessEqual(list(s.each()), []) + s = Spans() & s3 + self.failUnlessEqual(list(s.each()), []) + s = s3 & Spans() + self.failUnlessEqual(list(s.each()), []) + + s = s1 + s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9]) + s = s1 + s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9,10,11]) + s = s2 + s3 + self.failUnlessEqual(list(s.each()), [5,6,7,8,9,10,11]) + + s = Spans(s1) + s -= s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,8,9]) + s = Spans(s1) + s -= s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7]) + s = Spans(s2) + s -= s3 + self.failUnlessEqual(list(s.each()), [5,6,7]) + + s = Spans(s1) + s += s2 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9]) + s = Spans(s1) + s += s3 + self.failUnlessEqual(list(s.each()), [0,1,2,3,4,5,6,7,8,9,10,11]) + s = Spans(s2) + s += s3 + self.failUnlessEqual(list(s.each()), [5,6,7,8,9,10,11]) + + def test_random(self): + # attempt to increase coverage of corner cases by comparing behavior + # of a simple-but-slow model implementation against the + # complex-but-fast actual implementation, in a large number of random + # operations + S1 = SimpleSpans + S2 = Spans + s1 = S1(); s2 = S2() + seed = "" + def _create(subseed): + ns1 = S1(); ns2 = S2() + for i in range(10): + what = sha256(subseed+str(i)) + start = int(what[2:4], 16) + length = max(1,int(what[5:6], 16)) + ns1.add(start, length); ns2.add(start, length) + return ns1, ns2 + + #print + for i in range(1000): + what = sha256(seed+str(i)) + op = what[0] + subop = what[1] + start = int(what[2:4], 16) + length = max(1,int(what[5:6], 16)) + #print what + if op in "0": + if subop in "01234": + s1 = S1(); s2 = S2() + elif subop in "5678": + s1 = S1(start, length); s2 = S2(start, length) + else: + s1 = S1(s1); s2 = S2(s2) + #print "s2 = %s" % s2.dump() + elif op in "123": + #print "s2.add(%d,%d)" % (start, length) + s1.add(start, length); s2.add(start, length) + elif op in "456": + #print "s2.remove(%d,%d)" % (start, length) + s1.remove(start, length); s2.remove(start, length) + elif op in "78": + ns1, ns2 = _create(what[7:11]) + #print "s2 + %s" % ns2.dump() + s1 = s1 + ns1; s2 = s2 + ns2 + elif op in "9a": + ns1, ns2 = _create(what[7:11]) + #print "%s - %s" % (s2.dump(), ns2.dump()) + s1 = s1 - ns1; s2 = s2 - ns2 + elif op in "bc": + ns1, ns2 = _create(what[7:11]) + #print "s2 += %s" % ns2.dump() + s1 += ns1; s2 += ns2 + elif op in "de": + ns1, ns2 = _create(what[7:11]) + #print "%s -= %s" % (s2.dump(), ns2.dump()) + s1 -= ns1; s2 -= ns2 + else: + ns1, ns2 = _create(what[7:11]) + #print "%s &= %s" % (s2.dump(), ns2.dump()) + s1 = s1 & ns1; s2 = s2 & ns2 + #print "s2 now %s" % s2.dump() + self.failUnlessEqual(list(s1.each()), list(s2.each())) + self.failUnlessEqual(s1.len(), s2.len()) + self.failUnlessEqual(bool(s1), bool(s2)) + self.failUnlessEqual(list(s1), list(s2)) + for j in range(10): + what = sha256(what[12:14]+str(j)) + start = int(what[2:4], 16) + length = max(1, int(what[5:6], 16)) + span = (start, length) + self.failUnlessEqual(bool(span in s1), bool(span in s2)) + + + # s() + # s(start,length) + # s(s0) + # s.add(start,length) : returns s + # s.remove(start,length) + # s.each() -> list of byte offsets, mostly for testing + # list(s) -> list of (start,length) tuples, one per span + # (start,length) in s -> True if (start..start+length-1) are all members + # NOT equivalent to x in list(s) + # s.len() -> number of bytes, for testing, bool(), and accounting/limiting + # bool(s) (__nonzeron__) + # s = s1+s2, s1-s2, +=s1, -=s1 + + def test_overlap(self): + for a in range(20): + for b in range(10): + for c in range(20): + for d in range(10): + self._test_overlap(a,b,c,d) + + def _test_overlap(self, a, b, c, d): + s1 = set(range(a,a+b)) + s2 = set(range(c,c+d)) + #print "---" + #self._show_overlap(s1, "1") + #self._show_overlap(s2, "2") + o = overlap(a,b,c,d) + expected = s1.intersection(s2) + if not expected: + self.failUnlessEqual(o, None) + else: + start,length = o + so = set(range(start,start+length)) + #self._show(so, "o") + self.failUnlessEqual(so, expected) + + def _show_overlap(self, s, c): + import sys + out = sys.stdout + if s: + for i in range(max(s)): + if i in s: + out.write(c) + else: + out.write(" ") + out.write("\n") + +def extend(s, start, length, fill): + if len(s) >= start+length: + return s + assert len(fill) == 1 + return s + fill*(start+length-len(s)) + +def replace(s, start, data): + assert len(s) >= start+len(data) + return s[:start] + data + s[start+len(data):] + +class SimpleDataSpans(object): + def __init__(self, other=None): + self.missing = "" # "1" where missing, "0" where found + self.data = "" + if other: + for (start, data) in other.get_chunks(): + self.add(start, data) + + def __nonzero__(self): # this gets us bool() + return self.len() + def len(self): + return len(self.missing.replace("1", "")) + def _dump(self): + return [i for (i,c) in enumerate(self.missing) if c == "0"] + def _have(self, start, length): + m = self.missing[start:start+length] + if not m or len(m) list of byte offsets, mostly for testing - # list(s) -> list of (start,length) tuples, one per span - # (start,length) in s -> True if (start..start+length-1) are all members - # NOT equivalent to x in list(s) - # s.len() -> number of bytes, for testing, bool(), and accounting/limiting - # bool(s) (__nonzeron__) - # s = s1+s2, s1-s2, +=s1, -=s1 - - def test_overlap(self): - for a in range(20): - for b in range(10): - for c in range(20): - for d in range(10): - self._test_overlap(a,b,c,d) - - def _test_overlap(self, a, b, c, d): - s1 = set(range(a,a+b)) - s2 = set(range(c,c+d)) - #print "---" - #self._show_overlap(s1, "1") - #self._show_overlap(s2, "2") - o = overlap(a,b,c,d) - expected = s1.intersection(s2) - if not expected: - self.failUnlessEqual(o, None) - else: - start,length = o - so = set(range(start,start+length)) - #self._show(so, "o") - self.failUnlessEqual(so, expected) - - def _show_overlap(self, s, c): - import sys - out = sys.stdout - if s: - for i in range(max(s)): - if i in s: - out.write(c) - else: - out.write(" ") - out.write("\n") - -def extend(s, start, length, fill): - if len(s) >= start+length: - return s - assert len(fill) == 1 - return s + fill*(start+length-len(s)) - -def replace(s, start, data): - assert len(s) >= start+len(data) - return s[:start] + data + s[start+len(data):] - -class SimpleDataSpans(object): - def __init__(self, other=None): - self.missing = "" # "1" where missing, "0" where found - self.data = "" - if other: - for (start, data) in other.get_chunks(): - self.add(start, data) - - def __nonzero__(self): # this gets us bool() - return self.len() - def len(self): - return len(self.missing.replace("1", "")) - def _dump(self): - return [i for (i,c) in enumerate(self.missing) if c == "0"] - def _have(self, start, length): - m = self.missing[start:start+length] - if not m or len(m) Date: Tue, 21 Jul 2020 15:14:52 -0400 Subject: [PATCH 02/40] Manual porting to Python 3. --- src/allmydata/test/test_spans.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 9901a24be..1766bf353 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -231,7 +231,7 @@ class ByteSpans(unittest.TestCase): def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = sha256(subseed+str(i)) + what = sha256(subseed+bytes(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, length); ns2.add(start, length) @@ -239,7 +239,7 @@ class ByteSpans(unittest.TestCase): #print for i in range(1000): - what = sha256(seed+str(i)) + what = sha256(seed+bytes(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -285,7 +285,7 @@ class ByteSpans(unittest.TestCase): self.failUnlessEqual(bool(s1), bool(s2)) self.failUnlessEqual(list(s1), list(s2)) for j in range(10): - what = sha256(what[12:14]+str(j)) + what = sha256(what[12:14]+bytes(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) span = (start, length) @@ -549,19 +549,19 @@ class StringSpans(unittest.TestCase): S1 = SimpleDataSpans S2 = DataSpans s1 = S1(); s2 = S2() - seed = "" + seed = b"" def _randstr(length, seed): created = 0 pieces = [] while created < length: - piece = sha256(seed + str(created)) + piece = sha256(seed + bytes(created)) pieces.append(piece) created += len(piece) return "".join(pieces)[:length] def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): - what = sha256(subseed+str(i)) + what = sha256(subseed+bytes(i)) start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) ns1.add(start, _randstr(length, what[7:9])); @@ -570,7 +570,7 @@ class StringSpans(unittest.TestCase): #print for i in range(1000): - what = sha256(seed+str(i)) + what = sha256(seed+bytes(i)) op = what[0] subop = what[1] start = int(what[2:4], 16) @@ -598,7 +598,7 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(s1.len(), s2.len()) self.failUnlessEqual(list(s1._dump()), list(s2._dump())) for j in range(100): - what = sha256(what[12:14]+str(j)) + what = sha256(what[12:14]+bytes(j)) start = int(what[2:4], 16) length = max(1, int(what[5:6], 16)) d1 = s1.get(start, length); d2 = s2.get(start, length) From e9eb93468f03586b33ccb831807e71c6742212b8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jul 2020 15:25:23 -0400 Subject: [PATCH 03/40] Finish port to Python 3. --- src/allmydata/test/test_spans.py | 114 +++++++++++++++++-------------- src/allmydata/util/_python3.py | 2 + src/allmydata/util/spans.py | 13 +++- 3 files changed, 76 insertions(+), 53 deletions(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 1766bf353..1814e3382 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -3,6 +3,15 @@ Tests for allmydata.util.spans. """ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from past.builtins import long import binascii import hashlib @@ -63,8 +72,8 @@ class SimpleSpans(object): if prevstart is not None: yield (prevstart, prevend-prevstart+1) - def __nonzero__(self): # this gets us bool() - return self.len() + def __bool__(self): # this gets us bool() + return bool(self.len()) def len(self): return len(self._have) @@ -227,7 +236,7 @@ class ByteSpans(unittest.TestCase): S1 = SimpleSpans S2 = Spans s1 = S1(); s2 = S2() - seed = "" + seed = b"" def _create(subseed): ns1 = S1(); ns2 = S2() for i in range(10): @@ -240,38 +249,38 @@ class ByteSpans(unittest.TestCase): #print for i in range(1000): what = sha256(seed+bytes(i)) - op = what[0] - subop = what[1] + op = what[0:1] + subop = what[1:2] start = int(what[2:4], 16) length = max(1,int(what[5:6], 16)) #print what - if op in "0": - if subop in "01234": + if op in b"0": + if subop in b"01234": s1 = S1(); s2 = S2() - elif subop in "5678": + elif subop in b"5678": s1 = S1(start, length); s2 = S2(start, length) else: s1 = S1(s1); s2 = S2(s2) #print "s2 = %s" % s2.dump() - elif op in "123": + elif op in b"123": #print "s2.add(%d,%d)" % (start, length) s1.add(start, length); s2.add(start, length) - elif op in "456": + elif op in b"456": #print "s2.remove(%d,%d)" % (start, length) s1.remove(start, length); s2.remove(start, length) - elif op in "78": + elif op in b"78": ns1, ns2 = _create(what[7:11]) #print "s2 + %s" % ns2.dump() s1 = s1 + ns1; s2 = s2 + ns2 - elif op in "9a": + elif op in b"9a": ns1, ns2 = _create(what[7:11]) #print "%s - %s" % (s2.dump(), ns2.dump()) s1 = s1 - ns1; s2 = s2 - ns2 - elif op in "bc": + elif op in b"bc": ns1, ns2 = _create(what[7:11]) #print "s2 += %s" % ns2.dump() s1 += ns1; s2 += ns2 - elif op in "de": + elif op in b"de": ns1, ns2 = _create(what[7:11]) #print "%s -= %s" % (s2.dump(), ns2.dump()) s1 -= ns1; s2 -= ns2 @@ -352,17 +361,20 @@ def replace(s, start, data): class SimpleDataSpans(object): def __init__(self, other=None): self.missing = "" # "1" where missing, "0" where found - self.data = "" + self.data = b"" if other: for (start, data) in other.get_chunks(): self.add(start, data) - def __nonzero__(self): # this gets us bool() - return self.len() + def __bool__(self): # this gets us bool() + return bool(self.len()) + def len(self): return len(self.missing.replace("1", "")) + def _dump(self): return [i for (i,c) in enumerate(self.missing) if c == "0"] + def _have(self, start, length): m = self.missing[start:start+length] if not m or len(m) Date: Wed, 22 Jul 2020 09:59:26 -0400 Subject: [PATCH 04/40] Increase the Python 3 ratchet. --- misc/python3/ratchet-passing | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index a6e1de68b..f678ec619 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -26,3 +26,11 @@ allmydata.test.test_observer.Observer.test_oneshot_fireagain allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist +allmydata.test.test_spans.ByteSpans.test_basic +allmydata.test.test_spans.ByteSpans.test_large +allmydata.test.test_spans.ByteSpans.test_math +allmydata.test.test_spans.ByteSpans.test_overlap +allmydata.test.test_spans.ByteSpans.test_random +allmydata.test.test_spans.StringSpans.test_basic +allmydata.test.test_spans.StringSpans.test_random +allmydata.test.test_spans.StringSpans.test_test From 13306f70e74ff89382e39ebe87ab6784315a4f79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jul 2020 10:01:26 -0400 Subject: [PATCH 05/40] News file. --- newsfragments/3351.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3351.minor diff --git a/newsfragments/3351.minor b/newsfragments/3351.minor new file mode 100644 index 000000000..e69de29bb From 135ade02b159a8bde5c9083797d10720baf5c2f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 13:47:56 -0400 Subject: [PATCH 06/40] Start porting. --- src/allmydata/test/test_iputil.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index ebb77c3c8..7f26a53fb 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -10,7 +10,9 @@ from allmydata.util.namespace import Namespace DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") -MOCK_IPADDR_OUTPUT = """\ +# Mock output from subprocesses should be bytes, that's what happens on both +# Python 2 and Python 3: +MOCK_IPADDR_OUTPUT = b"""\ 1: lo: mtu 16436 qdisc noqueue state UNKNOWN \n\ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo @@ -28,7 +30,7 @@ MOCK_IPADDR_OUTPUT = """\ valid_lft forever preferred_lft forever """ -MOCK_IFCONFIG_OUTPUT = """\ +MOCK_IFCONFIG_OUTPUT = b"""\ eth1 Link encap:Ethernet HWaddr d4:3d:7e:01:b4:3e \n\ inet addr:192.168.0.6 Bcast:192.168.0.255 Mask:255.255.255.0 inet6 addr: fe80::d63d:7eff:fe01:b43e/64 Scope:Link @@ -59,7 +61,7 @@ wlan0 Link encap:Ethernet HWaddr 90:f6:52:27:15:0a \n\ """ # This is actually from a VirtualBox VM running XP. -MOCK_ROUTE_OUTPUT = """\ +MOCK_ROUTE_OUTPUT = b"""\ =========================================================================== Interface List 0x1 ........................... MS TCP Loopback interface @@ -98,6 +100,11 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): def test_get_local_ip_for(self): addr = iputil.get_local_ip_for('127.0.0.1') self.failUnless(DOTTED_QUAD_RE.match(addr)) + # Bytes can be taken as input: + bytes_addr = iputil.get_local_ip_for(b'127.0.0.1') + self.assertEqual(addr, bytes_addr) + # The output is a native string: + self.assertIsInstance(addr, str) def test_list_async(self): d = iputil.get_local_addresses_async() From 45a891114b033306dcbb66b3310d5d601970f628 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 13:48:11 -0400 Subject: [PATCH 07/40] Fix some things that prevent Python 3 imports. --- src/allmydata/util/encodingutil.py | 3 ++- src/allmydata/util/fileutil.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 568267b35..b31fd1205 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -4,7 +4,6 @@ unicode and back. """ import sys, os, re, locale -from types import NoneType from allmydata.util.assertutil import precondition, _assert from twisted.python import usage @@ -12,6 +11,8 @@ from twisted.python.filepath import FilePath from allmydata.util import log from allmydata.util.fileutil import abspath_expanduser_unicode +NoneType = type(None) + def canonical_encoding(encoding): if encoding is None: diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 269a8a356..36946ff02 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -4,7 +4,7 @@ from __future__ import print_function Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time, binascii +import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple from errno import ENOENT @@ -190,7 +190,7 @@ def make_dirs(dirname, mode=0o777): if not os.path.isdir(dirname): if tx: raise tx - raise exceptions.IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... + raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rm_dir(dirname): """ From 04db52b9e4a1881d03c946e40485c92464147273 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:01:14 -0400 Subject: [PATCH 08/40] Some manual tweaks to be more likely to work with Python 3. The Linux distribution porting code was obsolete (using Python APIs not present in Python 3, /etc/lsb-release isn't a thing on Fedora 31, for example), so replaced it with maintained third-party library. --- nix/tahoe-lafs.nix | 4 +- setup.py | 3 ++ src/allmydata/version_checks.py | 89 +++------------------------------ 3 files changed, 12 insertions(+), 84 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 097a463d1..49bf3075a 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, pyutil +, html5lib, pyutil, nix }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future pyutil + future pyutil nix ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index f22ca0f4c..9bb385cb4 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,9 @@ install_requires = [ # Utility code: "pyutil >= 3.3.0", + + # Linux distribution detection: + "distro >= 1.4.0", ] setup_requires = [ diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 7092a422b..c7f4d6c8a 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -10,10 +10,12 @@ __all__ = [ "normalized_version", ] -import os, platform, re, subprocess, sys, traceback, pkg_resources +import os, platform, re, sys, traceback, pkg_resources import six +import distro + from . import ( __appname__, full_version, @@ -80,7 +82,7 @@ def normalized_version(verstr, what=None): return verlib.NormalizedVersion(suggested) except verlib.IrrationalVersionError: raise - except StandardError: + except Exception: cls, value, trace = sys.exc_info() new_exc = PackagingError("could not parse %s due to %s: %s" % (what or repr(verstr), cls.__name__, value)) @@ -201,83 +203,6 @@ def _extract_openssl_version(ssl_module): return (version, None, comment if comment else None) -def _get_linux_distro(): - """ Tries to determine the name of the Linux OS distribution name. - - First, try to parse a file named "/etc/lsb-release". If it exists, and - contains the "DISTRIB_ID=" line and the "DISTRIB_RELEASE=" line, then return - the strings parsed from that file. - - If that doesn't work, then invoke platform.dist(). - - If that doesn't work, then try to execute "lsb_release", as standardized in - 2001: - - http://refspecs.freestandards.org/LSB_1.0.0/gLSB/lsbrelease.html - - The current version of the standard is here: - - http://refspecs.freestandards.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/lsbrelease.html - - that lsb_release emitted, as strings. - - Returns a tuple (distname,version). Distname is what LSB calls a - "distributor id", e.g. "Ubuntu". Version is what LSB calls a "release", - e.g. "8.04". - - A version of this has been submitted to python as a patch for the standard - library module "platform": - - http://bugs.python.org/issue3937 - """ - global _distname,_version - if _distname and _version: - return (_distname, _version) - - try: - with open("/etc/lsb-release", "rU") as etclsbrel: - for line in etclsbrel: - m = _distributor_id_file_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - m = _release_file_re.search(line) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - (_distname, _version) = platform.dist()[:2] - if _distname and _version: - return (_distname, _version) - - if os.path.isfile("/usr/bin/lsb_release") or os.path.isfile("/bin/lsb_release"): - try: - p = subprocess.Popen(["lsb_release", "--all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - rc = p.wait() - if rc == 0: - for line in p.stdout.readlines(): - m = _distributor_id_cmdline_re.search(line) - if m: - _distname = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - - m = _release_cmdline_re.search(p.stdout.read()) - if m: - _version = m.group(1).strip() - if _distname and _version: - return (_distname, _version) - except EnvironmentError: - pass - - if os.path.exists("/etc/arch-release"): - return ("Arch_Linux", "") - - return (_distname,_version) def _get_platform(): # Our version of platform.platform(), telling us both less and more than the @@ -288,7 +213,7 @@ def _get_platform(): if "linux" in platform.system().lower(): return ( platform.system() + "-" + - "_".join(_get_linux_distro()) + "-" + + "_".join(distro.linux_distribution()[:2]) + "-" + platform.machine() + "-" + "_".join([x for x in platform.architecture() if x]) ) @@ -321,7 +246,7 @@ def _get_package_versions_and_locations(): for modulename in warning_imports: try: __import__(modulename) - except ImportError: + except (ImportError, SyntaxError): pass finally: # Leave suppressions for UserWarnings and global_deprecation_messages active. @@ -355,7 +280,7 @@ def _get_package_versions_and_locations(): try: __import__(modulename) module = sys.modules[modulename] - except ImportError: + except (ImportError, SyntaxError): etype, emsg, etrace = sys.exc_info() trace_info = (etype, str(emsg), ([None] + traceback.extract_tb(etrace))[-1]) packages.append( (pkgname, (None, None, trace_info)) ) From f84d51d79528c87c118e254ee48d9e34b5c19aad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:12:10 -0400 Subject: [PATCH 09/40] Port to Python 3. --- src/allmydata/test/test_version.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index fd0cb0e20..5f397d9f9 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -1,3 +1,12 @@ +""" +Tests for allmydata.util.verlib and allmydata.version_checks. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import sys import pkg_resources diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a79f20cd0..4bb639996 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -38,6 +38,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", "allmydata.test.test_netstring", "allmydata.test.test_python3", + "allmydata.test.test_version", ] From 337a4381bba90f38f4630eeabf7eaa4a34d90764 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:31:36 -0400 Subject: [PATCH 10/40] Finish port to Python 3. --- misc/python3/ratchet-passing | 10 ++++++++++ newsfragments/3357.minor | 1 + src/allmydata/test/test_version.py | 8 ++++++-- src/allmydata/util/verlib.py | 12 +++++++++++- src/allmydata/version_checks.py | 14 ++++++++++++-- 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3357.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 293733b2b..522a9fc6d 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -45,3 +45,13 @@ allmydata.test.test_observer.Observer.test_oneshot_fireagain allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist +allmydata.test.test_version.CheckRequirement.test_cross_check +allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions +allmydata.test.test_version.CheckRequirement.test_extract_openssl_version +allmydata.test.test_version.CheckRequirement.test_packages_from_pkg_resources +allmydata.test.test_version.T.test_report_import_error +allmydata.test.test_version.VersionTestCase.test_basic_versions +allmydata.test.test_version.VersionTestCase.test_comparison +allmydata.test.test_version.VersionTestCase.test_from_parts +allmydata.test.test_version.VersionTestCase.test_irrational_versions +allmydata.test.test_version.VersionTestCase.test_suggest_normalized_version diff --git a/newsfragments/3357.minor b/newsfragments/3357.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3357.minor @@ -0,0 +1 @@ + diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 5f397d9f9..1b447e346 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -8,6 +8,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import sys import pkg_resources from operator import ( @@ -88,7 +92,7 @@ class CheckRequirement(unittest.TestCase): res = cross_check({}, [("foo", ("unparseable", "", None))]) self.failUnlessEqual(len(res), 1) - self.failUnlessIn("version 'unparseable'", res[0]) + self.assertTrue(("version 'unparseable'" in res[0]) or ("version u'unparseable'" in res[0])) self.failUnlessIn("was not found by pkg_resources", res[0]) res = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere", "distribute"))]) @@ -129,7 +133,7 @@ class CheckRequirement(unittest.TestCase): res = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("2.0", "/somewhere_different", None))]) self.failUnlessEqual(len(res), 1) - self.failUnlessIn("but version '2.0'", res[0]) + self.assertTrue(("but version '2.0'" in res[0]) or ("but version u'2.0'" in res[0])) def test_extract_openssl_version(self): self.failUnlessEqual(extract_openssl_version(MockSSL("")), diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py index 619f1a845..f69e34e3d 100644 --- a/src/allmydata/util/verlib.py +++ b/src/allmydata/util/verlib.py @@ -1,11 +1,21 @@ """ "Rational" version definition and parsing for DistutilsVersionFight discussion at PyCon 2009. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re + class IrrationalVersionError(Exception): """This is an irrational version.""" pass diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index c7f4d6c8a..51a49d78a 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -1,7 +1,17 @@ """ Produce reports about the versions of Python software in use by Tahoe-LAFS for debugging and auditing purposes. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 __all__ = [ "PackagingError", @@ -111,7 +121,7 @@ def _get_error_string(errors, debug=False): def _cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): """This function returns a list of errors due to any failed cross-checks.""" - from _auto_deps import not_import_versionable + from ._auto_deps import not_import_versionable errors = [] not_pkg_resourceable = ['python', 'platform', __appname__.lower(), 'openssl'] @@ -311,7 +321,7 @@ def _get_package_versions_and_locations(): imported_packages = set([p.lower() for (p, _) in packages]) extra_packages = [] - for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): + for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.items(): if pr_name not in imported_packages and pr_name not in ignorable: extra_packages.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) From 4a52061377a6e85f1aab1597ccd35539a23cdbe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:34:20 -0400 Subject: [PATCH 11/40] Fixing braino. --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 49bf3075a..0e941bce1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, pyutil, nix +, html5lib, pyutil, distro }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; @@ -50,7 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools - future pyutil nix + future pyutil distro ]; checkInputs = with python.pkgs; [ From e80f1388683505531338c3a56df506c769f5c56d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:48:56 -0400 Subject: [PATCH 12/40] Fix indent. --- src/allmydata/test/test_spans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 1814e3382..f62d6e684 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -530,7 +530,7 @@ class StringSpans(unittest.TestCase): d = b.get(t_start, t_len) if d is not None: which2 = "%s+(%d-%d)" % (which, t_start, - t_start+t_len-1) + t_start+t_len-1) self.failUnlessEqual(d, S[t_start:t_start+t_len], which2) # check that removing a subspan gives the right value From bf134019797b2fadc8f92405720b1956e4b2fbc5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Jul 2020 14:58:08 -0400 Subject: [PATCH 13/40] Move SignalMixin to common_py3.py. --- src/allmydata/test/common_py3.py | 25 +++++++++++++++++++++++++ src/allmydata/test/common_util.py | 23 ++--------------------- src/allmydata/test/test_iputil.py | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index e6303d2f2..97745e293 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -15,6 +15,9 @@ if PY2: import os import time +import signal + +from twisted.internet import reactor class TimezoneMixin(object): @@ -40,3 +43,25 @@ class TimezoneMixin(object): def have_working_tzset(self): return hasattr(time, 'tzset') + + +class SignalMixin(object): + # This class is necessary for any code which wants to use Processes + # outside the usual reactor.run() environment. It is copied from + # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses + # something rather different. + sigchldHandler = None + + def setUp(self): + # make sure SIGCHLD handler is installed, as it should be on + # reactor.run(). problem is reactor may not have been run when this + # test runs. + if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): + self.sigchldHandler = signal.signal(signal.SIGCHLD, + reactor._handleSigchld) + return super(SignalMixin, self).setUp() + + def tearDown(self): + if self.sigchldHandler: + signal.signal(signal.SIGCHLD, self.sigchldHandler) + return super(SignalMixin, self).tearDown() diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 996052692..83f37a11d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -12,6 +12,8 @@ from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) from ..scripts import runner +from .common_py3 import SignalMixin + def skip_if_cannot_represent_filename(u): precondition(isinstance(u, unicode)) @@ -88,27 +90,6 @@ class ReallyEqualMixin(object): self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg)) -class SignalMixin(object): - # This class is necessary for any code which wants to use Processes - # outside the usual reactor.run() environment. It is copied from - # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses - # something rather different. - sigchldHandler = None - - def setUp(self): - # make sure SIGCHLD handler is installed, as it should be on - # reactor.run(). problem is reactor may not have been run when this - # test runs. - if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): - self.sigchldHandler = signal.signal(signal.SIGCHLD, - reactor._handleSigchld) - return super(SignalMixin, self).setUp() - - def tearDown(self): - if self.sigchldHandler: - signal.signal(signal.SIGCHLD, self.sigchldHandler) - return super(SignalMixin, self).tearDown() - class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 7f26a53fb..0c0992430 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -4,7 +4,7 @@ import re, errno, subprocess, os from twisted.trial import unittest from allmydata.util import iputil -import allmydata.test.common_util as testutil +import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace From 1f34e6298790c6ff9db53743a3a0b441ffaa1313 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:14:01 -0400 Subject: [PATCH 14/40] Port test module to Python 3. --- src/allmydata/test/test_iputil.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 0c0992430..1113e3540 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -1,3 +1,11 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os @@ -8,7 +16,7 @@ import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace -DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") +DOTTED_QUAD_RE=re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") # Mock output from subprocesses should be bytes, that's what happens on both # Python 2 and Python 3: @@ -104,7 +112,7 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): bytes_addr = iputil.get_local_ip_for(b'127.0.0.1') self.assertEqual(addr, bytes_addr) # The output is a native string: - self.assertIsInstance(addr, str) + self.assertIsInstance(addr, native_str) def test_list_async(self): d = iputil.get_local_addresses_async() From fce7221481fd582a904bdb16ea3b81b57dfac970 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:42:20 -0400 Subject: [PATCH 15/40] Some steps to manually port to Python 3. --- src/allmydata/util/iputil.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 57a7e790a..71dded28f 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -171,11 +171,11 @@ def get_local_ip_for(target): # ... thus wrote Greg Smith in time immemorial... # Also, the Win32 APIs for this are really klunky and error-prone. --Daira -_win32_re = re.compile(r'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P
\d+\.\d+\.\d+\.\d+)\s+(?P\d+)\s*$', flags=re.M|re.I|re.S) +_win32_re = re.compile(br'^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P
\d+\.\d+\.\d+\.\d+)\s+(?P\d+)\s*$', flags=re.M|re.I|re.S) _win32_commands = (('route.exe', ('print',), _win32_re),) # These work in most Unices. -_addr_re = re.compile(r'^\s*inet [a-zA-Z]*:?(?P
\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S) +_addr_re = re.compile(br'^\s*inet [a-zA-Z]*:?(?P
\d+\.\d+\.\d+\.\d+)[\s/].+$', flags=re.M|re.I|re.S) _unix_commands = (('/bin/ip', ('addr',), _addr_re), ('/sbin/ip', ('addr',), _addr_re), ('/sbin/ifconfig', ('-a',), _addr_re), @@ -209,10 +209,13 @@ def _synchronously_find_addresses_via_config(): else: exes_to_try = which(pathtotool) + subprocess_error = getattr( + subprocess, "SubprocessError", subprocess.CalledProcessError + ) for exe in exes_to_try: try: addresses = _query(exe, args, regex) - except Exception: + except (IOError, OSError, ValueError, subprocess_error): addresses = [] if addresses: return addresses @@ -224,7 +227,7 @@ def _query(path, args, regex): return [] env = {'LANG': 'en_US.UTF-8'} TRIES = 5 - for trial in xrange(TRIES): + for trial in range(TRIES): try: p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) (output, err) = p.communicate() @@ -235,13 +238,13 @@ def _query(path, args, regex): raise addresses = [] - outputsplit = output.split('\n') + outputsplit = output.split(b'\n') for outline in outputsplit: m = regex.match(outline) if m: addr = m.group('address') if addr not in addresses: - addresses.append(addr) + addresses.append(addr.decode("utf-8")) return addresses From 436c2e77e422370c81ccdb650816ca8cd48ecca0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:44:58 -0400 Subject: [PATCH 16/40] Automated port to Python 3. --- src/allmydata/util/iputil.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 71dded28f..4623dbba6 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -1,4 +1,12 @@ -# from the Python Standard Library +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import os, re, socket, subprocess, errno from sys import platform From 9919d2c9a7fcb2887541d887d94daf2db8b08275 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:46:03 -0400 Subject: [PATCH 17/40] Record port status. --- src/allmydata/test/test_iputil.py | 6 ++++++ src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/iputil.py | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 1113e3540..3fac776c2 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -1,3 +1,9 @@ +""" +Tests for allmydata.util.iputil. + +Ported to Python 3. +""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ff6916a72..d939d08b3 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -23,6 +23,7 @@ PORTED_MODULES = [ "allmydata.util.deferredutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", + "allmydata.util.iputil", "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.netstring", @@ -42,6 +43,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", + "allmydata.test.test_iputil", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 4623dbba6..947430e35 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -1,3 +1,9 @@ +""" +Utilities for getting IP addresses. + +Ported to Python 3. +""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function From f9bda5bbd3975746dda392b636b80eb372d56370 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 11:46:22 -0400 Subject: [PATCH 18/40] News file. --- newsfragments/3356.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3356.minor diff --git a/newsfragments/3356.minor b/newsfragments/3356.minor new file mode 100644 index 000000000..e69de29bb From 817355d17e98fffe6bb5ee9a6e8cc03e6081de93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 13:06:41 -0400 Subject: [PATCH 19/40] Minimal testing for listenOnUnused. --- src/allmydata/test/test_iputil.py | 33 ++++++++++++++++++++++++++++++- src/allmydata/util/iputil.py | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 3fac776c2..a058da299 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -13,10 +13,12 @@ from future.utils import PY2, native_str if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -import re, errno, subprocess, os +import re, errno, subprocess, os, socket from twisted.trial import unittest +from foolscap.api import Tub + from allmydata.util import iputil import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace @@ -183,3 +185,32 @@ class ListAddresses(testutil.SignalMixin, unittest.TestCase): def test_list_async_mock_cygwin(self): self.patch(iputil, 'platform', "cygwin") return self._test_list_async_mock(None, None, CYGWIN_TEST_ADDRESSES) + + +class ListenOnUsed(unittest.TestCase): + """Tests for listenOnUnused.""" + + def create_tub(self, basedir): + os.makedirs(basedir) + tubfile = os.path.join(basedir, "tub.pem") + tub = Tub(certFile=tubfile) + tub.setOption("expose-remote-exception-types", False) + tub.startService() + self.addCleanup(tub.stopService) + return tub + + def test_random_port(self): + """A random port is selected if none is given.""" + tub = self.create_tub("utils/ListenOnUsed/test_randomport") + self.assertEqual(len(tub.getListeners()), 0) + portnum = iputil.listenOnUnused(tub) + # We can connect to this port: + s = socket.socket() + s.connect(("127.0.0.1", portnum)) + s.close() + self.assertEqual(len(tub.getListeners()), 1) + + # Listen on another port: + tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2") + portnum2 = iputil.listenOnUnused(tub2) + self.assertNotEqual(portnum, portnum2) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 947430e35..6a9639280 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -379,4 +379,5 @@ __all__ = ["allocate_tcp_port", "get_local_addresses_sync", "get_local_addresses_async", "get_local_ip_for", + "listenOnUnused", ] From 0071c1a48d0fa7ade416216456ff9f1c5446ef14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 13:08:25 -0400 Subject: [PATCH 20/40] More passing tests. --- misc/python3/ratchet-passing | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 6832b7fa7..bbe594998 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -44,6 +44,13 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for +allmydata.test.test_iputil.ListAddresses.test_list_async +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr +allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route +allmydata.test.test_iputil.ListenOnUsed.test_random_port allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested From d91b4f0e40ef6359213c73927e0102b8ce338f53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:12:09 -0400 Subject: [PATCH 21/40] Fix some tests and flake issues. --- src/allmydata/test/common_util.py | 2 +- src/allmydata/util/iputil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 83f37a11d..261d39af3 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -import os, signal +import os from random import randrange from six.moves import StringIO diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 6a9639280..940f215db 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -370,7 +370,7 @@ def listenOnUnused(tub, portnum=None): """ portnum, endpoint = _foolscapEndpointForPortNumber(portnum) tub.listenOn(endpoint) - tub.setLocation("localhost:%d" % (portnum,)) + tub.setLocation(native_str("localhost:%d" % (portnum,))) return portnum From ce8a57580846d659eb680c1f9efa429cabe3402d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:16:25 -0400 Subject: [PATCH 22/40] Test for specific ports. --- src/allmydata/test/test_iputil.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index a058da299..42677af2f 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -214,3 +214,13 @@ class ListenOnUsed(unittest.TestCase): tub2 = self.create_tub("utils/ListenOnUsed/test_randomport_2") portnum2 = iputil.listenOnUnused(tub2) self.assertNotEqual(portnum, portnum2) + + def test_specific_port(self): + """The given port is used.""" + tub = self.create_tub("utils/ListenOnUsed/test_givenport") + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + port2 = iputil.listenOnUnused(tub, port) + self.assertEqual(port, port2) From 74fe9ccf56a36902ff884bc298b4a85d4d4fc545 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:16:34 -0400 Subject: [PATCH 23/40] Foolscap expects a native string. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 940f215db..af982e908 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -321,7 +321,7 @@ def _foolscapEndpointForPortNumber(portnum): # approach is error prone for the reasons described on # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787 portnum = allocate_tcp_port() - return (portnum, "tcp:%d" % (portnum,)) + return (portnum, native_str("tcp:%d" % (portnum,))) @implementer(IStreamServerEndpoint) From d8bf811b92acaef6510e3e7503fb8424c2f57d5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:20:35 -0400 Subject: [PATCH 24/40] Native string all the things. --- src/allmydata/util/iputil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index af982e908..921c481ff 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -102,7 +102,7 @@ except ImportError: increase_rlimits = _increase_rlimits def get_local_addresses_sync(): - return _synchronously_find_addresses_via_config() + return [native_str(a) for a in _synchronously_find_addresses_via_config()] def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ @@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET addresses.append(addr) return addresses d.addCallback(_collect) - + d.addCallback(lambda addresses: [native_str(s) for s in addresses]) return d def get_local_ip_for(target): @@ -176,7 +176,7 @@ def get_local_ip_for(target): except (socket.error, CannotListenError): # no route to that host localip = None - return localip + return native_str(localip) # Wow, I'm really amazed at home much mileage we've gotten out of calling From b140d1f1afa837ff4c8cd064faf0176b2c4893c6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:29:01 -0400 Subject: [PATCH 25/40] Move statistics tests out. --- src/allmydata/test/test_statistics.py | 147 ++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 140 +----------------------- 2 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 src/allmydata/test/test_statistics.py diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py new file mode 100644 index 000000000..7fe23f88f --- /dev/null +++ b/src/allmydata/test/test_statistics.py @@ -0,0 +1,147 @@ +""" +Tests for allmydata.util.statistics. +""" + +from io import StringIO + +from twisted.trial import unittest + +from allmydata.util import statistics + + +class Statistics(unittest.TestCase): + def should_assert(self, msg, func, *args, **kwargs): + try: + func(*args, **kwargs) + self.fail(msg) + except AssertionError: + pass + + def failUnlessListEqual(self, a, b, msg = None): + self.failUnlessEqual(len(a), len(b)) + for i in range(len(a)): + self.failUnlessEqual(a[i], b[i], msg) + + def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None): + self.failUnlessEqual(len(a), len(b)) + for i in range(len(a)): + self.failUnlessAlmostEqual(a[i], b[i], places, msg) + + def test_binomial_coeff(self): + f = statistics.binomial_coeff + self.failUnlessEqual(f(20, 0), 1) + self.failUnlessEqual(f(20, 1), 20) + self.failUnlessEqual(f(20, 2), 190) + self.failUnlessEqual(f(20, 8), f(20, 12)) + self.should_assert("Should assert if n < k", f, 2, 3) + + def test_binomial_distribution_pmf(self): + f = statistics.binomial_distribution_pmf + + pmf_comp = f(2, .1) + pmf_stat = [0.81, 0.18, 0.01] + self.failUnlessListAlmostEqual(pmf_comp, pmf_stat) + + # Summing across a PMF should give the total probability 1 + self.failUnlessAlmostEqual(sum(pmf_comp), 1) + self.should_assert("Should assert if not 0<=p<=1", f, 1, -1) + self.should_assert("Should assert if n < 1", f, 0, .1) + + out = StringIO() + statistics.print_pmf(pmf_comp, out=out) + lines = out.getvalue().splitlines() + self.failUnlessEqual(lines[0], "i=0: 0.81") + self.failUnlessEqual(lines[1], "i=1: 0.18") + self.failUnlessEqual(lines[2], "i=2: 0.01") + + def test_survival_pmf(self): + f = statistics.survival_pmf + # Cross-check binomial-distribution method against convolution + # method. + p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20 + pmf1 = statistics.survival_pmf_via_conv(p_list) + pmf2 = statistics.survival_pmf_via_bd(p_list) + self.failUnlessListAlmostEqual(pmf1, pmf2) + self.failUnlessTrue(statistics.valid_pmf(pmf1)) + self.should_assert("Should assert if p_i > 1", f, [1.1]); + self.should_assert("Should assert if p_i < 0", f, [-.1]); + + def test_repair_count_pmf(self): + survival_pmf = statistics.binomial_distribution_pmf(5, .9) + repair_pmf = statistics.repair_count_pmf(survival_pmf, 3) + # repair_pmf[0] == sum(survival_pmf[0,1,2,5]) + # repair_pmf[1] == survival_pmf[4] + # repair_pmf[2] = survival_pmf[3] + self.failUnlessListAlmostEqual(repair_pmf, + [0.00001 + 0.00045 + 0.0081 + 0.59049, + .32805, + .0729, + 0, 0, 0]) + + def test_repair_cost(self): + survival_pmf = statistics.binomial_distribution_pmf(5, .9) + bwcost = statistics.bandwidth_cost_function + cost = statistics.mean_repair_cost(bwcost, 1000, + survival_pmf, 3, ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 558.90) + cost = statistics.mean_repair_cost(bwcost, 1000, + survival_pmf, 3, ul_dl_ratio=8.0) + self.failUnlessAlmostEqual(cost, 1664.55) + + # I haven't manually checked the math beyond here -warner + cost = statistics.eternal_repair_cost(bwcost, 1000, + survival_pmf, 3, + discount_rate=0, ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 65292.056074766246) + cost = statistics.eternal_repair_cost(bwcost, 1000, + survival_pmf, 3, + discount_rate=0.05, + ul_dl_ratio=1.0) + self.failUnlessAlmostEqual(cost, 9133.6097158191551) + + def test_convolve(self): + f = statistics.convolve + v1 = [ 1, 2, 3 ] + v2 = [ 4, 5, 6 ] + v3 = [ 7, 8 ] + v1v2result = [ 4, 13, 28, 27, 18 ] + # Convolution is commutative + r1 = f(v1, v2) + r2 = f(v2, v1) + self.failUnlessListEqual(r1, r2, "Convolution should be commutative") + self.failUnlessListEqual(r1, v1v2result, "Didn't match known result") + # Convolution is associative + r1 = f(f(v1, v2), v3) + r2 = f(v1, f(v2, v3)) + self.failUnlessListEqual(r1, r2, "Convolution should be associative") + # Convolution is distributive + r1 = f(v3, [ a + b for a, b in zip(v1, v2) ]) + tmp1 = f(v3, v1) + tmp2 = f(v3, v2) + r2 = [ a + b for a, b in zip(tmp1, tmp2) ] + self.failUnlessListEqual(r1, r2, "Convolution should be distributive") + # Convolution is scalar multiplication associative + tmp1 = f(v1, v2) + r1 = [ a * 4 for a in tmp1 ] + tmp2 = [ a * 4 for a in v1 ] + r2 = f(tmp2, v2) + self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative") + + def test_find_k(self): + f = statistics.find_k + g = statistics.pr_file_loss + plist = [.9] * 10 + [.8] * 10 # N=20 + t = .0001 + k = f(plist, t) + self.failUnlessEqual(k, 10) + self.failUnless(g(plist, k) < t) + + def test_pr_file_loss(self): + f = statistics.pr_file_loss + plist = [.5] * 10 + self.failUnlessEqual(f(plist, 3), .0546875) + + def test_pr_backup_file_loss(self): + f = statistics.pr_backup_file_loss + plist = [.5] * 10 + self.failUnlessEqual(f(plist, .5, 3), .02734375) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1519c42da..ff6a1650c 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -4,7 +4,6 @@ import six import os, time, sys import yaml -from six.moves import StringIO from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -12,7 +11,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin -from allmydata.util import statistics, dictutil, yamlutil +from allmydata.util import dictutil, yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -34,143 +33,6 @@ class Math(unittest.TestCase): f = mathutil.round_sigfigs self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002) -class Statistics(unittest.TestCase): - def should_assert(self, msg, func, *args, **kwargs): - try: - func(*args, **kwargs) - self.fail(msg) - except AssertionError: - pass - - def failUnlessListEqual(self, a, b, msg = None): - self.failUnlessEqual(len(a), len(b)) - for i in range(len(a)): - self.failUnlessEqual(a[i], b[i], msg) - - def failUnlessListAlmostEqual(self, a, b, places = 7, msg = None): - self.failUnlessEqual(len(a), len(b)) - for i in range(len(a)): - self.failUnlessAlmostEqual(a[i], b[i], places, msg) - - def test_binomial_coeff(self): - f = statistics.binomial_coeff - self.failUnlessEqual(f(20, 0), 1) - self.failUnlessEqual(f(20, 1), 20) - self.failUnlessEqual(f(20, 2), 190) - self.failUnlessEqual(f(20, 8), f(20, 12)) - self.should_assert("Should assert if n < k", f, 2, 3) - - def test_binomial_distribution_pmf(self): - f = statistics.binomial_distribution_pmf - - pmf_comp = f(2, .1) - pmf_stat = [0.81, 0.18, 0.01] - self.failUnlessListAlmostEqual(pmf_comp, pmf_stat) - - # Summing across a PMF should give the total probability 1 - self.failUnlessAlmostEqual(sum(pmf_comp), 1) - self.should_assert("Should assert if not 0<=p<=1", f, 1, -1) - self.should_assert("Should assert if n < 1", f, 0, .1) - - out = StringIO() - statistics.print_pmf(pmf_comp, out=out) - lines = out.getvalue().splitlines() - self.failUnlessEqual(lines[0], "i=0: 0.81") - self.failUnlessEqual(lines[1], "i=1: 0.18") - self.failUnlessEqual(lines[2], "i=2: 0.01") - - def test_survival_pmf(self): - f = statistics.survival_pmf - # Cross-check binomial-distribution method against convolution - # method. - p_list = [.9999] * 100 + [.99] * 50 + [.8] * 20 - pmf1 = statistics.survival_pmf_via_conv(p_list) - pmf2 = statistics.survival_pmf_via_bd(p_list) - self.failUnlessListAlmostEqual(pmf1, pmf2) - self.failUnlessTrue(statistics.valid_pmf(pmf1)) - self.should_assert("Should assert if p_i > 1", f, [1.1]); - self.should_assert("Should assert if p_i < 0", f, [-.1]); - - def test_repair_count_pmf(self): - survival_pmf = statistics.binomial_distribution_pmf(5, .9) - repair_pmf = statistics.repair_count_pmf(survival_pmf, 3) - # repair_pmf[0] == sum(survival_pmf[0,1,2,5]) - # repair_pmf[1] == survival_pmf[4] - # repair_pmf[2] = survival_pmf[3] - self.failUnlessListAlmostEqual(repair_pmf, - [0.00001 + 0.00045 + 0.0081 + 0.59049, - .32805, - .0729, - 0, 0, 0]) - - def test_repair_cost(self): - survival_pmf = statistics.binomial_distribution_pmf(5, .9) - bwcost = statistics.bandwidth_cost_function - cost = statistics.mean_repair_cost(bwcost, 1000, - survival_pmf, 3, ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 558.90) - cost = statistics.mean_repair_cost(bwcost, 1000, - survival_pmf, 3, ul_dl_ratio=8.0) - self.failUnlessAlmostEqual(cost, 1664.55) - - # I haven't manually checked the math beyond here -warner - cost = statistics.eternal_repair_cost(bwcost, 1000, - survival_pmf, 3, - discount_rate=0, ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 65292.056074766246) - cost = statistics.eternal_repair_cost(bwcost, 1000, - survival_pmf, 3, - discount_rate=0.05, - ul_dl_ratio=1.0) - self.failUnlessAlmostEqual(cost, 9133.6097158191551) - - def test_convolve(self): - f = statistics.convolve - v1 = [ 1, 2, 3 ] - v2 = [ 4, 5, 6 ] - v3 = [ 7, 8 ] - v1v2result = [ 4, 13, 28, 27, 18 ] - # Convolution is commutative - r1 = f(v1, v2) - r2 = f(v2, v1) - self.failUnlessListEqual(r1, r2, "Convolution should be commutative") - self.failUnlessListEqual(r1, v1v2result, "Didn't match known result") - # Convolution is associative - r1 = f(f(v1, v2), v3) - r2 = f(v1, f(v2, v3)) - self.failUnlessListEqual(r1, r2, "Convolution should be associative") - # Convolution is distributive - r1 = f(v3, [ a + b for a, b in zip(v1, v2) ]) - tmp1 = f(v3, v1) - tmp2 = f(v3, v2) - r2 = [ a + b for a, b in zip(tmp1, tmp2) ] - self.failUnlessListEqual(r1, r2, "Convolution should be distributive") - # Convolution is scalar multiplication associative - tmp1 = f(v1, v2) - r1 = [ a * 4 for a in tmp1 ] - tmp2 = [ a * 4 for a in v1 ] - r2 = f(tmp2, v2) - self.failUnlessListEqual(r1, r2, "Convolution should be scalar multiplication associative") - - def test_find_k(self): - f = statistics.find_k - g = statistics.pr_file_loss - plist = [.9] * 10 + [.8] * 10 # N=20 - t = .0001 - k = f(plist, t) - self.failUnlessEqual(k, 10) - self.failUnless(g(plist, k) < t) - - def test_pr_file_loss(self): - f = statistics.pr_file_loss - plist = [.5] * 10 - self.failUnlessEqual(f(plist, 3), .0546875) - - def test_pr_backup_file_loss(self): - f = statistics.pr_backup_file_loss - plist = [.5] * 10 - self.failUnlessEqual(f(plist, .5, 3), .02734375) - class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0o777): From 8d84be77d86f8af6e3025b237e3e681219ea2b7e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:31:31 -0400 Subject: [PATCH 26/40] Port to Python 3. --- src/allmydata/test/test_statistics.py | 10 +++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 7fe23f88f..a14fb55ab 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -1,8 +1,16 @@ """ Tests for allmydata.util.statistics. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from io import StringIO +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from six.moves import StringIO # native string StringIO from twisted.trial import unittest diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 53ccc6f63..f06f5bf3b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -48,6 +48,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_pipeline", "allmydata.test.test_python3", "allmydata.test.test_spans", + "allmydata.test.test_statistics", "allmydata.test.test_time_format", "allmydata.test.test_version", ] From 2ca223a67ca243890711846e039e3cd0f9259e93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:34:28 -0400 Subject: [PATCH 27/40] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/statistics.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f06f5bf3b..d5bdae2f4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -31,6 +31,7 @@ PORTED_MODULES = [ "allmydata.util.pollmixin", "allmydata.util._python3", "allmydata.util.spans", + "allmydata.util.statistics", "allmydata.util.time_format", "allmydata.test.common_py3", ] diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 3e50b39dc..a690eb235 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -1,3 +1,8 @@ +""" +Statistical utilities. + +Ported to Python 3. +""" # Copyright (c) 2009 Shawn Willden # mailto:shawn@willden.org # I hereby license all patches I have contributed or will contribute to the @@ -5,7 +10,18 @@ # either the GNU General Public License, version 2 or later, or under the # Transitive Grace Period Public License, version 1 or later. -from __future__ import division, print_function + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from functools import reduce + from allmydata.util.mathutil import round_sigfigs import math import sys @@ -78,7 +94,7 @@ def survival_pmf_via_bd(p_list): """ pmf_list = [ binomial_distribution_pmf(p_list.count(p), p) for p in set(p_list) ] - return reduce(convolve, pmf_list) + return list(reduce(convolve, pmf_list)) def survival_pmf_via_conv(p_list): """ @@ -89,7 +105,7 @@ def survival_pmf_via_conv(p_list): intended for internal use and testing only. """ pmf_list = [ [1 - p, p] for p in p_list ]; - return reduce(convolve, pmf_list) + return list(reduce(convolve, pmf_list)) def print_pmf(pmf, n=4, out=sys.stdout): """ From bde2f1394dae44d87b131f84335c2564f6710c46 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:53:26 -0400 Subject: [PATCH 28/40] Move dictutil tests out. --- src/allmydata/test/test_dictutil.py | 80 +++++++++++++++++++++++++++++ src/allmydata/test/test_util.py | 74 +------------------------- 2 files changed, 81 insertions(+), 73 deletions(-) create mode 100644 src/allmydata/test/test_dictutil.py diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py new file mode 100644 index 000000000..c22b2dfe3 --- /dev/null +++ b/src/allmydata/test/test_dictutil.py @@ -0,0 +1,80 @@ +""" +Tests for allmydata.util.dictutil. +""" + +from twisted.trial import unittest + +from allmydata.util import dictutil + + +class DictUtil(unittest.TestCase): + def test_dict_of_sets(self): + ds = dictutil.DictOfSets() + ds.add(1, "a") + ds.add(2, "b") + ds.add(2, "b") + ds.add(2, "c") + self.failUnlessEqual(ds[1], set(["a"])) + self.failUnlessEqual(ds[2], set(["b", "c"])) + ds.discard(3, "d") # should not raise an exception + ds.discard(2, "b") + self.failUnlessEqual(ds[2], set(["c"])) + ds.discard(2, "c") + self.failIf(2 in ds) + + ds.add(3, "f") + ds2 = dictutil.DictOfSets() + ds2.add(3, "f") + ds2.add(3, "g") + ds2.add(4, "h") + ds.update(ds2) + self.failUnlessEqual(ds[1], set(["a"])) + self.failUnlessEqual(ds[3], set(["f", "g"])) + self.failUnlessEqual(ds[4], set(["h"])) + + def test_auxdict(self): + d = dictutil.AuxValueDict() + # we put the serialized form in the auxdata + d.set_with_aux("key", ("filecap", "metadata"), "serialized") + + self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(d["key"], ("filecap", "metadata")) + self.failUnlessEqual(d.get_aux("key"), "serialized") + def _get_missing(key): + return d[key] + self.failUnlessRaises(KeyError, _get_missing, "nonkey") + self.failUnlessEqual(d.get("nonkey"), None) + self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue") + self.failUnlessEqual(d.get_aux("nonkey"), None) + self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue") + + d["key"] = ("filecap2", "metadata2") + self.failUnlessEqual(d["key"], ("filecap2", "metadata2")) + self.failUnlessEqual(d.get_aux("key"), None) + + d.set_with_aux("key2", "value2", "aux2") + self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) + del d["key2"] + self.failUnlessEqual(d.keys(), ["key"]) + self.failIf("key2" in d) + self.failUnlessRaises(KeyError, _get_missing, "key2") + self.failUnlessEqual(d.get("key2"), None) + self.failUnlessEqual(d.get_aux("key2"), None) + d["key2"] = "newvalue2" + self.failUnlessEqual(d.get("key2"), "newvalue2") + self.failUnlessEqual(d.get_aux("key2"), None) + + d = dictutil.AuxValueDict({1:2,3:4}) + self.failUnlessEqual(sorted(d.keys()), [1,3]) + self.failUnlessEqual(d[1], 2) + self.failUnlessEqual(d.get_aux(1), None) + + d = dictutil.AuxValueDict([ (1,2), (3,4) ]) + self.failUnlessEqual(sorted(d.keys()), [1,3]) + self.failUnlessEqual(d[1], 2) + self.failUnlessEqual(d.get_aux(1), None) + + d = dictutil.AuxValueDict(one=1, two=2) + self.failUnlessEqual(sorted(d.keys()), ["one","two"]) + self.failUnlessEqual(d["one"], 1) + self.failUnlessEqual(d.get_aux("one"), None) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index ff6a1650c..1e9dd0f49 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -11,7 +11,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin -from allmydata.util import dictutil, yamlutil +from allmydata.util import yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -527,78 +527,6 @@ class EqButNotIs(object): def __eq__(self, other): return self.x == other -class DictUtil(unittest.TestCase): - def test_dict_of_sets(self): - ds = dictutil.DictOfSets() - ds.add(1, "a") - ds.add(2, "b") - ds.add(2, "b") - ds.add(2, "c") - self.failUnlessEqual(ds[1], set(["a"])) - self.failUnlessEqual(ds[2], set(["b", "c"])) - ds.discard(3, "d") # should not raise an exception - ds.discard(2, "b") - self.failUnlessEqual(ds[2], set(["c"])) - ds.discard(2, "c") - self.failIf(2 in ds) - - ds.add(3, "f") - ds2 = dictutil.DictOfSets() - ds2.add(3, "f") - ds2.add(3, "g") - ds2.add(4, "h") - ds.update(ds2) - self.failUnlessEqual(ds[1], set(["a"])) - self.failUnlessEqual(ds[3], set(["f", "g"])) - self.failUnlessEqual(ds[4], set(["h"])) - - def test_auxdict(self): - d = dictutil.AuxValueDict() - # we put the serialized form in the auxdata - d.set_with_aux("key", ("filecap", "metadata"), "serialized") - - self.failUnlessEqual(d.keys(), ["key"]) - self.failUnlessEqual(d["key"], ("filecap", "metadata")) - self.failUnlessEqual(d.get_aux("key"), "serialized") - def _get_missing(key): - return d[key] - self.failUnlessRaises(KeyError, _get_missing, "nonkey") - self.failUnlessEqual(d.get("nonkey"), None) - self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue") - self.failUnlessEqual(d.get_aux("nonkey"), None) - self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue") - - d["key"] = ("filecap2", "metadata2") - self.failUnlessEqual(d["key"], ("filecap2", "metadata2")) - self.failUnlessEqual(d.get_aux("key"), None) - - d.set_with_aux("key2", "value2", "aux2") - self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) - del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) - self.failIf("key2" in d) - self.failUnlessRaises(KeyError, _get_missing, "key2") - self.failUnlessEqual(d.get("key2"), None) - self.failUnlessEqual(d.get_aux("key2"), None) - d["key2"] = "newvalue2" - self.failUnlessEqual(d.get("key2"), "newvalue2") - self.failUnlessEqual(d.get_aux("key2"), None) - - d = dictutil.AuxValueDict({1:2,3:4}) - self.failUnlessEqual(sorted(d.keys()), [1,3]) - self.failUnlessEqual(d[1], 2) - self.failUnlessEqual(d.get_aux(1), None) - - d = dictutil.AuxValueDict([ (1,2), (3,4) ]) - self.failUnlessEqual(sorted(d.keys()), [1,3]) - self.failUnlessEqual(d[1], 2) - self.failUnlessEqual(d.get_aux(1), None) - - d = dictutil.AuxValueDict(one=1, two=2) - self.failUnlessEqual(sorted(d.keys()), ["one","two"]) - self.failUnlessEqual(d["one"], 1) - self.failUnlessEqual(d.get_aux("one"), None) - class SampleError(Exception): pass From 79ae478a4806502f882641b4b51b018f7c32700a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:56:31 -0400 Subject: [PATCH 29/40] Port to Python 3. --- src/allmydata/test/test_dictutil.py | 14 ++++++++++++-- src/allmydata/util/_python3.py | 2 ++ src/allmydata/util/dictutil.py | 13 ++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index c22b2dfe3..0868db124 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -1,6 +1,16 @@ """ Tests for allmydata.util.dictutil. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest @@ -37,7 +47,7 @@ class DictUtil(unittest.TestCase): # we put the serialized form in the auxdata d.set_with_aux("key", ("filecap", "metadata"), "serialized") - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failUnlessEqual(d["key"], ("filecap", "metadata")) self.failUnlessEqual(d.get_aux("key"), "serialized") def _get_missing(key): @@ -55,7 +65,7 @@ class DictUtil(unittest.TestCase): d.set_with_aux("key2", "value2", "aux2") self.failUnlessEqual(sorted(d.keys()), ["key", "key2"]) del d["key2"] - self.failUnlessEqual(d.keys(), ["key"]) + self.failUnlessEqual(list(d.keys()), ["key"]) self.failIf("key2" in d) self.failUnlessRaises(KeyError, _get_missing, "key2") self.failUnlessEqual(d.get("key2"), None) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d5bdae2f4..66c0172ea 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ PORTED_MODULES = [ "allmydata.util.base32", "allmydata.util.base62", "allmydata.util.deferredutil", + "allmydata.util.dictutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.mathutil", @@ -41,6 +42,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_deferredutil", + "allmydata.test.test_dictutil", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d59e60dd6..9ed99d7c4 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -1,6 +1,17 @@ """ Tools to mess with dicts. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + class DictOfSets(dict): def add(self, key, value): @@ -10,7 +21,7 @@ class DictOfSets(dict): self[key] = set([value]) def update(self, otherdictofsets): - for key, values in otherdictofsets.iteritems(): + for key, values in otherdictofsets.items(): if key in self: self[key].update(values) else: From 058a76dc857545080fd81a98c8b147a73de7aeac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 15:58:28 -0400 Subject: [PATCH 30/40] Ratchet, and news file. --- misc/python3/ratchet-passing | 11 +++++++++++ newsfragments/3359.minor | 0 2 files changed, 11 insertions(+) create mode 100644 newsfragments/3359.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 6832b7fa7..dd0f8a7b0 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -28,6 +28,8 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls +allmydata.test.test_dictutil.DictUtil.test_auxdict +allmydata.test.test_dictutil.DictUtil.test_dict_of_sets allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes @@ -66,6 +68,15 @@ allmydata.test.test_spans.ByteSpans.test_random allmydata.test.test_spans.StringSpans.test_basic allmydata.test.test_spans.StringSpans.test_random allmydata.test.test_spans.StringSpans.test_test +allmydata.test.test_statistics.Statistics.test_binomial_coeff +allmydata.test.test_statistics.Statistics.test_binomial_distribution_pmf +allmydata.test.test_statistics.Statistics.test_convolve +allmydata.test.test_statistics.Statistics.test_find_k +allmydata.test.test_statistics.Statistics.test_pr_backup_file_loss +allmydata.test.test_statistics.Statistics.test_pr_file_loss +allmydata.test.test_statistics.Statistics.test_repair_cost +allmydata.test.test_statistics.Statistics.test_repair_count_pmf +allmydata.test.test_statistics.Statistics.test_survival_pmf allmydata.test.test_time_format.TimeFormat.test_epoch allmydata.test.test_time_format.TimeFormat.test_epoch_in_London allmydata.test.test_time_format.TimeFormat.test_format_delta diff --git a/newsfragments/3359.minor b/newsfragments/3359.minor new file mode 100644 index 000000000..e69de29bb From 94d489ff46883279f2994c03ca316f0c82f490d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 16:01:12 -0400 Subject: [PATCH 31/40] Fix trailing whitespace. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 921c481ff..ac5367cfc 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -131,7 +131,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET addresses.append(addr) return addresses d.addCallback(_collect) - d.addCallback(lambda addresses: [native_str(s) for s in addresses]) + d.addCallback(lambda addresses: [native_str(s) for s in addresses]) return d def get_local_ip_for(target): From cbbe260a4eafef224f3bf06f599030a463f0c7c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jul 2020 16:19:36 -0400 Subject: [PATCH 32/40] Windows doesn't like Unicode strings in os.environ in Python 2.7. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index ac5367cfc..36f4bb79b 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -239,7 +239,7 @@ def _synchronously_find_addresses_via_config(): def _query(path, args, regex): if not os.path.isfile(path): return [] - env = {'LANG': 'en_US.UTF-8'} + env = {native_str('LANG'): native_str('en_US.UTF-8')} TRIES = 5 for trial in range(TRIES): try: From 4fdcd06fbdb4ce1e2dc7d3ea10ccccb70ca5eea2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 10:48:17 -0400 Subject: [PATCH 33/40] Another passing test for the ratchet. --- misc/python3/ratchet-passing | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index bbe594998..68f1a968b 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -51,6 +51,7 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route allmydata.test.test_iputil.ListenOnUsed.test_random_port +allmydata.test.test_iputil.ListenOnUsed.test_specific_port allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested From 4dae5d867c8dd44d9ceb68b6732030f00b391c6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:08:38 -0400 Subject: [PATCH 34/40] Add retry logic, listenOnUnused is potentially flaky. --- setup.py | 1 + src/allmydata/test/test_iputil.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 9bb385cb4..a35023b8b 100644 --- a/setup.py +++ b/setup.py @@ -390,6 +390,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "beautifulsoup4", "html5lib", "junitxml", + "tenacity", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 42677af2f..c6caed7f9 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -17,6 +17,8 @@ import re, errno, subprocess, os, socket from twisted.trial import unittest +from tenacity import retry, stop_after_attempt + from foolscap.api import Tub from allmydata.util import iputil @@ -199,6 +201,7 @@ class ListenOnUsed(unittest.TestCase): self.addCleanup(tub.stopService) return tub + @retry(stop=stop_after_attempt(7)) def test_random_port(self): """A random port is selected if none is given.""" tub = self.create_tub("utils/ListenOnUsed/test_randomport") @@ -215,6 +218,7 @@ class ListenOnUsed(unittest.TestCase): portnum2 = iputil.listenOnUnused(tub2) self.assertNotEqual(portnum, portnum2) + @retry(stop=stop_after_attempt(7)) def test_specific_port(self): """The given port is used.""" tub = self.create_tub("utils/ListenOnUsed/test_givenport") From cc494e3d3978dd79eb67b3397bc8abcb29b7d1c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:11:05 -0400 Subject: [PATCH 35/40] Document that API returns native strings. --- src/allmydata/util/iputil.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 36f4bb79b..890ff98e7 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -102,13 +102,18 @@ except ImportError: increase_rlimits = _increase_rlimits def get_local_addresses_sync(): + """ + Return a list of IPv4 addresses (as dotted-quad native strings) that are + currently configured on this host, sorted in descending order of how likely + we think they are to work. + """ return [native_str(a) for a in _synchronously_find_addresses_via_config()] def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad - strings) that are currently configured on this host, sorted in descending - order of how likely we think they are to work. + native strings) that are currently configured on this host, sorted in + descending order of how likely we think they are to work. @param target: we want to learn an IP address they could try using to connect to us; The default value is fine, but it might help if you @@ -137,7 +142,7 @@ def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET def get_local_ip_for(target): """Find out what our IP address is for use by a given target. - @return: the IP address as a dotted-quad string which could be used by + @return: the IP address as a dotted-quad native string which could be used to connect to us. It might work for them, it might not. If there is no suitable address (perhaps we don't currently have an externally-visible interface), this will return None. @@ -172,7 +177,7 @@ def get_local_ip_for(target): return localip finally: d = port.stopListening() - d.addErrback(log.err) + 3 d.addErrback(log.err) except (socket.error, CannotListenError): # no route to that host localip = None From 8c9c691c02c420253a79393669856d0bea07f4e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:11:27 -0400 Subject: [PATCH 36/40] Fix typo. --- src/allmydata/util/iputil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 890ff98e7..8754fca53 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -177,7 +177,7 @@ def get_local_ip_for(target): return localip finally: d = port.stopListening() - 3 d.addErrback(log.err) + d.addErrback(log.err) except (socket.error, CannotListenError): # no route to that host localip = None From 9ca1fdef816263b61cdb51e801dd26aa7b992d28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:12:17 -0400 Subject: [PATCH 37/40] Note it's ported. --- src/allmydata/test/test_statistics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index a14fb55ab..cfbbc1dc0 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -1,5 +1,7 @@ """ Tests for allmydata.util.statistics. + +Ported to Python 3. """ from __future__ import absolute_import from __future__ import division From e954314fe52a6b3e40549b27f794aa00d0fd4375 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:23:40 -0400 Subject: [PATCH 38/40] Don't expose Python 3 dicts to innocent, unsuspecting Python 2 code. --- src/allmydata/util/dictutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 9ed99d7c4..d5f0da29b 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -10,7 +10,10 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + # IMPORTANT: We deliberately don't import dict. The issue is that we're + # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of + # code that doesn't support it. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 class DictOfSets(dict): From 17b15ae08577241df493aa972984b19a39fe12f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 11:25:34 -0400 Subject: [PATCH 39/40] Add another test. --- src/allmydata/test/test_statistics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index cfbbc1dc0..8d5837fc5 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -44,6 +44,7 @@ class Statistics(unittest.TestCase): self.failUnlessEqual(f(20, 2), 190) self.failUnlessEqual(f(20, 8), f(20, 12)) self.should_assert("Should assert if n < k", f, 2, 3) + self.assertEqual(f(5, 3), f(5, 2)) def test_binomial_distribution_pmf(self): f = statistics.binomial_distribution_pmf From 199c4f6acd350885e28e30b91216c77d1b739b0e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Jul 2020 13:00:44 -0400 Subject: [PATCH 40/40] Try to fix nix. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 0e941bce1..f2e61d6c2 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -59,6 +59,7 @@ python.pkgs.buildPythonPackage rec { fixtures beautifulsoup4 html5lib + tenacity ]; checkPhase = ''