From 3afa38adb4db82a264d434464b422b980357116d Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 12 Aug 2017 21:02:32 -0700 Subject: [PATCH 1/5] NummedObj: stop using dictutil.NumDict --- src/allmydata/util/nummedobj.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/nummedobj.py b/src/allmydata/util/nummedobj.py index de3c57d0e..1d338ec45 100644 --- a/src/allmydata/util/nummedobj.py +++ b/src/allmydata/util/nummedobj.py @@ -1,4 +1,6 @@ -from allmydata.util import dictutil +import collections, itertools + +objnums = collections.defaultdict(itertools.count) class NummedObj(object): """ @@ -10,7 +12,6 @@ class NummedObj(object): to diff outputs from separate runs to see what changed, without having to ignore a difference on every line due to different memory addresses of objects. """ - objnums = dictutil.NumDict() # key: class names, value: highest used object number def __init__(self, klass=None): """ @@ -20,8 +21,7 @@ class NummedObj(object): klass = self.__class__ self._classname = klass.__name__ - NummedObj.objnums.inc(self._classname) - self._objid = NummedObj.objnums[self._classname] + self._objid = objnums[self._classname].next() def __repr__(self): return "<%s #%d>" % (self._classname, self._objid,) From a76d9e084fe715abea6b6cf77b486b496ec03f5e Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 12 Aug 2017 21:21:04 -0700 Subject: [PATCH 2/5] dictutil: remove unused NumDict --- src/allmydata/test/test_util.py | 119 +---------------------- src/allmydata/util/dictutil.py | 164 +------------------------------- 2 files changed, 2 insertions(+), 281 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 81e1d2ef9..da8d4eb09 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,7 +1,7 @@ def foo(): pass # keep the line number constant -import os, time, sys, itertools, random +import os, time, sys import yaml from StringIO import StringIO from datetime import timedelta @@ -1409,13 +1409,10 @@ class DictUtil(unittest.TestCase): def test_all(self): self._help_test_eq_but_notis(dictutil.UtilDict) - self._help_test_eq_but_notis(dictutil.NumDict) self._help_test_eq_but_notis(dictutil.ValueOrderedDict) self._help_test_nonempty_dict(dictutil.UtilDict) - self._help_test_nonempty_dict(dictutil.NumDict) self._help_test_nonempty_dict(dictutil.ValueOrderedDict) self._help_test_eq_but_notis(dictutil.UtilDict) - self._help_test_eq_but_notis(dictutil.NumDict) self._help_test_eq_but_notis(dictutil.ValueOrderedDict) def test_dict_of_sets(self): @@ -1526,120 +1523,6 @@ class DictUtil(unittest.TestCase): self.failUnless(x in [(1, "b"), (2, "a")]) self.failUnlessRaises(KeyError, d.popitem) - def test_numdict(self): - d = dictutil.NumDict({"a": 1, "b": 2}) - - d.add_num("a", 10, 5) - d.add_num("c", 20, 5) - d.add_num("d", 30) - self.failUnlessEqual(d, {"a": 11, "b": 2, "c": 25, "d": 30}) - - d.subtract_num("a", 10) - d.subtract_num("e", 10) - d.subtract_num("f", 10, 15) - self.failUnlessEqual(d, {"a": 1, "b": 2, "c": 25, "d": 30, - "e": -10, "f": 5}) - - self.failUnlessEqual(d.sum(), sum([1, 2, 25, 30, -10, 5])) - - d = dictutil.NumDict() - d.inc("a") - d.inc("a") - d.inc("b", 5) - self.failUnlessEqual(d, {"a": 2, "b": 6}) - d.dec("a") - d.dec("c") - d.dec("d", 5) - self.failUnlessEqual(d, {"a": 1, "b": 6, "c": -1, "d": 4}) - self.failUnlessEqual(d.items_sorted_by_key(), - [("a", 1), ("b", 6), ("c", -1), ("d", 4)]) - self.failUnlessEqual(d.items_sorted_by_value(), - [("c", -1), ("a", 1), ("d", 4), ("b", 6)]) - self.failUnlessEqual(d.item_with_largest_value(), ("b", 6)) - - # to get full coverage of item_with_largest_value(), we need to - # exercise two situations: the first value (in iteritems() order) is - # larger than the second, and vice versa. Since iteration is not - # deterministic, we need to try a bunch of random dictionaries to - # exercise this - r = random.Random(0) # consistent seed - count = itertools.count() - found = set() - while count.next() < 1000: - a = r.randrange(100) - b = r.randrange(100) - larger = ("a",a) if a > b else ("b",b) - if a == b: - continue - d0 = dictutil.NumDict() - d0.add_num("a", a) - d0.add_num("b", b) - self.failUnlessEqual(d0, {"a": a, "b": b}) - items = list(d0.d.iteritems()) - if items[0][1] > items[1][1]: - found.add("first-larger") - else: - found.add("first-smaller") - self.failUnlessEqual(d0.item_with_largest_value(), larger) - if found == set(["first-larger", "first-smaller"]): - break - else: - self.fail("unable to exercise all cases of item_with_largest_value") - - d = dictutil.NumDict({"a": 1, "b": 2}) - self.failUnlessIn(repr(d), ("{'a': 1, 'b': 2}", - "{'b': 2, 'a': 1}")) - self.failUnless("a" in d) - - d2 = dictutil.NumDict({"c": 3, "d": 4}) - self.failUnless(d != d2) - self.failUnless(d2 > d) - self.failUnless(d2 >= d) - self.failUnless(d <= d2) - self.failUnless(d < d2) - self.failUnlessEqual(d["a"], 1) - self.failUnlessEqual(sorted(list([k for k in d])), ["a","b"]) - def eq(a, b): - return a == b - self.failUnlessRaises(TypeError, eq, d, "not a dict") - - d3 = d.copy() - self.failUnlessEqual(d, d3) - self.failUnless(isinstance(d3, dictutil.NumDict)) - - d4 = d.fromkeys(["a","b"], 5) - self.failUnlessEqual(d4, {"a": 5, "b": 5}) - - self.failUnlessEqual(d.get("a"), 1) - self.failUnlessEqual(d.get("c"), 0) - self.failUnlessEqual(d.get("c", 5), 5) - self.failUnlessEqual(sorted(list(d.items())), - [("a", 1), ("b", 2)]) - self.failUnlessEqual(sorted(list(d.iteritems())), - [("a", 1), ("b", 2)]) - self.failUnlessEqual(sorted(d.keys()), ["a", "b"]) - self.failUnlessEqual(sorted(d.values()), [1, 2]) - self.failUnless(d.has_key("a")) - self.failIf(d.has_key("c")) - - x = d.setdefault("c", 3) - self.failUnlessEqual(x, 3) - self.failUnlessEqual(d["c"], 3) - x = d.setdefault("c", 5) - self.failUnlessEqual(x, 3) - self.failUnlessEqual(d["c"], 3) - del d["c"] - - x = d.popitem() - self.failUnless(x in [("a", 1), ("b", 2)]) - x = d.popitem() - self.failUnless(x in [("a", 1), ("b", 2)]) - self.failUnlessRaises(KeyError, d.popitem) - - d.update({"c": 3}) - d.update({"c": 4, "d": 5}) - self.failUnlessEqual(d, {"c": 4, "d": 5}) - def test_del_if_present(self): d = {1: "a", 2: "b"} dictutil.del_if_present(d, 1) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 6715f47ae..fc2cdd445 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,7 +2,7 @@ Tools to mess with dicts. """ -import copy, operator +import operator from bisect import bisect_left, insort_left from allmydata.util.assertutil import _assert, precondition @@ -176,168 +176,6 @@ class UtilDict: def values(self, *args, **kwargs): return self.d.values(*args, **kwargs) -class NumDict: - def __init__(self, initialdict={}): - self.d = copy.deepcopy(initialdict) - - def add_num(self, key, val, default=0): - """ - If the key doesn't appear in self then it is created with value default - (before addition). - """ - self.d[key] = self.d.get(key, default) + val - - def subtract_num(self, key, val, default=0): - self.d[key] = self.d.get(key, default) - val - - def sum(self): - """ - @return: the sum of all values - """ - return reduce(operator.__add__, self.d.values()) - - def inc(self, key, default=0): - """ - Increment the value associated with key in dict. If there is no such - key, then one will be created with initial value 0 (before inc() -- - therefore value 1 after inc). - """ - self.add_num(key, 1, default) - - def dec(self, key, default=0): - """ - Decrement the value associated with key in dict. If there is no such - key, then one will be created with initial value 0 (before dec() -- - therefore value -1 after dec). - """ - self.subtract_num(key, 1, default) - - def items_sorted_by_value(self): - """ - @return a sequence of (key, value,) pairs sorted according to value - """ - l = [(x[1], x[0],) for x in self.d.iteritems()] - l.sort() - return [(x[1], x[0],) for x in l] - - def item_with_largest_value(self): - it = self.d.iteritems() - (winner, winnerval,) = it.next() - try: - while True: - n, nv = it.next() - if nv > winnerval: - winner = n - winnerval = nv - except StopIteration: - pass - return (winner, winnerval,) - - def items_sorted_by_key(self): - """ - @return a sequence of (key, value,) pairs sorted according to key - """ - l = self.d.items() - l.sort() - return l - - def __repr__(self, *args, **kwargs): - return self.d.__repr__(*args, **kwargs) - - def __str__(self, *args, **kwargs): - return self.d.__str__(*args, **kwargs) - - def __contains__(self, *args, **kwargs): - return self.d.__contains__(*args, **kwargs) - - def __len__(self, *args, **kwargs): - return self.d.__len__(*args, **kwargs) - - def __cmp__(self, other): - try: - return self.d.__cmp__(other) - except TypeError, le: - # maybe we should look for a .d member in other. I know this is insanely kludgey, but the Right Way To Do It is for dict.__cmp__ to use structural typing ("duck typing") - try: - return self.d.__cmp__(other.d) - except: - raise le - - def __eq__(self, *args, **kwargs): - return self.d.__eq__(*args, **kwargs) - - def __ne__(self, *args, **kwargs): - return self.d.__ne__(*args, **kwargs) - - def __gt__(self, *args, **kwargs): - return self.d.__gt__(*args, **kwargs) - - def __ge__(self, *args, **kwargs): - return self.d.__ge__(*args, **kwargs) - - def __le__(self, *args, **kwargs): - return self.d.__le__(*args, **kwargs) - - def __lt__(self, *args, **kwargs): - return self.d.__lt__(*args, **kwargs) - - def __getitem__(self, *args, **kwargs): - return self.d.__getitem__(*args, **kwargs) - - def __setitem__(self, *args, **kwargs): - return self.d.__setitem__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - return self.d.__delitem__(*args, **kwargs) - - def __iter__(self, *args, **kwargs): - return self.d.__iter__(*args, **kwargs) - - def clear(self, *args, **kwargs): - return self.d.clear(*args, **kwargs) - - def copy(self, *args, **kwargs): - return self.__class__(self.d.copy(*args, **kwargs)) - - def fromkeys(self, *args, **kwargs): - return self.__class__(self.d.fromkeys(*args, **kwargs)) - - def get(self, key, default=0): - return self.d.get(key, default) - - def has_key(self, *args, **kwargs): - return self.d.has_key(*args, **kwargs) - - def items(self, *args, **kwargs): - return self.d.items(*args, **kwargs) - - def iteritems(self, *args, **kwargs): - return self.d.iteritems(*args, **kwargs) - - def iterkeys(self, *args, **kwargs): - return self.d.iterkeys(*args, **kwargs) - - def itervalues(self, *args, **kwargs): - return self.d.itervalues(*args, **kwargs) - - def keys(self, *args, **kwargs): - return self.d.keys(*args, **kwargs) - - def pop(self, *args, **kwargs): - return self.d.pop(*args, **kwargs) - - def popitem(self, *args, **kwargs): - return self.d.popitem(*args, **kwargs) - - def setdefault(self, *args, **kwargs): - return self.d.setdefault(*args, **kwargs) - - def update(self, *args, **kwargs): - return self.d.update(*args, **kwargs) - - def values(self, *args, **kwargs): - return self.d.values(*args, **kwargs) - def del_if_present(d, k): if d.has_key(k): del d[k] From 4f493b1a030bb371d92d311b90f3bce9dd8b18a1 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 12 Aug 2017 21:31:47 -0700 Subject: [PATCH 3/5] remove unused UtilDict --- src/allmydata/test/test_util.py | 59 --------------- src/allmydata/util/dictutil.py | 122 -------------------------------- 2 files changed, 181 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index da8d4eb09..0a5c17149 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1408,11 +1408,8 @@ class DictUtil(unittest.TestCase): d[fake3] = 8 def test_all(self): - self._help_test_eq_but_notis(dictutil.UtilDict) self._help_test_eq_but_notis(dictutil.ValueOrderedDict) - self._help_test_nonempty_dict(dictutil.UtilDict) self._help_test_nonempty_dict(dictutil.ValueOrderedDict) - self._help_test_eq_but_notis(dictutil.UtilDict) self._help_test_eq_but_notis(dictutil.ValueOrderedDict) def test_dict_of_sets(self): @@ -1467,62 +1464,6 @@ class DictUtil(unittest.TestCase): d3 = dictutil.subtract(d1, d2) self.failUnlessEqual(d3, {1: "a", 3: "c"}) - def test_utildict(self): - d = dictutil.UtilDict({1: "a", 2: "b"}) - d.del_if_present(1) - d.del_if_present(3) - self.failUnlessEqual(d, {2: "b"}) - def eq(a, b): - return a == b - self.failUnlessRaises(TypeError, eq, d, "not a dict") - - d = dictutil.UtilDict({1: "b", 2: "a"}) - self.failUnlessEqual(d.items_sorted_by_value(), - [(2, "a"), (1, "b")]) - self.failUnlessEqual(d.items_sorted_by_key(), - [(1, "b"), (2, "a")]) - self.failUnlessEqual(repr(d), "{1: 'b', 2: 'a'}") - self.failUnless(1 in d) - - d2 = dictutil.UtilDict({3: "c", 4: "d"}) - self.failUnless(d != d2) - self.failUnless(d2 > d) - self.failUnless(d2 >= d) - self.failUnless(d <= d2) - self.failUnless(d < d2) - self.failUnlessEqual(d[1], "b") - self.failUnlessEqual(sorted(list([k for k in d])), [1,2]) - - d3 = d.copy() - self.failUnlessEqual(d, d3) - self.failUnless(isinstance(d3, dictutil.UtilDict)) - - d4 = d.fromkeys([3,4], "e") - self.failUnlessEqual(d4, {3: "e", 4: "e"}) - - self.failUnlessEqual(d.get(1), "b") - self.failUnlessEqual(d.get(3), None) - self.failUnlessEqual(d.get(3, "default"), "default") - self.failUnlessEqual(sorted(list(d.items())), - [(1, "b"), (2, "a")]) - self.failUnlessEqual(sorted(list(d.iteritems())), - [(1, "b"), (2, "a")]) - self.failUnlessEqual(sorted(d.keys()), [1, 2]) - self.failUnlessEqual(sorted(d.values()), ["a", "b"]) - x = d.setdefault(1, "new") - self.failUnlessEqual(x, "b") - self.failUnlessEqual(d[1], "b") - x = d.setdefault(3, "new") - self.failUnlessEqual(x, "new") - self.failUnlessEqual(d[3], "new") - del d[3] - - x = d.popitem() - self.failUnless(x in [(1, "b"), (2, "a")]) - x = d.popitem() - self.failUnless(x in [(1, "b"), (2, "a")]) - self.failUnlessRaises(KeyError, d.popitem) - def test_del_if_present(self): d = {1: "a", 2: "b"} dictutil.del_if_present(d, 1) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index fc2cdd445..9567941bb 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -54,128 +54,6 @@ class DictOfSets(dict): if not self[key]: del self[key] -class UtilDict: - def __init__(self, initialdata={}): - self.d = {} - self.update(initialdata) - - def del_if_present(self, key): - if self.has_key(key): - del self[key] - - def items_sorted_by_value(self): - """ - @return a sequence of (key, value,) pairs sorted according to value - """ - l = [(x[1], x[0],) for x in self.d.iteritems()] - l.sort() - return [(x[1], x[0],) for x in l] - - def items_sorted_by_key(self): - """ - @return a sequence of (key, value,) pairs sorted according to key - """ - l = self.d.items() - l.sort() - return l - - def __repr__(self, *args, **kwargs): - return self.d.__repr__(*args, **kwargs) - - def __str__(self, *args, **kwargs): - return self.d.__str__(*args, **kwargs) - - def __contains__(self, *args, **kwargs): - return self.d.__contains__(*args, **kwargs) - - def __len__(self, *args, **kwargs): - return self.d.__len__(*args, **kwargs) - - def __cmp__(self, other): - try: - return self.d.__cmp__(other) - except TypeError, le: - # maybe we should look for a .d member in other. I know this is insanely kludgey, but the Right Way To Do It is for dict.__cmp__ to use structural typing ("duck typing") - try: - return self.d.__cmp__(other.d) - except: - raise le - - def __eq__(self, *args, **kwargs): - return self.d.__eq__(*args, **kwargs) - - def __ne__(self, *args, **kwargs): - return self.d.__ne__(*args, **kwargs) - - def __gt__(self, *args, **kwargs): - return self.d.__gt__(*args, **kwargs) - - def __ge__(self, *args, **kwargs): - return self.d.__ge__(*args, **kwargs) - - def __le__(self, *args, **kwargs): - return self.d.__le__(*args, **kwargs) - - def __lt__(self, *args, **kwargs): - return self.d.__lt__(*args, **kwargs) - - def __getitem__(self, *args, **kwargs): - return self.d.__getitem__(*args, **kwargs) - - def __setitem__(self, *args, **kwargs): - return self.d.__setitem__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - return self.d.__delitem__(*args, **kwargs) - - def __iter__(self, *args, **kwargs): - return self.d.__iter__(*args, **kwargs) - - def clear(self, *args, **kwargs): - return self.d.clear(*args, **kwargs) - - def copy(self, *args, **kwargs): - return self.__class__(self.d.copy(*args, **kwargs)) - - def fromkeys(self, *args, **kwargs): - return self.__class__(self.d.fromkeys(*args, **kwargs)) - - def get(self, key, default=None): - return self.d.get(key, default) - - def has_key(self, *args, **kwargs): - return self.d.has_key(*args, **kwargs) - - def items(self, *args, **kwargs): - return self.d.items(*args, **kwargs) - - def iteritems(self, *args, **kwargs): - return self.d.iteritems(*args, **kwargs) - - def iterkeys(self, *args, **kwargs): - return self.d.iterkeys(*args, **kwargs) - - def itervalues(self, *args, **kwargs): - return self.d.itervalues(*args, **kwargs) - - def keys(self, *args, **kwargs): - return self.d.keys(*args, **kwargs) - - def pop(self, *args, **kwargs): - return self.d.pop(*args, **kwargs) - - def popitem(self, *args, **kwargs): - return self.d.popitem(*args, **kwargs) - - def setdefault(self, *args, **kwargs): - return self.d.setdefault(*args, **kwargs) - - def update(self, *args, **kwargs): - self.d.update(*args, **kwargs) - - def values(self, *args, **kwargs): - return self.d.values(*args, **kwargs) - def del_if_present(d, k): if d.has_key(k): del d[k] From 61b85dbf7846bc37b4782fc8a5f9ae80962c26c5 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 12 Aug 2017 21:34:24 -0700 Subject: [PATCH 4/5] dictutil: remove unused ValueOrderedDict --- src/allmydata/test/test_util.py | 67 -------- src/allmydata/util/dictutil.py | 270 -------------------------------- 2 files changed, 337 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 0a5c17149..14709ec4c 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1407,11 +1407,6 @@ class DictUtil(unittest.TestCase): self.failUnless(filter(lambda x: x is 3, d.iterkeys())) d[fake3] = 8 - def test_all(self): - self._help_test_eq_but_notis(dictutil.ValueOrderedDict) - self._help_test_nonempty_dict(dictutil.ValueOrderedDict) - self._help_test_eq_but_notis(dictutil.ValueOrderedDict) - def test_dict_of_sets(self): ds = dictutil.DictOfSets() ds.add(1, "a") @@ -1470,68 +1465,6 @@ class DictUtil(unittest.TestCase): dictutil.del_if_present(d, 3) self.failUnlessEqual(d, {2: "b"}) - def test_valueordereddict(self): - d = dictutil.ValueOrderedDict() - d["a"] = 3 - d["b"] = 2 - d["c"] = 1 - - self.failUnlessEqual(d, {"a": 3, "b": 2, "c": 1}) - self.failUnlessEqual(d.items(), [("c", 1), ("b", 2), ("a", 3)]) - self.failUnlessEqual(d.values(), [1, 2, 3]) - self.failUnlessEqual(d.keys(), ["c", "b", "a"]) - self.failUnlessEqual(repr(d), "") - self.failUnlessEqual(str(d), "") - # str() is supposed to only show the first 16 entries - large_d = dictutil.ValueOrderedDict() - for i in range(20): - large_d["k%d" % i] = i - large_d_repr = ("" % - ", ".join(["k%d: %d" % (i, i) for i in range(16)])) - self.failUnlessEqual(str(large_d), large_d_repr) - - def eq(a, b): - return a == b - self.failIf(d == {"a": 4}) - self.failUnless(d != {"a": 4}) - - x = d.setdefault("d", 0) - self.failUnlessEqual(x, 0) - self.failUnlessEqual(d["d"], 0) - x = d.setdefault("d", -1) - self.failUnlessEqual(x, 0) - self.failUnlessEqual(d["d"], 0) - - x = d.remove("e", "default", False) - self.failUnlessEqual(x, "default") - self.failUnlessRaises(KeyError, d.remove, "e", "default", True) - x = d.remove("d", 5) - self.failUnlessEqual(x, 0) - - x = d.__getitem__("c") - self.failUnlessEqual(x, 1) - x = d.__getitem__("e", "default", False) - self.failUnlessEqual(x, "default") - self.failUnlessRaises(KeyError, d.__getitem__, "e", "default", True) - - self.failUnlessEqual(d.popitem(), ("c", 1)) - self.failUnlessEqual(d.popitem(), ("b", 2)) - self.failUnlessEqual(d.popitem(), ("a", 3)) - self.failUnlessRaises(KeyError, d.popitem) - - d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1}) - x = d.pop("d", "default", False) - self.failUnlessEqual(x, "default") - self.failUnlessRaises(KeyError, d.pop, "d", "default", True) - x = d.pop("b") - self.failUnlessEqual(x, 2) - self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)]) - - d = dictutil.ValueOrderedDict({"a": 3, "b": 2, "c": 1}) - x = d.pop_from_list(1) # pop the second item, b/2 - self.failUnlessEqual(x, "b") - self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)]) - def test_auxdict(self): d = dictutil.AuxValueDict() # we put the serialized form in the auxdata diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 9567941bb..bbc28cde9 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,11 +2,6 @@ Tools to mess with dicts. """ -import operator -from bisect import bisect_left, insort_left - -from allmydata.util.assertutil import _assert, precondition - def move(k, d1, d2, strict=False): """ Move item with key k from d1 to d2. @@ -58,271 +53,6 @@ def del_if_present(d, k): if d.has_key(k): del d[k] -class ValueOrderedDict: - """ - Note: this implementation assumes that the values do not mutate and change - their sort order. That is, it stores the values in a sorted list and - as items are added and removed from the dict, it makes updates to the list - which will keep the list sorted. But if a value that is currently sitting - in the list changes its sort order, then the internal consistency of this - object will be lost. - - If that happens, and if assertion checking is turned on, then you will get - an assertion failure the very next time you try to do anything with this - ValueOrderedDict. However, those internal consistency checks are very slow - and almost certainly unacceptable to leave turned on in production code. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) - precondition((self.i == len(self.c.l)) or self.c.d.has_key(self.c.l[self.i][1]), "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) - if self.i == len(self.c.l): - raise StopIteration - le = self.c.l[self.i] - self.i += 1 - return (le[1], le[0],) - - def iteritems(self): - return ValueOrderedDict.ItemIterator(self) - - def items(self): - return zip(map(operator.__getitem__, self.l, [1]*len(self.l)), map(operator.__getitem__, self.l, [0]*len(self.l))) - - def values(self): - return map(operator.__getitem__, self.l, [0]*len(self.l)) - - def keys(self): - return map(operator.__getitem__, self.l, [1]*len(self.l)) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) - precondition((self.i == len(self.c.l)) or self.c.d.has_key(self.c.l[self.i][1]), "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) - if self.i == len(self.c.l): - raise StopIteration - le = self.c.l[self.i] - self.i += 1 - return le[1] - - def iterkeys(self): - return ValueOrderedDict.KeyIterator(self) - - class ValueIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) - precondition((self.i == len(self.c.l)) or self.c.d.has_key(self.c.l[self.i][1]), "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) - if self.i == len(self.c.l): - raise StopIteration - le = self.c.l[self.i] - self.i += 1 - return le[0] - - def itervalues(self): - return ValueOrderedDict.ValueIterator(self) - - def __init__(self, initialdata={}): - self.d = {} # k: key, v: val - self.l = [] # sorted list of tuples of (val, key,) - self.update(initialdata) - assert self._assert_invariants() - - def __len__(self): - return len(self.l) - - def __repr_n__(self, n=None): - s = ["{",] - try: - iter = self.iteritems() - x = iter.next() - s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - i = 1 - while (n is None) or (i < n): - i += 1 - x = iter.next() - s.append(", "); - s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - # if we get here, we're truncating the repr, so make that clear - s.append(", ...") - except StopIteration: - pass - s.append("}") - return ''.join(s) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) - - def __eq__(self, other): - for (k, v,) in other.iteritems(): - if not self.d.has_key(k) or self.d[k] != v: - return False - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def _assert_invariants(self): - iter = self.l.__iter__() - try: - oldx = iter.next() - while True: - x = iter.next() - # self.l is required to be sorted - _assert(x >= oldx, x, oldx) - # every element of self.l is required to appear in self.d - _assert(self.d.has_key(x[1]), x) - oldx =x - except StopIteration: - pass - for (k, v,) in self.d.iteritems(): - i = bisect_left(self.l, (v, k,)) - while (self.l[i][0] is not v) or (self.l[i][1] is not k): - i += 1 - _assert(i < len(self.l), i, len(self.l), k, v, self.l) - _assert(self.l[i][0] is v, i, v, l=self.l, d=self.d) - _assert(self.l[i][1] is k, i, k, l=self.l, d=self.d) - return True - - def insert(self, key, val=None): - assert self._assert_invariants() - result = self.__setitem__(key, val) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def __setitem__(self, key, val=None): - assert self._assert_invariants() - if self.d.has_key(key): - oldval = self.d[key] - if oldval != val: - # re-sort - i = bisect_left(self.l, (oldval, key,)) - while (self.l[i][0] is not oldval) or (self.l[i][1] is not key): - i += 1 - self.l.pop(i) - insort_left(self.l, (val, key,)) - elif oldval is not val: - # replace - i = bisect_left(self.l, (oldval, key,)) - while (self.l[i][0] is not oldval) or (self.l[i][1] is not key): - i += 1 - self.l[i] = (val, key,) - else: - insort_left(self.l, (val, key,)) - - self.d[key] = val - assert self._assert_invariants() - return val - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - - def __getitem__(self, key, default=None, strictkey=True): - if not self.d.has_key(key): - if strictkey: - raise KeyError, key - else: - return default - return self.d[key] - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the object removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if self.d.has_key(key): - val = self.d.pop(key) - i = bisect_left(self.l, (val, key,)) - while (self.l[i][0] is not val) or (self.l[i][1] is not key): - i += 1 - self.l.pop(i) - assert self._assert_invariants() - return val - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def clear(self): - assert self._assert_invariants() - self.d.clear() - del self.l[:] - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - for (k, v,) in otherdict.iteritems(): - self.insert(k, v) - assert self._assert_invariants() - return self - - def has_key(self, key): - assert self._assert_invariants() - return self.d.has_key(key) - - def popitem(self): - if not self.l: - raise KeyError, 'popitem(): dictionary is empty' - le = self.l.pop(0) - del self.d[le[1]] - return (le[1], le[0],) - - def pop(self, k, default=None, strictkey=False): - if not self.d.has_key(k): - if strictkey: - raise KeyError, k - else: - return default - v = self.d.pop(k) - i = bisect_left(self.l, (v, k,)) - while (self.l[i][0] is not v) or (self.l[i][1] is not k): - i += 1 - self.l.pop(i) - return v - - def pop_from_list(self, i=0): - le = self.l.pop(i) - del self.d[le[1]] - return le[1] - class AuxValueDict(dict): """I behave like a regular dict, but each key is associated with two values: the main value, and an auxilliary one. Setting the main value From 8dd0a5f772a43886f72182478a3bc54ebd2f4fbf Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 12 Aug 2017 21:48:05 -0700 Subject: [PATCH 5/5] dictutil: remove unused move, subtract, del_if_present --- src/allmydata/test/test_util.py | 34 --------------------------------- src/allmydata/util/dictutil.py | 30 ----------------------------- 2 files changed, 64 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 14709ec4c..b264dedfa 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1431,40 +1431,6 @@ class DictUtil(unittest.TestCase): self.failUnlessEqual(ds[3], set(["f", "g"])) self.failUnlessEqual(ds[4], set(["h"])) - def test_move(self): - d1 = {1: "a", 2: "b"} - d2 = {2: "c", 3: "d"} - dictutil.move(1, d1, d2) - self.failUnlessEqual(d1, {2: "b"}) - self.failUnlessEqual(d2, {1: "a", 2: "c", 3: "d"}) - - d1 = {1: "a", 2: "b"} - d2 = {2: "c", 3: "d"} - dictutil.move(2, d1, d2) - self.failUnlessEqual(d1, {1: "a"}) - self.failUnlessEqual(d2, {2: "b", 3: "d"}) - - d1 = {1: "a", 2: "b"} - d2 = {2: "c", 3: "d"} - self.failUnlessRaises(KeyError, dictutil.move, 5, d1, d2, strict=True) - - def test_subtract(self): - d1 = {1: "a", 2: "b"} - d2 = {2: "c", 3: "d"} - d3 = dictutil.subtract(d1, d2) - self.failUnlessEqual(d3, {1: "a"}) - - d1 = {1: "a", 2: "b", 3: "c"} - d2 = {2: "c", 4: "d"} - d3 = dictutil.subtract(d1, d2) - self.failUnlessEqual(d3, {1: "a", 3: "c"}) - - def test_del_if_present(self): - d = {1: "a", 2: "b"} - dictutil.del_if_present(d, 1) - dictutil.del_if_present(d, 3) - self.failUnlessEqual(d, {2: "b"}) - def test_auxdict(self): d = dictutil.AuxValueDict() # we put the serialized form in the auxdata diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index bbc28cde9..d59e60dd6 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,32 +2,6 @@ Tools to mess with dicts. """ -def move(k, d1, d2, strict=False): - """ - Move item with key k from d1 to d2. - """ - if strict and not d1.has_key(k): - raise KeyError, k - - d2[k] = d1[k] - del d1[k] - -def subtract(d1, d2): - """ - Remove all items from d1 whose key occurs in d2. - - @returns d1 - """ - if len(d1) > len(d2): - for k in d2.keys(): - if d1.has_key(k): - del d1[k] - else: - for k in d1.keys(): - if d2.has_key(k): - del d1[k] - return d1 - class DictOfSets(dict): def add(self, key, value): if key in self: @@ -49,10 +23,6 @@ class DictOfSets(dict): if not self[key]: del self[key] -def del_if_present(d, k): - if d.has_key(k): - del d[k] - class AuxValueDict(dict): """I behave like a regular dict, but each key is associated with two values: the main value, and an auxilliary one. Setting the main value