From 44143d1b083acf23b93bbfb5eb37a23e9af22934 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:37:44 -0400 Subject: [PATCH] Move tests for happinessutil.py into test_happiness.py. --- src/allmydata/test/common_py3.py | 26 +++- src/allmydata/test/common_util.py | 25 +--- src/allmydata/test/test_happiness.py | 198 ++++++++++++++++++++++++++- src/allmydata/test/test_upload.py | 191 +------------------------- 4 files changed, 224 insertions(+), 216 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 0dae05aa6..b29cb9691 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -17,7 +17,8 @@ import os import time import signal -from twisted.internet import reactor +from twisted.internet import defer, reactor +from twisted.python import failure class TimezoneMixin(object): @@ -65,3 +66,26 @@ class SignalMixin(object): if self.sigchldHandler: signal.signal(signal.SIGCHLD, self.sigchldHandler) return super(SignalMixin, self).tearDown() + + +class ShouldFailMixin(object): + + def shouldFail(self, expected_failure, which, substring, + callable, *args, **kwargs): + assert substring is None or isinstance(substring, str) + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + res.trap(expected_failure) + if substring: + self.failUnless(substring in str(res), + "%s: substring '%s' not in '%s'" + % (which, substring, str(res))) + # return the Failure for further analysis, but in a form that + # doesn't make the Deferred chain think that we failed. + return [res] + else: + self.fail("%s was supposed to raise %s, not get '%s'" % + (which, expected_failure, res)) + d.addBoth(done) + return d diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 261d39af3..684aee5ba 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -5,14 +5,13 @@ from random import randrange from six.moves import StringIO from twisted.internet import reactor, defer -from twisted.python import failure from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) from ..scripts import runner -from .common_py3 import SignalMixin +from .common_py3 import SignalMixin, ShouldFailMixin # noqa def skip_if_cannot_represent_filename(u): @@ -96,28 +95,6 @@ class StallMixin(object): reactor.callLater(delay, d.callback, res) return d -class ShouldFailMixin(object): - - def shouldFail(self, expected_failure, which, substring, - callable, *args, **kwargs): - assert substring is None or isinstance(substring, str) - d = defer.maybeDeferred(callable, *args, **kwargs) - def done(res): - if isinstance(res, failure.Failure): - res.trap(expected_failure) - if substring: - self.failUnless(substring in str(res), - "%s: substring '%s' not in '%s'" - % (which, substring, str(res))) - # return the Failure for further analysis, but in a form that - # doesn't make the Deferred chain think that we failed. - return [res] - else: - self.fail("%s was supposed to raise %s, not get '%s'" % - (which, expected_failure, res)) - d.addBoth(done) - return d - class TestMixin(SignalMixin): def setUp(self): diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index ffba402ef..0f7ee13d5 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -13,12 +13,17 @@ if PY2: from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets + from allmydata.immutable import happiness_upload +from allmydata.util.happinessutil import servers_of_happiness, \ + shares_by_server, merge_servers +from allmydata.test.common_py3 import ShouldFailMixin -class HappinessUtils(unittest.TestCase): +class HappinessUploadUtils(unittest.TestCase): """ - test-cases for utility functions augmenting_path_for and residual_network + test-cases for happiness_upload utility functions augmenting_path_for and + residual_network. """ def test_residual_0(self): @@ -279,3 +284,192 @@ class PlacementTests(unittest.TestCase): # peers; if we have fewer shares than peers happiness is capped at # # of peers. assert happiness == min(len(peers), len(shares)) + + +class FakeServerTracker(object): + def __init__(self, serverid, buckets): + self._serverid = serverid + self.buckets = buckets + def get_serverid(self): + return self._serverid + + +class HappinessUtilTests(unittest.TestCase, ShouldFailMixin): + """Tests for happinesutil.py.""" + + def test_merge_servers(self): + # merge_servers merges a list of upload_servers and a dict of + # shareid -> serverid mappings. + shares = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4", "server5"]), + 5 : set(["server1", "server2"]), + } + # if not provided with a upload_servers argument, it should just + # return the first argument unchanged. + self.failUnlessEqual(shares, merge_servers(shares, set([]))) + trackers = [] + for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]: + t = FakeServerTracker(server, [i]) + trackers.append(t) + expected = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4", "server5"]), + 5 : set(["server1", "server2", "server5"]), + 6 : set(["server6"]), + 7 : set(["server7"]), + 8 : set(["server8"]), + } + self.failUnlessEqual(expected, merge_servers(shares, set(trackers))) + shares2 = {} + expected = { + 5 : set(["server5"]), + 6 : set(["server6"]), + 7 : set(["server7"]), + 8 : set(["server8"]), + } + self.failUnlessEqual(expected, merge_servers(shares2, set(trackers))) + shares3 = {} + trackers = [] + expected = {} + for (i, server) in [(i, "server%d" % i) for i in range(10)]: + shares3[i] = set([server]) + t = FakeServerTracker(server, [i]) + trackers.append(t) + expected[i] = set([server]) + self.failUnlessEqual(expected, merge_servers(shares3, set(trackers))) + + + def test_servers_of_happiness_utility_function(self): + # These tests are concerned with the servers_of_happiness() + # utility function, and its underlying matching algorithm. Other + # aspects of the servers_of_happiness behavior are tested + # elsehwere These tests exist to ensure that + # servers_of_happiness doesn't under or overcount the happiness + # value for given inputs. + + # servers_of_happiness expects a dict of + # shnum => set(serverids) as a preexisting shares argument. + test1 = { + 1 : set(["server1"]), + 2 : set(["server2"]), + 3 : set(["server3"]), + 4 : set(["server4"]) + } + happy = servers_of_happiness(test1) + self.failUnlessEqual(4, happy) + test1[4] = set(["server1"]) + # We've added a duplicate server, so now servers_of_happiness + # should be 3 instead of 4. + happy = servers_of_happiness(test1) + self.failUnlessEqual(3, happy) + # The second argument of merge_servers should be a set of objects with + # serverid and buckets as attributes. In actual use, these will be + # ServerTracker instances, but for testing it is fine to make a + # FakeServerTracker whose job is to hold those instance variables to + # test that part. + trackers = [] + for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]: + t = FakeServerTracker(server, [i]) + trackers.append(t) + # Recall that test1 is a server layout with servers_of_happiness + # = 3. Since there isn't any overlap between the shnum -> + # set([serverid]) correspondences in test1 and those in trackers, + # the result here should be 7. + test2 = merge_servers(test1, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(7, happy) + # Now add an overlapping server to trackers. This is redundant, + # so it should not cause the previously reported happiness value + # to change. + t = FakeServerTracker("server1", [1]) + trackers.append(t) + test2 = merge_servers(test1, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(7, happy) + test = {} + happy = servers_of_happiness(test) + self.failUnlessEqual(0, happy) + # Test a more substantial overlap between the trackers and the + # existing assignments. + test = { + 1 : set(['server1']), + 2 : set(['server2']), + 3 : set(['server3']), + 4 : set(['server4']), + } + trackers = [] + t = FakeServerTracker('server5', [4]) + trackers.append(t) + t = FakeServerTracker('server6', [3, 5]) + trackers.append(t) + # The value returned by servers_of_happiness is the size + # of a maximum matching in the bipartite graph that + # servers_of_happiness() makes between serverids and share + # numbers. It should find something like this: + # (server 1, share 1) + # (server 2, share 2) + # (server 3, share 3) + # (server 5, share 4) + # (server 6, share 5) + # + # and, since there are 5 edges in this matching, it should + # return 5. + test2 = merge_servers(test, set(trackers)) + happy = servers_of_happiness(test2) + self.failUnlessEqual(5, happy) + # Zooko's first puzzle: + # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156) + # + # server 1: shares 0, 1 + # server 2: shares 1, 2 + # server 3: share 2 + # + # This should yield happiness of 3. + test = { + 0 : set(['server1']), + 1 : set(['server1', 'server2']), + 2 : set(['server2', 'server3']), + } + self.failUnlessEqual(3, servers_of_happiness(test)) + # Zooko's second puzzle: + # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158) + # + # server 1: shares 0, 1 + # server 2: share 1 + # + # This should yield happiness of 2. + test = { + 0 : set(['server1']), + 1 : set(['server1', 'server2']), + } + self.failUnlessEqual(2, servers_of_happiness(test)) + + + def test_shares_by_server(self): + test = dict([(i, set(["server%d" % i])) for i in range(1, 5)]) + sbs = shares_by_server(test) + self.failUnlessEqual(set([1]), sbs["server1"]) + self.failUnlessEqual(set([2]), sbs["server2"]) + self.failUnlessEqual(set([3]), sbs["server3"]) + self.failUnlessEqual(set([4]), sbs["server4"]) + test1 = { + 1 : set(["server1"]), + 2 : set(["server1"]), + 3 : set(["server1"]), + 4 : set(["server2"]), + 5 : set(["server2"]) + } + sbs = shares_by_server(test1) + self.failUnlessEqual(set([1, 2, 3]), sbs["server1"]) + self.failUnlessEqual(set([4, 5]), sbs["server2"]) + # This should fail unless the serverid part of the mapping is a set + test2 = {1: "server1"} + self.shouldFail(AssertionError, + "test_shares_by_server", + "", + shares_by_server, test2) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 165ca17d7..2ec426722 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -15,17 +15,15 @@ from allmydata.util import log, base32 from allmydata.util.assertutil import precondition from allmydata.util.deferredutil import DeferredListShouldSucceed from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_util import ShouldFailMixin -from allmydata.util.happinessutil import servers_of_happiness, \ - shares_by_server, merge_servers +from allmydata.test.common_py3 import ShouldFailMixin from allmydata.storage_client import StorageFarmBroker from allmydata.storage.server import storage_index_to_dir from allmydata.client import _Client - from .common import ( EMPTY_CLIENT_CONFIG, ) + MiB = 1024*1024 def extract_uri(results): @@ -864,12 +862,6 @@ def is_happy_enough(servertoshnums, h, k): return False return True -class FakeServerTracker(object): - def __init__(self, serverid, buckets): - self._serverid = serverid - self.buckets = buckets - def get_serverid(self): - return self._serverid class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, ShouldFailMixin): @@ -1499,185 +1491,6 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, self._do_upload_with_broken_servers, 2)) return d - - def test_merge_servers(self): - # merge_servers merges a list of upload_servers and a dict of - # shareid -> serverid mappings. - shares = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4", "server5"]), - 5 : set(["server1", "server2"]), - } - # if not provided with a upload_servers argument, it should just - # return the first argument unchanged. - self.failUnlessEqual(shares, merge_servers(shares, set([]))) - trackers = [] - for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]: - t = FakeServerTracker(server, [i]) - trackers.append(t) - expected = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4", "server5"]), - 5 : set(["server1", "server2", "server5"]), - 6 : set(["server6"]), - 7 : set(["server7"]), - 8 : set(["server8"]), - } - self.failUnlessEqual(expected, merge_servers(shares, set(trackers))) - shares2 = {} - expected = { - 5 : set(["server5"]), - 6 : set(["server6"]), - 7 : set(["server7"]), - 8 : set(["server8"]), - } - self.failUnlessEqual(expected, merge_servers(shares2, set(trackers))) - shares3 = {} - trackers = [] - expected = {} - for (i, server) in [(i, "server%d" % i) for i in xrange(10)]: - shares3[i] = set([server]) - t = FakeServerTracker(server, [i]) - trackers.append(t) - expected[i] = set([server]) - self.failUnlessEqual(expected, merge_servers(shares3, set(trackers))) - - - def test_servers_of_happiness_utility_function(self): - # These tests are concerned with the servers_of_happiness() - # utility function, and its underlying matching algorithm. Other - # aspects of the servers_of_happiness behavior are tested - # elsehwere These tests exist to ensure that - # servers_of_happiness doesn't under or overcount the happiness - # value for given inputs. - - # servers_of_happiness expects a dict of - # shnum => set(serverids) as a preexisting shares argument. - test1 = { - 1 : set(["server1"]), - 2 : set(["server2"]), - 3 : set(["server3"]), - 4 : set(["server4"]) - } - happy = servers_of_happiness(test1) - self.failUnlessEqual(4, happy) - test1[4] = set(["server1"]) - # We've added a duplicate server, so now servers_of_happiness - # should be 3 instead of 4. - happy = servers_of_happiness(test1) - self.failUnlessEqual(3, happy) - # The second argument of merge_servers should be a set of objects with - # serverid and buckets as attributes. In actual use, these will be - # ServerTracker instances, but for testing it is fine to make a - # FakeServerTracker whose job is to hold those instance variables to - # test that part. - trackers = [] - for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]: - t = FakeServerTracker(server, [i]) - trackers.append(t) - # Recall that test1 is a server layout with servers_of_happiness - # = 3. Since there isn't any overlap between the shnum -> - # set([serverid]) correspondences in test1 and those in trackers, - # the result here should be 7. - test2 = merge_servers(test1, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(7, happy) - # Now add an overlapping server to trackers. This is redundant, - # so it should not cause the previously reported happiness value - # to change. - t = FakeServerTracker("server1", [1]) - trackers.append(t) - test2 = merge_servers(test1, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(7, happy) - test = {} - happy = servers_of_happiness(test) - self.failUnlessEqual(0, happy) - # Test a more substantial overlap between the trackers and the - # existing assignments. - test = { - 1 : set(['server1']), - 2 : set(['server2']), - 3 : set(['server3']), - 4 : set(['server4']), - } - trackers = [] - t = FakeServerTracker('server5', [4]) - trackers.append(t) - t = FakeServerTracker('server6', [3, 5]) - trackers.append(t) - # The value returned by servers_of_happiness is the size - # of a maximum matching in the bipartite graph that - # servers_of_happiness() makes between serverids and share - # numbers. It should find something like this: - # (server 1, share 1) - # (server 2, share 2) - # (server 3, share 3) - # (server 5, share 4) - # (server 6, share 5) - # - # and, since there are 5 edges in this matching, it should - # return 5. - test2 = merge_servers(test, set(trackers)) - happy = servers_of_happiness(test2) - self.failUnlessEqual(5, happy) - # Zooko's first puzzle: - # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156) - # - # server 1: shares 0, 1 - # server 2: shares 1, 2 - # server 3: share 2 - # - # This should yield happiness of 3. - test = { - 0 : set(['server1']), - 1 : set(['server1', 'server2']), - 2 : set(['server2', 'server3']), - } - self.failUnlessEqual(3, servers_of_happiness(test)) - # Zooko's second puzzle: - # (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158) - # - # server 1: shares 0, 1 - # server 2: share 1 - # - # This should yield happiness of 2. - test = { - 0 : set(['server1']), - 1 : set(['server1', 'server2']), - } - self.failUnlessEqual(2, servers_of_happiness(test)) - - - def test_shares_by_server(self): - test = dict([(i, set(["server%d" % i])) for i in xrange(1, 5)]) - sbs = shares_by_server(test) - self.failUnlessEqual(set([1]), sbs["server1"]) - self.failUnlessEqual(set([2]), sbs["server2"]) - self.failUnlessEqual(set([3]), sbs["server3"]) - self.failUnlessEqual(set([4]), sbs["server4"]) - test1 = { - 1 : set(["server1"]), - 2 : set(["server1"]), - 3 : set(["server1"]), - 4 : set(["server2"]), - 5 : set(["server2"]) - } - sbs = shares_by_server(test1) - self.failUnlessEqual(set([1, 2, 3]), sbs["server1"]) - self.failUnlessEqual(set([4, 5]), sbs["server2"]) - # This should fail unless the serverid part of the mapping is a set - test2 = {1: "server1"} - self.shouldFail(AssertionError, - "test_shares_by_server", - "", - shares_by_server, test2) - - def test_existing_share_detection(self): self.basedir = self.mktemp() d = self._setup_and_upload()