mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-04-08 19:34:18 +00:00
Move tests for happinessutil.py into test_happiness.py.
This commit is contained in:
Itamar Turner-Trauring
parent
4d2193fe13
commit
44143d1b08
@ -17,7 +17,8 @@ import os
|
||||
import time
|
||||
import signal
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.python import failure
|
||||
|
||||
|
||||
class TimezoneMixin(object):
|
||||
@ -65,3 +66,26 @@ class SignalMixin(object):
|
||||
if self.sigchldHandler:
|
||||
signal.signal(signal.SIGCHLD, self.sigchldHandler)
|
||||
return super(SignalMixin, self).tearDown()
|
||||
|
||||
|
||||
class ShouldFailMixin(object):
|
||||
|
||||
def shouldFail(self, expected_failure, which, substring,
|
||||
callable, *args, **kwargs):
|
||||
assert substring is None or isinstance(substring, str)
|
||||
d = defer.maybeDeferred(callable, *args, **kwargs)
|
||||
def done(res):
|
||||
if isinstance(res, failure.Failure):
|
||||
res.trap(expected_failure)
|
||||
if substring:
|
||||
self.failUnless(substring in str(res),
|
||||
"%s: substring '%s' not in '%s'"
|
||||
% (which, substring, str(res)))
|
||||
# return the Failure for further analysis, but in a form that
|
||||
# doesn't make the Deferred chain think that we failed.
|
||||
return [res]
|
||||
else:
|
||||
self.fail("%s was supposed to raise %s, not get '%s'" %
|
||||
(which, expected_failure, res))
|
||||
d.addBoth(done)
|
||||
return d
|
||||
|
||||
@ -5,14 +5,13 @@ from random import randrange
|
||||
from six.moves import StringIO
|
||||
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.python import failure
|
||||
from twisted.trial import unittest
|
||||
|
||||
from ..util.assertutil import precondition
|
||||
from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
|
||||
get_io_encoding)
|
||||
from ..scripts import runner
|
||||
from .common_py3 import SignalMixin
|
||||
from .common_py3 import SignalMixin, ShouldFailMixin # noqa
|
||||
|
||||
|
||||
def skip_if_cannot_represent_filename(u):
|
||||
@ -96,28 +95,6 @@ class StallMixin(object):
|
||||
reactor.callLater(delay, d.callback, res)
|
||||
return d
|
||||
|
||||
class ShouldFailMixin(object):
|
||||
|
||||
def shouldFail(self, expected_failure, which, substring,
|
||||
callable, *args, **kwargs):
|
||||
assert substring is None or isinstance(substring, str)
|
||||
d = defer.maybeDeferred(callable, *args, **kwargs)
|
||||
def done(res):
|
||||
if isinstance(res, failure.Failure):
|
||||
res.trap(expected_failure)
|
||||
if substring:
|
||||
self.failUnless(substring in str(res),
|
||||
"%s: substring '%s' not in '%s'"
|
||||
% (which, substring, str(res)))
|
||||
# return the Failure for further analysis, but in a form that
|
||||
# doesn't make the Deferred chain think that we failed.
|
||||
return [res]
|
||||
else:
|
||||
self.fail("%s was supposed to raise %s, not get '%s'" %
|
||||
(which, expected_failure, res))
|
||||
d.addBoth(done)
|
||||
return d
|
||||
|
||||
|
||||
class TestMixin(SignalMixin):
|
||||
def setUp(self):
|
||||
|
||||
@ -13,12 +13,17 @@ if PY2:
|
||||
from twisted.trial import unittest
|
||||
from hypothesis import given
|
||||
from hypothesis.strategies import text, sets
|
||||
|
||||
from allmydata.immutable import happiness_upload
|
||||
from allmydata.util.happinessutil import servers_of_happiness, \
|
||||
shares_by_server, merge_servers
|
||||
from allmydata.test.common_py3 import ShouldFailMixin
|
||||
|
||||
|
||||
class HappinessUtils(unittest.TestCase):
|
||||
class HappinessUploadUtils(unittest.TestCase):
|
||||
"""
|
||||
test-cases for utility functions augmenting_path_for and residual_network
|
||||
test-cases for happiness_upload utility functions augmenting_path_for and
|
||||
residual_network.
|
||||
"""
|
||||
|
||||
def test_residual_0(self):
|
||||
@ -279,3 +284,192 @@ class PlacementTests(unittest.TestCase):
|
||||
# peers; if we have fewer shares than peers happiness is capped at
|
||||
# # of peers.
|
||||
assert happiness == min(len(peers), len(shares))
|
||||
|
||||
|
||||
class FakeServerTracker(object):
|
||||
def __init__(self, serverid, buckets):
|
||||
self._serverid = serverid
|
||||
self.buckets = buckets
|
||||
def get_serverid(self):
|
||||
return self._serverid
|
||||
|
||||
|
||||
class HappinessUtilTests(unittest.TestCase, ShouldFailMixin):
|
||||
"""Tests for happinesutil.py."""
|
||||
|
||||
def test_merge_servers(self):
|
||||
# merge_servers merges a list of upload_servers and a dict of
|
||||
# shareid -> serverid mappings.
|
||||
shares = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4", "server5"]),
|
||||
5 : set(["server1", "server2"]),
|
||||
}
|
||||
# if not provided with a upload_servers argument, it should just
|
||||
# return the first argument unchanged.
|
||||
self.failUnlessEqual(shares, merge_servers(shares, set([])))
|
||||
trackers = []
|
||||
for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]:
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
expected = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4", "server5"]),
|
||||
5 : set(["server1", "server2", "server5"]),
|
||||
6 : set(["server6"]),
|
||||
7 : set(["server7"]),
|
||||
8 : set(["server8"]),
|
||||
}
|
||||
self.failUnlessEqual(expected, merge_servers(shares, set(trackers)))
|
||||
shares2 = {}
|
||||
expected = {
|
||||
5 : set(["server5"]),
|
||||
6 : set(["server6"]),
|
||||
7 : set(["server7"]),
|
||||
8 : set(["server8"]),
|
||||
}
|
||||
self.failUnlessEqual(expected, merge_servers(shares2, set(trackers)))
|
||||
shares3 = {}
|
||||
trackers = []
|
||||
expected = {}
|
||||
for (i, server) in [(i, "server%d" % i) for i in range(10)]:
|
||||
shares3[i] = set([server])
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
expected[i] = set([server])
|
||||
self.failUnlessEqual(expected, merge_servers(shares3, set(trackers)))
|
||||
|
||||
|
||||
def test_servers_of_happiness_utility_function(self):
|
||||
# These tests are concerned with the servers_of_happiness()
|
||||
# utility function, and its underlying matching algorithm. Other
|
||||
# aspects of the servers_of_happiness behavior are tested
|
||||
# elsehwere These tests exist to ensure that
|
||||
# servers_of_happiness doesn't under or overcount the happiness
|
||||
# value for given inputs.
|
||||
|
||||
# servers_of_happiness expects a dict of
|
||||
# shnum => set(serverids) as a preexisting shares argument.
|
||||
test1 = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4"])
|
||||
}
|
||||
happy = servers_of_happiness(test1)
|
||||
self.failUnlessEqual(4, happy)
|
||||
test1[4] = set(["server1"])
|
||||
# We've added a duplicate server, so now servers_of_happiness
|
||||
# should be 3 instead of 4.
|
||||
happy = servers_of_happiness(test1)
|
||||
self.failUnlessEqual(3, happy)
|
||||
# The second argument of merge_servers should be a set of objects with
|
||||
# serverid and buckets as attributes. In actual use, these will be
|
||||
# ServerTracker instances, but for testing it is fine to make a
|
||||
# FakeServerTracker whose job is to hold those instance variables to
|
||||
# test that part.
|
||||
trackers = []
|
||||
for (i, server) in [(i, "server%d" % i) for i in range(5, 9)]:
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
# Recall that test1 is a server layout with servers_of_happiness
|
||||
# = 3. Since there isn't any overlap between the shnum ->
|
||||
# set([serverid]) correspondences in test1 and those in trackers,
|
||||
# the result here should be 7.
|
||||
test2 = merge_servers(test1, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(7, happy)
|
||||
# Now add an overlapping server to trackers. This is redundant,
|
||||
# so it should not cause the previously reported happiness value
|
||||
# to change.
|
||||
t = FakeServerTracker("server1", [1])
|
||||
trackers.append(t)
|
||||
test2 = merge_servers(test1, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(7, happy)
|
||||
test = {}
|
||||
happy = servers_of_happiness(test)
|
||||
self.failUnlessEqual(0, happy)
|
||||
# Test a more substantial overlap between the trackers and the
|
||||
# existing assignments.
|
||||
test = {
|
||||
1 : set(['server1']),
|
||||
2 : set(['server2']),
|
||||
3 : set(['server3']),
|
||||
4 : set(['server4']),
|
||||
}
|
||||
trackers = []
|
||||
t = FakeServerTracker('server5', [4])
|
||||
trackers.append(t)
|
||||
t = FakeServerTracker('server6', [3, 5])
|
||||
trackers.append(t)
|
||||
# The value returned by servers_of_happiness is the size
|
||||
# of a maximum matching in the bipartite graph that
|
||||
# servers_of_happiness() makes between serverids and share
|
||||
# numbers. It should find something like this:
|
||||
# (server 1, share 1)
|
||||
# (server 2, share 2)
|
||||
# (server 3, share 3)
|
||||
# (server 5, share 4)
|
||||
# (server 6, share 5)
|
||||
#
|
||||
# and, since there are 5 edges in this matching, it should
|
||||
# return 5.
|
||||
test2 = merge_servers(test, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(5, happy)
|
||||
# Zooko's first puzzle:
|
||||
# (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156)
|
||||
#
|
||||
# server 1: shares 0, 1
|
||||
# server 2: shares 1, 2
|
||||
# server 3: share 2
|
||||
#
|
||||
# This should yield happiness of 3.
|
||||
test = {
|
||||
0 : set(['server1']),
|
||||
1 : set(['server1', 'server2']),
|
||||
2 : set(['server2', 'server3']),
|
||||
}
|
||||
self.failUnlessEqual(3, servers_of_happiness(test))
|
||||
# Zooko's second puzzle:
|
||||
# (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158)
|
||||
#
|
||||
# server 1: shares 0, 1
|
||||
# server 2: share 1
|
||||
#
|
||||
# This should yield happiness of 2.
|
||||
test = {
|
||||
0 : set(['server1']),
|
||||
1 : set(['server1', 'server2']),
|
||||
}
|
||||
self.failUnlessEqual(2, servers_of_happiness(test))
|
||||
|
||||
|
||||
def test_shares_by_server(self):
|
||||
test = dict([(i, set(["server%d" % i])) for i in range(1, 5)])
|
||||
sbs = shares_by_server(test)
|
||||
self.failUnlessEqual(set([1]), sbs["server1"])
|
||||
self.failUnlessEqual(set([2]), sbs["server2"])
|
||||
self.failUnlessEqual(set([3]), sbs["server3"])
|
||||
self.failUnlessEqual(set([4]), sbs["server4"])
|
||||
test1 = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server1"]),
|
||||
3 : set(["server1"]),
|
||||
4 : set(["server2"]),
|
||||
5 : set(["server2"])
|
||||
}
|
||||
sbs = shares_by_server(test1)
|
||||
self.failUnlessEqual(set([1, 2, 3]), sbs["server1"])
|
||||
self.failUnlessEqual(set([4, 5]), sbs["server2"])
|
||||
# This should fail unless the serverid part of the mapping is a set
|
||||
test2 = {1: "server1"}
|
||||
self.shouldFail(AssertionError,
|
||||
"test_shares_by_server",
|
||||
"",
|
||||
shares_by_server, test2)
|
||||
|
||||
@ -15,17 +15,15 @@ from allmydata.util import log, base32
|
||||
from allmydata.util.assertutil import precondition
|
||||
from allmydata.util.deferredutil import DeferredListShouldSucceed
|
||||
from allmydata.test.no_network import GridTestMixin
|
||||
from allmydata.test.common_util import ShouldFailMixin
|
||||
from allmydata.util.happinessutil import servers_of_happiness, \
|
||||
shares_by_server, merge_servers
|
||||
from allmydata.test.common_py3 import ShouldFailMixin
|
||||
from allmydata.storage_client import StorageFarmBroker
|
||||
from allmydata.storage.server import storage_index_to_dir
|
||||
from allmydata.client import _Client
|
||||
|
||||
from .common import (
|
||||
EMPTY_CLIENT_CONFIG,
|
||||
)
|
||||
|
||||
|
||||
MiB = 1024*1024
|
||||
|
||||
def extract_uri(results):
|
||||
@ -864,12 +862,6 @@ def is_happy_enough(servertoshnums, h, k):
|
||||
return False
|
||||
return True
|
||||
|
||||
class FakeServerTracker(object):
|
||||
def __init__(self, serverid, buckets):
|
||||
self._serverid = serverid
|
||||
self.buckets = buckets
|
||||
def get_serverid(self):
|
||||
return self._serverid
|
||||
|
||||
class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
|
||||
ShouldFailMixin):
|
||||
@ -1499,185 +1491,6 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
|
||||
self._do_upload_with_broken_servers, 2))
|
||||
return d
|
||||
|
||||
|
||||
def test_merge_servers(self):
|
||||
# merge_servers merges a list of upload_servers and a dict of
|
||||
# shareid -> serverid mappings.
|
||||
shares = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4", "server5"]),
|
||||
5 : set(["server1", "server2"]),
|
||||
}
|
||||
# if not provided with a upload_servers argument, it should just
|
||||
# return the first argument unchanged.
|
||||
self.failUnlessEqual(shares, merge_servers(shares, set([])))
|
||||
trackers = []
|
||||
for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]:
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
expected = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4", "server5"]),
|
||||
5 : set(["server1", "server2", "server5"]),
|
||||
6 : set(["server6"]),
|
||||
7 : set(["server7"]),
|
||||
8 : set(["server8"]),
|
||||
}
|
||||
self.failUnlessEqual(expected, merge_servers(shares, set(trackers)))
|
||||
shares2 = {}
|
||||
expected = {
|
||||
5 : set(["server5"]),
|
||||
6 : set(["server6"]),
|
||||
7 : set(["server7"]),
|
||||
8 : set(["server8"]),
|
||||
}
|
||||
self.failUnlessEqual(expected, merge_servers(shares2, set(trackers)))
|
||||
shares3 = {}
|
||||
trackers = []
|
||||
expected = {}
|
||||
for (i, server) in [(i, "server%d" % i) for i in xrange(10)]:
|
||||
shares3[i] = set([server])
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
expected[i] = set([server])
|
||||
self.failUnlessEqual(expected, merge_servers(shares3, set(trackers)))
|
||||
|
||||
|
||||
def test_servers_of_happiness_utility_function(self):
|
||||
# These tests are concerned with the servers_of_happiness()
|
||||
# utility function, and its underlying matching algorithm. Other
|
||||
# aspects of the servers_of_happiness behavior are tested
|
||||
# elsehwere These tests exist to ensure that
|
||||
# servers_of_happiness doesn't under or overcount the happiness
|
||||
# value for given inputs.
|
||||
|
||||
# servers_of_happiness expects a dict of
|
||||
# shnum => set(serverids) as a preexisting shares argument.
|
||||
test1 = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server2"]),
|
||||
3 : set(["server3"]),
|
||||
4 : set(["server4"])
|
||||
}
|
||||
happy = servers_of_happiness(test1)
|
||||
self.failUnlessEqual(4, happy)
|
||||
test1[4] = set(["server1"])
|
||||
# We've added a duplicate server, so now servers_of_happiness
|
||||
# should be 3 instead of 4.
|
||||
happy = servers_of_happiness(test1)
|
||||
self.failUnlessEqual(3, happy)
|
||||
# The second argument of merge_servers should be a set of objects with
|
||||
# serverid and buckets as attributes. In actual use, these will be
|
||||
# ServerTracker instances, but for testing it is fine to make a
|
||||
# FakeServerTracker whose job is to hold those instance variables to
|
||||
# test that part.
|
||||
trackers = []
|
||||
for (i, server) in [(i, "server%d" % i) for i in xrange(5, 9)]:
|
||||
t = FakeServerTracker(server, [i])
|
||||
trackers.append(t)
|
||||
# Recall that test1 is a server layout with servers_of_happiness
|
||||
# = 3. Since there isn't any overlap between the shnum ->
|
||||
# set([serverid]) correspondences in test1 and those in trackers,
|
||||
# the result here should be 7.
|
||||
test2 = merge_servers(test1, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(7, happy)
|
||||
# Now add an overlapping server to trackers. This is redundant,
|
||||
# so it should not cause the previously reported happiness value
|
||||
# to change.
|
||||
t = FakeServerTracker("server1", [1])
|
||||
trackers.append(t)
|
||||
test2 = merge_servers(test1, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(7, happy)
|
||||
test = {}
|
||||
happy = servers_of_happiness(test)
|
||||
self.failUnlessEqual(0, happy)
|
||||
# Test a more substantial overlap between the trackers and the
|
||||
# existing assignments.
|
||||
test = {
|
||||
1 : set(['server1']),
|
||||
2 : set(['server2']),
|
||||
3 : set(['server3']),
|
||||
4 : set(['server4']),
|
||||
}
|
||||
trackers = []
|
||||
t = FakeServerTracker('server5', [4])
|
||||
trackers.append(t)
|
||||
t = FakeServerTracker('server6', [3, 5])
|
||||
trackers.append(t)
|
||||
# The value returned by servers_of_happiness is the size
|
||||
# of a maximum matching in the bipartite graph that
|
||||
# servers_of_happiness() makes between serverids and share
|
||||
# numbers. It should find something like this:
|
||||
# (server 1, share 1)
|
||||
# (server 2, share 2)
|
||||
# (server 3, share 3)
|
||||
# (server 5, share 4)
|
||||
# (server 6, share 5)
|
||||
#
|
||||
# and, since there are 5 edges in this matching, it should
|
||||
# return 5.
|
||||
test2 = merge_servers(test, set(trackers))
|
||||
happy = servers_of_happiness(test2)
|
||||
self.failUnlessEqual(5, happy)
|
||||
# Zooko's first puzzle:
|
||||
# (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:156)
|
||||
#
|
||||
# server 1: shares 0, 1
|
||||
# server 2: shares 1, 2
|
||||
# server 3: share 2
|
||||
#
|
||||
# This should yield happiness of 3.
|
||||
test = {
|
||||
0 : set(['server1']),
|
||||
1 : set(['server1', 'server2']),
|
||||
2 : set(['server2', 'server3']),
|
||||
}
|
||||
self.failUnlessEqual(3, servers_of_happiness(test))
|
||||
# Zooko's second puzzle:
|
||||
# (from http://allmydata.org/trac/tahoe-lafs/ticket/778#comment:158)
|
||||
#
|
||||
# server 1: shares 0, 1
|
||||
# server 2: share 1
|
||||
#
|
||||
# This should yield happiness of 2.
|
||||
test = {
|
||||
0 : set(['server1']),
|
||||
1 : set(['server1', 'server2']),
|
||||
}
|
||||
self.failUnlessEqual(2, servers_of_happiness(test))
|
||||
|
||||
|
||||
def test_shares_by_server(self):
|
||||
test = dict([(i, set(["server%d" % i])) for i in xrange(1, 5)])
|
||||
sbs = shares_by_server(test)
|
||||
self.failUnlessEqual(set([1]), sbs["server1"])
|
||||
self.failUnlessEqual(set([2]), sbs["server2"])
|
||||
self.failUnlessEqual(set([3]), sbs["server3"])
|
||||
self.failUnlessEqual(set([4]), sbs["server4"])
|
||||
test1 = {
|
||||
1 : set(["server1"]),
|
||||
2 : set(["server1"]),
|
||||
3 : set(["server1"]),
|
||||
4 : set(["server2"]),
|
||||
5 : set(["server2"])
|
||||
}
|
||||
sbs = shares_by_server(test1)
|
||||
self.failUnlessEqual(set([1, 2, 3]), sbs["server1"])
|
||||
self.failUnlessEqual(set([4, 5]), sbs["server2"])
|
||||
# This should fail unless the serverid part of the mapping is a set
|
||||
test2 = {1: "server1"}
|
||||
self.shouldFail(AssertionError,
|
||||
"test_shares_by_server",
|
||||
"",
|
||||
shares_by_server, test2)
|
||||
|
||||
|
||||
def test_existing_share_detection(self):
|
||||
self.basedir = self.mktemp()
|
||||
d = self._setup_and_upload()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user