ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-19 03:06:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Thread a _Config all the way down

Browse Source
This commit is contained in:
Jean-Paul Calderone 2019-08-19 16:09:26 -04:00
parent 05be6f2ef1
commit 9940beaae1
11 changed files with 90 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/allmydata/client.py
View File

@ -556,6 +556,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con
sb = storage_client.StorageFarmBroker(
permute_peers=True,
tub_maker=tub_creator,
node_config=config,
storage_client_config=storage_client_config,
)
for ic in introducer_clients:

21
src/allmydata/storage_client.py
View File

@ -153,6 +153,7 @@ class StorageFarmBroker(service.MultiService):
self,
permute_peers,
tub_maker,
node_config,
storage_client_config=None,
):
service.MultiService.__init__(self)
@ -160,6 +161,8 @@ class StorageFarmBroker(service.MultiService):
self.permute_peers = permute_peers
self._tub_maker = tub_maker
self.node_config = node_config
if storage_client_config is None:
storage_client_config = StorageClientConfig()
self.storage_client_config = storage_client_config
@ -233,6 +236,7 @@ class StorageFarmBroker(service.MultiService):
server["ann"],
self._tub_maker,
handler_overrides,
self.node_config,
self.storage_client_config,
)
s.on_status_changed(lambda _: self._got_connection())
@ -566,10 +570,13 @@ class AnnouncementNotMatched(Exception):
"""
def _storage_from_foolscap_plugin(config, announcement, get_rref):
def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref):
"""
Construct an ``IStorageServer`` from the most locally-preferred plugin
that is offered in the given announcement.
:param allmydata.node._Config node_config: The node configuration to
pass to the plugin.
"""
plugins = {
plugin.name: plugin
@ -586,7 +593,7 @@ def _storage_from_foolscap_plugin(config, announcement, get_rref):
if plugin_name == option[u"name"]:
furl = option[u"storage-server-FURL"]
return furl, plugin.get_storage_client(
plugin_config,
node_config,
option,
get_rref,
)
@ -621,7 +628,7 @@ class NativeStorageServer(service.MultiService):
"application-version": "unknown: no get_version()",
}
def __init__(self, server_id, ann, tub_maker, handler_overrides, config=StorageClientConfig()):
def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=StorageClientConfig()):
service.MultiService.__init__(self)
assert isinstance(server_id, str)
self._server_id = server_id
@ -629,7 +636,7 @@ class NativeStorageServer(service.MultiService):
self._tub_maker = tub_maker
self._handler_overrides = handler_overrides
self._storage = self._make_storage_system(config, ann)
self._storage = self._make_storage_system(node_config, config, ann)
self.last_connect_time = None
self.last_loss_time = None
@ -640,8 +647,11 @@ class NativeStorageServer(service.MultiService):
self._trigger_cb = None
self._on_status_changed = ObserverList()
def _make_storage_system(self, config, ann):
def _make_storage_system(self, node_config, config, ann):
"""
:param allmydata.node._Config node_config: The node configuration to pass
to any configured storage plugins.
:param StorageClientConfig config: Configuration specifying desired
storage client behavior.
@ -654,6 +664,7 @@ class NativeStorageServer(service.MultiService):
# Try to match the announcement against a plugin.
try:
furl, storage_server = _storage_from_foolscap_plugin(
node_config,
config,
ann,
# Pass in an accessor for our _rref attribute. The value of

6
src/allmydata/test/common.py
View File

@ -89,6 +89,12 @@ from .eliotutil import (
TEST_RSA_KEY_SIZE = 522
EMPTY_CLIENT_CONFIG = config_from_string(
b"/dev/null",
b"tub.port",
b""
)
@attr.s
class MemoryIntroducerClient(object):

7
src/allmydata/test/mutable/util.py
View File

@ -10,7 +10,10 @@ from allmydata.util.hashutil import tagged_hash
from allmydata.storage_client import StorageFarmBroker
from allmydata.mutable.layout import MDMFSlotReadProxy
from allmydata.mutable.publish import MutableData
from ..common import TEST_RSA_KEY_SIZE
from ..common import (
TEST_RSA_KEY_SIZE,
EMPTY_CLIENT_CONFIG,
)
def eventuaaaaaly(res=None):
d = fireEventually(res)
@ -254,7 +257,7 @@ def make_storagebroker_with_peers(peers):
:param list peers: The storage servers to associate with the storage
broker.
"""
storage_broker = StorageFarmBroker(True, None)
storage_broker = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
for peer in peers:
storage_broker.test_add_rref(
peer.peerid,

8
src/allmydata/test/test_checker.py
View File

@ -15,6 +15,10 @@ from allmydata.immutable.upload import Data
from allmydata.test.common_web import WebRenderingMixin
from allmydata.mutable.publish import MutableData
from .common import (
EMPTY_CLIENT_CONFIG,
)
class FakeClient(object):
def get_storage_broker(self):
return self.storage_broker
@ -22,7 +26,7 @@ class FakeClient(object):
class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
def create_fake_client(self):
sb = StorageFarmBroker(True, None)
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
@ -41,7 +45,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(server_id, ann, None, None)
s = NativeStorageServer(server_id, ann, None, None, None)
sb.test_add_server(server_id, s)
c = FakeClient()
c.storage_broker = sb

4
src/allmydata/test/test_client.py
View File

@ -61,6 +61,7 @@ from allmydata.interfaces import IFilesystemNode, IFileNode, \
from foolscap.api import flushEventualQueue
import allmydata.test.common_util as testutil
from .common import (
EMPTY_CLIENT_CONFIG,
SyncTestCase,
UseTestPlugins,
MemoryIntroducerClient,
@ -579,7 +580,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
return [ s.get_longname() for s in sb.get_servers_for_psi(key) ]
def test_permute(self):
sb = StorageFarmBroker(True, None)
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
for k in ["%d" % i for i in range(5)]:
ann = {"anonymous-storage-FURL": SOME_FURL,
"permutation-seed-base32": base32.b2a(k) }
@ -594,6 +595,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
sb = StorageFarmBroker(
True,
None,
EMPTY_CLIENT_CONFIG,
StorageClientConfig(preferred_peers=['1','4']),
)
for k in ["%d" % i for i in range(5)]:

10
src/allmydata/test/test_helper.py
View File

@ -12,6 +12,10 @@ from allmydata.immutable import offloaded, upload
from allmydata import uri, client
from allmydata.util import hashutil, fileutil, mathutil
from .common import (
EMPTY_CLIENT_CONFIG,
)
MiB = 1024*1024
DATA = "I need help\n" * 1000
@ -118,7 +122,11 @@ class AssistedUpload(unittest.TestCase):
self.tub = t = Tub()
t.setOption("expose-remote-exception-types", False)
self.s = FakeClient()
self.s.storage_broker = StorageFarmBroker(True, lambda h: self.tub)
self.s.storage_broker = StorageFarmBroker(
True,
lambda h: self.tub,
EMPTY_CLIENT_CONFIG,
)
self.s.secret_holder = client.SecretHolder("lease secret", "converge")
self.s.startService()

23
src/allmydata/test/test_storage_client.py
View File

@ -6,6 +6,9 @@ from json import (
from fixtures import (
TempDir,
)
from testtools.content import (
text_content,
)
from testtools.matchers import (
MatchesAll,
IsInstance,
@ -34,6 +37,7 @@ from foolscap.api import (
)
from .common import (
EMPTY_CLIENT_CONFIG,
SyncTestCase,
AsyncTestCase,
UseTestPlugins,
@ -94,7 +98,7 @@ class TestNativeStorageServer(unittest.TestCase):
ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
"permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
}
nss = NativeStorageServer("server_id", ann, None, {})
nss = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
self.assertEqual(nss.get_nickname(), "")
@ -110,7 +114,7 @@ class GetConnectionStatus(unittest.TestCase):
"""
# Pretty hard to recognize anything from an empty announcement.
ann = {}
nss = NativeStorageServer("server_id", ann, Tub, {})
nss = NativeStorageServer("server_id", ann, Tub, {}, EMPTY_CLIENT_CONFIG)
nss.start_connecting(lambda: None)
connection_status = nss.get_connection_status()
self.assertTrue(IConnectionStatus.providedBy(connection_status))
@ -144,6 +148,7 @@ class UnrecognizedAnnouncement(unittest.TestCase):
self.ann,
self._tub_maker,
{},
EMPTY_CLIENT_CONFIG,
)
def test_no_exceptions(self):
@ -351,6 +356,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
}
self.publish(server_id, ann, self.introducer_client)
storage = self.get_storage(server_id, self.node)
self.addDetail("storage", text_content(str(storage)))
self.expectThat(
storage.storage_server,
MatchesAll(
@ -449,8 +455,11 @@ class StoragePluginWebPresence(AsyncTestCase):
class TestStorageFarmBroker(unittest.TestCase):
def make_broker(self, tub_maker=lambda h: Mock()):
return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
def test_static_servers(self):
broker = StorageFarmBroker(True, lambda h: Mock())
broker = self.make_broker()
key_s = 'v0-1234-1'
servers_yaml = b"""\
@ -484,7 +493,7 @@ storage:
self.assertEqual(s2.get_permutation_seed(), permseed)
def test_static_permutation_seed_pubkey(self):
broker = StorageFarmBroker(True, lambda h: Mock())
broker = self.make_broker()
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
ann = {
@ -495,7 +504,7 @@ storage:
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_static_permutation_seed_explicit(self):
broker = StorageFarmBroker(True, lambda h: Mock())
broker = self.make_broker()
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7"
ann = {
@ -507,7 +516,7 @@ storage:
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_static_permutation_seed_hashed(self):
broker = StorageFarmBroker(True, lambda h: Mock())
broker = self.make_broker()
server_id = "unparseable"
ann = {
"anonymous-storage-FURL": SOME_FURL,
@ -523,7 +532,7 @@ storage:
new_tubs = []
def make_tub(*args, **kwargs):
return new_tubs.pop()
broker = StorageFarmBroker(True, make_tub)
broker = self.make_broker(make_tub)
done = broker.when_connected_enough(5)
broker.use_introducer(introducer)
# subscribes to "storage" to learn of new storage nodes

10
src/allmydata/test/test_upload.py
View File

@ -22,6 +22,10 @@ from allmydata.storage_client import StorageFarmBroker
from allmydata.storage.server import storage_index_to_dir
from allmydata.client import _Client
from .common import (
EMPTY_CLIENT_CONFIG,
)
MiB = 1024*1024
def extract_uri(results):
@ -217,7 +221,11 @@ class FakeClient(object):
("%20d" % fakeid, FakeStorageServer(mode[fakeid], reactor=reactor))
for fakeid in range(self.num_servers)
]
self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None)
self.storage_broker = StorageFarmBroker(
permute_peers=True,
tub_maker=None,
node_config=EMPTY_CLIENT_CONFIG,
)
for (serverid, rref) in servers:
ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid),
"permutation-seed-base32": base32.b2a(serverid) }

6
src/allmydata/test/web/test_root.py
View File

@ -4,6 +4,10 @@ from ...storage_client import NativeStorageServer
from ...web.root import Root
from ...util.connection_status import ConnectionStatus
from ..common import (
EMPTY_CLIENT_CONFIG,
)
class FakeRoot(Root):
def __init__(self):
pass
@ -26,7 +30,7 @@ class RenderServiceRow(unittest.TestCase):
ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
"permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
}
s = NativeStorageServer("server_id", ann, None, {})
s = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
cs = ConnectionStatus(False, "summary", {}, 0, 0)
s.get_connection_status = lambda: cs

18
src/allmydata/test/web/test_web.py
View File

@ -40,9 +40,15 @@ from allmydata.util import fileutil, base32, hashutil
from allmydata.util.consumer import download_to_data
from allmydata.util.encodingutil import to_str
from ...util.connection_status import ConnectionStatus
from ..common import FakeCHKFileNode, FakeMutableFileNode, \
create_chk_filenode, WebErrorMixin, \
make_mutable_file_uri, create_mutable_filenode
from ..common import (
EMPTY_CLIENT_CONFIG,
FakeCHKFileNode,
FakeMutableFileNode,
create_chk_filenode,
WebErrorMixin,
make_mutable_file_uri,
create_mutable_filenode,
)
from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
from allmydata.mutable import servermap, publish, retrieve
from .. import common_util as testutil
@ -280,7 +286,11 @@ class FakeClient(_Client):
self._secret_holder = SecretHolder("lease secret", "convergence secret")
self.helper = None
self.convergence = "some random string"
self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None)
self.storage_broker = StorageFarmBroker(
permute_peers=True,
tub_maker=None,
node_config=EMPTY_CLIENT_CONFIG,
)
# fake knowledge of another server
self.storage_broker.test_add_server("other_nodeid",
FakeDisplayableServer(