diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7739ea42c..3e9905821 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -556,6 +556,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, + node_config=config, storage_client_config=storage_client_config, ) for ic in introducer_clients: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 84d96cec0..fc03ae9ae 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -153,6 +153,7 @@ class StorageFarmBroker(service.MultiService): self, permute_peers, tub_maker, + node_config, storage_client_config=None, ): service.MultiService.__init__(self) @@ -160,6 +161,8 @@ class StorageFarmBroker(service.MultiService): self.permute_peers = permute_peers self._tub_maker = tub_maker + self.node_config = node_config + if storage_client_config is None: storage_client_config = StorageClientConfig() self.storage_client_config = storage_client_config @@ -233,6 +236,7 @@ class StorageFarmBroker(service.MultiService): server["ann"], self._tub_maker, handler_overrides, + self.node_config, self.storage_client_config, ) s.on_status_changed(lambda _: self._got_connection()) @@ -566,10 +570,13 @@ class AnnouncementNotMatched(Exception): """ -def _storage_from_foolscap_plugin(config, announcement, get_rref): +def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): """ Construct an ``IStorageServer`` from the most locally-preferred plugin that is offered in the given announcement. + + :param allmydata.node._Config node_config: The node configuration to + pass to the plugin. """ plugins = { plugin.name: plugin @@ -586,7 +593,7 @@ def _storage_from_foolscap_plugin(config, announcement, get_rref): if plugin_name == option[u"name"]: furl = option[u"storage-server-FURL"] return furl, plugin.get_storage_client( - plugin_config, + node_config, option, get_rref, ) @@ -621,7 +628,7 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides, config=StorageClientConfig()): + def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=StorageClientConfig()): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -629,7 +636,7 @@ class NativeStorageServer(service.MultiService): self._tub_maker = tub_maker self._handler_overrides = handler_overrides - self._storage = self._make_storage_system(config, ann) + self._storage = self._make_storage_system(node_config, config, ann) self.last_connect_time = None self.last_loss_time = None @@ -640,8 +647,11 @@ class NativeStorageServer(service.MultiService): self._trigger_cb = None self._on_status_changed = ObserverList() - def _make_storage_system(self, config, ann): + def _make_storage_system(self, node_config, config, ann): """ + :param allmydata.node._Config node_config: The node configuration to pass + to any configured storage plugins. + :param StorageClientConfig config: Configuration specifying desired storage client behavior. @@ -654,6 +664,7 @@ class NativeStorageServer(service.MultiService): # Try to match the announcement against a plugin. try: furl, storage_server = _storage_from_foolscap_plugin( + node_config, config, ann, # Pass in an accessor for our _rref attribute. The value of diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index ef1f95303..ff31ea939 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -89,6 +89,12 @@ from .eliotutil import ( TEST_RSA_KEY_SIZE = 522 +EMPTY_CLIENT_CONFIG = config_from_string( + b"/dev/null", + b"tub.port", + b"" +) + @attr.s class MemoryIntroducerClient(object): diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 115bc3b77..a664c1e08 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -10,7 +10,10 @@ from allmydata.util.hashutil import tagged_hash from allmydata.storage_client import StorageFarmBroker from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.mutable.publish import MutableData -from ..common import TEST_RSA_KEY_SIZE +from ..common import ( + TEST_RSA_KEY_SIZE, + EMPTY_CLIENT_CONFIG, +) def eventuaaaaaly(res=None): d = fireEventually(res) @@ -254,7 +257,7 @@ def make_storagebroker_with_peers(peers): :param list peers: The storage servers to associate with the storage broker. """ - storage_broker = StorageFarmBroker(True, None) + storage_broker = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) for peer in peers: storage_broker.test_add_rref( peer.peerid, diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index e1af90940..5eed6f21f 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -15,6 +15,10 @@ from allmydata.immutable.upload import Data from allmydata.test.common_web import WebRenderingMixin from allmydata.mutable.publish import MutableData +from .common import ( + EMPTY_CLIENT_CONFIG, +) + class FakeClient(object): def get_storage_broker(self): return self.storage_broker @@ -22,7 +26,7 @@ class FakeClient(object): class WebResultsRendering(unittest.TestCase, WebRenderingMixin): def create_fake_client(self): - sb = StorageFarmBroker(True, None) + sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) # s.get_name() (the "short description") will be "v0-00000000". # s.get_longname() will include the -long suffix. servers = [("v0-00000000-long", "\x00"*20, "peer-0"), @@ -41,7 +45,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(server_id, ann, None, None) + s = NativeStorageServer(server_id, ann, None, None, None) sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 1e397e802..098ab461b 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -61,6 +61,7 @@ from allmydata.interfaces import IFilesystemNode, IFileNode, \ from foolscap.api import flushEventualQueue import allmydata.test.common_util as testutil from .common import ( + EMPTY_CLIENT_CONFIG, SyncTestCase, UseTestPlugins, MemoryIntroducerClient, @@ -579,7 +580,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] def test_permute(self): - sb = StorageFarmBroker(True, None) + sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) for k in ["%d" % i for i in range(5)]: ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } @@ -594,6 +595,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test sb = StorageFarmBroker( True, None, + EMPTY_CLIENT_CONFIG, StorageClientConfig(preferred_peers=['1','4']), ) for k in ["%d" % i for i in range(5)]: diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 4b07f58ae..3774704c6 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -12,6 +12,10 @@ from allmydata.immutable import offloaded, upload from allmydata import uri, client from allmydata.util import hashutil, fileutil, mathutil +from .common import ( + EMPTY_CLIENT_CONFIG, +) + MiB = 1024*1024 DATA = "I need help\n" * 1000 @@ -118,7 +122,11 @@ class AssistedUpload(unittest.TestCase): self.tub = t = Tub() t.setOption("expose-remote-exception-types", False) self.s = FakeClient() - self.s.storage_broker = StorageFarmBroker(True, lambda h: self.tub) + self.s.storage_broker = StorageFarmBroker( + True, + lambda h: self.tub, + EMPTY_CLIENT_CONFIG, + ) self.s.secret_holder = client.SecretHolder("lease secret", "converge") self.s.startService() diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index b59b93347..1c7bcc3bc 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -6,6 +6,9 @@ from json import ( from fixtures import ( TempDir, ) +from testtools.content import ( + text_content, +) from testtools.matchers import ( MatchesAll, IsInstance, @@ -34,6 +37,7 @@ from foolscap.api import ( ) from .common import ( + EMPTY_CLIENT_CONFIG, SyncTestCase, AsyncTestCase, UseTestPlugins, @@ -94,7 +98,7 @@ class TestNativeStorageServer(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - nss = NativeStorageServer("server_id", ann, None, {}) + nss = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) self.assertEqual(nss.get_nickname(), "") @@ -110,7 +114,7 @@ class GetConnectionStatus(unittest.TestCase): """ # Pretty hard to recognize anything from an empty announcement. ann = {} - nss = NativeStorageServer("server_id", ann, Tub, {}) + nss = NativeStorageServer("server_id", ann, Tub, {}, EMPTY_CLIENT_CONFIG) nss.start_connecting(lambda: None) connection_status = nss.get_connection_status() self.assertTrue(IConnectionStatus.providedBy(connection_status)) @@ -144,6 +148,7 @@ class UnrecognizedAnnouncement(unittest.TestCase): self.ann, self._tub_maker, {}, + EMPTY_CLIENT_CONFIG, ) def test_no_exceptions(self): @@ -351,6 +356,7 @@ class PluginMatchedAnnouncement(SyncTestCase): } self.publish(server_id, ann, self.introducer_client) storage = self.get_storage(server_id, self.node) + self.addDetail("storage", text_content(str(storage))) self.expectThat( storage.storage_server, MatchesAll( @@ -449,8 +455,11 @@ class StoragePluginWebPresence(AsyncTestCase): class TestStorageFarmBroker(unittest.TestCase): + def make_broker(self, tub_maker=lambda h: Mock()): + return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) + def test_static_servers(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = self.make_broker() key_s = 'v0-1234-1' servers_yaml = b"""\ @@ -484,7 +493,7 @@ storage: self.assertEqual(s2.get_permutation_seed(), permseed) def test_static_permutation_seed_pubkey(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = self.make_broker() server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" ann = { @@ -495,7 +504,7 @@ storage: self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) def test_static_permutation_seed_explicit(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = self.make_broker() server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7" ann = { @@ -507,7 +516,7 @@ storage: self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) def test_static_permutation_seed_hashed(self): - broker = StorageFarmBroker(True, lambda h: Mock()) + broker = self.make_broker() server_id = "unparseable" ann = { "anonymous-storage-FURL": SOME_FURL, @@ -523,7 +532,7 @@ storage: new_tubs = [] def make_tub(*args, **kwargs): return new_tubs.pop() - broker = StorageFarmBroker(True, make_tub) + broker = self.make_broker(make_tub) done = broker.when_connected_enough(5) broker.use_introducer(introducer) # subscribes to "storage" to learn of new storage nodes diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index a7459c3a2..165ca17d7 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -22,6 +22,10 @@ from allmydata.storage_client import StorageFarmBroker from allmydata.storage.server import storage_index_to_dir from allmydata.client import _Client +from .common import ( + EMPTY_CLIENT_CONFIG, +) + MiB = 1024*1024 def extract_uri(results): @@ -217,7 +221,11 @@ class FakeClient(object): ("%20d" % fakeid, FakeStorageServer(mode[fakeid], reactor=reactor)) for fakeid in range(self.num_servers) ] - self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) for (serverid, rref) in servers: ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid), "permutation-seed-base32": base32.b2a(serverid) } diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 1e324c398..f9db6f145 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -4,6 +4,10 @@ from ...storage_client import NativeStorageServer from ...web.root import Root from ...util.connection_status import ConnectionStatus +from ..common import ( + EMPTY_CLIENT_CONFIG, +) + class FakeRoot(Root): def __init__(self): pass @@ -26,7 +30,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - s = NativeStorageServer("server_id", ann, None, {}) + s = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) cs = ConnectionStatus(False, "summary", {}, 0, 0) s.get_connection_status = lambda: cs diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 0b41f9b5a..4c01f6822 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -40,9 +40,15 @@ from allmydata.util import fileutil, base32, hashutil from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_str from ...util.connection_status import ConnectionStatus -from ..common import FakeCHKFileNode, FakeMutableFileNode, \ - create_chk_filenode, WebErrorMixin, \ - make_mutable_file_uri, create_mutable_filenode +from ..common import ( + EMPTY_CLIENT_CONFIG, + FakeCHKFileNode, + FakeMutableFileNode, + create_chk_filenode, + WebErrorMixin, + make_mutable_file_uri, + create_mutable_filenode, +) from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION from allmydata.mutable import servermap, publish, retrieve from .. import common_util as testutil @@ -280,7 +286,11 @@ class FakeClient(_Client): self._secret_holder = SecretHolder("lease secret", "convergence secret") self.helper = None self.convergence = "some random string" - self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) # fake knowledge of another server self.storage_broker.test_add_server("other_nodeid", FakeDisplayableServer(