Merge pull request #653 from tahoe-lafs/3248.pass-config-to-get_storage_client

Pass the full _Config to IFoolscapStoragePlugin.get_storage_client

Fixes: ticket:3248

src/allmydata/client.py

@@ -556,6 +556,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con
     sb = storage_client.StorageFarmBroker(
         permute_peers=True,
         tub_maker=tub_creator,
+        node_config=config,
         storage_client_config=storage_client_config,
     )
     for ic in introducer_clients:

src/allmydata/interfaces.py

@@ -3102,8 +3102,8 @@ class IFoolscapStoragePlugin(IPlugin):
         Get an ``IStorageServer`` provider that implements the client side of the
         storage protocol.
 
-        :param dict configuration: Any configuration given in the section for
+        :param allmydata.node._Config configuration: A representation of the
-            this plugin in the node's configuration file.
+            configuration for the node into which this plugin has been loaded.
 
         :param dict announcement: The announcement for the corresponding
             server portion of this plugin received from a storage server which

src/allmydata/storage_client.py

@@ -153,6 +153,7 @@ class StorageFarmBroker(service.MultiService):
             self,
             permute_peers,
             tub_maker,
+            node_config,
             storage_client_config=None,
     ):
         service.MultiService.__init__(self)
@@ -160,6 +161,8 @@ class StorageFarmBroker(service.MultiService):
         self.permute_peers = permute_peers
         self._tub_maker = tub_maker
 
+        self.node_config = node_config
+
         if storage_client_config is None:
             storage_client_config = StorageClientConfig()
         self.storage_client_config = storage_client_config
@@ -233,6 +236,7 @@ class StorageFarmBroker(service.MultiService):
             server["ann"],
             self._tub_maker,
             handler_overrides,
+            self.node_config,
             self.storage_client_config,
         )
         s.on_status_changed(lambda _: self._got_connection())
@@ -566,10 +570,13 @@ class AnnouncementNotMatched(Exception):
     """
 
 
-def _storage_from_foolscap_plugin(config, announcement, get_rref):
+def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref):
     """
     Construct an ``IStorageServer`` from the most locally-preferred plugin
     that is offered in the given announcement.
+
+    :param allmydata.node._Config node_config: The node configuration to
+        pass to the plugin.
     """
     plugins = {
         plugin.name: plugin
@@ -586,7 +593,7 @@ def _storage_from_foolscap_plugin(config, announcement, get_rref):
             if plugin_name == option[u"name"]:
                 furl = option[u"storage-server-FURL"]
                 return furl, plugin.get_storage_client(
-                    plugin_config,
+                    node_config,
                     option,
                     get_rref,
                 )
@@ -621,7 +628,7 @@ class NativeStorageServer(service.MultiService):
         "application-version": "unknown: no get_version()",
         }
 
-    def __init__(self, server_id, ann, tub_maker, handler_overrides, config=StorageClientConfig()):
+    def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=StorageClientConfig()):
         service.MultiService.__init__(self)
         assert isinstance(server_id, str)
         self._server_id = server_id
@@ -629,7 +636,7 @@ class NativeStorageServer(service.MultiService):
         self._tub_maker = tub_maker
         self._handler_overrides = handler_overrides
 
-        self._storage = self._make_storage_system(config, ann)
+        self._storage = self._make_storage_system(node_config, config, ann)
 
         self.last_connect_time = None
         self.last_loss_time = None
@@ -640,8 +647,11 @@ class NativeStorageServer(service.MultiService):
         self._trigger_cb = None
         self._on_status_changed = ObserverList()
 
-    def _make_storage_system(self, config, ann):
+    def _make_storage_system(self, node_config, config, ann):
         """
+        :param allmydata.node._Config node_config: The node configuration to pass
+            to any configured storage plugins.
+
         :param StorageClientConfig config: Configuration specifying desired
             storage client behavior.
 
@@ -654,6 +664,7 @@ class NativeStorageServer(service.MultiService):
         # Try to match the announcement against a plugin.
         try:
             furl, storage_server = _storage_from_foolscap_plugin(
+                node_config,
                 config,
                 ann,
                 # Pass in an accessor for our _rref attribute.  The value of

src/allmydata/test/common.py

@@ -89,6 +89,12 @@ from .eliotutil import (
 
 TEST_RSA_KEY_SIZE = 522
 
+EMPTY_CLIENT_CONFIG = config_from_string(
+    b"/dev/null",
+    b"tub.port",
+    b""
+)
+
 
 @attr.s
 class MemoryIntroducerClient(object):

src/allmydata/test/mutable/util.py

@@ -10,7 +10,10 @@ from allmydata.util.hashutil import tagged_hash
 from allmydata.storage_client import StorageFarmBroker
 from allmydata.mutable.layout import MDMFSlotReadProxy
 from allmydata.mutable.publish import MutableData
-from ..common import TEST_RSA_KEY_SIZE
+from ..common import (
+    TEST_RSA_KEY_SIZE,
+    EMPTY_CLIENT_CONFIG,
+)
 
 def eventuaaaaaly(res=None):
     d = fireEventually(res)
@@ -254,7 +257,7 @@ def make_storagebroker_with_peers(peers):
     :param list peers: The storage servers to associate with the storage
         broker.
     """
-    storage_broker = StorageFarmBroker(True, None)
+    storage_broker = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
     for peer in peers:
         storage_broker.test_add_rref(
             peer.peerid,

src/allmydata/test/storage_plugin.py

@@ -66,7 +66,11 @@ class DummyStorage(object):
         )
 
     def get_storage_client(self, configuration, announcement, get_rref):
-        return DummyStorageClient(get_rref, configuration, announcement)
+        return DummyStorageClient(
+            get_rref,
+            dict(configuration.items(self._client_section_name, [])),
+            announcement,
+        )
 
     def get_client_resource(self, configuration):
         """

src/allmydata/test/test_checker.py

@@ -15,6 +15,10 @@ from allmydata.immutable.upload import Data
 from allmydata.test.common_web import WebRenderingMixin
 from allmydata.mutable.publish import MutableData
 
+from .common import (
+    EMPTY_CLIENT_CONFIG,
+)
+
 class FakeClient(object):
     def get_storage_broker(self):
         return self.storage_broker
@@ -22,7 +26,7 @@ class FakeClient(object):
 class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
 
     def create_fake_client(self):
-        sb = StorageFarmBroker(True, None)
+        sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
         # s.get_name() (the "short description") will be "v0-00000000".
         # s.get_longname() will include the -long suffix.
         servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
@@ -41,7 +45,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
                     "my-version": "ver",
                     "oldest-supported": "oldest",
                     }
-            s = NativeStorageServer(server_id, ann, None, None)
+            s = NativeStorageServer(server_id, ann, None, None, None)
             sb.test_add_server(server_id, s)
         c = FakeClient()
         c.storage_broker = sb

src/allmydata/test/test_client.py

@@ -61,6 +61,7 @@ from allmydata.interfaces import IFilesystemNode, IFileNode, \
 from foolscap.api import flushEventualQueue
 import allmydata.test.common_util as testutil
 from .common import (
+    EMPTY_CLIENT_CONFIG,
     SyncTestCase,
     UseTestPlugins,
     MemoryIntroducerClient,
@@ -579,7 +580,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
         return [ s.get_longname() for s in sb.get_servers_for_psi(key) ]
 
     def test_permute(self):
-        sb = StorageFarmBroker(True, None)
+        sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
         for k in ["%d" % i for i in range(5)]:
             ann = {"anonymous-storage-FURL": SOME_FURL,
                    "permutation-seed-base32": base32.b2a(k) }
@@ -594,6 +595,7 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
         sb = StorageFarmBroker(
             True,
             None,
+            EMPTY_CLIENT_CONFIG,
             StorageClientConfig(preferred_peers=['1','4']),
         )
         for k in ["%d" % i for i in range(5)]:

src/allmydata/test/test_helper.py

@@ -12,6 +12,10 @@ from allmydata.immutable import offloaded, upload
 from allmydata import uri, client
 from allmydata.util import hashutil, fileutil, mathutil
 
+from .common import (
+    EMPTY_CLIENT_CONFIG,
+)
+
 MiB = 1024*1024
 
 DATA = "I need help\n" * 1000
@@ -118,7 +122,11 @@ class AssistedUpload(unittest.TestCase):
         self.tub = t = Tub()
         t.setOption("expose-remote-exception-types", False)
         self.s = FakeClient()
-        self.s.storage_broker = StorageFarmBroker(True, lambda h: self.tub)
+        self.s.storage_broker = StorageFarmBroker(
+            True,
+            lambda h: self.tub,
+            EMPTY_CLIENT_CONFIG,
+        )
         self.s.secret_holder = client.SecretHolder("lease secret", "converge")
         self.s.startService()
 

src/allmydata/test/test_storage_client.py

@@ -6,6 +6,9 @@ from json import (
 from fixtures import (
     TempDir,
 )
+from testtools.content import (
+    text_content,
+)
 from testtools.matchers import (
     MatchesAll,
     IsInstance,
@@ -37,6 +40,7 @@ from foolscap.api import (
 )
 
 from .common import (
+    EMPTY_CLIENT_CONFIG,
     SyncTestCase,
     AsyncTestCase,
     UseTestPlugins,
@@ -98,7 +102,7 @@ class TestNativeStorageServer(unittest.TestCase):
         ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
                "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
                }
-        nss = NativeStorageServer("server_id", ann, None, {})
+        nss = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
         self.assertEqual(nss.get_nickname(), "")
 
 
@@ -114,7 +118,7 @@ class GetConnectionStatus(unittest.TestCase):
         """
         # Pretty hard to recognize anything from an empty announcement.
         ann = {}
-        nss = NativeStorageServer("server_id", ann, Tub, {})
+        nss = NativeStorageServer("server_id", ann, Tub, {}, EMPTY_CLIENT_CONFIG)
         nss.start_connecting(lambda: None)
         connection_status = nss.get_connection_status()
         self.assertTrue(IConnectionStatus.providedBy(connection_status))
@@ -148,6 +152,7 @@ class UnrecognizedAnnouncement(unittest.TestCase):
             self.ann,
             self._tub_maker,
             {},
+            EMPTY_CLIENT_CONFIG,
         )
 
     def test_no_exceptions(self):
@@ -355,6 +360,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
         }
         self.publish(server_id, ann, self.introducer_client)
         storage = self.get_storage(server_id, self.node)
+        self.addDetail("storage", text_content(str(storage)))
         self.expectThat(
             storage.storage_server,
             MatchesAll(
@@ -454,10 +460,18 @@ class StoragePluginWebPresence(AsyncTestCase):
         self.assertThat(result, Equals(dumps({b"web": b"1"})))
 
 
+def make_broker(tub_maker=lambda h: Mock()):
+    """
+    Create a ``StorageFarmBroker`` with the given tub maker and an empty
+    client configuration.
+    """
+    return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
+
+
 class TestStorageFarmBroker(unittest.TestCase):
 
     def test_static_servers(self):
-        broker = StorageFarmBroker(True, lambda h: Mock())
+        broker = make_broker()
 
         key_s = 'v0-1234-1'
         servers_yaml = b"""\
@@ -491,7 +505,7 @@ storage:
         self.assertEqual(s2.get_permutation_seed(), permseed)
 
     def test_static_permutation_seed_pubkey(self):
-        broker = StorageFarmBroker(True, lambda h: Mock())
+        broker = make_broker()
         server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
         k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
         ann = {
@@ -502,7 +516,7 @@ storage:
         self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
 
     def test_static_permutation_seed_explicit(self):
-        broker = StorageFarmBroker(True, lambda h: Mock())
+        broker = make_broker()
         server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
         k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7"
         ann = {
@@ -514,7 +528,7 @@ storage:
         self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
 
     def test_static_permutation_seed_hashed(self):
-        broker = StorageFarmBroker(True, lambda h: Mock())
+        broker = make_broker()
         server_id = "unparseable"
         ann = {
             "anonymous-storage-FURL": SOME_FURL,
@@ -530,7 +544,7 @@ storage:
         new_tubs = []
         def make_tub(*args, **kwargs):
             return new_tubs.pop()
-        broker = StorageFarmBroker(True, make_tub)
+        broker = make_broker(make_tub)
         done = broker.when_connected_enough(5)
         broker.use_introducer(introducer)
         # subscribes to "storage" to learn of new storage nodes

src/allmydata/test/test_upload.py

@@ -22,6 +22,10 @@ from allmydata.storage_client import StorageFarmBroker
 from allmydata.storage.server import storage_index_to_dir
 from allmydata.client import _Client
 
+from .common import (
+    EMPTY_CLIENT_CONFIG,
+)
+
 MiB = 1024*1024
 
 def extract_uri(results):
@@ -217,7 +221,11 @@ class FakeClient(object):
             ("%20d" % fakeid, FakeStorageServer(mode[fakeid], reactor=reactor))
             for fakeid in range(self.num_servers)
         ]
-        self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None)
+        self.storage_broker = StorageFarmBroker(
+            permute_peers=True,
+            tub_maker=None,
+            node_config=EMPTY_CLIENT_CONFIG,
+        )
         for (serverid, rref) in servers:
             ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid),
                    "permutation-seed-base32": base32.b2a(serverid) }

src/allmydata/test/web/test_root.py

@@ -4,6 +4,10 @@ from ...storage_client import NativeStorageServer
 from ...web.root import Root
 from ...util.connection_status import ConnectionStatus
 
+from ..common import (
+    EMPTY_CLIENT_CONFIG,
+)
+
 class FakeRoot(Root):
     def __init__(self):
         pass
@@ -26,7 +30,7 @@ class RenderServiceRow(unittest.TestCase):
         ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
                "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
                }
-        s = NativeStorageServer("server_id", ann, None, {})
+        s = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
         cs = ConnectionStatus(False, "summary", {}, 0, 0)
         s.get_connection_status = lambda: cs
 

src/allmydata/test/web/test_web.py

@@ -40,9 +40,15 @@ from allmydata.util import fileutil, base32, hashutil
 from allmydata.util.consumer import download_to_data
 from allmydata.util.encodingutil import to_str
 from ...util.connection_status import ConnectionStatus
-from ..common import FakeCHKFileNode, FakeMutableFileNode, \
+from ..common import (
-     create_chk_filenode, WebErrorMixin, \
+    EMPTY_CLIENT_CONFIG,
-     make_mutable_file_uri, create_mutable_filenode
+    FakeCHKFileNode,
+    FakeMutableFileNode,
+    create_chk_filenode,
+    WebErrorMixin,
+    make_mutable_file_uri,
+    create_mutable_filenode,
+)
 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable import servermap, publish, retrieve
 from .. import common_util as testutil
@@ -280,7 +286,11 @@ class FakeClient(_Client):
         self._secret_holder = SecretHolder("lease secret", "convergence secret")
         self.helper = None
         self.convergence = "some random string"
-        self.storage_broker = StorageFarmBroker(permute_peers=True, tub_maker=None)
+        self.storage_broker = StorageFarmBroker(
+            permute_peers=True,
+            tub_maker=None,
+            node_config=EMPTY_CLIENT_CONFIG,
+        )
         # fake knowledge of another server
         self.storage_broker.test_add_server("other_nodeid",
             FakeDisplayableServer(