From d23fdcdb8ab32bd11530db344b0448ecaae2b041 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jan 2022 12:03:17 -0500 Subject: [PATCH] Sketch of first IStorageServer test with HTTP server/client. --- src/allmydata/storage_client.py | 38 ++++++++++++ src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++++-- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 526e4e70d..13b4dfd01 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,6 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.storage.http_client import StorageClient # who is responsible for de-duplication? @@ -1024,3 +1025,40 @@ class _StorageServer(object): shnum, reason, ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + + +# WORK IN PROGRESS, for now it doesn't actually implement whole thing. +@implementer(IStorageServer) # type: ignore +@attr.s +class _HTTPStorageServer(object): + """ + Talk to remote storage server over HTTP. + """ + _http_client = attr.ib(type=StorageClient) + + @staticmethod + def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer + """ + Create an ``IStorageServer`` from a HTTP ``StorageClient``. + """ + return _HTTPStorageServer(_http_client=http_client) + + def get_version(self): + return self._http_client.get_version() + + def allocate_buckets( + self, + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + pass + + def get_buckets( + self, + storage_index, + ): + pass diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a17264713..fb9765624 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1,6 +1,8 @@ """ Tests for the ``IStorageServer`` interface. +Keep in mind that ``IStorageServer`` is actually the storage _client_ interface. + Note that for performance, in the future we might want the same node to be reused across tests, so each test should be careful to generate unique storage indexes. @@ -22,6 +24,10 @@ from random import Random from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock +from twisted.internet import reactor +from twisted.web.server import Site +from hyperlink import DecodedURL +from treq.api import get_global_pool as get_treq_pool from foolscap.api import Referenceable, RemoteException @@ -29,6 +35,10 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! +from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_client import StorageClient +from allmydata.util.iputil import allocate_tcp_port + # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -998,11 +1008,11 @@ class IStorageServerMutableAPIsTestsMixin(object): self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) -class _FoolscapMixin(SystemTestMixin): - """Run tests on Foolscap version of ``IStorageServer.""" +class _SharedMixin(SystemTestMixin): + """Base class for Foolscap and HTTP mixins.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + def _get_istorage_server(self): + raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): @@ -1010,8 +1020,6 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_client = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_client)) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1021,6 +1029,7 @@ class _FoolscapMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1035,6 +1044,25 @@ class _FoolscapMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + raise NotImplementedError("implement in subclass") + + +class _FoolscapMixin(_SharedMixin): + """Run tests on Foolscap version of ``IStorageServer``.""" + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) + + def _get_istorage_server(self): + client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client + @inlineCallbacks def disconnect(self): """ @@ -1046,12 +1074,43 @@ class _FoolscapMixin(SystemTestMixin): assert self.storage_client is not current +class _HTTPMixin(_SharedMixin): + """Run tests on the HTTP version of ``IStorageServer``.""" + + def _get_istorage_server(self): + swissnum = b"1234" + self._http_storage_server = HTTPServer(self.server, swissnum) + self._port_number = allocate_tcp_port() + self._listening_port = reactor.listenTCP( + self._port_number, Site(self._http_storage_server.get_resource()), + interface="127.0.0.1" + ) + return StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum + ) + # Eventually should also: + # self.assertTrue(IStorageServer.providedBy(client)) + + @inlineCallbacks + def tearDown(self): + yield _SharedMixin.tearDown(self) + self._listening_port.stopListening() + yield get_treq_pool().closeCachedConnections() + + class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" +class HTTPSharedAPIsTests( + _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + + class FoolscapImmutableAPIsTests( _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ):