From d23fdcdb8ab32bd11530db344b0448ecaae2b041 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jan 2022 12:03:17 -0500 Subject: [PATCH 01/11] Sketch of first IStorageServer test with HTTP server/client. --- src/allmydata/storage_client.py | 38 ++++++++++++ src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++++-- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 526e4e70d..13b4dfd01 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,6 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.storage.http_client import StorageClient # who is responsible for de-duplication? @@ -1024,3 +1025,40 @@ class _StorageServer(object): shnum, reason, ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + + +# WORK IN PROGRESS, for now it doesn't actually implement whole thing. +@implementer(IStorageServer) # type: ignore +@attr.s +class _HTTPStorageServer(object): + """ + Talk to remote storage server over HTTP. + """ + _http_client = attr.ib(type=StorageClient) + + @staticmethod + def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer + """ + Create an ``IStorageServer`` from a HTTP ``StorageClient``. + """ + return _HTTPStorageServer(_http_client=http_client) + + def get_version(self): + return self._http_client.get_version() + + def allocate_buckets( + self, + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + pass + + def get_buckets( + self, + storage_index, + ): + pass diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a17264713..fb9765624 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1,6 +1,8 @@ """ Tests for the ``IStorageServer`` interface. +Keep in mind that ``IStorageServer`` is actually the storage _client_ interface. + Note that for performance, in the future we might want the same node to be reused across tests, so each test should be careful to generate unique storage indexes. @@ -22,6 +24,10 @@ from random import Random from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock +from twisted.internet import reactor +from twisted.web.server import Site +from hyperlink import DecodedURL +from treq.api import get_global_pool as get_treq_pool from foolscap.api import Referenceable, RemoteException @@ -29,6 +35,10 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! +from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_client import StorageClient +from allmydata.util.iputil import allocate_tcp_port + # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -998,11 +1008,11 @@ class IStorageServerMutableAPIsTestsMixin(object): self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) -class _FoolscapMixin(SystemTestMixin): - """Run tests on Foolscap version of ``IStorageServer.""" +class _SharedMixin(SystemTestMixin): + """Base class for Foolscap and HTTP mixins.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + def _get_istorage_server(self): + raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): @@ -1010,8 +1020,6 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_client = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_client)) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1021,6 +1029,7 @@ class _FoolscapMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1035,6 +1044,25 @@ class _FoolscapMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + raise NotImplementedError("implement in subclass") + + +class _FoolscapMixin(_SharedMixin): + """Run tests on Foolscap version of ``IStorageServer``.""" + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) + + def _get_istorage_server(self): + client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client + @inlineCallbacks def disconnect(self): """ @@ -1046,12 +1074,43 @@ class _FoolscapMixin(SystemTestMixin): assert self.storage_client is not current +class _HTTPMixin(_SharedMixin): + """Run tests on the HTTP version of ``IStorageServer``.""" + + def _get_istorage_server(self): + swissnum = b"1234" + self._http_storage_server = HTTPServer(self.server, swissnum) + self._port_number = allocate_tcp_port() + self._listening_port = reactor.listenTCP( + self._port_number, Site(self._http_storage_server.get_resource()), + interface="127.0.0.1" + ) + return StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum + ) + # Eventually should also: + # self.assertTrue(IStorageServer.providedBy(client)) + + @inlineCallbacks + def tearDown(self): + yield _SharedMixin.tearDown(self) + self._listening_port.stopListening() + yield get_treq_pool().closeCachedConnections() + + class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" +class HTTPSharedAPIsTests( + _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + + class FoolscapImmutableAPIsTests( _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): From e672029e6d1a1b2e6e209d6e5f6eca5f23842a40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 10:49:43 -0500 Subject: [PATCH 02/11] First HTTP test passes. --- src/allmydata/test/test_istorageserver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fb9765624..11758bf15 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -26,8 +26,9 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from twisted.internet import reactor from twisted.web.server import Site +from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import get_global_pool as get_treq_pool +from treq.api import set_global_pool as set_treq_pool from foolscap.api import Referenceable, RemoteException @@ -1078,6 +1079,7 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def _get_istorage_server(self): + set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() @@ -1095,8 +1097,7 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def tearDown(self): yield _SharedMixin.tearDown(self) - self._listening_port.stopListening() - yield get_treq_pool().closeCachedConnections() + yield self._listening_port.stopListening() class FoolscapSharedAPIsTests( From 66abe5dfca17b67e22474210d72eee92dcede3c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 12:02:52 -0500 Subject: [PATCH 03/11] First passing immutable-API-over-HTTP IStorageServer tests. --- src/allmydata/storage_client.py | 68 +++++++++++++++++++++-- src/allmydata/test/test_istorageserver.py | 20 +++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 13b4dfd01..ca977c9d9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -40,7 +40,7 @@ if PY2: from six import ensure_text import re, time, hashlib - +from os import urandom # On Python 2 this will be the backport. from configparser import NoSectionError @@ -75,7 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.storage.http_client import StorageClient +from allmydata.storage.http_client import StorageClient, StorageClientImmutables # who is responsible for de-duplication? @@ -1027,6 +1027,48 @@ class _StorageServer(object): ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + +@attr.s +class _FakeRemoteReference(object): + """ + Emulate a Foolscap RemoteReference, calling a local object instead. + """ + local_object = attr.ib(type=object) + + def callRemote(self, action, *args, **kwargs): + return getattr(self.local_object, action)(*args, **kwargs) + + +@attr.s +class _HTTPBucketWriter(object): + """ + Emulate a ``RIBucketWriter``. + """ + client = attr.ib(type=StorageClientImmutables) + storage_index = attr.ib(type=bytes) + share_number = attr.ib(type=int) + upload_secret = attr.ib(type=bytes) + finished = attr.ib(type=bool, default=False) + + def abort(self): + pass # TODO in later ticket + + @defer.inlineCallbacks + def write(self, offset, data): + result = yield self.client.write_share_chunk( + self.storage_index, self.share_number, self.upload_secret, offset, data + ) + if result.finished: + self.finished = True + defer.returnValue(None) + + def close(self): + # A no-op in HTTP protocol. + if not self.finished: + return defer.fail(RuntimeError("You didn't finish writing?!")) + return defer.succeed(None) + + # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @implementer(IStorageServer) # type: ignore @attr.s @@ -1041,11 +1083,12 @@ class _HTTPStorageServer(object): """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(_http_client=http_client) + return _HTTPStorageServer(http_client=http_client) def get_version(self): return self._http_client.get_version() + @defer.inlineCallbacks def allocate_buckets( self, storage_index, @@ -1055,7 +1098,24 @@ class _HTTPStorageServer(object): allocated_size, canary, ): - pass + upload_secret = urandom(20) + immutable_client = StorageClientImmutables(self._http_client) + result = immutable_client.create( + storage_index, sharenums, allocated_size, upload_secret, renew_secret, + cancel_secret + ) + result = yield result + defer.returnValue( + (result.already_have, { + share_num: _FakeRemoteReference(_HTTPBucketWriter( + client=immutable_client, + storage_index=storage_index, + share_number=share_num, + upload_secret=upload_secret + )) + for share_num in result.allocated + }) + ) def get_buckets( self, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 11758bf15..72c07ab82 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -39,6 +39,7 @@ from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient from allmydata.util.iputil import allocate_tcp_port +from allmydata.storage_client import _HTTPStorageServer # Use random generator with known seed, so results are reproducible if tests @@ -1084,12 +1085,15 @@ class _HTTPMixin(_SharedMixin): self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() self._listening_port = reactor.listenTCP( - self._port_number, Site(self._http_storage_server.get_resource()), - interface="127.0.0.1" + self._port_number, + Site(self._http_storage_server.get_resource()), + interface="127.0.0.1", ) - return StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum + return _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum, + ) ) # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) @@ -1118,6 +1122,12 @@ class FoolscapImmutableAPIsTests( """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" +class HTTPImmutableAPIsTests( + _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): From 5dfaa82ed294fe097fdd75e3b576b0ae7724e346 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:47:51 -0500 Subject: [PATCH 04/11] Skip tests that don't pass. --- src/allmydata/test/test_istorageserver.py | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 72c07ab82..65a67c586 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,6 +21,7 @@ if PY2: # fmt: on from random import Random +from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock @@ -1013,11 +1014,18 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" + SKIP_TESTS = set() + def _get_istorage_server(self): raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): + if self._testMethodName in self.SKIP_TESTS: + raise SkipTest( + "Test {} is still not supported".format(self._testMethodName) + ) + AsyncTestCase.setUp(self) self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) @@ -1127,6 +1135,23 @@ class HTTPImmutableAPIsTests( ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + # These will start passing in future PRs as HTTP protocol is implemented. + SKIP_TESTS = { + "test_abort", + "test_add_lease_renewal", + "test_add_new_lease", + "test_advise_corrupt_share", + "test_allocate_buckets_repeat", + "test_bucket_advise_corrupt_share", + "test_disconnection", + "test_get_buckets_skips_unfinished_buckets", + "test_matching_overlapping_writes", + "test_non_matching_overlapping_writes", + "test_read_bucket_at_offset", + "test_written_shares_are_readable", + "test_written_shares_are_allocated", + } + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase From 5cda7ad8b9221df9e5208a6592d39014354ded24 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:52:04 -0500 Subject: [PATCH 05/11] News file. --- newsfragments/3868.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3868.minor diff --git a/newsfragments/3868.minor b/newsfragments/3868.minor new file mode 100644 index 000000000..e69de29bb From c2e524ddb8fd3b655aad3577a6524b08f359ce4a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:55:13 -0500 Subject: [PATCH 06/11] Make mypy happy. --- src/allmydata/test/test_istorageserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 65a67c586..96d4f687f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,6 +19,8 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +else: + from typing import Set from random import Random from unittest import SkipTest @@ -1014,7 +1016,7 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" - SKIP_TESTS = set() + SKIP_TESTS = set() # type: Set[str] def _get_istorage_server(self): raise NotImplementedError("implement in subclass") From 0fbf746e27767c548219b23d25d883ca40a6ab82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:30:27 -0500 Subject: [PATCH 07/11] Skip on Python 2. --- src/allmydata/test/test_istorageserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 96d4f687f..9b21c6480 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1089,6 +1089,11 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + return _SharedMixin.setUp(self) + def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" From 1dfc0bde3679c7eaee8617b2dd9008ef8ae37053 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:43:49 -0500 Subject: [PATCH 08/11] Use better method to listen on random port. --- src/allmydata/test/test_istorageserver.py | 39 +++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9b21c6480..2a875b120 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -25,9 +25,10 @@ else: from random import Random from unittest import SkipTest -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor +from twisted.internet.endpoints import serverFromString from twisted.web.server import Site from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL @@ -37,11 +38,10 @@ from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin -from .common import AsyncTestCase +from .common import AsyncTestCase, SameProcessStreamEndpointAssigner from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient -from allmydata.util.iputil import allocate_tcp_port from allmydata.storage_client import _HTTPStorageServer @@ -1029,6 +1029,11 @@ class _SharedMixin(SystemTestMixin): ) AsyncTestCase.setUp(self) + + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) + self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -1041,7 +1046,7 @@ class _SharedMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock - self.storage_client = self._get_istorage_server() + self.storage_client = yield self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1073,7 +1078,7 @@ class _FoolscapMixin(_SharedMixin): def _get_istorage_server(self): client = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) - return client + return succeed(client) @inlineCallbacks def disconnect(self): @@ -1094,20 +1099,26 @@ class _HTTPMixin(_SharedMixin): self.skipTest("Not going to bother supporting Python 2") return _SharedMixin.setUp(self) + @inlineCallbacks def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) - self._port_number = allocate_tcp_port() - self._listening_port = reactor.listenTCP( - self._port_number, - Site(self._http_storage_server.get_resource()), - interface="127.0.0.1", + + # Listen on randomly assigned port: + tcp_address, endpoint_string = self._port_assigner.assign(reactor) + _, host, port = tcp_address.split(":") + port = int(port) + endpoint = serverFromString(reactor, endpoint_string) + self._listening_port = yield endpoint.listen( + Site(self._http_storage_server.get_resource()) ) - return _HTTPStorageServer.from_http_client( - StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum, + returnValue( + _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL().replace(scheme="http", host=host, port=port), + swissnum, + ) ) ) # Eventually should also: From 23c8bde9d597b83cc3f89ac9a781f3e385961cea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:44:55 -0500 Subject: [PATCH 09/11] Nicer cleanup. --- src/allmydata/test/test_istorageserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 2a875b120..cfd81feda 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1113,6 +1113,7 @@ class _HTTPMixin(_SharedMixin): self._listening_port = yield endpoint.listen( Site(self._http_storage_server.get_resource()) ) + self.addCleanup(self._listening_port.stopListening) returnValue( _HTTPStorageServer.from_http_client( StorageClient( @@ -1124,11 +1125,6 @@ class _HTTPMixin(_SharedMixin): # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) - @inlineCallbacks - def tearDown(self): - yield _SharedMixin.tearDown(self) - yield self._listening_port.stopListening() - class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase From 6b3722d3f664d9274ce70812a485eb99a8e5f5a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:50:29 -0500 Subject: [PATCH 10/11] Avoid using possibly-private API. --- src/allmydata/test/test_istorageserver.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cfd81feda..7ed8a1b65 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -30,9 +30,9 @@ from twisted.internet.task import Clock from twisted.internet import reactor from twisted.internet.endpoints import serverFromString from twisted.web.server import Site -from twisted.web.client import HTTPConnectionPool +from twisted.web.client import Agent, HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import set_global_pool as set_treq_pool +from treq.client import HTTPClient from foolscap.api import Referenceable, RemoteException @@ -1101,24 +1101,29 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def _get_istorage_server(self): - set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" - self._http_storage_server = HTTPServer(self.server, swissnum) + http_storage_server = HTTPServer(self.server, swissnum) # Listen on randomly assigned port: tcp_address, endpoint_string = self._port_assigner.assign(reactor) _, host, port = tcp_address.split(":") port = int(port) endpoint = serverFromString(reactor, endpoint_string) - self._listening_port = yield endpoint.listen( - Site(self._http_storage_server.get_resource()) + listening_port = yield endpoint.listen(Site(http_storage_server.get_resource())) + self.addCleanup(listening_port.stopListening) + + # Create HTTP client with non-persistent connections, so we don't leak + # state across tests: + treq_client = HTTPClient( + Agent(reactor, HTTPConnectionPool(reactor, persistent=False)) ) - self.addCleanup(self._listening_port.stopListening) + returnValue( _HTTPStorageServer.from_http_client( StorageClient( DecodedURL().replace(scheme="http", host=host, port=port), swissnum, + treq=treq_client, ) ) ) From c2c3411dc40155e31743089d99465ad9041cdafc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:57:48 -0500 Subject: [PATCH 11/11] Try to fix Python 2. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ae70a3bb..f65890d37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,7 +217,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -277,7 +277,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }}