Merge branch '4028-more-integration' into 4027-invalid-unicode

.github/workflows/ci.yml

@@ -53,9 +53,9 @@ jobs:
           - "3.11"
         include:
           # On macOS don't bother with 3.8, just to get faster builds.
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.9"
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.11"
           # We only support PyPy on Linux at the moment.
           - os: ubuntu-latest
@@ -165,7 +165,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.9"
             force-foolscap: false
           - os: windows-latest
@@ -248,7 +248,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - macos-10.15
+          - macos-12
           - windows-latest
           - ubuntu-latest
         python-version:

integration/conftest.py

@@ -401,9 +401,6 @@ def alice(
             reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
             web_port="tcp:9980:interface=localhost",
             storage=False,
-            # We're going to kill this ourselves, so no need for finalizer to
-            # do it:
-            finalize=False,
         )
     )
     pytest_twisted.blockon(await_client_ready(process))

integration/test_get_put.py

@@ -6,8 +6,9 @@ and stdout.
 from subprocess import Popen, PIPE, check_output, check_call
 
 import pytest
-from pytest_twisted import ensureDeferred
 from twisted.internet import reactor
+from twisted.internet.threads import blockingCallFromThread
+from twisted.internet.defer import Deferred
 
 from .util import run_in_thread, cli, reconfigure
 
@@ -86,8 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
     assert outfile.read_bytes() == tempfile.read_bytes()
 
 
-@ensureDeferred
-async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
+@run_in_thread
+def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
     """
     Tahoe-LAFS used to have a default max segment size of 128KB, and is now
     1MB.  Test that an upload created when 128KB was the default can be
@@ -100,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
     with tempfile.open("wb") as f:
         f.write(large_data)
 
-    async def set_segment_size(segment_size):
-        await reconfigure(
+    def set_segment_size(segment_size):
+        return blockingCallFromThread(
             reactor,
-            request,
-            alice,
-            (1, 1, 1),
-            None,
-            max_segment_size=segment_size
-    )
+            lambda: Deferred.fromCoroutine(reconfigure(
+                reactor,
+                request,
+                alice,
+                (1, 1, 1),
+                None,
+                max_segment_size=segment_size
+            ))
+        )
 
     # 1. Upload file 1 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
     cli(alice, "put", str(tempfile), "getput:seg1024kb")
 
     # 2. Download file 1 with default segment size set to 128KB
-    await set_segment_size(128 * 1024)
+    set_segment_size(128 * 1024)
     assert large_data == check_output(
         ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
     )
@@ -124,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
     cli(alice, "put", str(tempfile), "getput:seg128kb")
 
     # 4. Download file 2 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
     assert large_data == check_output(
         ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
     )

integration/test_web.py

@@ -14,6 +14,8 @@ from __future__ import annotations
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote
 
+from twisted.internet.threads import deferToThread
+
 import allmydata.uri
 from allmydata.util import jsonbytes as json
 
@@ -24,7 +26,7 @@ import requests
 import html5lib
 from bs4 import BeautifulSoup
 
-from pytest_twisted import ensureDeferred
+import pytest_twisted
 
 @run_in_thread
 def test_index(alice):
@@ -185,7 +187,7 @@ def test_deep_stats(alice):
         time.sleep(.5)
 
 
-@util.run_in_thread
+@run_in_thread
 def test_status(alice):
     """
     confirm we get something sensible from /status and the various sub-types
@@ -251,7 +253,7 @@ def test_status(alice):
     assert found_download, "Failed to find the file we downloaded in the status-page"
 
 
-@ensureDeferred
+@pytest_twisted.ensureDeferred
 async def test_directory_deep_check(reactor, request, alice):
     """
     use deep-check and confirm the result pages work
@@ -263,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice):
     total = 4
 
     await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
+    await deferToThread(_test_directory_deep_check_blocking, alice)
 
+
+def _test_directory_deep_check_blocking(alice):
     # create a directory
     resp = requests.post(
         util.node_url(alice.node_dir, u"uri"),

src/allmydata/storage/http_client.py

@@ -310,6 +310,7 @@ class StorageClient(object):
     _base_url: DecodedURL
     _swissnum: bytes
     _treq: Union[treq, StubTreq, HTTPClient]
+    _pool: Optional[HTTPConnectionPool]
     _clock: IReactorTime
 
     @classmethod
@@ -325,7 +326,7 @@ class StorageClient(object):
         certificate_hash = nurl.user.encode("ascii")
         if pool is None:
             pool = HTTPConnectionPool(reactor)
-            pool.maxPersistentPerHost = 20
+            pool.maxPersistentPerHost = 10
 
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
             cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
@@ -339,7 +340,7 @@ class StorageClient(object):
         )
 
         https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
-        return cls(https_url, swissnum, treq_client, reactor)
+        return cls(https_url, swissnum, treq_client, pool, reactor)
 
     def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
@@ -479,6 +480,11 @@ class StorageClient(object):
                 ).read()
                 raise ClientException(response.code, response.phrase, data)
 
+    def shutdown(self) -> Deferred:
+        """Shutdown any connections."""
+        if self._pool is not None:
+            return self._pool.closeCachedConnections()
+
 
 @define(hash=True)
 class StorageClientGeneral(object):

src/allmydata/storage_client.py

@@ -1271,6 +1271,11 @@ class HTTPNativeStorageServer(service.MultiService):
         if self._lc.running:
             self._lc.stop()
         self._failed_to_connect("shut down")
+
+        if self._istorage_server is not None:
+            client_shutting_down = self._istorage_server._http_client.shutdown()
+            result.addCallback(lambda _: client_shutting_down)
+
         return result
 
 

src/allmydata/test/test_storage_http.py

@@ -335,6 +335,7 @@ class CustomHTTPServerTests(SyncTestCase):
             DecodedURL.from_text("http://127.0.0.1"),
             SWISSNUM_FOR_TEST,
             treq=treq,
+            pool=None,
             # We're using a Treq private API to get the reactor, alas, but only
             # in a test, so not going to worry about it too much. This would be
             # fixed if https://github.com/twisted/treq/issues/226 were ever
@@ -555,6 +556,7 @@ class HttpTestFixture(Fixture):
             DecodedURL.from_text("http://127.0.0.1"),
             SWISSNUM_FOR_TEST,
             treq=self.treq,
+            pool=None,
             clock=self.clock,
         )
 
@@ -667,6 +669,7 @@ class GenericHTTPAPITests(SyncTestCase):
                 DecodedURL.from_text("http://127.0.0.1"),
                 b"something wrong",
                 treq=StubTreq(self.http.http_server.get_resource()),
+                pool=None,
                 clock=self.http.clock,
             )
         )