diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe911e34d..dc9854ae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,9 @@ jobs: - "3.11" include: # On macOS don't bother with 3.8, just to get faster builds. - - os: macos-latest + - os: macos-12 python-version: "3.9" - - os: macos-latest + - os: macos-12 python-version: "3.11" # We only support PyPy on Linux at the moment. - os: ubuntu-latest @@ -165,7 +165,7 @@ jobs: fail-fast: false matrix: include: - - os: macos-latest + - os: macos-12 python-version: "3.9" force-foolscap: false - os: windows-latest @@ -248,7 +248,7 @@ jobs: fail-fast: false matrix: os: - - macos-10.15 + - macos-12 - windows-latest - ubuntu-latest python-version: diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 65b390f26..665b53178 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,5 +1,10 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.10" + python: install: - requirements: docs/requirements.txt diff --git a/integration/test_get_put.py b/integration/test_get_put.py index f121d6284..e30a34f97 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -6,8 +6,9 @@ and stdout. from subprocess import Popen, PIPE, check_output, check_call import pytest -from pytest_twisted import ensureDeferred from twisted.internet import reactor +from twisted.internet.threads import blockingCallFromThread +from twisted.internet.defer import Deferred from .util import run_in_thread, cli, reconfigure @@ -86,8 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path): assert outfile.read_bytes() == tempfile.read_bytes() -@ensureDeferred -async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): +@run_in_thread +def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): """ Tahoe-LAFS used to have a default max segment size of 128KB, and is now 1MB. Test that an upload created when 128KB was the default can be @@ -100,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic with tempfile.open("wb") as f: f.write(large_data) - async def set_segment_size(segment_size): - await reconfigure( + def set_segment_size(segment_size): + return blockingCallFromThread( reactor, - request, - alice, - (1, 1, 1), - None, - max_segment_size=segment_size - ) + lambda: Deferred.fromCoroutine(reconfigure( + reactor, + request, + alice, + (1, 1, 1), + None, + max_segment_size=segment_size + )) + ) # 1. Upload file 1 with default segment size set to 1MB - await set_segment_size(1024 * 1024) + set_segment_size(1024 * 1024) cli(alice, "put", str(tempfile), "getput:seg1024kb") # 2. Download file 1 with default segment size set to 128KB - await set_segment_size(128 * 1024) + set_segment_size(128 * 1024) assert large_data == check_output( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] ) @@ -124,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic cli(alice, "put", str(tempfile), "getput:seg128kb") # 4. Download file 2 with default segment size set to 1MB - await set_segment_size(1024 * 1024) + set_segment_size(1024 * 1024) assert large_data == check_output( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] ) diff --git a/integration/test_web.py b/integration/test_web.py index b3c4a8e5f..fd29504f8 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -14,6 +14,8 @@ from __future__ import annotations import time from urllib.parse import unquote as url_unquote, quote as url_quote +from twisted.internet.threads import deferToThread + import allmydata.uri from allmydata.util import jsonbytes as json @@ -24,7 +26,7 @@ import requests import html5lib from bs4 import BeautifulSoup -from pytest_twisted import ensureDeferred +import pytest_twisted @run_in_thread def test_index(alice): @@ -185,7 +187,7 @@ def test_deep_stats(alice): time.sleep(.5) -@util.run_in_thread +@run_in_thread def test_status(alice): """ confirm we get something sensible from /status and the various sub-types @@ -251,7 +253,7 @@ def test_status(alice): assert found_download, "Failed to find the file we downloaded in the status-page" -@ensureDeferred +@pytest_twisted.ensureDeferred async def test_directory_deep_check(reactor, request, alice): """ use deep-check and confirm the result pages work @@ -263,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice): total = 4 await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) + await deferToThread(_test_directory_deep_check_blocking, alice) + +def _test_directory_deep_check_blocking(alice): # create a directory resp = requests.post( util.node_url(alice.node_dir, u"uri"), diff --git a/newsfragments/4016.minor b/newsfragments/4016.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/4023.minor b/newsfragments/4023.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/4026.minor b/newsfragments/4026.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 481988758..549fc9719 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -347,7 +347,7 @@ class StorageClient(object): certificate_hash = nurl.user.encode("ascii") if pool is None: pool = HTTPConnectionPool(reactor) - pool.maxPersistentPerHost = 20 + pool.maxPersistentPerHost = 10 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) @@ -465,11 +465,20 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return await self._treq.request( + response = await self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs ) - async def decode_cbor(self, response, schema: Schema) -> object: + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: + if response.code != 404: + # We're doing API queries, HTML is never correct except in 404, but + # it's the default for Twisted's web server so make sure nothing + # unexpected happened. + assert get_content_type(response.headers) != "text/html" + + return response + + async def decode_cbor(self, response: IResponse, schema: Schema) -> object: """Given HTTP response, return decoded CBOR body.""" with start_action(action_type="allmydata:storage:http-client:decode-cbor"): if response.code > 199 and response.code < 300: @@ -627,6 +636,12 @@ def read_share_chunk( if response.code == http.NO_CONTENT: return b"" + content_type = get_content_type(response.headers) + if content_type != "application/octet-stream": + raise ValueError( + f"Content-type was wrong: {content_type}, should be application/octet-stream" + ) + if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( response.headers.getRawHeaders("content-range")[0] or "" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8647274f8..7d7398b1e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -106,6 +106,9 @@ def _authorization_decorator(required_secrets): def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): + # Don't set text/html content type by default: + request.defaultContentType = None + with start_action( action_type="allmydata:storage:http-server:handle-request", method=request.method, @@ -114,9 +117,9 @@ def _authorization_decorator(required_secrets): try: # Check Authorization header: if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( - "utf-8" - ), + request.requestHeaders.getRawHeaders("Authorization", [""])[ + 0 + ].encode("utf-8"), swissnum_auth_header(self._swissnum), ): raise _HTTPError(http.UNAUTHORIZED) @@ -491,6 +494,7 @@ def read_range( def _add_error_handling(app: Klein): """Add exception handlers to a Klein app.""" + @app.handle_errors(_HTTPError) def _http_error(_, request, failure): """Handle ``_HTTPError`` exceptions.""" @@ -775,6 +779,7 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" + request.setHeader("content-type", "application/octet-stream") try: bucket = self._storage_server.get_buckets(storage_index)[share_number] except KeyError: @@ -880,6 +885,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" + request.setHeader("content-type", "application/octet-stream") try: share_length = self._storage_server.get_mutable_share_length(