From 5f196050753b70c9224f1a10427d13649553b66a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:41:51 -0400 Subject: [PATCH 1/6] During testing, ensure we're not getting text/html unexpectedly. --- src/allmydata/storage/http_client.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f786b8f30..7314adf38 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -443,11 +443,20 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return await self._treq.request( + response = await self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs ) - async def decode_cbor(self, response, schema: Schema) -> object: + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: + if response.code != 404: + # We're doing API queries, HTML is never correct except in 404, but + # it's the default for Twisted's web server so make sure nothing + # unexpected happened. + assert get_content_type(response.headers) != "text/html" + + return response + + async def decode_cbor(self, response: IResponse, schema: Schema) -> object: """Given HTTP response, return decoded CBOR body.""" with start_action(action_type="allmydata:storage:http-client:decode-cbor"): if response.code > 199 and response.code < 300: From fbd6dbda47fec79f016ff5ba4609f1b03203a248 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:42:02 -0400 Subject: [PATCH 2/6] text/html is a bad default content type. --- src/allmydata/storage/http_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8647274f8..e0040d377 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -106,6 +106,9 @@ def _authorization_decorator(required_secrets): def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): + # Don't set text/html content type by default: + request.defaultContentType = None + with start_action( action_type="allmydata:storage:http-server:handle-request", method=request.method, From 2292d64fcddc5d585551a0310a6b3076eb68caf3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:49:09 -0400 Subject: [PATCH 3/6] Set a better content type for data downloads. --- src/allmydata/storage/http_client.py | 6 ++++++ src/allmydata/storage/http_server.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7314adf38..64962e7b6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -596,6 +596,12 @@ def read_share_chunk( if response.code == http.NO_CONTENT: return b"" + content_type = get_content_type(response.headers) + if content_type != "application/octet-stream": + raise ValueError( + f"Content-type was wrong: {content_type}, should be application/octet-stream" + ) + if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( response.headers.getRawHeaders("content-range")[0] or "" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e0040d377..0791c3389 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -778,6 +778,7 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" + request.setHeader("content-type", "application/octet-stream") try: bucket = self._storage_server.get_buckets(storage_index)[share_number] except KeyError: @@ -883,7 +884,8 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - + request.setHeader("content-type", "application/octet-stream") + try: share_length = self._storage_server.get_mutable_share_length( storage_index, share_number From 5632e82e1338541d3c8f574070f402fabdc3523c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:49:29 -0400 Subject: [PATCH 4/6] News fragment. --- newsfragments/4016.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4016.minor diff --git a/newsfragments/4016.minor b/newsfragments/4016.minor new file mode 100644 index 000000000..e69de29bb From 8c8e24a3b9c7655c197d24ece14e559511727610 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:50:05 -0400 Subject: [PATCH 5/6] Black reformat. --- src/allmydata/storage/http_server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0791c3389..7d7398b1e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -117,9 +117,9 @@ def _authorization_decorator(required_secrets): try: # Check Authorization header: if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( - "utf-8" - ), + request.requestHeaders.getRawHeaders("Authorization", [""])[ + 0 + ].encode("utf-8"), swissnum_auth_header(self._swissnum), ): raise _HTTPError(http.UNAUTHORIZED) @@ -494,6 +494,7 @@ def read_range( def _add_error_handling(app: Klein): """Add exception handlers to a Klein app.""" + @app.handle_errors(_HTTPError) def _http_error(_, request, failure): """Handle ``_HTTPError`` exceptions.""" @@ -885,7 +886,7 @@ class HTTPServer(object): def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" request.setHeader("content-type", "application/octet-stream") - + try: share_length = self._storage_server.get_mutable_share_length( storage_index, share_number From 3d6b3b3b74f2fc4878629e9838471f3d12a1eb6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:10:23 -0400 Subject: [PATCH 6/6] Use modern Docker image. --- .readthedocs.yaml | 5 +++++ newsfragments/4026.minor | 0 2 files changed, 5 insertions(+) create mode 100644 newsfragments/4026.minor diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 65b390f26..665b53178 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,5 +1,10 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.10" + python: install: - requirements: docs/requirements.txt diff --git a/newsfragments/4026.minor b/newsfragments/4026.minor new file mode 100644 index 000000000..e69de29bb