From b4b4a1aabfb471912c40bab980d9a2e78a36ab8c Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 29 Dec 2020 12:39:00 -0500 Subject: [PATCH 1/3] Port webish/test_webish to Python 3 --- newsfragments/3577.minor | 0 src/allmydata/test/web/test_webish.py | 16 +++++++++--- src/allmydata/util/_python3.py | 2 ++ src/allmydata/webish.py | 36 ++++++++++++++++++++------- 4 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3577.minor diff --git a/newsfragments/3577.minor b/newsfragments/3577.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index e680acd04..12a04a6eb 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.webish``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from uuid import ( uuid4, @@ -96,7 +106,7 @@ class TahoeLAFSRequestTests(SyncTestCase): ]) self._fields_test( b"POST", - {b"content-type": b"multipart/form-data; boundary={}".format(boundary)}, + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, form_data.encode("ascii"), AfterPreprocessing( lambda fs: { @@ -105,8 +115,8 @@ class TahoeLAFSRequestTests(SyncTestCase): in fs.keys() }, Equals({ - b"foo": b"bar", - b"baz": b"some file contents", + "foo": "bar", + "baz": b"some file contents", }), ), ) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a7b77001a..b2bc47153 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -110,6 +110,7 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.webish", ] PORTED_TEST_MODULES = [ @@ -179,6 +180,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_upload", "allmydata.test.test_uri", "allmydata.test.test_util", + "allmydata.test.test_webish", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", "allmydata.test.web.test_util", diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index f32f56714..b3b819b6a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str import re, time, tempfile @@ -65,18 +77,24 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == 'POST': + if self.method == b'POST': # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. - self.fields = FieldStorage( - self.content, - { - name.lower(): value[-1] - for (name, value) - in self.requestHeaders.getAllRawHeaders() - }, - environ={'REQUEST_METHOD': 'POST'}) + + headers = { + ensure_str(name.lower()): ensure_str(value[-1]) + for (name, value) + in self.requestHeaders.getAllRawHeaders() + } + + if 'content-length' not in headers: + # Python 3's cgi module would really, really like us to set + # Content-Length. This seems likely to shoot performance in + # the foot. + headers['content-length'] = len(self.content.getvalue()) + + self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'}) self.content.seek(0) self._tahoeLAFSSecurityPolicy() From 2a8fa4da7a475eeb34c3f19d8e9eb30dc30d1b33 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 29 Dec 2020 12:54:19 -0500 Subject: [PATCH 2/3] Fix test module reference in util._python3 --- src/allmydata/util/_python3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b2bc47153..b54fed188 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -180,9 +180,9 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_upload", "allmydata.test.test_uri", "allmydata.test.test_util", - "allmydata.test.test_webish", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", - "allmydata.test.web.test_util", "allmydata.test.web.test_status", + "allmydata.test.web.test_util", + "allmydata.test.web.test_webish", ] From bd402ce1f411cc97b7dd81a210ff897049e2ce67 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 5 Jan 2021 06:27:46 -0500 Subject: [PATCH 3/3] Compute Content-Length more betterly --- src/allmydata/webish.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index b3b819b6a..e90fa573a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -89,10 +89,10 @@ class TahoeLAFSRequest(Request, object): } if 'content-length' not in headers: - # Python 3's cgi module would really, really like us to set - # Content-Length. This seems likely to shoot performance in - # the foot. - headers['content-length'] = len(self.content.getvalue()) + # Python 3's cgi module would really, really like us to set Content-Length. + self.content.seek(0, 2) + headers['content-length'] = str(self.content.tell()) + self.content.seek(0) self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'}) self.content.seek(0)