Merge pull request #1289 from tahoe-lafs/4012.add-timeout-to-http-client

Add timeout to the blocking http client

Fixes ticket:4012

integration/conftest.py

@@ -4,6 +4,7 @@ Ported to Python 3.
 
 from __future__ import annotations
 
+import os
 import sys
 import shutil
 from time import sleep
@@ -49,6 +50,11 @@ from .util import (
 )
 
 
+# No reason for HTTP requests to take longer than two minutes in the
+# integration tests. See allmydata/scripts/common_http.py for usage.
+os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
+
+
 # pytest customization hooks
 
 def pytest_addoption(parser):

src/allmydata/scripts/common_http.py

@@ -1,19 +1,11 @@
 """
-Ported to Python 3.
+Blocking HTTP client APIs.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 from io import BytesIO
-from six.moves import urllib, http_client
-import six
+from http import client as http_client
+import urllib
 import allmydata # for __full_version__
 
 from allmydata.util.encodingutil import quote_output
@@ -51,7 +43,7 @@ class BadResponse(object):
 def do_http(method, url, body=b""):
     if isinstance(body, bytes):
         body = BytesIO(body)
-    elif isinstance(body, six.text_type):
+    elif isinstance(body, str):
         raise TypeError("do_http body must be a bytestring, not unicode")
     else:
         # We must give a Content-Length header to twisted.web, otherwise it
@@ -61,10 +53,17 @@ def do_http(method, url, body=b""):
         assert body.seek
         assert body.read
     scheme, host, port, path = parse_url(url)
+
+    # For testing purposes, allow setting a timeout on HTTP requests. If this
+    # ever become a user-facing feature, this should probably be a CLI option?
+    timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None)
+    if timeout is not None:
+        timeout = float(timeout)
+
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port)
+        c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port)
+        c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)
@@ -85,7 +84,7 @@ def do_http(method, url, body=b""):
         return BadResponse(url, err)
 
     while True:
-        data = body.read(8192)
+        data = body.read(65536)
         if not data:
             break
         c.send(data)
@@ -94,16 +93,14 @@ def do_http(method, url, body=b""):
 
 
 def format_http_success(resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s %s" % (resp.status, six.ensure_text(resp.reason)),
+        "%s %s" % (resp.status, resp.reason),
         quotemarks=False)
 
 def format_http_error(msg, resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason),
-                           six.ensure_text(resp.read())),
+        "%s: %s %s\n%r" % (msg, resp.status, resp.reason,
+                           resp.read()),
         quotemarks=False)
 
 def check_http_error(resp, stderr):