merge PR428: fix client.getPage deprecation warnings

closes ticket:2857

src/allmydata/test/cli/test_status.py

@@ -8,7 +8,6 @@ from UserDict import UserDict
 
 from twisted.trial import unittest
 from twisted.internet import defer
-from twisted.web.client import getPage
 
 from allmydata.mutable.publish import MutableData
 from allmydata.scripts.common_http import BadResponse
@@ -18,6 +17,7 @@ from allmydata.scripts.tahoe_status import pretty_progress
 from allmydata.scripts.tahoe_status import do_status
 
 from ..no_network import GridTestMixin
+from ..common_web import do_http
 from .common import CLITestMixin
 
 
@@ -83,7 +83,7 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.uri = filenode.get_uri()
 
         # make sure our web-port is actually answering
-        yield getPage('http://127.0.0.1:{}/status?t=json'.format(self.client_webports[0]))
+        yield do_http("get", 'http://127.0.0.1:{}/status?t=json'.format(self.client_webports[0]))
 
     def test_simple(self):
         d = self.do_cli('status')# '--verbose')

src/allmydata/test/common.py

@@ -1,6 +1,8 @@
 import os, random, struct
+import treq
 from zope.interface import implementer
 from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.internet.interfaces import IPullProducer
 from twisted.python import failure
 from twisted.application import service
@@ -509,6 +511,18 @@ class WebErrorMixin:
         d.addBoth(self._shouldHTTPError, which, _validate)
         return d
 
+    @inlineCallbacks
+    def assertHTTPError(self, url, code, response_substring,
+                        method="get", persistent=False,
+                        **args):
+        response = yield treq.request(method, url, persistent=persistent,
+                                      **args)
+        body = yield response.content()
+        self.assertEquals(response.code, code)
+        if response_substring is not None:
+            self.assertIn(response_substring, body)
+        returnValue(body)
+
 class ErrorMixin(WebErrorMixin):
     def explain_error(self, f):
         if f.check(defer.FirstError):

src/allmydata/test/common_web.py

@@ -2,7 +2,6 @@
 import re
 import treq
 from twisted.internet import defer
-from twisted.web import client
 from twisted.web.error import Error
 from nevow.testutil import FakeRequest
 from nevow import inevow, context
@@ -61,29 +60,6 @@ class WebRenderingMixin:
         s = re.sub(r'\s+', ' ', s)
         return s
 
-
-class MyGetter(client.HTTPPageGetter):
-    handleStatus_206 = lambda self: self.handleStatus_200() # PARTIAL_CONTENT
-    handleStatus_304 = lambda self: self.handleStatus_200() # NOT_MODIFIED
-
-class HTTPClientHEADFactory(client.HTTPClientFactory):
-    protocol = MyGetter
-
-    def noPage(self, reason):
-        # Twisted-2.5.0 and earlier had a bug, in which they would raise an
-        # exception when the response to a HEAD request had no body (when in
-        # fact they are defined to never have a body). This was fixed in
-        # Twisted-8.0 . To work around this, we catch the
-        # PartialDownloadError and make it disappear.
-        if (reason.check(client.PartialDownloadError)
-            and self.method.upper() == "HEAD"):
-            self.page("")
-            return
-        return client.HTTPClientFactory.noPage(self, reason)
-
-class HTTPClientGETFactory(client.HTTPClientFactory):
-    protocol = MyGetter
-
 @defer.inlineCallbacks
 def do_http(method, url, **kwargs):
     response = yield treq.request(method, url, persistent=False, **kwargs)

src/allmydata/test/no_network.py

@@ -16,10 +16,12 @@
 import os
 from zope.interface import implementer
 from twisted.application import service
-from twisted.internet import defer, reactor
+from twisted.internet import defer
 from twisted.python.failure import Failure
+from twisted.web.error import Error
 from foolscap.api import Referenceable, fireEventually, RemoteException
 from base64 import b32encode
+import treq
 
 from allmydata.util.assertutil import _assert
 
@@ -28,7 +30,6 @@ from allmydata.client import Client
 from allmydata.storage.server import StorageServer, storage_index_to_dir
 from allmydata.util import fileutil, idlib, hashutil
 from allmydata.util.hashutil import permute_server_hash
-from allmydata.test.common_web import HTTPClientGETFactory
 from allmydata.interfaces import IStorageBroker, IServer
 from .common import TEST_RSA_KEY_SIZE
 
@@ -479,21 +480,28 @@ class GridTestMixin:
             with open(i_sharefile, "wb") as f:
                 f.write(corruptdata)
 
+    @defer.inlineCallbacks
     def GET(self, urlpath, followRedirect=False, return_response=False,
             method="GET", clientnum=0, **kwargs):
         # if return_response=True, this fires with (data, statuscode,
         # respheaders) instead of just data.
         assert not isinstance(urlpath, unicode)
         url = self.client_baseurls[clientnum] + urlpath
-        factory = HTTPClientGETFactory(url, method=method,
-                                       followRedirect=followRedirect, **kwargs)
-        reactor.connectTCP("localhost", self.client_webports[clientnum],factory)
-        d = factory.deferred
-        def _got_data(data):
-            return (data, factory.status, factory.response_headers)
+
+        response = yield treq.request(method, url, persistent=False,
+                                      allow_redirects=followRedirect,
+                                      **kwargs)
+        data = yield response.content()
         if return_response:
-            d.addCallback(_got_data)
-        return factory.deferred
+            # we emulate the old HTTPClientGetFactory-based response, which
+            # wanted a tuple of (bytestring of data, bytestring of response
+            # code like "200" or "404", and a
+            # twisted.web.http_headers.Headers instance). Fortunately treq's
+            # response.headers has one.
+            defer.returnValue( (data, str(response.code), response.headers) )
+        if 400 <= response.code < 600:
+            raise Error(response.code, response=data)
+        defer.returnValue(data)
 
     def PUT(self, urlpath, **kwargs):
         return self.GET(urlpath, method="PUT", **kwargs)