From bd1e4507ee21d676307b5aef91611a40ccab6a1c Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 23 Jul 2017 15:48:14 -0500 Subject: [PATCH 01/14] remove unused HTTPClientHEADFactory --- src/allmydata/test/common_web.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index 162b1bba8..33bcd19fb 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -66,21 +66,6 @@ class MyGetter(client.HTTPPageGetter): handleStatus_206 = lambda self: self.handleStatus_200() # PARTIAL_CONTENT handleStatus_304 = lambda self: self.handleStatus_200() # NOT_MODIFIED -class HTTPClientHEADFactory(client.HTTPClientFactory): - protocol = MyGetter - - def noPage(self, reason): - # Twisted-2.5.0 and earlier had a bug, in which they would raise an - # exception when the response to a HEAD request had no body (when in - # fact they are defined to never have a body). This was fixed in - # Twisted-8.0 . To work around this, we catch the - # PartialDownloadError and make it disappear. - if (reason.check(client.PartialDownloadError) - and self.method.upper() == "HEAD"): - self.page("") - return - return client.HTTPClientFactory.noPage(self, reason) - class HTTPClientGETFactory(client.HTTPClientFactory): protocol = MyGetter From 3311f9520d098ccd001d300999320eac1e966aff Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 23 Jul 2017 15:48:36 -0500 Subject: [PATCH 02/14] test.cli.test_status: replace getPage with treq --- src/allmydata/test/cli/test_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index f18cbeab2..0ae420a7c 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -8,7 +8,6 @@ from UserDict import UserDict from twisted.trial import unittest from twisted.internet import defer -from twisted.web.client import getPage from allmydata.mutable.publish import MutableData from allmydata.scripts.common_http import BadResponse @@ -18,6 +17,7 @@ from allmydata.scripts.tahoe_status import pretty_progress from allmydata.scripts.tahoe_status import do_status from ..no_network import GridTestMixin +from ..common_web import do_http from .common import CLITestMixin @@ -83,7 +83,7 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): self.uri = filenode.get_uri() # make sure our web-port is actually answering - yield getPage('http://127.0.0.1:{}/status?t=json'.format(self.client_webports[0])) + yield do_http("get", 'http://127.0.0.1:{}/status?t=json'.format(self.client_webports[0])) def test_simple(self): d = self.do_cli('status')# '--verbose') From 5a895b5fb6ff3eacf666e1e6ae9471a74ea49ca9 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 14:06:28 -0500 Subject: [PATCH 03/14] GridTestMixin.GET: use treq instead of HTTPClientGETFactory --- src/allmydata/test/no_network.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1a06d1ac5..90e92f6e2 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -16,10 +16,12 @@ import os from zope.interface import implementer from twisted.application import service -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python.failure import Failure +from twisted.web.error import Error from foolscap.api import Referenceable, fireEventually, RemoteException from base64 import b32encode +import treq from allmydata.util.assertutil import _assert @@ -28,7 +30,6 @@ from allmydata.client import Client from allmydata.storage.server import StorageServer, storage_index_to_dir from allmydata.util import fileutil, idlib, hashutil from allmydata.util.hashutil import permute_server_hash -from allmydata.test.common_web import HTTPClientGETFactory from allmydata.interfaces import IStorageBroker, IServer from .common import TEST_RSA_KEY_SIZE @@ -479,21 +480,28 @@ class GridTestMixin: with open(i_sharefile, "wb") as f: f.write(corruptdata) + @defer.inlineCallbacks def GET(self, urlpath, followRedirect=False, return_response=False, method="GET", clientnum=0, **kwargs): # if return_response=True, this fires with (data, statuscode, # respheaders) instead of just data. assert not isinstance(urlpath, unicode) url = self.client_baseurls[clientnum] + urlpath - factory = HTTPClientGETFactory(url, method=method, - followRedirect=followRedirect, **kwargs) - reactor.connectTCP("localhost", self.client_webports[clientnum],factory) - d = factory.deferred - def _got_data(data): - return (data, factory.status, factory.response_headers) + + response = yield treq.request(method, url, persistent=False, + allow_redirects=followRedirect, + **kwargs) + data = yield response.content() if return_response: - d.addCallback(_got_data) - return factory.deferred + # we emulate the old HTTPClientGetFactory-based response, which + # wanted a tuple of (bytestring of data, bytestring of response + # code like "200" or "404", and a + # twisted.web.http_headers.Headers instance). Fortunately treq's + # response.headers has one. + defer.returnValue( (data, str(response.code), response.headers) ) + if 400 <= response.code < 600: + raise Error(response.code, response=data) + defer.returnValue(data) def PUT(self, urlpath, **kwargs): return self.GET(urlpath, method="PUT", **kwargs) From bee05e883c60caafa48a9732f66bfac514526705 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 15:51:58 -0500 Subject: [PATCH 04/14] test_web: remove HTTPClientGETFactory This also changes the tests to handle the new API: URL can be unicode, and the returned Headers object is not a dictionary. --- src/allmydata/test/common_web.py | 9 -- src/allmydata/test/web/test_web.py | 131 ++++++++++++++++------------- 2 files changed, 73 insertions(+), 67 deletions(-) diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index 33bcd19fb..fc8fe6e3b 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -2,7 +2,6 @@ import re import treq from twisted.internet import defer -from twisted.web import client from twisted.web.error import Error from nevow.testutil import FakeRequest from nevow import inevow, context @@ -61,14 +60,6 @@ class WebRenderingMixin: s = re.sub(r'\s+', ' ', s) return s - -class MyGetter(client.HTTPPageGetter): - handleStatus_206 = lambda self: self.handleStatus_200() # PARTIAL_CONTENT - handleStatus_304 = lambda self: self.handleStatus_200() # NOT_MODIFIED - -class HTTPClientGETFactory(client.HTTPClientFactory): - protocol = MyGetter - @defer.inlineCallbacks def do_http(method, url, **kwargs): response = yield treq.request(method, url, persistent=False, **kwargs) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index fc7155d39..6e5837313 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4,7 +4,7 @@ import treq from twisted.application import service from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred from twisted.internet.task import Clock from twisted.web import client, error, http @@ -43,7 +43,6 @@ from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION from allmydata.mutable import servermap, publish, retrieve from .. import common_util as testutil from ..common_web import ( - HTTPClientGETFactory, do_http, Error, ) @@ -497,21 +496,28 @@ class WebMixin(testutil.TimezoneMixin): self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]), self._quux_txt_readonly_uri) + @inlineCallbacks def GET(self, urlpath, followRedirect=False, return_response=False, **kwargs): # if return_response=True, this fires with (data, statuscode, # respheaders) instead of just data. - assert not isinstance(urlpath, unicode) + + # treq can accept unicode URLs, unlike the old client.getPage url = self.webish_url + urlpath - factory = HTTPClientGETFactory(url, method="GET", - followRedirect=followRedirect, **kwargs) - reactor.connectTCP("localhost", self.webish_port, factory) - d = factory.deferred - def _got_data(data): - return (data, factory.status, factory.response_headers) + response = yield treq.request("get", url, persistent=False, + allow_redirects=followRedirect, + **kwargs) + data = yield response.content() if return_response: - d.addCallback(_got_data) - return factory.deferred + # we emulate the old HTTPClientGetFactory-based response, which + # wanted a tuple of (bytestring of data, bytestring of response + # code like "200" or "404", and a + # twisted.web.http_headers.Headers instance). Fortunately treq's + # response.headers has one. + returnValue( (data, str(response.code), response.headers) ) + if 400 <= response.code < 600: + raise Error(response.code, response=data) + returnValue(data) @inlineCallbacks def HEAD(self, urlpath, return_response=False, headers={}): @@ -520,10 +526,7 @@ class WebMixin(testutil.TimezoneMixin): headers=headers) if 400 <= response.code < 600: raise Error(response.code, response="") - resp_headers = {} - for (key, values) in response.headers.getAllRawHeaders(): - resp_headers[key.lower()] = values - returnValue( ("", response.code, resp_headers) ) + returnValue( ("", response.code, response.headers) ) def PUT(self, urlpath, data, headers={}): url = self.webish_url + urlpath @@ -1036,8 +1039,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11]) d.addCallback(_got) @@ -1050,8 +1053,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes 5-%d/%d" % (length-1, length)) self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:]) d.addCallback(_got) @@ -1064,8 +1067,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes %d-%d/%d" % (length-5, length-1, length)) self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:]) d.addCallback(_got) @@ -1087,8 +1090,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got((res, status, headers)): self.failUnlessReallyEqual(res, "") self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) d.addCallback(_got) return d @@ -1100,8 +1103,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes 5-%d/%d" % (length-1, length)) d.addCallback(_got) return d @@ -1113,8 +1116,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 206) - self.failUnless(headers.has_key("content-range")) - self.failUnlessReallyEqual(headers["content-range"][0], + self.failUnless(headers.hasHeader("content-range")) + self.failUnlessReallyEqual(headers.getRawHeaders("content-range")[0], "bytes %d-%d/%d" % (length-5, length-1, length)) d.addCallback(_got) return d @@ -1134,7 +1137,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(int(status), 200) - self.failUnless(not headers.has_key("content-range")) + self.failUnless(not headers.hasHeader("content-range")) self.failUnlessReallyEqual(res, self.BAR_CONTENTS) d.addCallback(_got) return d @@ -1143,9 +1146,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True) def _got((res, status, headers)): self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers["content-length"][0], + self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], str(len(self.BAR_CONTENTS))) - self.failUnlessReallyEqual(headers["content-type"], ["text/plain"]) + self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), + ["text/plain"]) d.addCallback(_got) return d @@ -1325,7 +1329,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d = self.GET(targetbase, return_response=True, followRedirect=True) def _just_the_etag(result): data, response, headers = result - etag = headers['etag'][0] + etag = headers.getRawHeaders('etag')[0] if uri.startswith('URI:DIR'): self.failUnless(etag.startswith('DIR:'), etag) return etag @@ -1350,7 +1354,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d = self.GET(uri, return_response=True) # extract the ETag d.addCallback(lambda (data, code, headers): - headers['etag'][0]) + headers.getRawHeaders('etag')[0]) # do a GET that's supposed to match the ETag d.addCallback(lambda etag: self.GET(uri, return_response=True, @@ -1365,13 +1369,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi target = "/uri/%s?t=%s" % (uri, t) d = self.GET(target, return_response=True, followRedirect=True) d.addCallback(lambda (data, code, headers): - self.failIf("etag" in headers, target)) + self.failIf(headers.hasHeader("etag"), target)) return d def _yes_etag(uri, t): target = "/uri/%s?t=%s" % (uri, t) d = self.GET(target, return_response=True, followRedirect=True) d.addCallback(lambda (data, code, headers): - self.failUnless("etag" in headers, target)) + self.failUnless(headers.hasHeader("etag"), target)) return d d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "")) @@ -1394,7 +1398,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true", return_response=True) def _got((res, statuscode, headers)): - content_disposition = headers["content-disposition"][0] + content_disposition = headers.getRawHeaders("content-disposition")[0] self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition) self.failUnlessIsBarDotTxt(res) d.addCallback(_got) @@ -2433,7 +2437,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) - target_url = self.public_url + "/foo/" + filename.encode("utf-8") + target_url = self.public_url + u"/foo/" + filename d.addCallback(lambda res: self.GET(target_url)) d.addCallback(lambda contents: self.failUnlessReallyEqual(contents, self.NEWFILE_CONTENTS, @@ -2450,7 +2454,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) - target_url = self.public_url + "/foo/" + filename.encode("utf-8") + target_url = self.public_url + u"/foo/" + filename d.addCallback(lambda res: self.GET(target_url)) d.addCallback(lambda contents: self.failUnlessReallyEqual(contents, self.NEWFILE_CONTENTS, @@ -2736,9 +2740,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return_response=True)) def _got_headers((res, status, headers)): self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers["content-length"][0], + self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], str(len(NEW2_CONTENTS))) - self.failUnlessReallyEqual(headers["content-type"], ["text/plain"]) + self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), + ["text/plain"]) d.addCallback(_got_headers) # make sure that outdated size limits aren't enforced anymore. @@ -4045,26 +4050,36 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi "%s: wrong target" % which) return res.value.location - def test_GET_URI_form(self): - base = "/uri?uri=%s" % self._bar_txt_uri - # this is supposed to give us a redirect to /uri/$URI, plus arguments - targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri) - d = self.GET(base) - d.addBoth(self.shouldRedirect, targetbase) - d.addCallback(lambda res: self.GET(base+"&filename=bar.txt")) - d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt") - d.addCallback(lambda res: self.GET(base+"&t=json")) - d.addBoth(self.shouldRedirect, targetbase+"?t=json") - d.addCallback(self.log, "about to get file by uri") - d.addCallback(lambda res: self.GET(base, followRedirect=True)) - d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(self.log, "got file by uri, about to get dir by uri") - d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri, - followRedirect=True)) - d.addCallback(self.failUnlessIsFooJSON) - d.addCallback(self.log, "got dir by uri") + @inlineCallbacks + def shouldRedirectTo(self, url, target_location): + response = yield treq.request("get", url, persistent=False, + allow_redirects=False) + self.assertIn(response.code, [http.MOVED_PERMANENTLY, + http.FOUND, + http.TEMPORARY_REDIRECT]) + location = response.headers.getRawHeaders(b"location")[0] + self.assertEquals(location, target_location) - return d + @inlineCallbacks + def test_GET_URI_form(self): + relbase = "/uri?uri=%s" % self._bar_txt_uri + base = self.webish_url + relbase + # this is supposed to give us a redirect to /uri/$URI, plus arguments + targetbase = self.webish_url + "/uri/%s" % urllib.quote(self._bar_txt_uri) + yield self.shouldRedirectTo(base, targetbase) + yield self.shouldRedirectTo(base+"&filename=bar.txt", + targetbase+"?filename=bar.txt") + yield self.shouldRedirectTo(base+"&t=json", + targetbase+"?t=json") + + self.log(None, "about to get file by uri") + data = yield self.GET(relbase, followRedirect=True) + self.failUnlessIsBarDotTxt(data) + self.log(None, "got file by uri, about to get dir by uri") + data = yield self.GET("/uri?uri=%s&t=json" % self._foo_uri, + followRedirect=True) + self.failUnlessIsFooJSON(data) + self.log(None, "got dir by uri") def test_GET_URI_form_bad(self): d = self.shouldFail2(error.Error, "test_GET_URI_form_bad", From 202a9714c41e39d856687619f0871f703aca3c51 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 16:02:31 -0500 Subject: [PATCH 05/14] test_web test_bad_method: remove a client.getPage Add WebErrorMixin.assertHTTPError, to replace (getPage + shouldHTTPError) --- src/allmydata/test/common.py | 14 ++++++++++++++ src/allmydata/test/web/test_web.py | 9 ++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 3ad3ef3d1..f0cccb48b 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,6 +1,8 @@ import os, random, struct +import treq from zope.interface import implementer from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.interfaces import IPullProducer from twisted.python import failure from twisted.application import service @@ -509,6 +511,18 @@ class WebErrorMixin: d.addBoth(self._shouldHTTPError, which, _validate) return d + @inlineCallbacks + def assertHTTPError(self, url, code, response_substring, + method="get", persistent=False, + **args): + response = yield treq.request(method, url, persistent=persistent, + **args) + body = yield response.content() + self.assertEquals(response.code, code) + if response_substring is not None: + self.assertIn(response_substring, body) + returnValue(body) + class ErrorMixin(WebErrorMixin): def explain_error(self, f): if f.check(defer.FirstError): diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 6e5837313..0e7dcc8db 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4437,13 +4437,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d + @inlineCallbacks def test_bad_method(self): url = self.webish_url + self.public_url + "/foo/bar.txt" - d = self.shouldHTTPError("bad_method", - 501, "Not Implemented", - "I don't know how to treat a BOGUS request.", - client.getPage, url, method="BOGUS") - return d + yield self.assertHTTPError(url, 501, + "I don't know how to treat a BOGUS request.", + method="BOGUS") def test_short_url(self): url = self.webish_url + "/uri" From 73d09082d7d205f3b511606f5c8c1875dbc86beb Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 16:24:14 -0500 Subject: [PATCH 06/14] test_web.web.Web.POST: split out build_form() helper function --- src/allmydata/test/web/test_web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 0e7dcc8db..5bcc5b8bb 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -536,7 +536,7 @@ class WebMixin(testutil.TimezoneMixin): url = self.webish_url + urlpath return do_http("delete", url) - def POST(self, urlpath, followRedirect=False, **fields): + def build_form(self, **fields): sepbase = "boogabooga" sep = "--" + sepbase form = [] @@ -566,6 +566,10 @@ class WebMixin(testutil.TimezoneMixin): if fields: body = "\r\n".join(form) + "\r\n" headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase + return (body, headers) + + def POST(self, urlpath, followRedirect=False, **fields): + body, headers = self.build_form(**fields) return self.POST2(urlpath, body, headers, followRedirect) def POST2(self, urlpath, body="", headers={}, followRedirect=False): From 4745239c2cd355720b2919578a0aa9a8c9d3c484 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 16:40:11 -0500 Subject: [PATCH 07/14] test_web: use inlineCallbacks in many functions specifically everywhere we use self.shouldRedirect2 --- src/allmydata/test/web/test_web.py | 218 ++++++++++++----------------- 1 file changed, 92 insertions(+), 126 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5bcc5b8bb..46cd394c8 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -2501,20 +2501,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addBoth(done) return d + @inlineCallbacks def test_POST_upload_no_link_whendone_results(self): def check(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnless(target.startswith(self.webish_url), target) return client.getPage(target, method="GET") # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860. - d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results", - check, - self.POST, "/uri", t="upload", - when_done="/%75ri/%(uri)s", - file=("new.txt", self.NEWFILE_CONTENTS)) - d.addCallback(lambda res: - self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS)) - return d + res = yield self.shouldRedirect2("test_POST_upload_no_link_whendone_results", + check, + self.POST, "/uri", t="upload", + when_done="/%75ri/%(uri)s", + file=("new.txt", self.NEWFILE_CONTENTS)) + self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS) def test_POST_upload_no_link_mutable(self): d = self.POST("/uri", t="upload", mutable="true", @@ -2861,123 +2860,95 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi u"sub"])) return d + @inlineCallbacks def test_POST_FILEURL_check(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check") - def _check(res): - self.failUnlessIn("Healthy :", res) - d.addCallback(_check) + res = yield self.POST(bar_url, t="check") + self.failUnlessIn("Healthy :", res) + redir_url = "http://allmydata.org/TARGET" def _check2(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnlessReallyEqual(target, redir_url) - d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check", - _check2, - self.POST, bar_url, - t="check", - when_done=redir_url)) - d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) - def _check3(res): - self.failUnlessIn("Healthy :", res) - self.failUnlessIn("Return to file", res) - self.failUnlessIn(redir_url, res) - d.addCallback(_check3) + yield self.shouldRedirect2("test_POST_FILEURL_check", + _check2, + self.POST, bar_url, + t="check", + when_done=redir_url) + res = yield self.POST(bar_url, t="check", return_to=redir_url) + self.failUnlessIn("Healthy :", res) + self.failUnlessIn("Return to file", res) + self.failUnlessIn(redir_url, res) - d.addCallback(lambda res: - self.POST(bar_url, t="check", output="JSON")) - def _check_json(res): - data = json.loads(res) - self.failUnlessIn("storage-index", data) - self.failUnless(data["results"]["healthy"]) - d.addCallback(_check_json) - - return d + res = yield self.POST(bar_url, t="check", output="JSON") + data = json.loads(res) + self.failUnlessIn("storage-index", data) + self.failUnless(data["results"]["healthy"]) + @inlineCallbacks def test_POST_FILEURL_check_and_repair(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check", repair="true") - def _check(res): - self.failUnlessIn("Healthy :", res) - d.addCallback(_check) + res = yield self.POST(bar_url, t="check", repair="true") + self.failUnlessIn("Healthy :", res) + redir_url = "http://allmydata.org/TARGET" def _check2(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnlessReallyEqual(target, redir_url) - d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check_and_repair", - _check2, - self.POST, bar_url, - t="check", repair="true", - when_done=redir_url)) - d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) - def _check3(res): - self.failUnlessIn("Healthy :", res) - self.failUnlessIn("Return to file", res) - self.failUnlessIn(redir_url, res) - d.addCallback(_check3) - return d + yield self.shouldRedirect2("test_POST_FILEURL_check_and_repair", + _check2, + self.POST, bar_url, + t="check", repair="true", + when_done=redir_url) + res = yield self.POST(bar_url, t="check", return_to=redir_url) + self.failUnlessIn("Healthy :", res) + self.failUnlessIn("Return to file", res) + self.failUnlessIn(redir_url, res) + @inlineCallbacks def test_POST_DIRURL_check(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check") - def _check(res): - self.failUnlessIn("Healthy :", res) - d.addCallback(_check) + res = yield self.POST(foo_url, t="check") + self.failUnlessIn("Healthy :", res) + redir_url = "http://allmydata.org/TARGET" def _check2(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnlessReallyEqual(target, redir_url) - d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check", - _check2, - self.POST, foo_url, - t="check", - when_done=redir_url)) - d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) - def _check3(res): - self.failUnlessIn("Healthy :", res) - self.failUnlessIn("Return to file/directory", res) - self.failUnlessIn(redir_url, res) - d.addCallback(_check3) + yield self.shouldRedirect2("test_POST_DIRURL_check", + _check2, + self.POST, foo_url, + t="check", + when_done=redir_url) + res = yield self.POST(foo_url, t="check", return_to=redir_url) + self.failUnlessIn("Healthy :", res) + self.failUnlessIn("Return to file/directory", res) + self.failUnlessIn(redir_url, res) - d.addCallback(lambda res: - self.POST(foo_url, t="check", output="JSON")) - def _check_json(res): - data = json.loads(res) - self.failUnlessIn("storage-index", data) - self.failUnless(data["results"]["healthy"]) - d.addCallback(_check_json) - - return d + res = yield self.POST(foo_url, t="check", output="JSON") + data = json.loads(res) + self.failUnlessIn("storage-index", data) + self.failUnless(data["results"]["healthy"]) + @inlineCallbacks def test_POST_DIRURL_check_and_repair(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check", repair="true") - def _check(res): - self.failUnlessIn("Healthy :", res) - d.addCallback(_check) + res = yield self.POST(foo_url, t="check", repair="true") + self.failUnlessIn("Healthy :", res) + redir_url = "http://allmydata.org/TARGET" def _check2(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnlessReallyEqual(target, redir_url) - d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check_and_repair", - _check2, - self.POST, foo_url, - t="check", repair="true", - when_done=redir_url)) - d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) - def _check3(res): - self.failUnlessIn("Healthy :", res) - self.failUnlessIn("Return to file/directory", res) - self.failUnlessIn(redir_url, res) - d.addCallback(_check3) - return d + yield self.shouldRedirect2("test_POST_DIRURL_check_and_repair", + _check2, + self.POST, foo_url, + t="check", repair="true", + when_done=redir_url) + res = yield self.POST(foo_url, t="check", return_to=redir_url) + self.failUnlessIn("Healthy :", res) + self.failUnlessIn("Return to file/directory", res) + self.failUnlessIn(redir_url, res) def test_POST_FILEURL_mdmf_check(self): quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri) @@ -3038,45 +3009,40 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.POST, self.public_url, t="start-deep-check") return d + @inlineCallbacks def test_POST_DIRURL_deepcheck(self): def _check_redirect(statuscode, target): self.failUnlessReallyEqual(statuscode, str(http.FOUND)) self.failUnless(target.endswith("/operations/123")) - d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect, - self.POST, self.public_url, - t="start-deep-check", ophandle="123") - d.addCallback(self.wait_for_operation, "123") - def _check_json(data): - self.failUnlessReallyEqual(data["finished"], True) - self.failUnlessReallyEqual(data["count-objects-checked"], 11) - self.failUnlessReallyEqual(data["count-objects-healthy"], 11) - d.addCallback(_check_json) - d.addCallback(self.get_operation_results, "123", "html") - def _check_html(res): - self.failUnlessIn("Objects Checked: 11", res) - self.failUnlessIn("Objects Healthy: 11", res) - self.failUnlessIn(FAVICON_MARKUP, res) - d.addCallback(_check_html) + yield self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect, + self.POST, self.public_url, + t="start-deep-check", ophandle="123") + data = yield self.wait_for_operation(None, "123") + self.failUnlessReallyEqual(data["finished"], True) + self.failUnlessReallyEqual(data["count-objects-checked"], 11) + self.failUnlessReallyEqual(data["count-objects-healthy"], 11) - d.addCallback(lambda res: - self.GET("/operations/123/")) - d.addCallback(_check_html) # should be the same as without the slash + res = yield self.get_operation_results(None, "123", "html") + self.failUnlessIn("Objects Checked: 11", res) + self.failUnlessIn("Objects Healthy: 11", res) + self.failUnlessIn(FAVICON_MARKUP, res) - d.addCallback(lambda res: - self.shouldFail2(error.Error, "one", "404 Not Found", - "No detailed results for SI bogus", - self.GET, "/operations/123/bogus")) + res = yield self.GET("/operations/123/") + # should be the same as without the slash + self.failUnlessIn("Objects Checked: 11", res) + self.failUnlessIn("Objects Healthy: 11", res) + self.failUnlessIn(FAVICON_MARKUP, res) + + yield self.shouldFail2(error.Error, "one", "404 Not Found", + "No detailed results for SI bogus", + self.GET, "/operations/123/bogus") foo_si = self._foo_node.get_storage_index() foo_si_s = base32.b2a(foo_si) - d.addCallback(lambda res: - self.GET("/operations/123/%s?output=JSON" % foo_si_s)) - def _check_foo_json(res): - data = json.loads(res) - self.failUnlessEqual(data["storage-index"], foo_si_s) - self.failUnless(data["results"]["healthy"]) - d.addCallback(_check_foo_json) - return d + res = yield self.GET("/operations/123/%s?output=JSON" % foo_si_s) + data = json.loads(res) + self.failUnlessEqual(data["storage-index"], foo_si_s) + self.failUnless(data["results"]["healthy"]) def test_POST_DIRURL_deepcheck_and_repair(self): d = self.POST(self.public_url, t="start-deep-check", repair="true", From 6be91e369c38cf9713d37e4c92a8b2594c12f7d3 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 17:02:23 -0500 Subject: [PATCH 08/14] test_web: replace shouldReplace2 with shouldReplaceTo This removes many uses of self.POST (since shouldReplaceTo does its own HTTP), which will make it easier to remove client.getPage from POST. --- src/allmydata/test/web/test_web.py | 117 ++++++++++++----------------- 1 file changed, 49 insertions(+), 68 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 46cd394c8..e83a631ea 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -2488,31 +2488,18 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addBoth(self.shouldRedirect, "/") return d - def shouldRedirect2(self, which, checker, callable, *args, **kwargs): - d = defer.maybeDeferred(callable, *args, **kwargs) - def done(res): - if isinstance(res, failure.Failure): - res.trap(error.PageRedirect) - statuscode = res.value.status - target = res.value.location - return checker(statuscode, target) - self.fail("%s: callable was supposed to redirect, not return '%s'" - % (which, res)) - d.addBoth(done) - return d - @inlineCallbacks def test_POST_upload_no_link_whendone_results(self): - def check(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnless(target.startswith(self.webish_url), target) - return client.getPage(target, method="GET") - # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860. - res = yield self.shouldRedirect2("test_POST_upload_no_link_whendone_results", - check, - self.POST, "/uri", t="upload", - when_done="/%75ri/%(uri)s", - file=("new.txt", self.NEWFILE_CONTENTS)) + # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860 + body, headers = self.build_form(t="upload", + when_done="/%75ri/%(uri)s", + file=("new.txt", self.NEWFILE_CONTENTS), + ) + redir_url = yield self.shouldRedirectTo(self.webish_url + "/uri", None, + method="post", + data=body, headers=headers, + code=http.FOUND) + res = yield do_http("get", redir_url) self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS) def test_POST_upload_no_link_mutable(self): @@ -2867,14 +2854,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Healthy :", res) redir_url = "http://allmydata.org/TARGET" - def _check2(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnlessReallyEqual(target, redir_url) - yield self.shouldRedirect2("test_POST_FILEURL_check", - _check2, - self.POST, bar_url, - t="check", - when_done=redir_url) + body, headers = self.build_form(t="check", when_done=redir_url) + yield self.shouldRedirectTo(self.webish_url + bar_url, redir_url, + method="post", data=body, headers=headers, + code=http.FOUND) + res = yield self.POST(bar_url, t="check", return_to=redir_url) self.failUnlessIn("Healthy :", res) self.failUnlessIn("Return to file", res) @@ -2892,14 +2876,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Healthy :", res) redir_url = "http://allmydata.org/TARGET" - def _check2(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnlessReallyEqual(target, redir_url) - yield self.shouldRedirect2("test_POST_FILEURL_check_and_repair", - _check2, - self.POST, bar_url, - t="check", repair="true", - when_done=redir_url) + body, headers = self.build_form(t="check", repair="true", + when_done=redir_url) + yield self.shouldRedirectTo(self.webish_url + bar_url, redir_url, + method="post", data=body, headers=headers, + code=http.FOUND) + res = yield self.POST(bar_url, t="check", return_to=redir_url) self.failUnlessIn("Healthy :", res) self.failUnlessIn("Return to file", res) @@ -2912,14 +2894,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Healthy :", res) redir_url = "http://allmydata.org/TARGET" - def _check2(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnlessReallyEqual(target, redir_url) - yield self.shouldRedirect2("test_POST_DIRURL_check", - _check2, - self.POST, foo_url, - t="check", - when_done=redir_url) + body, headers = self.build_form(t="check", when_done=redir_url) + yield self.shouldRedirectTo(self.webish_url + foo_url, redir_url, + method="post", data=body, headers=headers, + code=http.FOUND) + res = yield self.POST(foo_url, t="check", return_to=redir_url) self.failUnlessIn("Healthy :", res) self.failUnlessIn("Return to file/directory", res) @@ -2937,14 +2916,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessIn("Healthy :", res) redir_url = "http://allmydata.org/TARGET" - def _check2(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnlessReallyEqual(target, redir_url) - yield self.shouldRedirect2("test_POST_DIRURL_check_and_repair", - _check2, - self.POST, foo_url, - t="check", repair="true", - when_done=redir_url) + body, headers = self.build_form(t="check", repair="true", + when_done=redir_url) + yield self.shouldRedirectTo(self.webish_url + foo_url, redir_url, + method="post", data=body, headers=headers, + code=http.FOUND) res = yield self.POST(foo_url, t="check", return_to=redir_url) self.failUnlessIn("Healthy :", res) self.failUnlessIn("Return to file/directory", res) @@ -3011,12 +2987,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_POST_DIRURL_deepcheck(self): - def _check_redirect(statuscode, target): - self.failUnlessReallyEqual(statuscode, str(http.FOUND)) - self.failUnless(target.endswith("/operations/123")) - yield self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect, - self.POST, self.public_url, - t="start-deep-check", ophandle="123") + body, headers = self.build_form(t="start-deep-check", ophandle="123") + yield self.shouldRedirectTo(self.webish_url + self.public_url, + self.webish_url + "/operations/123", + method="post", data=body, headers=headers, + code=http.FOUND) + data = yield self.wait_for_operation(None, "123") self.failUnlessReallyEqual(data["finished"], True) self.failUnlessReallyEqual(data["count-objects-checked"], 11) @@ -4021,14 +3997,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return res.value.location @inlineCallbacks - def shouldRedirectTo(self, url, target_location): - response = yield treq.request("get", url, persistent=False, - allow_redirects=False) - self.assertIn(response.code, [http.MOVED_PERMANENTLY, - http.FOUND, - http.TEMPORARY_REDIRECT]) + def shouldRedirectTo(self, url, target_location, method="get", + code=None, **args): + response = yield treq.request(method, url, persistent=False, + allow_redirects=False, **args) + codes = [http.MOVED_PERMANENTLY, + http.FOUND, + http.TEMPORARY_REDIRECT, + ] if code is None else [code] + self.assertIn(response.code, codes) location = response.headers.getRawHeaders(b"location")[0] - self.assertEquals(location, target_location) + if target_location is not None: + self.assertEquals(location, target_location) + returnValue(location) @inlineCallbacks def test_GET_URI_form(self): From 76063b1c12ca7f96e9b264bbdfb7254d2798e043 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 17:16:24 -0500 Subject: [PATCH 09/14] test_web: replace some instances of shouldRedirect with shouldRedirectTo --- src/allmydata/test/web/test_web.py | 56 ++++++++++++++++-------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e83a631ea..773f3009c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -2482,11 +2482,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS) return d + @inlineCallbacks def test_POST_upload_no_link_whendone(self): - d = self.POST("/uri", t="upload", when_done="/", - file=("new.txt", self.NEWFILE_CONTENTS)) - d.addBoth(self.shouldRedirect, "/") - return d + body, headers = self.build_form(t="upload", when_done="/", + file=("new.txt", self.NEWFILE_CONTENTS)) + yield self.shouldRedirectTo(self.webish_url + "/uri", + self.webish_url + "/", + method="post", data=body, headers=headers, + code=http.FOUND) @inlineCallbacks def test_POST_upload_no_link_whendone_results(self): @@ -2810,15 +2813,17 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(self.failUnlessIsBarDotTxt) return d + @inlineCallbacks def test_POST_upload_whendone(self): - d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE", - file=("new.txt", self.NEWFILE_CONTENTS)) - d.addBoth(self.shouldRedirect, "/THERE") + body, headers = self.build_form(t="upload", when_done="/THERE", + file=("new.txt", self.NEWFILE_CONTENTS)) + yield self.shouldRedirectTo(self.webish_url + self.public_url + "/foo", + self.webish_url + "/THERE", + method="post", data=body, headers=headers, + code=http.FOUND) fn = self._foo_node - d.addCallback(lambda res: - self.failUnlessChildContentsAre(fn, u"new.txt", - self.NEWFILE_CONTENTS)) - return d + yield self.failUnlessChildContentsAre(fn, u"new.txt", + self.NEWFILE_CONTENTS) def test_POST_upload_named(self): fn = self._foo_node @@ -3230,24 +3235,23 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addErrback(self.explain_web_error) return d + @inlineCallbacks def test_POST_mkdir_no_parentdir_redirect(self): - d = self.POST("/uri?t=mkdir&redirect_to_result=true") - d.addBoth(self.shouldRedirect, None, statuscode='303') - def _check_target(target): - target = urllib.unquote(target) - self.failUnless(target.startswith("uri/URI:DIR2:"), target) - d.addCallback(_check_target) - return d + url = self.webish_url + "/uri?t=mkdir&redirect_to_result=true" + target = yield self.shouldRedirectTo(url, None, method="post", + code=http.SEE_OTHER) + target = urllib.unquote(target) + self.failUnless(target.startswith("uri/URI:DIR2:"), target) + @inlineCallbacks def test_POST_mkdir_no_parentdir_redirect2(self): - d = self.POST("/uri", t="mkdir", redirect_to_result="true") - d.addBoth(self.shouldRedirect, None, statuscode='303') - def _check_target(target): - target = urllib.unquote(target) - self.failUnless(target.startswith("uri/URI:DIR2:"), target) - d.addCallback(_check_target) - d.addErrback(self.explain_web_error) - return d + body, headers = self.build_form(t="mkdir", redirect_to_result="true") + target = yield self.shouldRedirectTo(self.webish_url + "/uri", None, + method="post", + data=body, headers=headers, + code=http.SEE_OTHER) + target = urllib.unquote(target) + self.failUnless(target.startswith("uri/URI:DIR2:"), target) def _make_readonly(self, u): ro_uri = uri.from_string(u).get_readonly() From 3f03367d2f29d3789416cde51ad730e387d8b871 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 17:26:07 -0500 Subject: [PATCH 10/14] test_web: remove all other uses of shouldRedirect --- src/allmydata/test/web/test_web.py | 93 +++++++++++++----------------- 1 file changed, 39 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 773f3009c..8b7726991 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -3410,30 +3410,30 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi json.dumps(newkids)) return d + @inlineCallbacks def test_welcome_page_mkdir_button(self): # Fetch the welcome page. - d = self.GET("/") - def _after_get_welcome_page(res): - MKDIR_BUTTON_RE = re.compile( - '.*' - '[ ]*' - '[ ]*' - '') - html = res.replace('\n', ' ') - mo = MKDIR_BUTTON_RE.search(html) - self.failUnless(mo, html) - formaction = mo.group(1) - formt = mo.group(2) - formaname = mo.group(3) - formavalue = mo.group(4) - return (formaction, formt, formaname, formavalue) - d.addCallback(_after_get_welcome_page) - def _after_parse_form(res): - (formaction, formt, formaname, formavalue) = res - return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue)) - d.addCallback(_after_parse_form) - d.addBoth(self.shouldRedirect, None, statuscode='303') - return d + res = yield self.GET("/") + MKDIR_BUTTON_RE = re.compile( + '.*' + '[ ]*' + '[ ]*' + '') + html = res.replace('\n', ' ') + mo = MKDIR_BUTTON_RE.search(html) + self.failUnless(mo, html) + formaction = mo.group(1) + formt = mo.group(2) + formaname = mo.group(3) + formavalue = mo.group(4) + + url = self.webish_url + "/%s?t=%s&%s=%s" % (formaction, formt, + formaname, formavalue) + target = yield self.shouldRedirectTo(url, None, + method="post", + code=http.SEE_OTHER) + target = urllib.unquote(target) + self.failUnless(target.startswith("uri/URI:DIR2:"), target) def test_POST_mkdir_replace(self): # return value? d = self.POST(self.public_url + "/foo", t="mkdir", name="sub") @@ -3463,21 +3463,26 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"]) return d + @inlineCallbacks def test_POST_mkdir_whendone_field(self): - d = self.POST(self.public_url + "/foo", - t="mkdir", name="newdir", when_done="/THERE") - d.addBoth(self.shouldRedirect, "/THERE") - d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessNodeKeysAre, []) - return d + body, headers = self.build_form(t="mkdir", name="newdir", + when_done="/THERE") + yield self.shouldRedirectTo(self.webish_url + self.public_url + "/foo", + self.webish_url + "/THERE", + method="post", data=body, headers=headers, + code=http.FOUND) + res = yield self._foo_node.get(u"newdir") + self.failUnlessNodeKeysAre(res, []) + @inlineCallbacks def test_POST_mkdir_whendone_queryarg(self): - d = self.POST(self.public_url + "/foo?when_done=/THERE", - t="mkdir", name="newdir") - d.addBoth(self.shouldRedirect, "/THERE") - d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessNodeKeysAre, []) - return d + body, headers = self.build_form(t="mkdir", name="newdir") + url = self.webish_url + self.public_url + "/foo?when_done=/THERE" + yield self.shouldRedirectTo(url, self.webish_url + "/THERE", + method="post", data=body, headers=headers, + code=http.FOUND) + res = yield self._foo_node.get(u"newdir") + self.failUnlessNodeKeysAre(res, []) def test_POST_bad_t(self): d = self.shouldFail2(error.Error, "POST_bad_t", @@ -3980,26 +3985,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(self.failUnlessIsBarDotTxt) return d - def shouldRedirect(self, res, target=None, statuscode=None, which=""): - """ If target is not None then the redirection has to go to target. If - statuscode is not None then the redirection has to be accomplished with - that HTTP status code.""" - if not isinstance(res, failure.Failure): - to_where = (target is None) and "somewhere" or ("to " + target) - self.fail("%s: we were expecting to get redirected %s, not get an" - " actual page: %s" % (which, to_where, res)) - res.trap(error.PageRedirect) - if statuscode is not None: - self.failUnlessReallyEqual(res.value.status, statuscode, - "%s: not a redirect" % which) - if target is not None: - # the PageRedirect does not seem to capture the uri= query arg - # properly, so we can't check for it. - realtarget = self.webish_url + target - self.failUnlessReallyEqual(res.value.location, realtarget, - "%s: wrong target" % which) - return res.value.location - @inlineCallbacks def shouldRedirectTo(self, url, target_location, method="get", code=None, **args): From 0deb90388862905be44557154fbe06dd99f65515 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 18:12:35 -0500 Subject: [PATCH 11/14] test_web: rewrite all POST(followRedirect=True) calls with do_http Since POST() is about to lose followRedirect=True --- src/allmydata/test/web/test_web.py | 77 ++++++++++++++++++------------ 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 8b7726991..f12c03eb7 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1898,8 +1898,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_DIRURL_manifest(self): d = defer.succeed(None) def getman(ignored, output): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=125" + d = do_http("post", url, allow_redirects=True, + browser_like_redirects=True) d.addCallback(self.wait_for_operation, "125") d.addCallback(self.get_operation_results, "125", output) return d @@ -1948,8 +1949,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_DIRURL_deepsize(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-deep-size&ophandle=126" + d = do_http("post", url, allow_redirects=True, + browser_like_redirects=True) d.addCallback(self.wait_for_operation, "126") d.addCallback(self.get_operation_results, "126", "json") def _got_json(data): @@ -1976,8 +1978,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_DIRURL_deepstats(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-deep-stats&ophandle=127" + d = do_http("post", url, + allow_redirects=True, browser_like_redirects=True) d.addCallback(self.wait_for_operation, "127") d.addCallback(self.get_operation_results, "127", "json") def _got_json(stats): @@ -3026,8 +3029,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnless(data["results"]["healthy"]) def test_POST_DIRURL_deepcheck_and_repair(self): - d = self.POST(self.public_url, t="start-deep-check", repair="true", - ophandle="124", output="json", followRedirect=True) + url = self.webish_url + self.public_url + body, headers = self.build_form(t="start-deep-check", repair="true", + ophandle="124", output="json") + d = do_http("post", url, data=body, headers=headers, + allow_redirects=True, + browser_like_redirects=True) d.addCallback(self.wait_for_operation, "124") def _check_json(data): self.failUnlessReallyEqual(data["finished"], True) @@ -4399,8 +4406,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_ophandle_cancel(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=128" + d = do_http("post", url, + allow_redirects=True, browser_like_redirects=True) d.addCallback(lambda ignored: self.GET("/operations/128?t=status&output=JSON")) def _check1(res): @@ -4425,8 +4433,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_ophandle_retainfor(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60" + d = do_http("post", url, + allow_redirects=True, browser_like_redirects=True) d.addCallback(lambda ignored: self.GET("/operations/129?t=status&output=JSON&retain-for=0")) def _check1(res): @@ -4445,8 +4454,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_ophandle_release_after_complete(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130", - followRedirect=True) + url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=130" + d = do_http("post", url, + allow_redirects=True, browser_like_redirects=True) d.addCallback(self.wait_for_operation, "130") d.addCallback(lambda ignored: self.GET("/operations/130?t=status&output=JSON&release-after-complete=true")) @@ -4462,19 +4472,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_uncollected_ophandle_expiration(self): # uncollected ophandles should expire after 4 days def _make_uncollected_ophandle(ophandle): - d = self.POST(self.public_url + - "/foo/?t=start-manifest&ophandle=%d" % ophandle, - followRedirect=False) - # When we start the operation, the webapi server will want - # to redirect us to the page for the ophandle, so we get - # confirmation that the operation has started. If the - # manifest operation has finished by the time we get there, - # following that redirect (by setting followRedirect=True - # above) has the side effect of collecting the ophandle that - # we've just created, which means that we can't use the - # ophandle to test the uncollected timeout anymore. So, - # instead, catch the 302 here and don't follow it. - d.addBoth(self.should302, "uncollected_ophandle_creation") + url = (self.webish_url + self.public_url + + "/foo/?t=start-manifest&ophandle=%d" % ophandle) + # When we start the operation, the webapi server will want to + # redirect us to the page for the ophandle, so we get + # confirmation that the operation has started. If the manifest + # operation has finished by the time we get there, following that + # redirect would have the side effect of collecting the ophandle + # that we've just created, which means that we can't use the + # ophandle to test the uncollected timeout anymore. So, instead, + # catch+ignore any 302 here and don't follow it. + d = treq.request("post", url, persistent=False) + def _ignore_redirect(f): + f.trap(client.ResponseFailed) + e = f.value + reasons = e.reasons + r0 = reasons[0] + r0.trap(error.PageRedirect) + d.addErrback(_ignore_redirect) return d # Create an ophandle, don't collect it, then advance the clock by # 4 days - 1 second and make sure that the ophandle is still there. @@ -4504,12 +4519,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_collected_ophandle_expiration(self): # collected ophandles should expire after 1 day def _make_collected_ophandle(ophandle): - d = self.POST(self.public_url + - "/foo/?t=start-manifest&ophandle=%d" % ophandle, - followRedirect=True) + url = (self.webish_url + self.public_url + + "/foo/?t=start-manifest&ophandle=%d" % ophandle) # By following the initial redirect, we collect the ophandle # we've just created. - return d + return do_http("post", url, + allow_redirects=True, browser_like_redirects=True) # Create a collected ophandle, then collect it after 23 hours # and 59 seconds to make sure that it is still there. d = _make_collected_ophandle(133) From 965e974ce84bef16b97ed008c03732f698b0bc85 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 18:22:36 -0500 Subject: [PATCH 12/14] test_web: fix POST(url,body) calls that should have been POST2() and remove the followRedirect= argument from POST --- src/allmydata/test/web/test_web.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f12c03eb7..b15922236 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -568,9 +568,9 @@ class WebMixin(testutil.TimezoneMixin): headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase return (body, headers) - def POST(self, urlpath, followRedirect=False, **fields): + def POST(self, urlpath, **fields): body, headers = self.build_form(**fields) - return self.POST2(urlpath, body, headers, followRedirect) + return self.POST2(urlpath, body, headers) def POST2(self, urlpath, body="", headers={}, followRedirect=False): url = self.webish_url + urlpath @@ -3150,7 +3150,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi (newkids, caps) = self._create_initial_children() return self.shouldHTTPError("POST_mkdir_initial_children_bad_format", 400, "Bad Request", "Unknown format: foo", - self.POST, self.public_url + \ + self.POST2, self.public_url + \ "/foo?t=mkdir-with-children&name=newdir&format=foo", json.dumps(newkids)) @@ -3187,7 +3187,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_mkdir_2(self): - d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -3195,7 +3195,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_mkdirs_2(self): - d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "") + d = self.POST2(self.public_url + "/foo/bardir/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bardir")) d.addCallback(lambda res: self._foo_node.get(u"bardir")) @@ -3916,7 +3916,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d def test_POST_relink_file_multi_level(self): - d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "") + d = self.POST2(self.public_url + "/foo/sub/level2?t=mkdir", "") d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink", from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2")) d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) From 2687ee90c5940becf92053a9f75247b97956ded2 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 18:31:44 -0500 Subject: [PATCH 13/14] test_web: remove last traces of deprecated client.getPage --- src/allmydata/test/web/test_web.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index b15922236..6a7899de8 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -574,8 +574,8 @@ class WebMixin(testutil.TimezoneMixin): def POST2(self, urlpath, body="", headers={}, followRedirect=False): url = self.webish_url + urlpath - return client.getPage(url, method="POST", postdata=body, - headers=headers, followRedirect=followRedirect) + return do_http("POST", url, allow_redirects=followRedirect, + headers=headers, data=body) def shouldFail(self, res, expected_failure, which, substring=None, response_substring=None): @@ -3529,7 +3529,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name - d = client.getPage(url, method="POST", postdata=reqbody) + d = do_http("post", url, data=reqbody) def _then(res): self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1") self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2") @@ -4391,19 +4391,18 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi "I don't know how to treat a BOGUS request.", method="BOGUS") + @inlineCallbacks def test_short_url(self): url = self.webish_url + "/uri" - d = self.shouldHTTPError("short_url", 501, "Not Implemented", - "I don't know how to treat a DELETE request.", - client.getPage, url, method="DELETE") - return d + yield self.assertHTTPError(url, 501, + "I don't know how to treat a DELETE request.", + method="DELETE") + @inlineCallbacks def test_ophandle_bad(self): url = self.webish_url + "/operations/bogus?t=status" - d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found", - "unknown/expired handle 'bogus'", - client.getPage, url) - return d + yield self.assertHTTPError(url, 404, + "unknown/expired handle 'bogus'") def test_ophandle_cancel(self): url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=128" From 05317d4d128a1f4e7790c81a8c03c3d1d266eacf Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Jul 2017 19:53:48 -0500 Subject: [PATCH 14/14] test_web: replace all shouldHTTPError with assertHTTPError and update to inlineCallbacks while we're in there --- src/allmydata/test/web/test_web.py | 345 ++++++++++++----------------- 1 file changed, 144 insertions(+), 201 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 6a7899de8..4a1d3a33e 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1511,12 +1511,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(lambda json: self.failUnlessIn("SDMF", json)) return d + @inlineCallbacks def test_PUT_NEWFILEURL_unlinked_bad_format(self): contents = self.NEWFILE_CONTENTS * 300000 - return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.PUT, "/uri?format=foo", - contents) + yield self.assertHTTPError(self.webish_url + "/uri?format=foo", 400, + "Unknown format: foo", + method="put", data=contents) def test_PUT_NEWFILEURL_range_bad(self): headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)} @@ -1695,12 +1695,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check2) return d + @inlineCallbacks def test_GET_FILEURL_badtype(self): - d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request", - "bad t=bogus", - self.GET, - self.public_url + "/foo/bar.txt?t=bogus") - return d + url = self.webish_url + self.public_url + "/foo/bar.txt?t=bogus" + yield self.assertHTTPError(url, 400, "bad t=bogus") def test_CSS_FILE(self): d = self.GET("/tahoe.css", followRedirect=True) @@ -1844,13 +1842,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check5) return d + @inlineCallbacks def test_GET_DIRURL_badtype(self): - d = self.shouldHTTPError("test_GET_DIRURL_badtype", - 400, "Bad Request", - "bad t=bogus", - self.GET, - self.public_url + "/foo?t=bogus") - return d + url = self.webish_url + self.public_url + "/foo?t=bogus" + yield self.assertHTTPError(url, 400, "bad t=bogus") def test_GET_DIRURL_json(self): d = self.GET(self.public_url + "/foo?t=json") @@ -2068,11 +2063,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(node._node.get_version(), SDMF_VERSION)) return d + @inlineCallbacks def test_PUT_NEWDIRURL_bad_format(self): - return self.shouldHTTPError("PUT_NEWDIRURL_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.PUT, self.public_url + - "/foo/newdir=?t=mkdir&format=foo", "") + url = (self.webish_url + self.public_url + + "/foo/newdir=?t=mkdir&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="put", data="") def test_POST_NEWDIRURL(self): d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "") @@ -2100,11 +2096,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(node._node.get_version(), SDMF_VERSION)) return d + @inlineCallbacks def test_POST_NEWDIRURL_bad_format(self): - return self.shouldHTTPError("POST_NEWDIRURL_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST2, self.public_url + \ - "/foo/newdir?t=mkdir&format=foo", "") + url = (self.webish_url + self.public_url + + "/foo/newdir?t=mkdir&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post", data="") def test_POST_NEWDIRURL_emptyname(self): # an empty pathname component (i.e. a double-slash) is disallowed @@ -2175,13 +2172,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_NEWDIRURL_initial_children_sdmf(self): return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION) + @inlineCallbacks def test_POST_NEWDIRURL_initial_children_bad_format(self): (newkids, caps) = self._create_initial_children() - return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST2, self.public_url + \ - "/foo/newdir?t=mkdir-with-children&format=foo", - json.dumps(newkids)) + url = (self.webish_url + self.public_url + + "/foo/newdir?t=mkdir-with-children&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post", data=json.dumps(newkids)) def test_POST_NEWDIRURL_immutable(self): (newkids, caps) = self._create_immutable_children() @@ -2300,12 +2297,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION)) return d + @inlineCallbacks def test_PUT_NEWDIRURL_mkdirs_bad_format(self): - return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.PUT, self.public_url + \ - "/foo/subdir/newdir?t=mkdir&format=foo", - "") + url = (self.webish_url + self.public_url + + "/foo/subdir/newdir?t=mkdir&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="put", data="") def test_DELETE_DIRURL(self): d = self.DELETE(self.public_url + "/foo") @@ -2575,12 +2572,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF") return d + @inlineCallbacks def test_POST_upload_bad_format_unlinked(self): - return self.shouldHTTPError("POST_upload_bad_format_unlinked", - 400, "Bad Request", "Unknown format: foo", - self.POST, - "/uri?t=upload&format=foo", - file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) + url = self.webish_url + "/uri?t=upload&format=foo" + body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) + yield self.assertHTTPError(url, 400, + "Unknown format: foo", + method="post", data=body, headers=headers) def test_POST_upload_format(self): def _check_upload(ign, format, uri_prefix, fn=None): @@ -2610,12 +2608,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_check_upload, "MDMF", "URI:MDMF") return d + @inlineCallbacks def test_POST_upload_bad_format(self): - return self.shouldHTTPError("POST_upload_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST, self.public_url + \ - "/foo?t=upload&format=foo", - file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) + url = self.webish_url + self.public_url + "/foo?t=upload&format=foo" + body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post", data=body, headers=headers) def test_POST_upload_mutable(self): # this creates a mutable file @@ -3096,11 +3094,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(node._node.get_version(), SDMF_VERSION)) return d + @inlineCallbacks def test_POST_mkdir_bad_format(self): - return self.shouldHTTPError("POST_mkdir_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST, self.public_url + - "/foo?t=mkdir&name=newdir&format=foo") + url = (self.webish_url + self.public_url + + "/foo?t=mkdir&name=newdir&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post") def test_POST_mkdir_initial_children(self): (newkids, caps) = self._create_initial_children() @@ -3146,13 +3145,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi caps['filecap1']) return d + @inlineCallbacks def test_POST_mkdir_initial_children_bad_format(self): (newkids, caps) = self._create_initial_children() - return self.shouldHTTPError("POST_mkdir_initial_children_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST2, self.public_url + \ - "/foo?t=mkdir-with-children&name=newdir&format=foo", - json.dumps(newkids)) + url = (self.webish_url + self.public_url + + "/foo?t=mkdir-with-children&name=newdir&format=foo") + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post", data=json.dumps(newkids)) def test_POST_mkdir_immutable(self): (newkids, caps) = self._create_immutable_children() @@ -3227,11 +3226,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_after_mkdir) return d + @inlineCallbacks def test_POST_mkdir_no_parentdir_noredirect_bad_format(self): - return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.POST, self.public_url + - "/uri?t=mkdir&format=foo") + url = self.webish_url + self.public_url + "/uri?t=mkdir&format=foo" + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="post") def test_POST_mkdir_no_parentdir_noredirect2(self): # make sure form-based arguments (as on the welcome page) still work @@ -3360,25 +3359,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_after_mkdir) return d + @inlineCallbacks def test_POST_mkdir_no_parentdir_unexpected_children(self): # the regular /uri?t=mkdir operation is specified to ignore its body. # Only t=mkdir-with-children pays attention to it. (newkids, caps) = self._create_initial_children() - d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children", - 400, "Bad Request", - "t=mkdir does not accept children=, " - "try t=mkdir-with-children instead", - self.POST2, "/uri?t=mkdir", # without children - json.dumps(newkids)) - return d + url = self.webish_url + "/uri?t=mkdir" # without children + yield self.assertHTTPError(url, 400, + "t=mkdir does not accept children=, " + "try t=mkdir-with-children instead", + method="post", data=json.dumps(newkids)) + @inlineCallbacks def test_POST_noparent_bad(self): - d = self.shouldHTTPError("POST_noparent_bad", - 400, "Bad Request", - "/uri accepts only PUT, PUT?t=mkdir, " - "POST?t=upload, and POST?t=mkdir", - self.POST, "/uri?t=bogus") - return d + url = self.webish_url + "/uri?t=bogus" + yield self.assertHTTPError(url, 400, + "/uri accepts only PUT, PUT?t=mkdir, " + "POST?t=upload, and POST?t=mkdir", + method="post") def test_POST_mkdir_no_parentdir_immutable(self): (newkids, caps) = self._create_immutable_children() @@ -4065,16 +4063,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(self.failUnlessIsFooJSON) return d + @inlineCallbacks def test_GET_URI_URL_missing(self): base = "/uri/%s" % self._bad_file_uri - d = self.shouldHTTPError("test_GET_URI_URL_missing", - http.GONE, None, "NotEnoughSharesError", - self.GET, base) + url = self.webish_url + base + yield self.assertHTTPError(url, http.GONE, "NotEnoughSharesError") # TODO: how can we exercise both sides of WebDownloadTarget.fail # here? we must arrange for a download to fail after target.open() # has been called, and then inspect the response to see that it is # shorter than we expected. - return d def test_PUT_DIRURL_uri(self): d = self.s.create_dirnode() @@ -4162,13 +4159,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_got_json) return d + @inlineCallbacks def test_PUT_NEWFILEURL_bad_format(self): new_contents = self.NEWFILE_CONTENTS * 300000 - return self.shouldHTTPError("PUT_NEWFILEURL_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.PUT, self.public_url + \ - "/foo/foo.txt?format=foo", - new_contents) + url = self.webish_url + self.public_url + "/foo/foo.txt?format=foo" + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="put", data=new_contents) def test_PUT_NEWFILEURL_uri_replace(self): contents, n, new_uri = self.makefile(8) @@ -4297,11 +4293,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi d.addCallback(_got) return d + @inlineCallbacks def test_PUT_mkdir_bad_format(self): - return self.shouldHTTPError("PUT_mkdir_bad_format", - 400, "Bad Request", "Unknown format: foo", - self.PUT, "/uri?t=mkdir&format=foo", - "") + url = self.webish_url + "/uri?t=mkdir&format=foo" + yield self.assertHTTPError(url, 400, "Unknown format: foo", + method="put", data="") def test_POST_check(self): d = self.POST(self.public_url + "/foo", t="check", name="bar.txt") @@ -4352,37 +4348,22 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100))) return d + @inlineCallbacks def test_PUT_update_at_invalid_offset(self): file_contents = "test file" * 100000 # about 900 KiB - d = self.PUT("/uri?mutable=true", file_contents) - def _then(filecap): - self.filecap = filecap - d.addCallback(_then) + filecap = yield self.PUT("/uri?mutable=true", file_contents) # Negative offsets should cause an error. - d.addCallback(lambda ignored: - self.shouldHTTPError("PUT_update_at_invalid_offset", - 400, "Bad Request", - "Invalid offset", - self.PUT, - "/uri/%s?offset=-1" % self.filecap, - "foo")) - return d + url = self.webish_url + "/uri/%s?offset=-1" % filecap + yield self.assertHTTPError(url, 400, "Invalid offset", + method="put", data="foo") + @inlineCallbacks def test_PUT_update_at_offset_immutable(self): file_contents = "Test file" * 100000 - d = self.PUT("/uri", file_contents) - def _then(filecap): - self.filecap = filecap - d.addCallback(_then) - d.addCallback(lambda ignored: - self.shouldHTTPError("PUT_update_at_offset_immutable", - 400, "Bad Request", - "immutable", - self.PUT, - "/uri/%s?offset=50" % self.filecap, - "foo")) - return d - + filecap = yield self.PUT("/uri", file_contents) + url = self.webish_url + "/uri/%s?offset=50" % filecap + yield self.assertHTTPError(url, 400, "immutable", + method="put", data="foo") @inlineCallbacks def test_bad_method(self): @@ -4404,70 +4385,51 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi yield self.assertHTTPError(url, 404, "unknown/expired handle 'bogus'") + @inlineCallbacks def test_ophandle_cancel(self): url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=128" - d = do_http("post", url, - allow_redirects=True, browser_like_redirects=True) - d.addCallback(lambda ignored: - self.GET("/operations/128?t=status&output=JSON")) - def _check1(res): - data = json.loads(res) - self.failUnless("finished" in data, res) - monitor = self.ws.root.child_operations.handles["128"][0] - d = self.POST("/operations/128?t=cancel&output=JSON") - def _check2(res): - data = json.loads(res) - self.failUnless("finished" in data, res) - # t=cancel causes the handle to be forgotten - self.failUnless(monitor.is_cancelled()) - d.addCallback(_check2) - return d - d.addCallback(_check1) - d.addCallback(lambda ignored: - self.shouldHTTPError("ophandle_cancel", - 404, "404 Not Found", - "unknown/expired handle '128'", - self.GET, - "/operations/128?t=status&output=JSON")) - return d + yield do_http("post", url, + allow_redirects=True, browser_like_redirects=True) + res = yield self.GET("/operations/128?t=status&output=JSON") + data = json.loads(res) + self.failUnless("finished" in data, res) + monitor = self.ws.root.child_operations.handles["128"][0] + res = yield self.POST("/operations/128?t=cancel&output=JSON") + data = json.loads(res) + self.failUnless("finished" in data, res) + # t=cancel causes the handle to be forgotten + self.failUnless(monitor.is_cancelled()) + + url = self.webish_url + "/operations/128?t=status&output=JSON" + yield self.assertHTTPError(url, 404, "unknown/expired handle '128'") + + @inlineCallbacks def test_ophandle_retainfor(self): url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60" - d = do_http("post", url, - allow_redirects=True, browser_like_redirects=True) - d.addCallback(lambda ignored: - self.GET("/operations/129?t=status&output=JSON&retain-for=0")) - def _check1(res): - data = json.loads(res) - self.failUnless("finished" in data, res) - d.addCallback(_check1) - # the retain-for=0 will cause the handle to be expired very soon - d.addCallback(lambda ign: - self.clock.advance(2.0)) - d.addCallback(lambda ignored: - self.shouldHTTPError("ophandle_retainfor", - 404, "404 Not Found", - "unknown/expired handle '129'", - self.GET, - "/operations/129?t=status&output=JSON")) - return d + yield do_http("post", url, + allow_redirects=True, browser_like_redirects=True) + res = yield self.GET("/operations/129?t=status&output=JSON&retain-for=0") + data = json.loads(res) + self.failUnless("finished" in data, res) + # the retain-for=0 will cause the handle to be expired very soon + yield self.clock.advance(2.0) + url = self.webish_url + "/operations/129?t=status&output=JSON" + yield self.assertHTTPError(url, 404, "unknown/expired handle '129'") + + @inlineCallbacks def test_ophandle_release_after_complete(self): url = self.webish_url + self.public_url + "/foo/?t=start-manifest&ophandle=130" - d = do_http("post", url, - allow_redirects=True, browser_like_redirects=True) - d.addCallback(self.wait_for_operation, "130") - d.addCallback(lambda ignored: - self.GET("/operations/130?t=status&output=JSON&release-after-complete=true")) + yield do_http("post", url, + allow_redirects=True, browser_like_redirects=True) + yield self.wait_for_operation(None, "130") + yield self.GET("/operations/130?t=status&output=JSON&release-after-complete=true") # the release-after-complete=true will cause the handle to be expired - d.addCallback(lambda ignored: - self.shouldHTTPError("ophandle_release_after_complete", - 404, "404 Not Found", - "unknown/expired handle '130'", - self.GET, - "/operations/130?t=status&output=JSON")) - return d + op_url = self.webish_url + "/operations/130?t=status&output=JSON" + yield self.assertHTTPError(op_url, 404, "unknown/expired handle '130'") + @inlineCallbacks def test_uncollected_ophandle_expiration(self): # uncollected ophandles should expire after 4 days def _make_uncollected_ophandle(ophandle): @@ -4492,29 +4454,20 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return d # Create an ophandle, don't collect it, then advance the clock by # 4 days - 1 second and make sure that the ophandle is still there. - d = _make_uncollected_ophandle(131) - d.addCallback(lambda ign: - self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days - d.addCallback(lambda ign: - self.GET("/operations/131?t=status&output=JSON")) - def _check1(res): - data = json.loads(res) - self.failUnless("finished" in data, res) - d.addCallback(_check1) + yield _make_uncollected_ophandle(131) + yield self.clock.advance((96*60*60) - 1) # 96 hours = 4 days + res = yield self.GET("/operations/131?t=status&output=JSON") + data = json.loads(res) + self.failUnless("finished" in data, res) + # Create an ophandle, don't collect it, then try to collect it # after 4 days. It should be gone. - d.addCallback(lambda ign: - _make_uncollected_ophandle(132)) - d.addCallback(lambda ign: - self.clock.advance(96*60*60)) - d.addCallback(lambda ign: - self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours", - 404, "404 Not Found", - "unknown/expired handle '132'", - self.GET, - "/operations/132?t=status&output=JSON")) - return d + yield _make_uncollected_ophandle(132) + yield self.clock.advance(96*60*60) + op_url = self.webish_url + "/operations/132?t=status&output=JSON" + yield self.assertHTTPError(op_url, 404, "unknown/expired handle '132'") + @inlineCallbacks def test_collected_ophandle_expiration(self): # collected ophandles should expire after 1 day def _make_collected_ophandle(ophandle): @@ -4526,28 +4479,18 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi allow_redirects=True, browser_like_redirects=True) # Create a collected ophandle, then collect it after 23 hours # and 59 seconds to make sure that it is still there. - d = _make_collected_ophandle(133) - d.addCallback(lambda ign: - self.clock.advance((24*60*60) - 1)) - d.addCallback(lambda ign: - self.GET("/operations/133?t=status&output=JSON")) - def _check1(res): - data = json.loads(res) - self.failUnless("finished" in data, res) - d.addCallback(_check1) + yield _make_collected_ophandle(133) + yield self.clock.advance((24*60*60) - 1) + res = yield self.GET("/operations/133?t=status&output=JSON") + data = json.loads(res) + self.failUnless("finished" in data, res) + # Create another uncollected ophandle, then try to collect it # after 24 hours to make sure that it is gone. - d.addCallback(lambda ign: - _make_collected_ophandle(134)) - d.addCallback(lambda ign: - self.clock.advance(24*60*60)) - d.addCallback(lambda ign: - self.shouldHTTPError("collected_ophandle_expired_after_1_day", - 404, "404 Not Found", - "unknown/expired handle '134'", - self.GET, - "/operations/134?t=status&output=JSON")) - return d + yield _make_collected_ophandle(134) + yield self.clock.advance(24*60*60) + op_url = self.webish_url + "/operations/134?t=status&output=JSON" + yield self.assertHTTPError(op_url, 404, "unknown/expired handle '134'") def test_incident(self): d = self.POST("/report_incident", details="eek")