diff --git a/misc/build_helpers/show-tool-versions.py b/misc/build_helpers/show-tool-versions.py index c4fb79eff..f70183ae1 100644 --- a/misc/build_helpers/show-tool-versions.py +++ b/misc/build_helpers/show-tool-versions.py @@ -143,7 +143,6 @@ print_py_pkg_ver('coverage') print_py_pkg_ver('cryptography') print_py_pkg_ver('foolscap') print_py_pkg_ver('mock') -print_py_pkg_ver('Nevow', 'nevow') print_py_pkg_ver('pyasn1') print_py_pkg_ver('pycparser') print_py_pkg_ver('cryptography') diff --git a/newsfragments/3432.minor b/newsfragments/3432.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3433.installation b/newsfragments/3433.installation new file mode 100644 index 000000000..3c06e53d3 --- /dev/null +++ b/newsfragments/3433.installation @@ -0,0 +1 @@ +Tahoe-LAFS no longer depends on Nevow. \ No newline at end of file diff --git a/newsfragments/3434.minor b/newsfragments/3434.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3435.minor b/newsfragments/3435.minor new file mode 100644 index 000000000..e69de29bb diff --git a/nix/nevow.nix b/nix/nevow.nix deleted file mode 100644 index 202a59722..000000000 --- a/nix/nevow.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ stdenv, buildPythonPackage, fetchPypi, isPy3k, twisted }: - -buildPythonPackage rec { - pname = "Nevow"; - version = "0.14.5"; - name = "${pname}-${version}"; - disabled = isPy3k; - - src = fetchPypi { - inherit pname; - inherit version; - sha256 = "1wr3fai01h1bcp4qpia6indg4qmxvywwv3q1iibm669mln2vmdmg"; - }; - - propagatedBuildInputs = [ twisted ]; - - checkInputs = [ twisted ]; - - checkPhase = '' - trial formless nevow - ''; - - meta = with stdenv.lib; { - description = "Nevow, a web application construction kit for Python"; - longDescription = '' - Nevow - Pronounced as the French "nouveau", or "noo-voh", Nevow - is a web application construction kit written in Python. It is - designed to allow the programmer to express as much of the view - logic as desired in Python, and includes a pure Python XML - expression syntax named stan to facilitate this. However it - also provides rich support for designer-edited templates, using - a very small XML attribute language to provide bi-directional - template manipulation capability. - - Nevow also includes formless, a declarative syntax for - specifying the types of method parameters and exposing these - methods to the web. Forms can be rendered automatically, and - form posts will be validated and input coerced, rendering error - pages if appropriate. Once a form post has validated - successfully, the method will be called with the coerced values. - ''; - homepage = https://github.com/twisted/nevow; - license = licenses.mit; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix index ba3c9c885..4ee63a412 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -3,10 +3,7 @@ self: super: { packageOverrides = python-self: python-super: { # eliot is not part of nixpkgs at all at this time. eliot = python-self.callPackage ./eliot.nix { }; - # The packaged version of Nevow is very slightly out of date but also - # conflicts with the packaged version of Twisted. Supply our own - # slightly newer version. - nevow = python-super.callPackage ./nevow.nix { }; + # NixOS autobahn package has trollius as a dependency, although # it is optional. Trollius is unmaintained and fails on CI. autobahn = python-super.callPackage ./autobahn.nix { }; diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f2e61d6c2..a7f8fcbf7 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,6 +1,6 @@ { fetchFromGitHub, lib , nettools, python -, twisted, foolscap, nevow, zfec +, twisted, foolscap, zfec , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography @@ -46,7 +46,7 @@ python.pkgs.buildPythonPackage rec { ]; propagatedBuildInputs = with python.pkgs; [ - twisted foolscap nevow zfec appdirs + twisted foolscap zfec appdirs setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools diff --git a/setup.py b/setup.py index 4151545f7..874cc1258 100644 --- a/setup.py +++ b/setup.py @@ -38,8 +38,7 @@ install_requires = [ "zfec >= 1.1.0", # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. - # zope.interface 3.6.3 and 3.6.4 are incompatible with Nevow (#1435). - "zope.interface >= 3.6.0, != 3.6.3, != 3.6.4", + "zope.interface >= 3.6.0", # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for # transferring large mutable files of size N. @@ -70,7 +69,6 @@ install_requires = [ # rekeying bug # * The FTP frontend depends on Twisted >= 11.1.0 for # filepath.Permissions - # * Nevow 0.11.1 depends on Twisted >= 13.0.0. # * The SFTP frontend and manhole depend on the conch extra. However, we # can't explicitly declare that without an undesirable dependency on gmpy, # as explained in ticket #2740. @@ -102,9 +100,6 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. "Twisted[tls,conch] >= 18.4.0", - # We need Nevow >= 0.11.1 which can be installed using pip. - "Nevow >= 0.11.1", - "PyYAML >= 3.11", "six >= 1.10.0", diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py index cf98aae96..17f39cfcc 100644 --- a/src/allmydata/_auto_deps.py +++ b/src/allmydata/_auto_deps.py @@ -11,7 +11,6 @@ package_imports = [ ('foolscap', 'foolscap'), ('zfec', 'zfec'), ('Twisted', 'twisted'), - ('Nevow', 'nevow'), ('zope.interface', 'zope.interface'), ('python', None), ('platform', None), @@ -72,7 +71,6 @@ runtime_warning_messages = [ ] warning_imports = [ - 'nevow', 'twisted.persisted.sob', 'twisted.python.filepath', ] diff --git a/src/allmydata/test/common_nevow.py b/src/allmydata/test/common_nevow.py deleted file mode 100644 index d6327f9c6..000000000 --- a/src/allmydata/test/common_nevow.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -General helpers related to Nevow. -""" - -from twisted.internet.defer import ( - maybeDeferred, -) - -from nevow.context import WebContext -from nevow.testutil import FakeRequest -from nevow.appserver import ( - processingFailed, - DefaultExceptionHandler, -) -from nevow.inevow import ( - ICanHandleException, - IRequest, - IResource as INevowResource, - IData, -) - -def render(resource, query_args): - """ - Render (in the manner of the Nevow appserver) a Nevow ``Page`` or a - Twisted ``Resource`` against a request with the given query arguments . - - :param resource: The page or resource to render. - - :param query_args: The query arguments to put into the request being - rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. - - :return Deferred: A Deferred that fires with the rendered response body as - ``bytes``. - """ - ctx = WebContext(tag=resource) - req = FakeRequest(args=query_args) - ctx.remember(DefaultExceptionHandler(), ICanHandleException) - ctx.remember(req, IRequest) - ctx.remember(None, IData) - - def maybe_concat(res): - if isinstance(res, bytes): - return req.v + res - return req.v - - resource = INevowResource(resource) - d = maybeDeferred(resource.renderHTTP, ctx) - d.addErrback(processingFailed, req, ctx) - d.addCallback(maybe_concat) - return d diff --git a/src/allmydata/test/common_tweb.py b/src/allmydata/test/common_tweb.py deleted file mode 100644 index 37e24b5b8..000000000 --- a/src/allmydata/test/common_tweb.py +++ /dev/null @@ -1,54 +0,0 @@ -from twisted.python.reflect import ( - fullyQualifiedName, -) -from twisted.internet.defer import ( - succeed, -) -from twisted.web.test.requesthelper import ( - DummyChannel, -) -from twisted.web.server import ( - Request, -) -from twisted.web.server import ( - NOT_DONE_YET, -) - -def render(resource, query_args): - """ - Render (in the manner of the Twisted Web Site) a Twisted ``Resource`` - against a request with the given query arguments . - - :param resource: The page or resource to render. - - :param query_args: The query arguments to put into the request being - rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. - - :return Deferred: A Deferred that fires with the rendered response body as - ``bytes``. - """ - channel = DummyChannel() - request = Request(channel) - request.args = query_args - result = resource.render(request) - if isinstance(result, bytes): - request.write(result) - done = succeed(None) - elif result == NOT_DONE_YET: - if request.finished: - done = succeed(None) - else: - done = request.notifyFinish() - else: - raise ValueError( - "{!r} returned {!r}, required bytes or NOT_DONE_YET.".format( - fullyQualifiedName(resource.render), - result, - ), - ) - def get_body(ignored): - complete_response = channel.transport.written.getvalue() - header, body = complete_response.split(b"\r\n\r\n", 1) - return body - done.addCallback(get_body) - return done diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index 08b356cd0..033b77c98 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -4,14 +4,37 @@ __all__ = [ "render", ] -from future.utils import PY2 - -import treq from twisted.internet.defer import ( inlineCallbacks, returnValue, ) -from twisted.web.error import Error +from twisted.web.error import ( + Error, +) +from twisted.python.reflect import ( + fullyQualifiedName, +) +from twisted.internet.defer import ( + succeed, +) +from twisted.web.test.requesthelper import ( + DummyChannel, +) +from twisted.web.error import ( + UnsupportedMethod, +) +from twisted.web.http import ( + NOT_ALLOWED, +) +from twisted.web.server import ( + NOT_DONE_YET, +) + +import treq + +from ..webish import ( + TahoeLAFSRequest, +) @inlineCallbacks def do_http(method, url, **kwargs): @@ -24,16 +47,49 @@ def do_http(method, url, **kwargs): returnValue(body) -if PY2: - # We can only use Nevow on Python 2 and Tahoe-LAFS still *does* use Nevow - # so prefer the Nevow-based renderer if we can get it. - from .common_nevow import ( - render, - ) -else: - # However, Tahoe-LAFS *will* use Twisted Web before too much longer so go - # ahead and let some tests run against the Twisted Web-based renderer on - # Python 3. Later this will become the only codepath. - from .common_tweb import ( - render, - ) +def render(resource, query_args): + """ + Render (in the manner of the Twisted Web Site) a Twisted ``Resource`` + against a request with the given query arguments . + + :param resource: The page or resource to render. + + :param query_args: The query arguments to put into the request being + rendered. A mapping from ``bytes`` to ``list`` of ``bytes``. + + :return Deferred: A Deferred that fires with the rendered response body as + ``bytes``. + """ + channel = DummyChannel() + request = TahoeLAFSRequest(channel) + request.method = b"GET" + request.args = query_args + request.prepath = [b""] + request.postpath = [] + try: + result = resource.render(request) + except UnsupportedMethod: + request.setResponseCode(NOT_ALLOWED) + result = b"" + + if isinstance(result, bytes): + request.write(result) + done = succeed(None) + elif result == NOT_DONE_YET: + if request.finished: + done = succeed(None) + else: + done = request.notifyFinish() + else: + raise ValueError( + "{!r} returned {!r}, required bytes or NOT_DONE_YET.".format( + fullyQualifiedName(resource.render), + result, + ), + ) + def get_body(ignored): + complete_response = channel.transport.written.getvalue() + header, body = complete_response.split(b"\r\n\r\n", 1) + return body + done.addCallback(get_body) + return done diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 3813ac199..85b894b1f 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -20,18 +20,13 @@ from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.internet import defer -# We need to use `nevow.inevow.IRequest` for now for compatibility -# with the code in web/common.py. Once nevow bits are gone from -# web/common.py, we can use `twisted.web.iweb.IRequest` here. -if PY2: - from nevow.inevow import IRequest -else: - from twisted.web.iweb import IRequest - from zope.interface import implementer -from twisted.web.server import Request -from twisted.web.test.requesthelper import DummyChannel -from twisted.web.template import flattenString +from twisted.web.resource import ( + Resource, +) +from twisted.web.template import ( + renderElement, +) from allmydata import check_results, uri from allmydata import uri as tahoe_uri @@ -65,24 +60,6 @@ class FakeClient(object): def get_storage_broker(self): return self.storage_broker -@implementer(IRequest) -class TestRequest(Request, object): - """ - A minimal Request class to use in tests. - - XXX: We have to have this class because `common.get_arg()` expects - a `nevow.inevow.IRequest`, which `twisted.web.server.Request` - isn't. The request needs to have `args`, `fields`, `prepath`, and - `postpath` properties so that `allmydata.web.common.get_arg()` - won't complain. - """ - def __init__(self, args=None, fields=None): - super(TestRequest, self).__init__(DummyChannel()) - self.args = args or {} - self.fields = fields or {} - self.prepath = [b""] - self.postpath = [b""] - @implementer(IServer) class FakeServer(object): @@ -154,6 +131,15 @@ class FakeCheckAndRepairResults(object): return self._repair_success +class ElementResource(Resource, object): + def __init__(self, element): + Resource.__init__(self) + self.element = element + + def render(self, request): + return renderElement(request, self.element) + + class WebResultsRendering(unittest.TestCase): @staticmethod @@ -190,8 +176,9 @@ class WebResultsRendering(unittest.TestCase): return self.successResultOf(render(resource, {"output": ["json"]})) def render_element(self, element, args=None): - d = flattenString(TestRequest(args), element) - return self.successResultOf(d) + if args is None: + args = {} + return self.successResultOf(render(ElementResource(element), args)) def test_literal(self): lcr = web_check_results.LiteralCheckResultsRendererElement() diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index aa1f19936..ca0cd85fc 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - # Omitted list sinc it broke a test on Python 2. Shouldn't require further + # Omitted list since it broke a test on Python 2. Shouldn't require further # work, when we switch to Python 3 we'll be dropping this, anyway. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min # noqa: F401 @@ -26,18 +26,6 @@ from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString -# We need to use `nevow.inevow.IRequest` for now for compatibility -# with the code in web/common.py. Once nevow bits are gone from -# web/common.py, we can use `twisted.web.iweb.IRequest` here. -if PY2: - from nevow.inevow import IRequest -else: - from twisted.web.iweb import IRequest - -from twisted.web.server import Request -from twisted.web.test.requesthelper import DummyChannel -from zope.interface import implementer - from foolscap.api import fireEventually from allmydata.util import fileutil, hashutil, base32, pollmixin from allmydata.storage.common import storage_index_to_dir, \ @@ -52,6 +40,10 @@ from allmydata.web.storage import ( ) from .common_util import FakeCanary +from .common_web import ( + render, +) + def remove_tags(s): s = re.sub(br'<[^>]*>', b' ', s) s = re.sub(br'\s+', b' ', s) @@ -75,20 +67,10 @@ def renderDeferred(ss): return flattenString(None, elem) def renderJSON(resource): - """Render a JSON from the given resource.""" - - @implementer(IRequest) - class JSONRequest(Request): - """ - A Request with t=json argument added to it. This is useful to - invoke a Resouce.render_JSON() method. - """ - def __init__(self): - Request.__init__(self, DummyChannel()) - self.args = {"t": ["json"]} - self.fields = {} - - return resource.render(JSONRequest()) + """ + Render a JSON from the given resource. + """ + return render(resource, {"t": ["json"]}) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): diff --git a/src/allmydata/test/web/test_common.py b/src/allmydata/test/web/test_common.py index 6431ba610..2a0ebd3d8 100644 --- a/src/allmydata/test/web/test_common.py +++ b/src/allmydata/test/web/test_common.py @@ -2,6 +2,8 @@ Tests for ``allmydata.web.common``. """ +import gc + from bs4 import ( BeautifulSoup, ) @@ -13,13 +15,22 @@ from testtools.matchers import ( Equals, Contains, MatchesPredicate, + AfterPreprocessing, ) from testtools.twistedsupport import ( + failed, succeeded, has_no_result, ) +from twisted.python.failure import ( + Failure, +) +from twisted.internet.error import ( + ConnectionDone, +) from twisted.internet.defer import ( + Deferred, fail, ) from twisted.web.server import ( @@ -52,9 +63,11 @@ class StaticResource(Resource, object): def __init__(self, response): Resource.__init__(self) self._response = response + self._request = None @render_exception def render(self, request): + self._request = request return self._response @@ -214,3 +227,31 @@ class RenderExceptionTests(SyncTestCase): Equals(b"Internal Server Error"), ), ) + + def test_disconnected(self): + """ + If the transport is disconnected before the response is available, no + ``RuntimeError`` is logged for finishing a disconnected request. + """ + result = Deferred() + resource = StaticResource(result) + d = render(resource, {}) + + resource._request.connectionLost(Failure(ConnectionDone())) + result.callback(b"Some result") + + self.assertThat( + d, + failed( + AfterPreprocessing( + lambda reason: reason.type, + Equals(ConnectionDone), + ), + ), + ) + + # Since we're not a trial TestCase we don't have flushLoggedErrors. + # The next best thing is to make sure any dangling Deferreds have been + # garbage collected and then let the generic trial logic for failing + # tests with logged errors kick in. + gc.collect() diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 508fc82d4..aa6d44ea4 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4767,11 +4767,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_static_missing(self): # self.staticdir does not exist yet, because we used self.mktemp() d = self.assertFailure(self.GET("/static"), error.Error) - # nevow.static throws an exception when it tries to os.stat the - # missing directory, which gives the client a 500 Internal Server - # Error, and the traceback reveals the parent directory name. By - # switching to plain twisted.web.static, this gives a normal 404 that - # doesn't reveal anything. This addresses #1720. + # If os.stat raises an exception for the missing directory and the + # traceback reveals the parent directory name we don't want to see + # that parent directory name in the response. This addresses #1720. d.addCallback(lambda e: self.assertEquals(str(e), "404 Not Found")) return d diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py new file mode 100644 index 000000000..1e659812f --- /dev/null +++ b/src/allmydata/test/web/test_webish.py @@ -0,0 +1,208 @@ +""" +Tests for ``allmydata.webish``. +""" + +from uuid import ( + uuid4, +) + +from testtools.matchers import ( + AfterPreprocessing, + Contains, + Equals, + MatchesAll, + Not, +) + +from twisted.python.filepath import ( + FilePath, +) +from twisted.web.test.requesthelper import ( + DummyChannel, +) +from twisted.web.resource import ( + Resource, +) + +from ..common import ( + SyncTestCase, +) + +from ...webish import ( + TahoeLAFSRequest, + tahoe_lafs_site, +) + + +class TahoeLAFSRequestTests(SyncTestCase): + """ + Tests for ``TahoeLAFSRequest``. + """ + def _fields_test(self, method, request_headers, request_body, match_fields): + channel = DummyChannel() + request = TahoeLAFSRequest( + channel, + ) + for (k, v) in request_headers.items(): + request.requestHeaders.setRawHeaders(k, [v]) + request.gotLength(len(request_body)) + request.handleContentChunk(request_body) + request.requestReceived(method, b"/", b"HTTP/1.1") + + # We don't really care what happened to the request. What we do care + # about is what the `fields` attribute is set to. + self.assertThat( + request.fields, + match_fields, + ) + + def test_no_form_fields(self): + """ + When a ``GET`` request is received, ``TahoeLAFSRequest.fields`` is None. + """ + self._fields_test(b"GET", {}, b"", Equals(None)) + + def test_form_fields(self): + """ + When a ``POST`` request is received, form fields are parsed into + ``TahoeLAFSRequest.fields``. + """ + form_data, boundary = multipart_formdata([ + [param(u"name", u"foo"), + body(u"bar"), + ], + [param(u"name", u"baz"), + param(u"filename", u"quux"), + body(u"some file contents"), + ], + ]) + self._fields_test( + b"POST", + {b"content-type": b"multipart/form-data; boundary={}".format(boundary)}, + form_data.encode("ascii"), + AfterPreprocessing( + lambda fs: { + k: fs.getvalue(k) + for k + in fs.keys() + }, + Equals({ + b"foo": b"bar", + b"baz": b"some file contents", + }), + ), + ) + + +class TahoeLAFSSiteTests(SyncTestCase): + """ + Tests for the ``Site`` created by ``tahoe_lafs_site``. + """ + def _test_censoring(self, path, censored): + """ + Verify that the event logged for a request for ``path`` does not include + ``path`` but instead includes ``censored``. + + :param bytes path: A request path. + + :param bytes censored: A replacement value for the request path in the + access log. + + :return: ``None`` if the logging looks good. + """ + logPath = self.mktemp() + + site = tahoe_lafs_site(Resource(), logPath=logPath) + site.startFactory() + + channel = DummyChannel() + channel.factory = site + request = TahoeLAFSRequest(channel) + + request.gotLength(None) + request.requestReceived(b"GET", path, b"HTTP/1.1") + + self.assertThat( + FilePath(logPath).getContent(), + MatchesAll( + Contains(censored), + Not(Contains(path)), + ), + ) + + def test_uri_censoring(self): + """ + The log event for a request for **/uri/** has the capability value + censored. + """ + self._test_censoring( + b"/uri/URI:CHK:aaa:bbb", + b"/uri/[CENSORED]", + ) + + def test_file_censoring(self): + """ + The log event for a request for **/file/** has the capability value + censored. + """ + self._test_censoring( + b"/file/URI:CHK:aaa:bbb", + b"/file/[CENSORED]", + ) + + def test_named_censoring(self): + """ + The log event for a request for **/named/** has the capability value + censored. + """ + self._test_censoring( + b"/named/URI:CHK:aaa:bbb", + b"/named/[CENSORED]", + ) + + def test_uri_queryarg_censoring(self): + """ + The log event for a request for **/uri?cap=** has the capability + value censored. + """ + self._test_censoring( + b"/uri?uri=URI:CHK:aaa:bbb", + b"/uri?uri=[CENSORED]", + ) + +def param(name, value): + return u"; {}={}".format(name, value) + + +def body(value): + return u"\r\n\r\n{}".format(value) + + +def _field(field): + yield u"Content-Disposition: form-data" + for param in field: + yield param + + +def _multipart_formdata(fields): + for field in fields: + yield u"".join(_field(field)) + u"\r\n" + + +def multipart_formdata(fields): + """ + Serialize some simple fields into a multipart/form-data string. + + :param fields: A list of lists of unicode strings to assemble into the + result. See ``param`` and ``body``. + + :return unicode: The given fields combined into a multipart/form-data + string. + """ + boundary = str(uuid4()) + parts = list(_multipart_formdata(fields)) + parts.insert(0, u"") + return ( + (u"--" + boundary + u"\r\n").join(parts), + boundary, + ) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index c9d59e7ae..d970cc918 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,4 +1,3 @@ -from future.utils import PY2 from past.builtins import unicode import time @@ -22,12 +21,14 @@ from twisted.web import ( resource, template, ) +from twisted.web.iweb import ( + IRequest, +) from twisted.web.template import ( tags, ) from twisted.web.server import ( NOT_DONE_YET, - UnsupportedMethod, ) from twisted.web.util import ( DeferredResource, @@ -42,20 +43,12 @@ from twisted.python.failure import ( Failure, ) from twisted.internet.defer import ( + CancelledError, maybeDeferred, ) from twisted.web.resource import ( IResource, ) -from twisted.web.iweb import IRequest as ITwistedRequest -if PY2: - from nevow.appserver import DefaultExceptionHandler - from nevow.inevow import IRequest as INevowRequest -else: - class DefaultExceptionHandler: - def __init__(self, *args, **kwargs): - raise NotImplementedError("Still not ported to Python 3") - INevowRequest = None from allmydata import blacklist from allmydata.interfaces import ( @@ -161,11 +154,22 @@ def parse_offset_arg(offset): return offset -def get_root(ctx_or_req): - if PY2: - req = INevowRequest(ctx_or_req) - else: - req = ITwistedRequest(ctx_or_req) +def get_root(req): + """ + Get a relative path with parent directory segments that refers to the root + location known to the given request. This seems a lot like the constant + absolute path **/** but it will behave differently if the Tahoe-LAFS HTTP + server is reverse-proxied and mounted somewhere other than at the root. + + :param twisted.web.iweb.IRequest req: The request to consider. + + :return: A string like ``../../..`` with the correct number of segments to + reach the root. + """ + if not IRequest.providedBy(req): + raise TypeError( + "get_root requires IRequest provider, got {!r}".format(req), + ) depth = len(req.prepath) + len(req.postpath) link = "/".join([".."] * depth) return link @@ -366,20 +370,13 @@ def humanize_failure(f): return humanize_exception(f.value) -class MyExceptionHandler(DefaultExceptionHandler, object): - def renderHTTP_exception(self, ctx, f): - req = INevowRequest(ctx) - req.write(_renderHTTP_exception(req, f)) - req.finishRequest(False) - - class NeedOperationHandleError(WebError): pass class SlotsSequenceElement(template.Element): """ - ``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for + ``SlotsSequenceElement` is a minimal port of Nevow's sequence renderer for twisted.web.template. Tags passed in to be templated will have two renderers available: ``item`` @@ -493,7 +490,17 @@ def render_exception(render): # Apply `_finish` all of our result handling logic to whatever it # returned. result.addBoth(_finish, bound_render, request) - result.addActionFinish() + d = result.addActionFinish() + + # If the connection is lost then there's no point running our _finish + # logic because it has nowhere to send anything. There may also be no + # point in finishing whatever operation was being performed because + # the client cannot be informed of its result. Also, Twisted Web + # raises exceptions from some Request methods if they're used after + # the connection is lost. + request.notifyFinish().addErrback( + lambda ignored: d.cancel(), + ) return NOT_DONE_YET return g @@ -518,6 +525,8 @@ def _finish(result, render, request): :return: ``None`` """ if isinstance(result, Failure): + if result.check(CancelledError): + return Message.log( message_type=u"allmydata:web:common-render:failure", message=result.getErrorMessage(), @@ -590,17 +599,6 @@ def _renderHTTP_exception(request, failure): if code is not None: return _renderHTTP_exception_simple(request, text, code) - if failure.check(UnsupportedMethod): - # twisted.web.server.Request.render() has support for transforming - # this into an appropriate 501 NOT_IMPLEMENTED or 405 NOT_ALLOWED - # return code, but nevow does not. - method = request.method - return _renderHTTP_exception_simple( - request, - "I don't know how to treat a %s request." % (method,), - http.NOT_IMPLEMENTED, - ) - accept = request.getHeader("accept") if not accept: accept = "*/*" diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 73130cbab..22f235790 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -4,15 +4,7 @@ Common utilities that are available from Python 3. Can eventually be merged back into allmydata.web.common. """ -from future.utils import PY2 - -if PY2: - from nevow.inevow import IRequest as INevowRequest -else: - INevowRequest = None - from twisted.web import resource, http -from twisted.web.iweb import IRequest from allmydata.util import abbreviate @@ -23,24 +15,20 @@ class WebError(Exception): self.code = code -def get_arg(ctx_or_req, argname, default=None, multiple=False): +def get_arg(req, argname, default=None, multiple=False): """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take precedence. If multiple=True, this returns a tuple of arguments (possibly empty), starting with all those in the query args. + + :param TahoeLAFSRequest req: The request to consider. """ results = [] - if PY2: - req = INevowRequest(ctx_or_req) - if argname in req.args: - results.extend(req.args[argname]) - if req.fields and argname in req.fields: - results.append(req.fields[argname].value) - else: - req = IRequest(ctx_or_req) - if argname in req.args: - results.extend(req.args[argname]) + if argname in req.args: + results.extend(req.args[argname]) + if req.fields and argname in req.fields: + results.append(req.fields[argname].value) if multiple: return tuple(results) if results: diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 0d521951f..f83defd6a 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -114,7 +114,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # Rejecting URIs that contain empty path pieces (for example: # "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was - # the old nevow behavior and it is encoded in the test suite; + # the old Nevow behavior and it is encoded in the test suite; # we will follow suit. for segment in req.prepath: if not segment: @@ -398,8 +398,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): d.addBoth(_maybe_got_node) # now we have a placeholder or a filenodehandler, and we can just # delegate to it. We could return the resource back out of - # DirectoryNodeHandler.renderHTTP, and nevow would recurse into it, - # but the addCallback() that handles when_done= would break. + # DirectoryNodeHandler.render_POST and it would get rendered but the + # addCallback() that handles when_done= would break. def render_child(child): req.dont_apply_extra_processing = True return child.render(req) @@ -522,9 +522,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): d.addCallback(self._maybe_literal, CheckResultsRenderer) return d - def _start_operation(self, monitor, renderer, ctx): - self._operations.add_monitor(ctx, monitor, renderer) - return self._operations.redirect_to(ctx) + def _start_operation(self, monitor, renderer, req): + self._operations.add_monitor(req, monitor, renderer) + return self._operations.redirect_to(req) def _POST_start_deep_check(self, req): # check this directory and everything reachable from it diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 7686b35b5..f65977460 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -480,7 +480,10 @@ class FileDownloader(Resource, object): d = self.filenode.read(req, first, size) def _error(f): - req._tahoe_request_had_error = f # for HTTP-style logging + if f.check(defer.CancelledError): + # The HTTP connection was lost and we no longer have anywhere + # to send our result. Let this pass through. + return f if req.startedWriting: # The content-type is already set, and the response code has # already been sent, so we can't provide a clean error diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 0e53d075d..2ba87c5ec 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -152,8 +152,6 @@ class ReloadMixin(object): def refresh(self, req, tag): if self.monitor.is_finished(): return "" - # dreid suggests ctx.tag(**dict([("http-equiv", "refresh")])) - # but I can't tell if he's joking or not tag.attributes["http-equiv"] = "refresh" tag.attributes["content"] = str(self.REFRESH_TIME) return tag diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 78daadef4..91f14bd91 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -189,7 +189,7 @@ class FileHandler(resource.Resource, object): return filenode.FileNodeDownloadHandler(self.client, node) @render_exception - def render_GET(self, ctx): + def render_GET(self, req): raise WebError("/file must be followed by a file-cap and a name", http.NOT_FOUND) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index cf3264dac..51624a409 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,3 +1,4 @@ +from future.utils import PY2 import time, json from twisted.python.filepath import FilePath @@ -317,4 +318,7 @@ class StorageStatus(MultiFormatResource): "lease-checker": self._storage.lease_checker.get_state(), "lease-checker-progress": self._storage.lease_checker.get_progress(), } - return json.dumps(d, indent=1) + "\n" + result = json.dumps(d, indent=1) + "\n" + if PY2: + result = result.decode("utf-8") + return result.encode("utf-8") diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 432a8cf2f..b07d06be1 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,43 +1,57 @@ import re, time + +from functools import ( + partial, +) +from cgi import ( + FieldStorage, +) + from twisted.application import service, strports, internet -from twisted.web import http, static +from twisted.web import static +from twisted.web.http import ( + parse_qs, +) +from twisted.web.server import ( + Request, + Site, +) from twisted.internet import defer from twisted.internet.address import ( IPv4Address, IPv6Address, ) -from nevow import appserver, inevow from allmydata.util import log, fileutil from allmydata.web import introweb, root -from allmydata.web.common import MyExceptionHandler from allmydata.web.operations import OphandleTable from .web.storage_plugins import ( StoragePlugins, ) -# we must override twisted.web.http.Request.requestReceived with a version -# that doesn't use cgi.parse_multipart() . Since we actually use Nevow, we -# override the nevow-specific subclass, nevow.appserver.NevowRequest . This -# is an exact copy of twisted.web.http.Request (from SVN HEAD on 10-Aug-2007) -# that modifies the way form arguments are parsed. Note that this sort of -# surgery may induce a dependency upon a particular version of twisted.web +class TahoeLAFSRequest(Request, object): + """ + ``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request`` + that are useful for Tahoe-LAFS. -parse_qs = http.parse_qs -class MyRequest(appserver.NevowRequest, object): + :ivar NoneType|FieldStorage fields: For POST requests, a structured + representation of the contents of the request body. For anything + else, ``None``. + """ fields = None - _tahoe_request_had_error = None def requestReceived(self, command, path, version): - """Called by channel when all data has been received. - - This method is not intended for users. """ - self.content.seek(0,0) + Called by channel when all data has been received. + + Override the base implementation to apply certain site-wide policies + and to provide less memory-intensive multipart/form-post handling for + large file uploads. + """ + self.content.seek(0) self.args = {} self.stack = [] - self.setHeader("Referrer-Policy", "no-referrer") self.method, self.uri = command, path self.clientproto = version @@ -49,93 +63,36 @@ class MyRequest(appserver.NevowRequest, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - # Adding security headers. These will be sent for *all* HTTP requests. - # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options - self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"]) + if self.method == 'POST': + # We use FieldStorage here because it performs better than + # cgi.parse_multipart(self.content, pdict) which is what + # twisted.web.http.Request uses. + self.fields = FieldStorage( + self.content, + { + name.lower(): value[-1] + for (name, value) + in self.requestHeaders.getAllRawHeaders() + }, + environ={'REQUEST_METHOD': 'POST'}) + self.content.seek(0) - # Argument processing. + self._tahoeLAFSSecurityPolicy() -## The original twisted.web.http.Request.requestReceived code parsed the -## content and added the form fields it found there to self.args . It -## did this with cgi.parse_multipart, which holds the arguments in RAM -## and is thus unsuitable for large file uploads. The Nevow subclass -## (nevow.appserver.NevowRequest) uses cgi.FieldStorage instead (putting -## the results in self.fields), which is much more memory-efficient. -## Since we know we're using Nevow, we can anticipate these arguments -## appearing in self.fields instead of self.args, and thus skip the -## parse-content-into-self.args step. - -## args = self.args -## ctype = self.getHeader('content-type') -## if self.method == "POST" and ctype: -## mfd = 'multipart/form-data' -## key, pdict = cgi.parse_header(ctype) -## if key == 'application/x-www-form-urlencoded': -## args.update(parse_qs(self.content.read(), 1)) -## elif key == mfd: -## try: -## args.update(cgi.parse_multipart(self.content, pdict)) -## except KeyError, e: -## if e.args[0] == 'content-disposition': -## # Parse_multipart can't cope with missing -## # content-dispostion headers in multipart/form-data -## # parts, so we catch the exception and tell the client -## # it was a bad request. -## self.channel.transport.write( -## "HTTP/1.1 400 Bad Request\r\n\r\n") -## self.channel.transport.loseConnection() -## return -## raise self.processing_started_timestamp = time.time() self.process() - def _logger(self): - # we build up a log string that hides most of the cap, to preserve - # user privacy. We retain the query args so we can identify things - # like t=json. Then we send it to the flog. We make no attempt to - # match apache formatting. TODO: when we move to DSA dirnodes and - # shorter caps, consider exposing a few characters of the cap, or - # maybe a few characters of its hash. - x = self.uri.split("?", 1) - if len(x) == 1: - # no query args - path = self.uri - queryargs = "" - else: - path, queryargs = x - # there is a form handler which redirects POST /uri?uri=FOO into - # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make - # sure we censor these too. - if queryargs.startswith("uri="): - queryargs = "[uri=CENSORED]" - queryargs = "?" + queryargs - if path.startswith("/uri"): - path = "/uri/[CENSORED].." - elif path.startswith("/file"): - path = "/file/[CENSORED].." - elif path.startswith("/named"): - path = "/named/[CENSORED].." - - uri = path + queryargs - - error = "" - if self._tahoe_request_had_error: - error = " [ERROR]" - - log.msg( - format=( - "web: %(clientip)s %(method)s %(uri)s %(code)s " - "%(length)s%(error)s" - ), - clientip=_get_client_ip(self), - method=self.method, - uri=uri, - code=self.code, - length=(self.sentLength or "-"), - error=error, - facility="tahoe.webish", - level=log.OPERATIONAL, - ) + def _tahoeLAFSSecurityPolicy(self): + """ + Set response properties related to Tahoe-LAFS-imposed security policy. + This will ensure that all HTTP requests received by the Tahoe-LAFS + HTTP server have this policy imposed, regardless of other + implementation details. + """ + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options + self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"]) + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy + self.setHeader("Referrer-Policy", "no-referrer") def _get_client_ip(request): @@ -150,6 +107,54 @@ def _get_client_ip(request): return None +def _logFormatter(logDateTime, request): + # we build up a log string that hides most of the cap, to preserve + # user privacy. We retain the query args so we can identify things + # like t=json. Then we send it to the flog. We make no attempt to + # match apache formatting. TODO: when we move to DSA dirnodes and + # shorter caps, consider exposing a few characters of the cap, or + # maybe a few characters of its hash. + x = request.uri.split("?", 1) + if len(x) == 1: + # no query args + path = request.uri + queryargs = "" + else: + path, queryargs = x + # there is a form handler which redirects POST /uri?uri=FOO into + # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make + # sure we censor these too. + if queryargs.startswith("uri="): + queryargs = "uri=[CENSORED]" + queryargs = "?" + queryargs + if path.startswith("/uri/"): + path = "/uri/[CENSORED]" + elif path.startswith("/file/"): + path = "/file/[CENSORED]" + elif path.startswith("/named/"): + path = "/named/[CENSORED]" + + uri = path + queryargs + + template = "web: %(clientip)s %(method)s %(uri)s %(code)s %(length)s" + return template % dict( + clientip=_get_client_ip(request), + method=request.method, + uri=uri, + code=request.code, + length=(request.sentLength or "-"), + facility="tahoe.webish", + level=log.OPERATIONAL, + ) + + +tahoe_lafs_site = partial( + Site, + requestFactory=TahoeLAFSRequest, + logFormatter=_logFormatter, +) + + class WebishServer(service.MultiService): name = "webish" @@ -175,15 +180,13 @@ class WebishServer(service.MultiService): def buildServer(self, webport, nodeurl_path, staticdir): self.webport = webport - self.site = site = appserver.NevowSite(self.root) - self.site.requestFactory = MyRequest - self.site.remember(MyExceptionHandler(), inevow.ICanHandleException) + self.site = tahoe_lafs_site(self.root) self.staticdir = staticdir # so tests can check if staticdir: self.root.putChild("static", static.File(staticdir)) if re.search(r'^\d', webport): webport = "tcp:"+webport # twisted warns about bare "0" or "3456" - s = strports.service(webport, site) + s = strports.service(webport, self.site) s.setServiceParent(self) self._scheme = None diff --git a/static/tahoe.py b/static/tahoe.py index cac53bdfa..c18f60e2c 100644 --- a/static/tahoe.py +++ b/static/tahoe.py @@ -3,23 +3,19 @@ # Import this first to suppress deprecation warnings. import allmydata -# nevow requires all these for its voodoo module import time adaptor registrations -from nevow import accessors, appserver, static, rend, url, util, query, i18n, flat -from nevow import guard, stan, testutil, context -from nevow.flat import flatmdom, flatstan, twist -from formless import webform, processors, annotate, iformless from decimal import Decimal from xml.dom import minidom import allmydata.web -# junk to appease pyflakes's outrage -[ - accessors, appserver, static, rend, url, util, query, i18n, flat, guard, stan, testutil, - context, flatmdom, flatstan, twist, webform, processors, annotate, iformless, Decimal, - minidom, allmydata, -] +# We import these things to give PyInstaller's dependency resolver some hints +# about what it needs to include. We don't use them otherwise _here_ but +# other parts of the codebase do. pyflakes points out that they are unused +# unless we use them. So ... use them. +Decimal +minidom +allmydata from allmydata.scripts import runner -runner.run() \ No newline at end of file +runner.run()