ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-24 07:06:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #869 from tahoe-lafs/3433.twebish

Browse Source 
Remove Nevow dependency

Fixes: ticket:3432
Fixes: ticket:3433
Fixes: ticket:3434
Fixes: ticket:3435
This commit is contained in:
Jean-Paul Calderone 2020-10-22 08:20:13 -04:00 committed by GitHub
commit 03948ba6a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 528 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
misc/build_helpers/show-tool-versions.py
View File

@ -143,7 +143,6 @@ print_py_pkg_ver('coverage')
print_py_pkg_ver('cryptography')
print_py_pkg_ver('foolscap')
print_py_pkg_ver('mock')
print_py_pkg_ver('Nevow', 'nevow')
print_py_pkg_ver('pyasn1')
print_py_pkg_ver('pycparser')
print_py_pkg_ver('cryptography')

0
newsfragments/3432.minor Normal file
View File

1
newsfragments/3433.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS no longer depends on Nevow.

0
newsfragments/3434.minor Normal file
View File

0
newsfragments/3435.minor Normal file
View File

45
nix/nevow.nix
View File

@ -1,45 +0,0 @@
{ stdenv, buildPythonPackage, fetchPypi, isPy3k, twisted }:
buildPythonPackage rec {
pname = "Nevow";
version = "0.14.5";
name = "${pname}-${version}";
disabled = isPy3k;
src = fetchPypi {
inherit pname;
inherit version;
sha256 = "1wr3fai01h1bcp4qpia6indg4qmxvywwv3q1iibm669mln2vmdmg";
};
propagatedBuildInputs = [ twisted ];
checkInputs = [ twisted ];
checkPhase = ''
trial formless nevow
'';
meta = with stdenv.lib; {
description = "Nevow, a web application construction kit for Python";
longDescription = ''
Nevow - Pronounced as the French "nouveau", or "noo-voh", Nevow
is a web application construction kit written in Python. It is
designed to allow the programmer to express as much of the view
logic as desired in Python, and includes a pure Python XML
expression syntax named stan to facilitate this. However it
also provides rich support for designer-edited templates, using
a very small XML attribute language to provide bi-directional
template manipulation capability.
Nevow also includes formless, a declarative syntax for
specifying the types of method parameters and exposing these
methods to the web. Forms can be rendered automatically, and
form posts will be validated and input coerced, rendering error
pages if appropriate. Once a form post has validated
successfully, the method will be called with the coerced values.
'';
homepage = https://github.com/twisted/nevow;
license = licenses.mit;
};
}

5
nix/overlays.nix
View File

@ -3,10 +3,7 @@ self: super: {
packageOverrides = python-self: python-super: {
# eliot is not part of nixpkgs at all at this time.
eliot = python-self.callPackage ./eliot.nix { };
# The packaged version of Nevow is very slightly out of date but also
# conflicts with the packaged version of Twisted. Supply our own
# slightly newer version.
nevow = python-super.callPackage ./nevow.nix { };
# NixOS autobahn package has trollius as a dependency, although
# it is optional. Trollius is unmaintained and fails on CI.
autobahn = python-super.callPackage ./autobahn.nix { };

4
nix/tahoe-lafs.nix
View File

@ -1,6 +1,6 @@
{ fetchFromGitHub, lib
, nettools, python
, twisted, foolscap, nevow, zfec
, twisted, foolscap, zfec
, setuptools, setuptoolsTrial, pyasn1, zope_interface
, service-identity, pyyaml, magic-wormhole, treq, appdirs
, beautifulsoup4, eliot, autobahn, cryptography
@ -46,7 +46,7 @@ python.pkgs.buildPythonPackage rec {
];
propagatedBuildInputs = with python.pkgs; [
twisted foolscap nevow zfec appdirs
twisted foolscap zfec appdirs
setuptoolsTrial pyasn1 zope_interface
service-identity pyyaml magic-wormhole treq
eliot autobahn cryptography setuptools

7
setup.py
View File

@ -38,8 +38,7 @@ install_requires = [
"zfec >= 1.1.0",
# zope.interface >= 3.6.0 is required for Twisted >= 12.1.0.
# zope.interface 3.6.3 and 3.6.4 are incompatible with Nevow (#1435).
"zope.interface >= 3.6.0, != 3.6.3, != 3.6.4",
"zope.interface >= 3.6.0",
# * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for
# transferring large mutable files of size N.
@ -70,7 +69,6 @@ install_requires = [
# rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
# * The FTP frontend depends on Twisted >= 11.1.0 for
# filepath.Permissions
# * Nevow 0.11.1 depends on Twisted >= 13.0.0.
# * The SFTP frontend and manhole depend on the conch extra. However, we
# can't explicitly declare that without an undesirable dependency on gmpy,
# as explained in ticket #2740.
@ -102,9 +100,6 @@ install_requires = [
# an sftp extra in Tahoe-LAFS, there is no point in having one.
"Twisted[tls,conch] >= 18.4.0",
# We need Nevow >= 0.11.1 which can be installed using pip.
"Nevow >= 0.11.1",
"PyYAML >= 3.11",
"six >= 1.10.0",

2
src/allmydata/_auto_deps.py
View File

@ -11,7 +11,6 @@ package_imports = [
('foolscap', 'foolscap'),
('zfec', 'zfec'),
('Twisted', 'twisted'),
('Nevow', 'nevow'),
('zope.interface', 'zope.interface'),
('python', None),
('platform', None),
@ -72,7 +71,6 @@ runtime_warning_messages = [
]
warning_imports = [
'nevow',
'twisted.persisted.sob',
'twisted.python.filepath',
]

50
src/allmydata/test/common_nevow.py
View File

@ -1,50 +0,0 @@
"""
General helpers related to Nevow.
"""
from twisted.internet.defer import (
maybeDeferred,
)
from nevow.context import WebContext
from nevow.testutil import FakeRequest
from nevow.appserver import (
processingFailed,
DefaultExceptionHandler,
)
from nevow.inevow import (
ICanHandleException,
IRequest,
IResource as INevowResource,
IData,
)
def render(resource, query_args):
"""
Render (in the manner of the Nevow appserver) a Nevow ``Page`` or a
Twisted ``Resource`` against a request with the given query arguments .
:param resource: The page or resource to render.
:param query_args: The query arguments to put into the request being
rendered. A mapping from ``bytes`` to ``list`` of ``bytes``.
:return Deferred: A Deferred that fires with the rendered response body as
``bytes``.
"""
ctx = WebContext(tag=resource)
req = FakeRequest(args=query_args)
ctx.remember(DefaultExceptionHandler(), ICanHandleException)
ctx.remember(req, IRequest)
ctx.remember(None, IData)
def maybe_concat(res):
if isinstance(res, bytes):
return req.v + res
return req.v
resource = INevowResource(resource)
d = maybeDeferred(resource.renderHTTP, ctx)
d.addErrback(processingFailed, req, ctx)
d.addCallback(maybe_concat)
return d

54
src/allmydata/test/common_tweb.py
View File

@ -1,54 +0,0 @@
from twisted.python.reflect import (
fullyQualifiedName,
)
from twisted.internet.defer import (
succeed,
)
from twisted.web.test.requesthelper import (
DummyChannel,
)
from twisted.web.server import (
Request,
)
from twisted.web.server import (
NOT_DONE_YET,
)
def render(resource, query_args):
"""
Render (in the manner of the Twisted Web Site) a Twisted ``Resource``
against a request with the given query arguments .
:param resource: The page or resource to render.
:param query_args: The query arguments to put into the request being
rendered. A mapping from ``bytes`` to ``list`` of ``bytes``.
:return Deferred: A Deferred that fires with the rendered response body as
``bytes``.
"""
channel = DummyChannel()
request = Request(channel)
request.args = query_args
result = resource.render(request)
if isinstance(result, bytes):
request.write(result)
done = succeed(None)
elif result == NOT_DONE_YET:
if request.finished:
done = succeed(None)
else:
done = request.notifyFinish()
else:
raise ValueError(
"{!r} returned {!r}, required bytes or NOT_DONE_YET.".format(
fullyQualifiedName(resource.render),
result,
),
)
def get_body(ignored):
complete_response = channel.transport.written.getvalue()
header, body = complete_response.split(b"\r\n\r\n", 1)
return body
done.addCallback(get_body)
return done

90
src/allmydata/test/common_web.py
View File

@ -4,14 +4,37 @@ __all__ = [
"render",
]
from future.utils import PY2
import treq
from twisted.internet.defer import (
inlineCallbacks,
returnValue,
)
from twisted.web.error import Error
from twisted.web.error import (
Error,
)
from twisted.python.reflect import (
fullyQualifiedName,
)
from twisted.internet.defer import (
succeed,
)
from twisted.web.test.requesthelper import (
DummyChannel,
)
from twisted.web.error import (
UnsupportedMethod,
)
from twisted.web.http import (
NOT_ALLOWED,
)
from twisted.web.server import (
NOT_DONE_YET,
)
import treq
from ..webish import (
TahoeLAFSRequest,
)
@inlineCallbacks
def do_http(method, url, **kwargs):
@ -24,16 +47,49 @@ def do_http(method, url, **kwargs):
returnValue(body)
if PY2:
# We can only use Nevow on Python 2 and Tahoe-LAFS still *does* use Nevow
# so prefer the Nevow-based renderer if we can get it.
from .common_nevow import (
render,
)
else:
# However, Tahoe-LAFS *will* use Twisted Web before too much longer so go
# ahead and let some tests run against the Twisted Web-based renderer on
# Python 3. Later this will become the only codepath.
from .common_tweb import (
render,
)
def render(resource, query_args):
"""
Render (in the manner of the Twisted Web Site) a Twisted ``Resource``
against a request with the given query arguments .
:param resource: The page or resource to render.
:param query_args: The query arguments to put into the request being
rendered. A mapping from ``bytes`` to ``list`` of ``bytes``.
:return Deferred: A Deferred that fires with the rendered response body as
``bytes``.
"""
channel = DummyChannel()
request = TahoeLAFSRequest(channel)
request.method = b"GET"
request.args = query_args
request.prepath = [b""]
request.postpath = []
try:
result = resource.render(request)
except UnsupportedMethod:
request.setResponseCode(NOT_ALLOWED)
result = b""
if isinstance(result, bytes):
request.write(result)
done = succeed(None)
elif result == NOT_DONE_YET:
if request.finished:
done = succeed(None)
else:
done = request.notifyFinish()
else:
raise ValueError(
"{!r} returned {!r}, required bytes or NOT_DONE_YET.".format(
fullyQualifiedName(resource.render),
result,
),
)
def get_body(ignored):
complete_response = channel.transport.written.getvalue()
header, body = complete_response.split(b"\r\n\r\n", 1)
return body
done.addCallback(get_body)
return done

49
src/allmydata/test/test_checker.py
View File

@ -20,18 +20,13 @@ from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.internet import defer
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from zope.interface import implementer
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
from twisted.web.template import flattenString
from twisted.web.resource import (
Resource,
)
from twisted.web.template import (
renderElement,
)
from allmydata import check_results, uri
from allmydata import uri as tahoe_uri
@ -65,24 +60,6 @@ class FakeClient(object):
def get_storage_broker(self):
return self.storage_broker
@implementer(IRequest)
class TestRequest(Request, object):
"""
A minimal Request class to use in tests.
XXX: We have to have this class because `common.get_arg()` expects
a `nevow.inevow.IRequest`, which `twisted.web.server.Request`
isn't. The request needs to have `args`, `fields`, `prepath`, and
`postpath` properties so that `allmydata.web.common.get_arg()`
won't complain.
"""
def __init__(self, args=None, fields=None):
super(TestRequest, self).__init__(DummyChannel())
self.args = args or {}
self.fields = fields or {}
self.prepath = [b""]
self.postpath = [b""]
@implementer(IServer)
class FakeServer(object):
@ -154,6 +131,15 @@ class FakeCheckAndRepairResults(object):
return self._repair_success
class ElementResource(Resource, object):
def __init__(self, element):
Resource.__init__(self)
self.element = element
def render(self, request):
return renderElement(request, self.element)
class WebResultsRendering(unittest.TestCase):
@staticmethod
@ -190,8 +176,9 @@ class WebResultsRendering(unittest.TestCase):
return self.successResultOf(render(resource, {"output": ["json"]}))
def render_element(self, element, args=None):
d = flattenString(TestRequest(args), element)
return self.successResultOf(d)
if args is None:
args = {}
return self.successResultOf(render(ElementResource(element), args))
def test_literal(self):
lcr = web_check_results.LiteralCheckResultsRendererElement()

36
src/allmydata/test/test_storage_web.py
View File

@ -11,7 +11,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Omitted list sinc it broke a test on Python 2. Shouldn't require further
# Omitted list since it broke a test on Python 2. Shouldn't require further
# work, when we switch to Python 3 we'll be dropping this, anyway.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min # noqa: F401
@ -26,18 +26,6 @@ from twisted.internet import defer
from twisted.application import service
from twisted.web.template import flattenString
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
from zope.interface import implementer
from foolscap.api import fireEventually
from allmydata.util import fileutil, hashutil, base32, pollmixin
from allmydata.storage.common import storage_index_to_dir, \
@ -52,6 +40,10 @@ from allmydata.web.storage import (
)
from .common_util import FakeCanary
from .common_web import (
render,
)
def remove_tags(s):
s = re.sub(br'<[^>]*>', b' ', s)
s = re.sub(br'\s+', b' ', s)
@ -75,20 +67,10 @@ def renderDeferred(ss):
return flattenString(None, elem)
def renderJSON(resource):
"""Render a JSON from the given resource."""
@implementer(IRequest)
class JSONRequest(Request):
"""
A Request with t=json argument added to it. This is useful to
invoke a Resouce.render_JSON() method.
"""
def __init__(self):
Request.__init__(self, DummyChannel())
self.args = {"t": ["json"]}
self.fields = {}
return resource.render(JSONRequest())
"""
Render a JSON from the given resource.
"""
return render(resource, {"t": ["json"]})
class MyBucketCountingCrawler(BucketCountingCrawler):
def finished_prefix(self, cycle, prefix):

41
src/allmydata/test/web/test_common.py
View File

@ -2,6 +2,8 @@
Tests for ``allmydata.web.common``.
"""
import gc
from bs4 import (
BeautifulSoup,
)
@ -13,13 +15,22 @@ from testtools.matchers import (
Equals,
Contains,
MatchesPredicate,
AfterPreprocessing,
)
from testtools.twistedsupport import (
failed,
succeeded,
has_no_result,
)
from twisted.python.failure import (
Failure,
)
from twisted.internet.error import (
ConnectionDone,
)
from twisted.internet.defer import (
Deferred,
fail,
)
from twisted.web.server import (
@ -52,9 +63,11 @@ class StaticResource(Resource, object):
def __init__(self, response):
Resource.__init__(self)
self._response = response
self._request = None
@render_exception
def render(self, request):
self._request = request
return self._response
@ -214,3 +227,31 @@ class RenderExceptionTests(SyncTestCase):
Equals(b"Internal Server Error"),
),
)
def test_disconnected(self):
"""
If the transport is disconnected before the response is available, no
``RuntimeError`` is logged for finishing a disconnected request.
"""
result = Deferred()
resource = StaticResource(result)
d = render(resource, {})
resource._request.connectionLost(Failure(ConnectionDone()))
result.callback(b"Some result")
self.assertThat(
d,
failed(
AfterPreprocessing(
lambda reason: reason.type,
Equals(ConnectionDone),
),
),
)
# Since we're not a trial TestCase we don't have flushLoggedErrors.
# The next best thing is to make sure any dangling Deferreds have been
# garbage collected and then let the generic trial logic for failing
# tests with logged errors kick in.
gc.collect()

8
src/allmydata/test/web/test_web.py
View File

@ -4767,11 +4767,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_static_missing(self):
# self.staticdir does not exist yet, because we used self.mktemp()
d = self.assertFailure(self.GET("/static"), error.Error)
# nevow.static throws an exception when it tries to os.stat the
# missing directory, which gives the client a 500 Internal Server
# Error, and the traceback reveals the parent directory name. By
# switching to plain twisted.web.static, this gives a normal 404 that
# doesn't reveal anything. This addresses #1720.
# If os.stat raises an exception for the missing directory and the
# traceback reveals the parent directory name we don't want to see
# that parent directory name in the response. This addresses #1720.
d.addCallback(lambda e: self.assertEquals(str(e), "404 Not Found"))
return d

208
src/allmydata/test/web/test_webish.py Normal file
View File

@ -0,0 +1,208 @@
"""
Tests for ``allmydata.webish``.
"""
from uuid import (
uuid4,
)
from testtools.matchers import (
AfterPreprocessing,
Contains,
Equals,
MatchesAll,
Not,
)
from twisted.python.filepath import (
FilePath,
)
from twisted.web.test.requesthelper import (
DummyChannel,
)
from twisted.web.resource import (
Resource,
)
from ..common import (
SyncTestCase,
)
from ...webish import (
TahoeLAFSRequest,
tahoe_lafs_site,
)
class TahoeLAFSRequestTests(SyncTestCase):
"""
Tests for ``TahoeLAFSRequest``.
"""
def _fields_test(self, method, request_headers, request_body, match_fields):
channel = DummyChannel()
request = TahoeLAFSRequest(
channel,
)
for (k, v) in request_headers.items():
request.requestHeaders.setRawHeaders(k, [v])
request.gotLength(len(request_body))
request.handleContentChunk(request_body)
request.requestReceived(method, b"/", b"HTTP/1.1")
# We don't really care what happened to the request. What we do care
# about is what the `fields` attribute is set to.
self.assertThat(
request.fields,
match_fields,
)
def test_no_form_fields(self):
"""
When a ``GET`` request is received, ``TahoeLAFSRequest.fields`` is None.
"""
self._fields_test(b"GET", {}, b"", Equals(None))
def test_form_fields(self):
"""
When a ``POST`` request is received, form fields are parsed into
``TahoeLAFSRequest.fields``.
"""
form_data, boundary = multipart_formdata([
[param(u"name", u"foo"),
body(u"bar"),
],
[param(u"name", u"baz"),
param(u"filename", u"quux"),
body(u"some file contents"),
],
])
self._fields_test(
b"POST",
{b"content-type": b"multipart/form-data; boundary={}".format(boundary)},
form_data.encode("ascii"),
AfterPreprocessing(
lambda fs: {
k: fs.getvalue(k)
for k
in fs.keys()
},
Equals({
b"foo": b"bar",
b"baz": b"some file contents",
}),
),
)
class TahoeLAFSSiteTests(SyncTestCase):
"""
Tests for the ``Site`` created by ``tahoe_lafs_site``.
"""
def _test_censoring(self, path, censored):
"""
Verify that the event logged for a request for ``path`` does not include
``path`` but instead includes ``censored``.
:param bytes path: A request path.
:param bytes censored: A replacement value for the request path in the
access log.
:return: ``None`` if the logging looks good.
"""
logPath = self.mktemp()
site = tahoe_lafs_site(Resource(), logPath=logPath)
site.startFactory()
channel = DummyChannel()
channel.factory = site
request = TahoeLAFSRequest(channel)
request.gotLength(None)
request.requestReceived(b"GET", path, b"HTTP/1.1")
self.assertThat(
FilePath(logPath).getContent(),
MatchesAll(
Contains(censored),
Not(Contains(path)),
),
)
def test_uri_censoring(self):
"""
The log event for a request for **/uri/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/uri/URI:CHK:aaa:bbb",
b"/uri/[CENSORED]",
)
def test_file_censoring(self):
"""
The log event for a request for **/file/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/file/URI:CHK:aaa:bbb",
b"/file/[CENSORED]",
)
def test_named_censoring(self):
"""
The log event for a request for **/named/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/named/URI:CHK:aaa:bbb",
b"/named/[CENSORED]",
)
def test_uri_queryarg_censoring(self):
"""
The log event for a request for **/uri?cap=<CAP>** has the capability
value censored.
"""
self._test_censoring(
b"/uri?uri=URI:CHK:aaa:bbb",
b"/uri?uri=[CENSORED]",
)
def param(name, value):
return u"; {}={}".format(name, value)
def body(value):
return u"\r\n\r\n{}".format(value)
def _field(field):
yield u"Content-Disposition: form-data"
for param in field:
yield param
def _multipart_formdata(fields):
for field in fields:
yield u"".join(_field(field)) + u"\r\n"
def multipart_formdata(fields):
"""
Serialize some simple fields into a multipart/form-data string.
:param fields: A list of lists of unicode strings to assemble into the
result. See ``param`` and ``body``.
:return unicode: The given fields combined into a multipart/form-data
string.
"""
boundary = str(uuid4())
parts = list(_multipart_formdata(fields))
parts.insert(0, u"")
return (
(u"--" + boundary + u"\r\n").join(parts),
boundary,
)

70
src/allmydata/web/common.py
View File

@ -1,4 +1,3 @@
from future.utils import PY2
from past.builtins import unicode
import time
@ -22,12 +21,14 @@ from twisted.web import (
resource,
template,
)
from twisted.web.iweb import (
IRequest,
)
from twisted.web.template import (
tags,
)
from twisted.web.server import (
NOT_DONE_YET,
UnsupportedMethod,
)
from twisted.web.util import (
DeferredResource,
@ -42,20 +43,12 @@ from twisted.python.failure import (
Failure,
)
from twisted.internet.defer import (
CancelledError,
maybeDeferred,
)
from twisted.web.resource import (
IResource,
)
from twisted.web.iweb import IRequest as ITwistedRequest
if PY2:
from nevow.appserver import DefaultExceptionHandler
from nevow.inevow import IRequest as INevowRequest
else:
class DefaultExceptionHandler:
def __init__(self, *args, **kwargs):
raise NotImplementedError("Still not ported to Python 3")
INevowRequest = None
from allmydata import blacklist
from allmydata.interfaces import (
@ -161,11 +154,22 @@ def parse_offset_arg(offset):
return offset
def get_root(ctx_or_req):
if PY2:
req = INevowRequest(ctx_or_req)
else:
req = ITwistedRequest(ctx_or_req)
def get_root(req):
"""
Get a relative path with parent directory segments that refers to the root
location known to the given request. This seems a lot like the constant
absolute path **/** but it will behave differently if the Tahoe-LAFS HTTP
server is reverse-proxied and mounted somewhere other than at the root.
:param twisted.web.iweb.IRequest req: The request to consider.
:return: A string like ``../../..`` with the correct number of segments to
reach the root.
"""
if not IRequest.providedBy(req):
raise TypeError(
"get_root requires IRequest provider, got {!r}".format(req),
)
depth = len(req.prepath) + len(req.postpath)
link = "/".join([".."] * depth)
return link
@ -366,20 +370,13 @@ def humanize_failure(f):
return humanize_exception(f.value)
class MyExceptionHandler(DefaultExceptionHandler, object):
def renderHTTP_exception(self, ctx, f):
req = INevowRequest(ctx)
req.write(_renderHTTP_exception(req, f))
req.finishRequest(False)
class NeedOperationHandleError(WebError):
pass
class SlotsSequenceElement(template.Element):
"""
``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for
``SlotsSequenceElement` is a minimal port of Nevow's sequence renderer for
twisted.web.template.
Tags passed in to be templated will have two renderers available: ``item``
@ -493,7 +490,17 @@ def render_exception(render):
# Apply `_finish` all of our result handling logic to whatever it
# returned.
result.addBoth(_finish, bound_render, request)
result.addActionFinish()
d = result.addActionFinish()
# If the connection is lost then there's no point running our _finish
# logic because it has nowhere to send anything. There may also be no
# point in finishing whatever operation was being performed because
# the client cannot be informed of its result. Also, Twisted Web
# raises exceptions from some Request methods if they're used after
# the connection is lost.
request.notifyFinish().addErrback(
lambda ignored: d.cancel(),
)
return NOT_DONE_YET
return g
@ -518,6 +525,8 @@ def _finish(result, render, request):
:return: ``None``
"""
if isinstance(result, Failure):
if result.check(CancelledError):
return
Message.log(
message_type=u"allmydata:web:common-render:failure",
message=result.getErrorMessage(),
@ -590,17 +599,6 @@ def _renderHTTP_exception(request, failure):
if code is not None:
return _renderHTTP_exception_simple(request, text, code)
if failure.check(UnsupportedMethod):
# twisted.web.server.Request.render() has support for transforming
# this into an appropriate 501 NOT_IMPLEMENTED or 405 NOT_ALLOWED
# return code, but nevow does not.
method = request.method
return _renderHTTP_exception_simple(
request,
"I don't know how to treat a %s request." % (method,),
http.NOT_IMPLEMENTED,
)
accept = request.getHeader("accept")
if not accept:
accept = "*/*"

26
src/allmydata/web/common_py3.py
View File

@ -4,15 +4,7 @@ Common utilities that are available from Python 3.
Can eventually be merged back into allmydata.web.common.
"""
from future.utils import PY2
if PY2:
from nevow.inevow import IRequest as INevowRequest
else:
INevowRequest = None
from twisted.web import resource, http
from twisted.web.iweb import IRequest
from allmydata.util import abbreviate
@ -23,24 +15,20 @@ class WebError(Exception):
self.code = code
def get_arg(ctx_or_req, argname, default=None, multiple=False):
def get_arg(req, argname, default=None, multiple=False):
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
precedence. If multiple=True, this returns a tuple of arguments (possibly
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
"""
results = []
if PY2:
req = INevowRequest(ctx_or_req)
if argname in req.args:
results.extend(req.args[argname])
if req.fields and argname in req.fields:
results.append(req.fields[argname].value)
else:
req = IRequest(ctx_or_req)
if argname in req.args:
results.extend(req.args[argname])
if argname in req.args:
results.extend(req.args[argname])
if req.fields and argname in req.fields:
results.append(req.fields[argname].value)
if multiple:
return tuple(results)
if results:

12
src/allmydata/web/directory.py
View File

@ -114,7 +114,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
# Rejecting URIs that contain empty path pieces (for example:
# "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was
# the old nevow behavior and it is encoded in the test suite;
# the old Nevow behavior and it is encoded in the test suite;
# we will follow suit.
for segment in req.prepath:
if not segment:
@ -398,8 +398,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
d.addBoth(_maybe_got_node)
# now we have a placeholder or a filenodehandler, and we can just
# delegate to it. We could return the resource back out of
# DirectoryNodeHandler.renderHTTP, and nevow would recurse into it,
# but the addCallback() that handles when_done= would break.
# DirectoryNodeHandler.render_POST and it would get rendered but the
# addCallback() that handles when_done= would break.
def render_child(child):
req.dont_apply_extra_processing = True
return child.render(req)
@ -522,9 +522,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
d.addCallback(self._maybe_literal, CheckResultsRenderer)
return d
def _start_operation(self, monitor, renderer, ctx):
self._operations.add_monitor(ctx, monitor, renderer)
return self._operations.redirect_to(ctx)
def _start_operation(self, monitor, renderer, req):
self._operations.add_monitor(req, monitor, renderer)
return self._operations.redirect_to(req)
def _POST_start_deep_check(self, req):
# check this directory and everything reachable from it

5
src/allmydata/web/filenode.py
View File

@ -480,7 +480,10 @@ class FileDownloader(Resource, object):
d = self.filenode.read(req, first, size)
def _error(f):
req._tahoe_request_had_error = f # for HTTP-style logging
if f.check(defer.CancelledError):
# The HTTP connection was lost and we no longer have anywhere
# to send our result. Let this pass through.
return f
if req.startedWriting:
# The content-type is already set, and the response code has
# already been sent, so we can't provide a clean error

2
src/allmydata/web/operations.py
View File

@ -152,8 +152,6 @@ class ReloadMixin(object):
def refresh(self, req, tag):
if self.monitor.is_finished():
return ""
# dreid suggests ctx.tag(**dict([("http-equiv", "refresh")]))
# but I can't tell if he's joking or not
tag.attributes["http-equiv"] = "refresh"
tag.attributes["content"] = str(self.REFRESH_TIME)
return tag

2
src/allmydata/web/root.py
View File

@ -189,7 +189,7 @@ class FileHandler(resource.Resource, object):
return filenode.FileNodeDownloadHandler(self.client, node)
@render_exception
def render_GET(self, ctx):
def render_GET(self, req):
raise WebError("/file must be followed by a file-cap and a name",
http.NOT_FOUND)

6
src/allmydata/web/storage.py
View File

@ -1,3 +1,4 @@
from future.utils import PY2
import time, json
from twisted.python.filepath import FilePath
@ -317,4 +318,7 @@ class StorageStatus(MultiFormatResource):
"lease-checker": self._storage.lease_checker.get_state(),
"lease-checker-progress": self._storage.lease_checker.get_progress(),
}
return json.dumps(d, indent=1) + "\n"
result = json.dumps(d, indent=1) + "\n"
if PY2:
result = result.decode("utf-8")
return result.encode("utf-8")

209
src/allmydata/webish.py
View File

@ -1,43 +1,57 @@
import re, time
from functools import (
partial,
)
from cgi import (
FieldStorage,
)
from twisted.application import service, strports, internet
from twisted.web import http, static
from twisted.web import static
from twisted.web.http import (
parse_qs,
)
from twisted.web.server import (
Request,
Site,
)
from twisted.internet import defer
from twisted.internet.address import (
IPv4Address,
IPv6Address,
)
from nevow import appserver, inevow
from allmydata.util import log, fileutil
from allmydata.web import introweb, root
from allmydata.web.common import MyExceptionHandler
from allmydata.web.operations import OphandleTable
from .web.storage_plugins import (
StoragePlugins,
)
# we must override twisted.web.http.Request.requestReceived with a version
# that doesn't use cgi.parse_multipart() . Since we actually use Nevow, we
# override the nevow-specific subclass, nevow.appserver.NevowRequest . This
# is an exact copy of twisted.web.http.Request (from SVN HEAD on 10-Aug-2007)
# that modifies the way form arguments are parsed. Note that this sort of
# surgery may induce a dependency upon a particular version of twisted.web
class TahoeLAFSRequest(Request, object):
"""
``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
that are useful for Tahoe-LAFS.
parse_qs = http.parse_qs
class MyRequest(appserver.NevowRequest, object):
:ivar NoneType|FieldStorage fields: For POST requests, a structured
representation of the contents of the request body. For anything
else, ``None``.
"""
fields = None
_tahoe_request_had_error = None
def requestReceived(self, command, path, version):
"""Called by channel when all data has been received.
This method is not intended for users.
"""
self.content.seek(0,0)
Called by channel when all data has been received.
Override the base implementation to apply certain site-wide policies
and to provide less memory-intensive multipart/form-post handling for
large file uploads.
"""
self.content.seek(0)
self.args = {}
self.stack = []
self.setHeader("Referrer-Policy", "no-referrer")
self.method, self.uri = command, path
self.clientproto = version
@ -49,93 +63,36 @@ class MyRequest(appserver.NevowRequest, object):
self.path, argstring = x
self.args = parse_qs(argstring, 1)
# Adding security headers. These will be sent for *all* HTTP requests.
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"])
if self.method == 'POST':
# We use FieldStorage here because it performs better than
# cgi.parse_multipart(self.content, pdict) which is what
# twisted.web.http.Request uses.
self.fields = FieldStorage(
self.content,
{
name.lower(): value[-1]
for (name, value)
in self.requestHeaders.getAllRawHeaders()
},
environ={'REQUEST_METHOD': 'POST'})
self.content.seek(0)
# Argument processing.
self._tahoeLAFSSecurityPolicy()
## The original twisted.web.http.Request.requestReceived code parsed the
## content and added the form fields it found there to self.args . It
## did this with cgi.parse_multipart, which holds the arguments in RAM
## and is thus unsuitable for large file uploads. The Nevow subclass
## (nevow.appserver.NevowRequest) uses cgi.FieldStorage instead (putting
## the results in self.fields), which is much more memory-efficient.
## Since we know we're using Nevow, we can anticipate these arguments
## appearing in self.fields instead of self.args, and thus skip the
## parse-content-into-self.args step.
## args = self.args
## ctype = self.getHeader('content-type')
## if self.method == "POST" and ctype:
## mfd = 'multipart/form-data'
## key, pdict = cgi.parse_header(ctype)
## if key == 'application/x-www-form-urlencoded':
## args.update(parse_qs(self.content.read(), 1))
## elif key == mfd:
## try:
## args.update(cgi.parse_multipart(self.content, pdict))
## except KeyError, e:
## if e.args[0] == 'content-disposition':
## # Parse_multipart can't cope with missing
## # content-dispostion headers in multipart/form-data
## # parts, so we catch the exception and tell the client
## # it was a bad request.
## self.channel.transport.write(
## "HTTP/1.1 400 Bad Request\r\n\r\n")
## self.channel.transport.loseConnection()
## return
## raise
self.processing_started_timestamp = time.time()
self.process()
def _logger(self):
# we build up a log string that hides most of the cap, to preserve
# user privacy. We retain the query args so we can identify things
# like t=json. Then we send it to the flog. We make no attempt to
# match apache formatting. TODO: when we move to DSA dirnodes and
# shorter caps, consider exposing a few characters of the cap, or
# maybe a few characters of its hash.
x = self.uri.split("?", 1)
if len(x) == 1:
# no query args
path = self.uri
queryargs = ""
else:
path, queryargs = x
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these too.
if queryargs.startswith("uri="):
queryargs = "[uri=CENSORED]"
queryargs = "?" + queryargs
if path.startswith("/uri"):
path = "/uri/[CENSORED].."
elif path.startswith("/file"):
path = "/file/[CENSORED].."
elif path.startswith("/named"):
path = "/named/[CENSORED].."
uri = path + queryargs
error = ""
if self._tahoe_request_had_error:
error = " [ERROR]"
log.msg(
format=(
"web: %(clientip)s %(method)s %(uri)s %(code)s "
"%(length)s%(error)s"
),
clientip=_get_client_ip(self),
method=self.method,
uri=uri,
code=self.code,
length=(self.sentLength or "-"),
error=error,
facility="tahoe.webish",
level=log.OPERATIONAL,
)
def _tahoeLAFSSecurityPolicy(self):
"""
Set response properties related to Tahoe-LAFS-imposed security policy.
This will ensure that all HTTP requests received by the Tahoe-LAFS
HTTP server have this policy imposed, regardless of other
implementation details.
"""
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"])
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
self.setHeader("Referrer-Policy", "no-referrer")
def _get_client_ip(request):
@ -150,6 +107,54 @@ def _get_client_ip(request):
return None
def _logFormatter(logDateTime, request):
# we build up a log string that hides most of the cap, to preserve
# user privacy. We retain the query args so we can identify things
# like t=json. Then we send it to the flog. We make no attempt to
# match apache formatting. TODO: when we move to DSA dirnodes and
# shorter caps, consider exposing a few characters of the cap, or
# maybe a few characters of its hash.
x = request.uri.split("?", 1)
if len(x) == 1:
# no query args
path = request.uri
queryargs = ""
else:
path, queryargs = x
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these too.
if queryargs.startswith("uri="):
queryargs = "uri=[CENSORED]"
queryargs = "?" + queryargs
if path.startswith("/uri/"):
path = "/uri/[CENSORED]"
elif path.startswith("/file/"):
path = "/file/[CENSORED]"
elif path.startswith("/named/"):
path = "/named/[CENSORED]"
uri = path + queryargs
template = "web: %(clientip)s %(method)s %(uri)s %(code)s %(length)s"
return template % dict(
clientip=_get_client_ip(request),
method=request.method,
uri=uri,
code=request.code,
length=(request.sentLength or "-"),
facility="tahoe.webish",
level=log.OPERATIONAL,
)
tahoe_lafs_site = partial(
Site,
requestFactory=TahoeLAFSRequest,
logFormatter=_logFormatter,
)
class WebishServer(service.MultiService):
name = "webish"
@ -175,15 +180,13 @@ class WebishServer(service.MultiService):
def buildServer(self, webport, nodeurl_path, staticdir):
self.webport = webport
self.site = site = appserver.NevowSite(self.root)
self.site.requestFactory = MyRequest
self.site.remember(MyExceptionHandler(), inevow.ICanHandleException)
self.site = tahoe_lafs_site(self.root)
self.staticdir = staticdir # so tests can check
if staticdir:
self.root.putChild("static", static.File(staticdir))
if re.search(r'^\d', webport):
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
s = strports.service(webport, site)
s = strports.service(webport, self.site)
s.setServiceParent(self)
self._scheme = None

20
static/tahoe.py
View File

@ -3,23 +3,19 @@
# Import this first to suppress deprecation warnings.
import allmydata
# nevow requires all these for its voodoo module import time adaptor registrations
from nevow import accessors, appserver, static, rend, url, util, query, i18n, flat
from nevow import guard, stan, testutil, context
from nevow.flat import flatmdom, flatstan, twist
from formless import webform, processors, annotate, iformless
from decimal import Decimal
from xml.dom import minidom
import allmydata.web
# junk to appease pyflakes's outrage
[
accessors, appserver, static, rend, url, util, query, i18n, flat, guard, stan, testutil,
context, flatmdom, flatstan, twist, webform, processors, annotate, iformless, Decimal,
minidom, allmydata,
]
# We import these things to give PyInstaller's dependency resolver some hints
# about what it needs to include. We don't use them otherwise _here_ but
# other parts of the codebase do. pyflakes points out that they are unused
# unless we use them. So ... use them.
Decimal
minidom
allmydata
from allmydata.scripts import runner
runner.run()
runner.run()