ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-05 09:59:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into 3468.offloaded-test-coverage

Browse Source
This commit is contained in:
Chad Whitacre 2020-10-23 12:08:48 -04:00
commit 725291c2aa
43 changed files with 1210 additions and 728 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
.circleci/config.yml
View File

@ -14,44 +14,73 @@ version: 2.1
workflows:
ci:
jobs:
# Platforms
- "debian-9"
# Start with jobs testing various platforms.
# Every job that pulls a Docker image from Docker Hub needs to provide
# credentials for that pull operation to avoid being subjected to
# unauthenticated pull limits shared across all of CircleCI. Use this
# first job to define a yaml anchor that can be used to supply a
# CircleCI job context which makes Docker Hub credentials available in
# the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "debian-9": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "debian-8":
<<: *DOCKERHUB_CONTEXT
requires:
- "debian-9"
- "ubuntu-20-04"
- "ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "ubuntu-16-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "fedora-29"
- "fedora-29":
<<: *DOCKERHUB_CONTEXT
- "fedora-28":
<<: *DOCKERHUB_CONTEXT
requires:
- "fedora-29"
- "centos-8"
- "centos-8":
<<: *DOCKERHUB_CONTEXT
- "nixos-19-09"
- "nixos-19-09":
<<: *DOCKERHUB_CONTEXT
# Test against PyPy 2.7
- "pypy27-buster"
- "pypy27-buster":
<<: *DOCKERHUB_CONTEXT
# Just one Python 3.6 configuration while the port is in-progress.
- "python36"
- "python36":
<<: *DOCKERHUB_CONTEXT
# Other assorted tasks and configurations
- "lint"
- "pyinstaller"
- "deprecations"
- "c-locale"
- "lint":
<<: *DOCKERHUB_CONTEXT
- "pyinstaller":
<<: *DOCKERHUB_CONTEXT
- "deprecations":
<<: *DOCKERHUB_CONTEXT
- "c-locale":
<<: *DOCKERHUB_CONTEXT
# Any locale other than C or UTF-8.
- "another-locale"
- "another-locale":
<<: *DOCKERHUB_CONTEXT
- "integration":
<<: *DOCKERHUB_CONTEXT
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
@ -59,7 +88,8 @@ workflows:
# Generate the underlying data for a visualization to aid with Python 3
# porting.
- "build-porting-depgraph"
- "build-porting-depgraph":
<<: *DOCKERHUB_CONTEXT
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
@ -74,22 +104,55 @@ workflows:
- "master"
jobs:
- "build-image-debian-8"
- "build-image-debian-9"
- "build-image-ubuntu-16-04"
- "build-image-ubuntu-18-04"
- "build-image-ubuntu-20-04"
- "build-image-fedora-28"
- "build-image-fedora-29"
- "build-image-centos-8"
- "build-image-pypy27-buster"
- "build-image-python36-ubuntu"
- "build-image-debian-8":
<<: *DOCKERHUB_CONTEXT
- "build-image-debian-9":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-16-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-28":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-29":
<<: *DOCKERHUB_CONTEXT
- "build-image-centos-8":
<<: *DOCKERHUB_CONTEXT
- "build-image-pypy27-buster":
<<: *DOCKERHUB_CONTEXT
- "build-image-python36-ubuntu":
<<: *DOCKERHUB_CONTEXT
jobs:
dockerhub-auth-template:
# This isn't a real job. It doesn't get scheduled as part of any
# workflow. Instead, it's just a place we can hang a yaml anchor to
# finish the Docker Hub authentication configuration. Workflow jobs using
# the DOCKERHUB_CONTEXT anchor will have access to the environment
# variables used here. These variables will allow the Docker Hub image
# pull to be authenticated and hopefully avoid hitting and rate limits.
docker: &DOCKERHUB_AUTH
- image: "null"
auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
steps:
- run:
name: "CircleCI YAML schema conformity"
command: |
# This isn't a real command. We have to have something in this
# space, though, or the CircleCI yaml schema validator gets angry.
# Since this job is never scheduled this step is never run so the
# actual value here is irrelevant.
lint:
docker:
- image: "circleci/python:2"
- <<: *DOCKERHUB_AUTH
image: "circleci/python:2"
steps:
- "checkout"
@ -106,7 +169,8 @@ jobs:
pyinstaller:
docker:
- image: "circleci/python:2"
- <<: *DOCKERHUB_AUTH
image: "circleci/python:2"
steps:
- "checkout"
@ -131,7 +195,8 @@ jobs:
debian-9: &DEBIAN
docker:
- image: "tahoelafsci/debian:9-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:9-py2.7"
user: "nobody"
environment: &UTF_8_ENVIRONMENT
@ -212,14 +277,16 @@ jobs:
debian-8:
<<: *DEBIAN
docker:
- image: "tahoelafsci/debian:8-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:8-py2.7"
user: "nobody"
pypy27-buster:
<<: *DEBIAN
docker:
- image: "tahoelafsci/pypy:buster-py2"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/pypy:buster-py2"
user: "nobody"
environment:
@ -280,21 +347,24 @@ jobs:
ubuntu-16-04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:16.04-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:16.04-py2.7"
user: "nobody"
ubuntu-18-04: &UBUNTU_18_04
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:18.04-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py2.7"
user: "nobody"
python36:
<<: *UBUNTU_18_04
docker:
- image: "tahoelafsci/ubuntu:18.04-py3"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3"
user: "nobody"
environment:
@ -309,13 +379,15 @@ jobs:
ubuntu-20-04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:20.04"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:20.04"
user: "nobody"
centos-8: &RHEL_DERIV
docker:
- image: "tahoelafsci/centos:8-py2"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/centos:8-py2"
user: "nobody"
environment: *UTF_8_ENVIRONMENT
@ -337,21 +409,24 @@ jobs:
fedora-28:
<<: *RHEL_DERIV
docker:
- image: "tahoelafsci/fedora:28-py"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/fedora:28-py"
user: "nobody"
fedora-29:
<<: *RHEL_DERIV
docker:
- image: "tahoelafsci/fedora:29-py"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/fedora:29-py"
user: "nobody"
nixos-19-09:
docker:
# Run in a highly Nix-capable environment.
- image: "nixorg/nix:circleci"
- <<: *DOCKERHUB_AUTH
image: "nixorg/nix:circleci"
environment:
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
@ -408,7 +483,8 @@ jobs:
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- image: "docker:17.05.0-ce-git"
- <<: *DOCKERHUB_AUTH
image: "docker:17.05.0-ce-git"
environment:
DISTRO: "tahoelafsci/<DISTRO>:foo-py2"
@ -418,47 +494,10 @@ jobs:
steps:
- "checkout"
- "setup_remote_docker"
- run:
name: "Get openssl"
command: |
apk add --no-cache openssl
- run:
name: "Get Dockerhub secrets"
command: |
# If you create an encryption key like this:
#
# openssl enc -aes-256-cbc -k secret -P -md sha256
# From the output that looks like:
#
# salt=...
# key=...
# iv =...
#
# extract just the value for ``key``.
# then you can re-generate ``secret-env-cipher`` locally using the
# command:
#
# openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
#
# Make sure the key is set as the KEY environment variable in the
# CircleCI web interface. You can do this by visiting
# <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
# after logging in to CircleCI with an account in the tahoe-lafs
# CircleCI team.
#
# Then you can recover the environment plaintext (for example, to
# change and re-encrypt it) like just like CircleCI recovers it
# here:
#
openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
- run:
name: "Log in to Dockerhub"
command: |
. ~/.env
# TAHOELAFSCI_PASSWORD come from the secret env.
docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
- run:
name: "Build image"
command: |

1
.circleci/secret-env-cipher
View File

@ -1 +0,0 @@
Salted__ •GPÁøÊ)|!÷[©U[‡ûvSÚ,F¿–m:ö š~ÓY[Uú_¸Fx×’¤Ÿ%<25>“4l×Ö»Š8¼œ¹„1öø‰/lƒÌ`nÆ^·Z]óqš¬æ¢&ø°÷£Ý‚‚ß%T¡n

14
integration/conftest.py
View File

@ -8,6 +8,10 @@ from os.path import join, exists
from tempfile import mkdtemp, mktemp
from functools import partial
from foolscap.furl import (
decode_furl,
)
from eliot import (
to_file,
log_call,
@ -226,6 +230,16 @@ def introducer_furl(introducer, temp_dir):
print("Don't see {} yet".format(furl_fname))
sleep(.1)
furl = open(furl_fname, 'r').read()
tubID, location_hints, name = decode_furl(furl)
if not location_hints:
# If there are no location hints then nothing can ever possibly
# connect to it and the only thing that can happen next is something
# will hang or time out. So just give up right now.
raise ValueError(
"Introducer ({!r}) fURL has no location hints!".format(
introducer_furl,
),
)
return furl

12
integration/test_streaming_logs.py
View File

@ -53,7 +53,12 @@ class _StreamingLogClientProtocol(WebSocketClientProtocol):
self.factory.on_open.callback(self)
def onMessage(self, payload, isBinary):
self.on_message.callback(payload)
if self.on_message is None:
# Already did our job, ignore it
return
on_message = self.on_message
self.on_message = None
on_message.callback(payload)
def onClose(self, wasClean, code, reason):
self.on_close.callback(reason)
@ -131,10 +136,13 @@ def _test_streaming_logs(reactor, temp_dir, alice):
client.on_close = Deferred()
client.on_message = Deferred()
# Capture this now before on_message perhaps goes away.
racing = _race(client.on_close, client.on_message)
# Provoke _some_ log event.
yield treq.get(node_url)
result = yield _race(client.on_close, client.on_message)
result = yield racing
assert isinstance(result, Right)
json.loads(result.value)

3
integration/util.py
View File

@ -1,7 +1,7 @@
import sys
import time
import json
from os import mkdir
from os import mkdir, environ
from os.path import exists, join
from six.moves import StringIO
from functools import partial
@ -145,6 +145,7 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
proto,
sys.executable,
args,
env=environ,
)

1
misc/build_helpers/show-tool-versions.py
View File

@ -143,7 +143,6 @@ print_py_pkg_ver('coverage')
print_py_pkg_ver('cryptography')
print_py_pkg_ver('foolscap')
print_py_pkg_ver('mock')
print_py_pkg_ver('Nevow', 'nevow')
print_py_pkg_ver('pyasn1')
print_py_pkg_ver('pycparser')
print_py_pkg_ver('cryptography')

0
newsfragments/3314.minor Normal file
View File

0
newsfragments/3428.minor Normal file
View File

0
newsfragments/3432.minor Normal file
View File

1
newsfragments/3433.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS no longer depends on Nevow.

0
newsfragments/3434.minor Normal file
View File

0
newsfragments/3435.minor Normal file
View File

0
newsfragments/3466.minor Normal file
View File

0
newsfragments/3481.minor Normal file
View File

0
newsfragments/3482.minor Normal file
View File

0
newsfragments/3483.minor Normal file
View File

45
nix/nevow.nix
View File

@ -1,45 +0,0 @@
{ stdenv, buildPythonPackage, fetchPypi, isPy3k, twisted }:
buildPythonPackage rec {
pname = "Nevow";
version = "0.14.5";
name = "${pname}-${version}";
disabled = isPy3k;
src = fetchPypi {
inherit pname;
inherit version;
sha256 = "1wr3fai01h1bcp4qpia6indg4qmxvywwv3q1iibm669mln2vmdmg";
};
propagatedBuildInputs = [ twisted ];
checkInputs = [ twisted ];
checkPhase = ''
trial formless nevow
'';
meta = with stdenv.lib; {
description = "Nevow, a web application construction kit for Python";
longDescription = ''
Nevow - Pronounced as the French "nouveau", or "noo-voh", Nevow
is a web application construction kit written in Python. It is
designed to allow the programmer to express as much of the view
logic as desired in Python, and includes a pure Python XML
expression syntax named stan to facilitate this. However it
also provides rich support for designer-edited templates, using
a very small XML attribute language to provide bi-directional
template manipulation capability.
Nevow also includes formless, a declarative syntax for
specifying the types of method parameters and exposing these
methods to the web. Forms can be rendered automatically, and
form posts will be validated and input coerced, rendering error
pages if appropriate. Once a form post has validated
successfully, the method will be called with the coerced values.
'';
homepage = https://github.com/twisted/nevow;
license = licenses.mit;
};
}

5
nix/overlays.nix
View File

@ -3,10 +3,7 @@ self: super: {
packageOverrides = python-self: python-super: {
# eliot is not part of nixpkgs at all at this time.
eliot = python-self.callPackage ./eliot.nix { };
# The packaged version of Nevow is very slightly out of date but also
# conflicts with the packaged version of Twisted. Supply our own
# slightly newer version.
nevow = python-super.callPackage ./nevow.nix { };
# NixOS autobahn package has trollius as a dependency, although
# it is optional. Trollius is unmaintained and fails on CI.
autobahn = python-super.callPackage ./autobahn.nix { };

4
nix/tahoe-lafs.nix
View File

@ -1,6 +1,6 @@
{ fetchFromGitHub, lib
, nettools, python
, twisted, foolscap, nevow, zfec
, twisted, foolscap, zfec
, setuptools, setuptoolsTrial, pyasn1, zope_interface
, service-identity, pyyaml, magic-wormhole, treq, appdirs
, beautifulsoup4, eliot, autobahn, cryptography
@ -46,7 +46,7 @@ python.pkgs.buildPythonPackage rec {
];
propagatedBuildInputs = with python.pkgs; [
twisted foolscap nevow zfec appdirs
twisted foolscap zfec appdirs
setuptoolsTrial pyasn1 zope_interface
service-identity pyyaml magic-wormhole treq
eliot autobahn cryptography setuptools

7
setup.py
View File

@ -38,8 +38,7 @@ install_requires = [
"zfec >= 1.1.0",
# zope.interface >= 3.6.0 is required for Twisted >= 12.1.0.
# zope.interface 3.6.3 and 3.6.4 are incompatible with Nevow (#1435).
"zope.interface >= 3.6.0, != 3.6.3, != 3.6.4",
"zope.interface >= 3.6.0",
# * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for
# transferring large mutable files of size N.
@ -70,7 +69,6 @@ install_requires = [
# rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
# * The FTP frontend depends on Twisted >= 11.1.0 for
# filepath.Permissions
# * Nevow 0.11.1 depends on Twisted >= 13.0.0.
# * The SFTP frontend and manhole depend on the conch extra. However, we
# can't explicitly declare that without an undesirable dependency on gmpy,
# as explained in ticket #2740.
@ -102,9 +100,6 @@ install_requires = [
# an sftp extra in Tahoe-LAFS, there is no point in having one.
"Twisted[tls,conch] >= 18.4.0",
# We need Nevow >= 0.11.1 which can be installed using pip.
"Nevow >= 0.11.1",
"PyYAML >= 3.11",
"six >= 1.10.0",

2
src/allmydata/_auto_deps.py
View File

@ -11,7 +11,6 @@ package_imports = [
('foolscap', 'foolscap'),
('zfec', 'zfec'),
('Twisted', 'twisted'),
('Nevow', 'nevow'),
('zope.interface', 'zope.interface'),
('python', None),
('platform', None),
@ -72,7 +71,6 @@ runtime_warning_messages = [
]
warning_imports = [
'nevow',
'twisted.persisted.sob',
'twisted.python.filepath',
]

15
src/allmydata/immutable/offloaded.py
View File

@ -1,3 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os, stat, time, weakref
from zope.interface import implementer
@ -135,9 +146,9 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader):
peer selection, encoding, and share pushing. I read ciphertext from the
remote AssistedUploader.
"""
VERSION = { "http://allmydata.org/tahoe/protocols/helper/chk-upload/v1" :
VERSION = { b"http://allmydata.org/tahoe/protocols/helper/chk-upload/v1" :
{ },
"application-version": str(allmydata.__full_version__),
b"application-version": allmydata.__full_version__.encode("utf-8"),
}
def __init__(self, storage_index,

95
src/allmydata/test/common_web.py
View File

@ -1,23 +1,39 @@
import treq
__all__ = [
"do_http",
"render",
]
from twisted.internet.defer import (
maybeDeferred,
inlineCallbacks,
returnValue,
)
from twisted.web.error import Error
from nevow.context import WebContext
from nevow.testutil import FakeRequest
from nevow.appserver import (
processingFailed,
DefaultExceptionHandler,
from twisted.web.error import (
Error,
)
from nevow.inevow import (
ICanHandleException,
IRequest,
IResource as INevowResource,
IData,
from twisted.python.reflect import (
fullyQualifiedName,
)
from twisted.internet.defer import (
succeed,
)
from twisted.web.test.requesthelper import (
DummyChannel,
)
from twisted.web.error import (
UnsupportedMethod,
)
from twisted.web.http import (
NOT_ALLOWED,
)
from twisted.web.server import (
NOT_DONE_YET,
)
import treq
from ..webish import (
TahoeLAFSRequest,
)
@inlineCallbacks
@ -33,8 +49,8 @@ def do_http(method, url, **kwargs):
def render(resource, query_args):
"""
Render (in the manner of the Nevow appserver) a Nevow ``Page`` or a
Twisted ``Resource`` against a request with the given query arguments .
Render (in the manner of the Twisted Web Site) a Twisted ``Resource``
against a request with the given query arguments .
:param resource: The page or resource to render.
@ -44,19 +60,36 @@ def render(resource, query_args):
:return Deferred: A Deferred that fires with the rendered response body as
``bytes``.
"""
ctx = WebContext(tag=resource)
req = FakeRequest(args=query_args)
ctx.remember(DefaultExceptionHandler(), ICanHandleException)
ctx.remember(req, IRequest)
ctx.remember(None, IData)
channel = DummyChannel()
request = TahoeLAFSRequest(channel)
request.method = b"GET"
request.args = query_args
request.prepath = [b""]
request.postpath = []
try:
result = resource.render(request)
except UnsupportedMethod:
request.setResponseCode(NOT_ALLOWED)
result = b""
def maybe_concat(res):
if isinstance(res, bytes):
return req.v + res
return req.v
resource = INevowResource(resource)
d = maybeDeferred(resource.renderHTTP, ctx)
d.addErrback(processingFailed, req, ctx)
d.addCallback(maybe_concat)
return d
if isinstance(result, bytes):
request.write(result)
done = succeed(None)
elif result == NOT_DONE_YET:
if request.finished:
done = succeed(None)
else:
done = request.notifyFinish()
else:
raise ValueError(
"{!r} returned {!r}, required bytes or NOT_DONE_YET.".format(
fullyQualifiedName(resource.render),
result,
),
)
def get_body(ignored):
complete_response = channel.transport.written.getvalue()
header, body = complete_response.split(b"\r\n\r\n", 1)
return body
done.addCallback(get_body)
return done

54
src/allmydata/test/test_checker.py
View File

@ -20,18 +20,13 @@ from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.internet import defer
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from zope.interface import implementer
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
from twisted.web.template import flattenString
from twisted.web.resource import (
Resource,
)
from twisted.web.template import (
renderElement,
)
from allmydata import check_results, uri
from allmydata import uri as tahoe_uri
@ -52,6 +47,9 @@ from allmydata.mutable.publish import MutableData
from .common import (
EMPTY_CLIENT_CONFIG,
)
from .common_web import (
render,
)
from .web.common import (
assert_soup_has_favicon,
@ -62,24 +60,6 @@ class FakeClient(object):
def get_storage_broker(self):
return self.storage_broker
@implementer(IRequest)
class TestRequest(Request, object):
"""
A minimal Request class to use in tests.
XXX: We have to have this class because `common.get_arg()` expects
a `nevow.inevow.IRequest`, which `twisted.web.server.Request`
isn't. The request needs to have `args`, `fields`, `prepath`, and
`postpath` properties so that `allmydata.web.common.get_arg()`
won't complain.
"""
def __init__(self, args=None, fields=None):
super(TestRequest, self).__init__(DummyChannel())
self.args = args or {}
self.fields = fields or {}
self.prepath = [b""]
self.postpath = [b""]
@implementer(IServer)
class FakeServer(object):
@ -151,6 +131,15 @@ class FakeCheckAndRepairResults(object):
return self._repair_success
class ElementResource(Resource, object):
def __init__(self, element):
Resource.__init__(self)
self.element = element
def render(self, request):
return renderElement(request, self.element)
class WebResultsRendering(unittest.TestCase):
@staticmethod
@ -184,11 +173,12 @@ class WebResultsRendering(unittest.TestCase):
return c
def render_json(self, resource):
return resource.render(TestRequest(args={"output": ["json"]}))
return self.successResultOf(render(resource, {"output": ["json"]}))
def render_element(self, element, args=None):
d = flattenString(TestRequest(args), element)
return unittest.TestCase().successResultOf(d)
if args is None:
args = {}
return self.successResultOf(render(ElementResource(element), args))
def test_literal(self):
lcr = web_check_results.LiteralCheckResultsRendererElement()

3
src/allmydata/test/test_helper.py
View File

@ -1,3 +1,6 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

36
src/allmydata/test/test_storage_web.py
View File

@ -11,7 +11,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Omitted list sinc it broke a test on Python 2. Shouldn't require further
# Omitted list since it broke a test on Python 2. Shouldn't require further
# work, when we switch to Python 3 we'll be dropping this, anyway.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min # noqa: F401
@ -26,18 +26,6 @@ from twisted.internet import defer
from twisted.application import service
from twisted.web.template import flattenString
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
from zope.interface import implementer
from foolscap.api import fireEventually
from allmydata.util import fileutil, hashutil, base32, pollmixin
from allmydata.storage.common import storage_index_to_dir, \
@ -52,6 +40,10 @@ from allmydata.web.storage import (
)
from .common_util import FakeCanary
from .common_web import (
render,
)
def remove_tags(s):
s = re.sub(br'<[^>]*>', b' ', s)
s = re.sub(br'\s+', b' ', s)
@ -75,20 +67,10 @@ def renderDeferred(ss):
return flattenString(None, elem)
def renderJSON(resource):
"""Render a JSON from the given resource."""
@implementer(IRequest)
class JSONRequest(Request):
"""
A Request with t=json argument added to it. This is useful to
invoke a Resouce.render_JSON() method.
"""
def __init__(self):
Request.__init__(self, DummyChannel())
self.args = {"t": ["json"]}
self.fields = {}
return resource.render(JSONRequest())
"""
Render a JSON from the given resource.
"""
return render(resource, {"t": ["json"]})
class MyBucketCountingCrawler(BucketCountingCrawler):
def finished_prefix(self, cycle, prefix):

257
src/allmydata/test/web/test_common.py Normal file
View File

@ -0,0 +1,257 @@
"""
Tests for ``allmydata.web.common``.
"""
import gc
from bs4 import (
BeautifulSoup,
)
from hyperlink import (
DecodedURL,
)
from testtools.matchers import (
Equals,
Contains,
MatchesPredicate,
AfterPreprocessing,
)
from testtools.twistedsupport import (
failed,
succeeded,
has_no_result,
)
from twisted.python.failure import (
Failure,
)
from twisted.internet.error import (
ConnectionDone,
)
from twisted.internet.defer import (
Deferred,
fail,
)
from twisted.web.server import (
NOT_DONE_YET,
)
from twisted.web.resource import (
Resource,
)
from ...web.common import (
render_exception,
)
from ..common import (
SyncTestCase,
)
from ..common_web import (
render,
)
from .common import (
assert_soup_has_tag_with_attributes,
)
class StaticResource(Resource, object):
"""
``StaticResource`` is a resource that returns whatever Python object it is
given from its render method. This is useful for testing
``render_exception``\\ 's handling of different render results.
"""
def __init__(self, response):
Resource.__init__(self)
self._response = response
self._request = None
@render_exception
def render(self, request):
self._request = request
return self._response
class RenderExceptionTests(SyncTestCase):
"""
Tests for ``render_exception`` (including the private helper ``_finish``).
"""
def test_exception(self):
"""
If the decorated method raises an exception then the exception is rendered
into the response.
"""
class R(Resource):
@render_exception
def render(self, request):
raise Exception("synthetic exception")
self.assertThat(
render(R(), {}),
succeeded(
Contains(b"synthetic exception"),
),
)
def test_failure(self):
"""
If the decorated method returns a ``Deferred`` that fires with a
``Failure`` then the exception the ``Failure`` wraps is rendered into
the response.
"""
resource = StaticResource(fail(Exception("synthetic exception")))
self.assertThat(
render(resource, {}),
succeeded(
Contains(b"synthetic exception"),
),
)
def test_resource(self):
"""
If the decorated method returns an ``IResource`` provider then that
resource is used to render the response.
"""
resource = StaticResource(StaticResource(b"static result"))
self.assertThat(
render(resource, {}),
succeeded(
Equals(b"static result"),
),
)
def test_unicode(self):
"""
If the decorated method returns a ``unicode`` string then that string is
UTF-8 encoded and rendered into the response.
"""
text = u"\N{SNOWMAN}"
resource = StaticResource(text)
self.assertThat(
render(resource, {}),
succeeded(
Equals(text.encode("utf-8")),
),
)
def test_bytes(self):
"""
If the decorated method returns a ``bytes`` string then that string is
rendered into the response.
"""
data = b"hello world"
resource = StaticResource(data)
self.assertThat(
render(resource, {}),
succeeded(
Equals(data),
),
)
def test_decodedurl(self):
"""
If the decorated method returns a ``DecodedURL`` then a redirect to that
location is rendered into the response.
"""
loc = u"http://example.invalid/foo?bar=baz"
resource = StaticResource(DecodedURL.from_text(loc))
self.assertThat(
render(resource, {}),
succeeded(
MatchesPredicate(
lambda value: assert_soup_has_tag_with_attributes(
self,
BeautifulSoup(value),
"meta",
{"http-equiv": "refresh",
"content": "0;URL={}".format(loc.encode("ascii")),
},
)
# The assertion will raise if it has a problem, otherwise
# return None. Turn the None into something
# MatchesPredicate recognizes as success.
or True,
"did not find meta refresh tag in %r",
),
),
)
def test_none(self):
"""
If the decorated method returns ``None`` then the response is finished
with no additional content.
"""
self.assertThat(
render(StaticResource(None), {}),
succeeded(
Equals(b""),
),
)
def test_not_done_yet(self):
"""
If the decorated method returns ``NOT_DONE_YET`` then the resource is
responsible for finishing the request itself.
"""
the_request = []
class R(Resource):
@render_exception
def render(self, request):
the_request.append(request)
return NOT_DONE_YET
d = render(R(), {})
self.assertThat(
d,
has_no_result(),
)
the_request[0].write(b"some content")
the_request[0].finish()
self.assertThat(
d,
succeeded(
Equals(b"some content"),
),
)
def test_unknown(self):
"""
If the decorated method returns something which is not explicitly
supported, an internal server error is rendered into the response.
"""
self.assertThat(
render(StaticResource(object()), {}),
succeeded(
Equals(b"Internal Server Error"),
),
)
def test_disconnected(self):
"""
If the transport is disconnected before the response is available, no
``RuntimeError`` is logged for finishing a disconnected request.
"""
result = Deferred()
resource = StaticResource(result)
d = render(resource, {})
resource._request.connectionLost(Failure(ConnectionDone()))
result.callback(b"Some result")
self.assertThat(
d,
failed(
AfterPreprocessing(
lambda reason: reason.type,
Equals(ConnectionDone),
),
),
)
# Since we're not a trial TestCase we don't have flushLoggedErrors.
# The next best thing is to make sure any dangling Deferreds have been
# garbage collected and then let the generic trial logic for failing
# tests with logged errors kick in.
gc.collect()

5
src/allmydata/test/web/test_grid.py
View File

@ -18,6 +18,10 @@ from allmydata.storage.shares import get_share_file
from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
from allmydata.immutable import upload
from allmydata.mutable import publish
from ...web.common import (
render_exception,
)
from .. import common_util as testutil
from ..common import WebErrorMixin, ShouldFailMixin
from ..no_network import GridTestMixin
@ -34,6 +38,7 @@ class CompletelyUnhandledError(Exception):
pass
class ErrorBoom(object, resource.Resource):
@render_exception
def render(self, req):
raise CompletelyUnhandledError("whoops")

69
src/allmydata/test/web/test_root.py
View File

@ -2,8 +2,6 @@ from mock import Mock
import time
from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.web.template import Tag
from twisted.web.test.requesthelper import DummyRequest
@ -18,13 +16,9 @@ from ...util.connection_status import ConnectionStatus
from allmydata.web.root import URIHandler
from allmydata.client import _Client
from hypothesis import given
from hypothesis.strategies import text
from .common import (
assert_soup_has_tag_with_content,
from ..common_web import (
render,
)
from ..common import (
EMPTY_CLIENT_CONFIG,
)
@ -36,13 +30,6 @@ class RenderSlashUri(unittest.TestCase):
"""
def setUp(self):
self.request = DummyRequest(b"/uri")
self.request.fields = {}
def prepathURL():
return b"http://127.0.0.1.99999/" + b"/".join(self.request.prepath)
self.request.prePathURL = prepathURL
self.client = Mock()
self.res = URIHandler(self.client)
@ -50,51 +37,29 @@ class RenderSlashUri(unittest.TestCase):
"""
A valid capbility does not result in error
"""
self.request.args[b"uri"] = [(
query_args = {b"uri": [
b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:"
b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882"
)]
self.res.render_GET(self.request)
]}
response_body = self.successResultOf(
render(self.res, query_args),
)
self.assertNotEqual(
response_body,
"Invalid capability",
)
def test_invalid(self):
"""
A (trivially) invalid capbility is an error
"""
self.request.args[b"uri"] = [b"not a capability"]
response_body = self.res.render_GET(self.request)
soup = BeautifulSoup(response_body, 'html5lib')
assert_soup_has_tag_with_content(
self, soup, "title", "400 - Error",
query_args = {b"uri": [b"not a capability"]}
response_body = self.successResultOf(
render(self.res, query_args),
)
assert_soup_has_tag_with_content(
self, soup, "h1", "Error",
)
assert_soup_has_tag_with_content(
self, soup, "p", "Invalid capability",
)
@given(
text()
)
def test_hypothesis_error_caps(self, cap):
"""
Let hypothesis try a bunch of invalid capabilities
"""
self.request.args[b"uri"] = [cap.encode('utf8')]
response_body = self.res.render_GET(self.request)
soup = BeautifulSoup(response_body, 'html5lib')
assert_soup_has_tag_with_content(
self, soup, "title", "400 - Error",
)
assert_soup_has_tag_with_content(
self, soup, "h1", "Error",
)
assert_soup_has_tag_with_content(
self, soup, "p", "Invalid capability",
self.assertEqual(
response_body,
"Invalid capability",
)

105
src/allmydata/test/web/test_token.py
View File

@ -1,105 +0,0 @@
from zope.interface import implementer
from twisted.trial import unittest
from twisted.web import server
from nevow.inevow import IRequest
from allmydata.web import common
# XXX FIXME when we introduce "mock" as a dependency, these can
# probably just be Mock instances
@implementer(IRequest)
class FakeRequest(object):
def __init__(self):
self.method = "POST"
self.fields = dict()
self.args = dict()
class FakeField(object):
def __init__(self, *values):
if len(values) == 1:
self.value = values[0]
else:
self.value = list(values)
class FakeClientWithToken(object):
token = 'a' * 32
def get_auth_token(self):
return self.token
class TestTokenOnlyApi(unittest.TestCase):
def setUp(self):
self.client = FakeClientWithToken()
self.page = common.TokenOnlyWebApi(self.client)
def test_not_post(self):
req = FakeRequest()
req.method = "GET"
self.assertRaises(
server.UnsupportedMethod,
self.page.render, req,
)
def test_missing_token(self):
req = FakeRequest()
exc = self.assertRaises(
common.WebError,
self.page.render, req,
)
self.assertEquals(exc.text, "Missing token")
self.assertEquals(exc.code, 401)
def test_token_in_get_args(self):
req = FakeRequest()
req.args['token'] = 'z' * 32
exc = self.assertRaises(
common.WebError,
self.page.render, req,
)
self.assertEquals(exc.text, "Do not pass 'token' as URL argument")
self.assertEquals(exc.code, 400)
def test_invalid_token(self):
wrong_token = 'b' * 32
req = FakeRequest()
req.fields['token'] = FakeField(wrong_token)
exc = self.assertRaises(
common.WebError,
self.page.render, req,
)
self.assertEquals(exc.text, "Invalid token")
self.assertEquals(exc.code, 401)
def test_valid_token_no_t_arg(self):
req = FakeRequest()
req.fields['token'] = FakeField(self.client.token)
with self.assertRaises(common.WebError) as exc:
self.page.render(req)
self.assertEquals(exc.exception.text, "Must provide 't=' argument")
self.assertEquals(exc.exception.code, 400)
def test_valid_token_invalid_t_arg(self):
req = FakeRequest()
req.fields['token'] = FakeField(self.client.token)
req.args['t'] = 'not at all json'
with self.assertRaises(common.WebError) as exc:
self.page.render(req)
self.assertTrue("invalid type" in exc.exception.text)
self.assertEquals(exc.exception.code, 400)
def test_valid(self):
req = FakeRequest()
req.fields['token'] = FakeField(self.client.token)
req.args['t'] = ['json']
result = self.page.render(req)
self.assertTrue(result == NotImplemented)

8
src/allmydata/test/web/test_web.py
View File

@ -4767,11 +4767,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_static_missing(self):
# self.staticdir does not exist yet, because we used self.mktemp()
d = self.assertFailure(self.GET("/static"), error.Error)
# nevow.static throws an exception when it tries to os.stat the
# missing directory, which gives the client a 500 Internal Server
# Error, and the traceback reveals the parent directory name. By
# switching to plain twisted.web.static, this gives a normal 404 that
# doesn't reveal anything. This addresses #1720.
# If os.stat raises an exception for the missing directory and the
# traceback reveals the parent directory name we don't want to see
# that parent directory name in the response. This addresses #1720.
d.addCallback(lambda e: self.assertEquals(str(e), "404 Not Found"))
return d

208
src/allmydata/test/web/test_webish.py Normal file
View File

@ -0,0 +1,208 @@
"""
Tests for ``allmydata.webish``.
"""
from uuid import (
uuid4,
)
from testtools.matchers import (
AfterPreprocessing,
Contains,
Equals,
MatchesAll,
Not,
)
from twisted.python.filepath import (
FilePath,
)
from twisted.web.test.requesthelper import (
DummyChannel,
)
from twisted.web.resource import (
Resource,
)
from ..common import (
SyncTestCase,
)
from ...webish import (
TahoeLAFSRequest,
tahoe_lafs_site,
)
class TahoeLAFSRequestTests(SyncTestCase):
"""
Tests for ``TahoeLAFSRequest``.
"""
def _fields_test(self, method, request_headers, request_body, match_fields):
channel = DummyChannel()
request = TahoeLAFSRequest(
channel,
)
for (k, v) in request_headers.items():
request.requestHeaders.setRawHeaders(k, [v])
request.gotLength(len(request_body))
request.handleContentChunk(request_body)
request.requestReceived(method, b"/", b"HTTP/1.1")
# We don't really care what happened to the request. What we do care
# about is what the `fields` attribute is set to.
self.assertThat(
request.fields,
match_fields,
)
def test_no_form_fields(self):
"""
When a ``GET`` request is received, ``TahoeLAFSRequest.fields`` is None.
"""
self._fields_test(b"GET", {}, b"", Equals(None))
def test_form_fields(self):
"""
When a ``POST`` request is received, form fields are parsed into
``TahoeLAFSRequest.fields``.
"""
form_data, boundary = multipart_formdata([
[param(u"name", u"foo"),
body(u"bar"),
],
[param(u"name", u"baz"),
param(u"filename", u"quux"),
body(u"some file contents"),
],
])
self._fields_test(
b"POST",
{b"content-type": b"multipart/form-data; boundary={}".format(boundary)},
form_data.encode("ascii"),
AfterPreprocessing(
lambda fs: {
k: fs.getvalue(k)
for k
in fs.keys()
},
Equals({
b"foo": b"bar",
b"baz": b"some file contents",
}),
),
)
class TahoeLAFSSiteTests(SyncTestCase):
"""
Tests for the ``Site`` created by ``tahoe_lafs_site``.
"""
def _test_censoring(self, path, censored):
"""
Verify that the event logged for a request for ``path`` does not include
``path`` but instead includes ``censored``.
:param bytes path: A request path.
:param bytes censored: A replacement value for the request path in the
access log.
:return: ``None`` if the logging looks good.
"""
logPath = self.mktemp()
site = tahoe_lafs_site(Resource(), logPath=logPath)
site.startFactory()
channel = DummyChannel()
channel.factory = site
request = TahoeLAFSRequest(channel)
request.gotLength(None)
request.requestReceived(b"GET", path, b"HTTP/1.1")
self.assertThat(
FilePath(logPath).getContent(),
MatchesAll(
Contains(censored),
Not(Contains(path)),
),
)
def test_uri_censoring(self):
"""
The log event for a request for **/uri/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/uri/URI:CHK:aaa:bbb",
b"/uri/[CENSORED]",
)
def test_file_censoring(self):
"""
The log event for a request for **/file/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/file/URI:CHK:aaa:bbb",
b"/file/[CENSORED]",
)
def test_named_censoring(self):
"""
The log event for a request for **/named/<CAP>** has the capability value
censored.
"""
self._test_censoring(
b"/named/URI:CHK:aaa:bbb",
b"/named/[CENSORED]",
)
def test_uri_queryarg_censoring(self):
"""
The log event for a request for **/uri?cap=<CAP>** has the capability
value censored.
"""
self._test_censoring(
b"/uri?uri=URI:CHK:aaa:bbb",
b"/uri?uri=[CENSORED]",
)
def param(name, value):
return u"; {}={}".format(name, value)
def body(value):
return u"\r\n\r\n{}".format(value)
def _field(field):
yield u"Content-Disposition: form-data"
for param in field:
yield param
def _multipart_formdata(fields):
for field in fields:
yield u"".join(_field(field)) + u"\r\n"
def multipart_formdata(fields):
"""
Serialize some simple fields into a multipart/form-data string.
:param fields: A list of lists of unicode strings to assemble into the
result. See ``param`` and ``body``.
:return unicode: The given fields combined into a multipart/form-data
string.
"""
boundary = str(uuid4())
parts = list(_multipart_formdata(fields))
parts.insert(0, u"")
return (
(u"--" + boundary + u"\r\n").join(parts),
boundary,
)

1
src/allmydata/util/_python3.py
View File

@ -46,6 +46,7 @@ PORTED_MODULES = [
"allmydata.immutable.happiness_upload",
"allmydata.immutable.layout",
"allmydata.immutable.literal",
"allmydata.immutable.offloaded",
"allmydata.immutable.upload",
"allmydata.interfaces",
"allmydata.introducer.interfaces",

423
src/allmydata/web/common.py
View File

@ -1,26 +1,54 @@
from future.utils import PY2
from past.builtins import unicode
import time
import json
from functools import wraps
from hyperlink import (
DecodedURL,
)
from eliot import (
Message,
start_action,
)
from eliot.twisted import (
DeferredContext,
)
from twisted.web import (
http,
resource,
server,
template,
)
from twisted.web.iweb import IRequest as ITwistedRequest
from twisted.web.iweb import (
IRequest,
)
from twisted.web.template import (
tags,
)
from twisted.web.server import (
NOT_DONE_YET,
)
from twisted.web.util import (
DeferredResource,
FailureElement,
redirectTo,
)
from twisted.python.reflect import (
fullyQualifiedName,
)
from twisted.python import log
if PY2:
from nevow.appserver import DefaultExceptionHandler
from nevow.inevow import IRequest as INevowRequest
else:
class DefaultExceptionHandler:
def __init__(self, *args, **kwargs):
raise NotImplementedError("Still not ported to Python 3")
INevowRequest = None
from twisted.python.failure import (
Failure,
)
from twisted.internet.defer import (
CancelledError,
maybeDeferred,
)
from twisted.web.resource import (
IResource,
)
from allmydata import blacklist
from allmydata.interfaces import (
@ -37,7 +65,6 @@ from allmydata.interfaces import (
SDMF_VERSION,
)
from allmydata.mutable.common import UnrecoverableFileError
from allmydata.util.hashutil import timing_safe_compare
from allmydata.util.time_format import (
format_delta,
format_time,
@ -127,11 +154,22 @@ def parse_offset_arg(offset):
return offset
def get_root(ctx_or_req):
if PY2:
req = INevowRequest(ctx_or_req)
else:
req = ITwistedRequest(ctx_or_req)
def get_root(req):
"""
Get a relative path with parent directory segments that refers to the root
location known to the given request. This seems a lot like the constant
absolute path **/** but it will behave differently if the Tahoe-LAFS HTTP
server is reverse-proxied and mounted somewhere other than at the root.
:param twisted.web.iweb.IRequest req: The request to consider.
:return: A string like ``../../..`` with the correct number of segments to
reach the root.
"""
if not IRequest.providedBy(req):
raise TypeError(
"get_root requires IRequest provider, got {!r}".format(req),
)
depth = len(req.prepath) + len(req.postpath)
link = "/".join([".."] * depth)
return link
@ -332,58 +370,13 @@ def humanize_failure(f):
return humanize_exception(f.value)
class MyExceptionHandler(DefaultExceptionHandler, object):
def simple(self, ctx, text, code=http.BAD_REQUEST):
req = INevowRequest(ctx)
req.setResponseCode(code)
#req.responseHeaders.setRawHeaders("content-encoding", [])
#req.responseHeaders.setRawHeaders("content-disposition", [])
req.setHeader("content-type", "text/plain;charset=utf-8")
if isinstance(text, unicode):
text = text.encode("utf-8")
req.setHeader("content-length", b"%d" % len(text))
req.write(text)
# TODO: consider putting the requested URL here
req.finishRequest(False)
def renderHTTP_exception(self, ctx, f):
try:
text, code = humanize_failure(f)
except:
log.msg("exception in humanize_failure")
log.msg("argument was %s" % (f,))
log.err()
text, code = str(f), None
if code is not None:
return self.simple(ctx, text, code)
if f.check(server.UnsupportedMethod):
# twisted.web.server.Request.render() has support for transforming
# this into an appropriate 501 NOT_IMPLEMENTED or 405 NOT_ALLOWED
# return code, but nevow does not.
req = INevowRequest(ctx)
method = req.method
return self.simple(ctx,
"I don't know how to treat a %s request." % method,
http.NOT_IMPLEMENTED)
req = INevowRequest(ctx)
accept = req.getHeader("accept")
if not accept:
accept = "*/*"
if "*/*" in accept or "text/*" in accept or "text/html" in accept:
super = DefaultExceptionHandler
return super.renderHTTP_exception(self, ctx, f)
# use plain text
traceback = f.getTraceback()
return self.simple(ctx, traceback, http.INTERNAL_SERVER_ERROR)
class NeedOperationHandleError(WebError):
pass
class SlotsSequenceElement(template.Element):
"""
``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for
``SlotsSequenceElement` is a minimal port of Nevow's sequence renderer for
twisted.web.template.
Tags passed in to be templated will have two renderers available: ``item``
@ -426,84 +419,254 @@ class SlotsSequenceElement(template.Element):
return tag
class TokenOnlyWebApi(resource.Resource, object):
"""
I provide a rend.Page implementation that only accepts POST calls,
and only if they have a 'token=' arg with the correct
authentication token (see
:meth:`allmydata.client.Client.get_auth_token`). Callers must also
provide the "t=" argument to indicate the return-value (the only
valid value for this is "json")
Subclasses should override 'post_json' which should process the
API call and return a string which encodes a valid JSON
object. This will only be called if the correct token is present
and valid (during renderHTTP processing).
"""
def __init__(self, client):
self.client = client
def post_json(self, req):
return NotImplemented
def render(self, req):
if req.method != 'POST':
raise server.UnsupportedMethod(('POST',))
if req.args.get('token', False):
raise WebError("Do not pass 'token' as URL argument", http.BAD_REQUEST)
# not using get_arg() here because we *don't* want the token
# argument to work if you passed it as a GET-style argument
token = None
if req.fields and 'token' in req.fields:
token = req.fields['token'].value.strip()
if not token:
raise WebError("Missing token", http.UNAUTHORIZED)
if not timing_safe_compare(token, self.client.get_auth_token()):
raise WebError("Invalid token", http.UNAUTHORIZED)
t = get_arg(req, "t", "").strip()
if not t:
raise WebError("Must provide 't=' argument")
if t == u'json':
try:
return self.post_json(req)
except WebError as e:
req.setResponseCode(e.code)
return json.dumps({"error": e.text})
except Exception as e:
message, code = humanize_exception(e)
req.setResponseCode(500 if code is None else code)
return json.dumps({"error": message})
else:
raise WebError("'%s' invalid type for 't' arg" % (t,), http.BAD_REQUEST)
def exception_to_child(f):
def exception_to_child(getChild):
"""
Decorate ``getChild`` method with exception handling behavior to render an
error page reflecting the exception.
"""
@wraps(f)
@wraps(getChild)
def g(self, name, req):
try:
return f(self, name, req)
except Exception as e:
description, status = humanize_exception(e)
return resource.ErrorPage(status, "Error", description)
# Bind the method to the instance so it has a better
# fullyQualifiedName later on. This is not necessary on Python 3.
bound_getChild = getChild.__get__(self, type(self))
action = start_action(
action_type=u"allmydata:web:common-getChild",
uri=req.uri,
method=req.method,
name=name,
handler=fullyQualifiedName(bound_getChild),
)
with action.context():
result = DeferredContext(maybeDeferred(bound_getChild, name, req))
result.addCallbacks(
_getChild_done,
_getChild_failed,
callbackArgs=(self,),
)
result = result.addActionFinish()
return DeferredResource(result)
return g
def render_exception(f):
def _getChild_done(child, parent):
Message.log(
message_type=u"allmydata:web:common-getChild:result",
result=fullyQualifiedName(type(child)),
)
if child is None:
return resource.NoResource()
return child
def _getChild_failed(reason):
text, code = humanize_failure(reason)
return resource.ErrorPage(code, "Error", text)
def render_exception(render):
"""
Decorate a ``render_*`` method with exception handling behavior to render
an error page reflecting the exception.
"""
@wraps(f)
@wraps(render)
def g(self, request):
try:
return f(self, request)
except Exception as e:
description, status = humanize_exception(e)
return resource.ErrorPage(status, "Error", description).render(request)
# Bind the method to the instance so it has a better
# fullyQualifiedName later on. This is not necessary on Python 3.
bound_render = render.__get__(self, type(self))
action = start_action(
action_type=u"allmydata:web:common-render",
uri=request.uri,
method=request.method,
handler=fullyQualifiedName(bound_render),
)
if getattr(request, "dont_apply_extra_processing", False):
with action:
return bound_render(request)
with action.context():
result = DeferredContext(maybeDeferred(bound_render, request))
# Apply `_finish` all of our result handling logic to whatever it
# returned.
result.addBoth(_finish, bound_render, request)
d = result.addActionFinish()
# If the connection is lost then there's no point running our _finish
# logic because it has nowhere to send anything. There may also be no
# point in finishing whatever operation was being performed because
# the client cannot be informed of its result. Also, Twisted Web
# raises exceptions from some Request methods if they're used after
# the connection is lost.
request.notifyFinish().addErrback(
lambda ignored: d.cancel(),
)
return NOT_DONE_YET
return g
def _finish(result, render, request):
"""
Try to finish rendering the response to a request.
This implements extra convenience functionality not provided by Twisted
Web. Various resources in Tahoe-LAFS made use of this functionality when
it was provided by Nevow. Rather than making that application code do the
more tedious thing itself, we duplicate the functionality here.
:param result: Something returned by a render method which we can turn
into a response.
:param render: The original render method which produced the result.
:param request: The request being responded to.
:return: ``None``
"""
if isinstance(result, Failure):
if result.check(CancelledError):
return
Message.log(
message_type=u"allmydata:web:common-render:failure",
message=result.getErrorMessage(),
)
_finish(
_renderHTTP_exception(request, result),
render,
request,
)
elif IResource.providedBy(result):
# If result is also using @render_exception then we don't want to
# double-apply the logic. This leads to an attempt to double-finish
# the request. If it isn't using @render_exception then you should
# fix it so it is.
Message.log(
message_type=u"allmydata:web:common-render:resource",
resource=fullyQualifiedName(type(result)),
)
result.render(request)
elif isinstance(result, unicode):
Message.log(
message_type=u"allmydata:web:common-render:unicode",
)
request.write(result.encode("utf-8"))
request.finish()
elif isinstance(result, bytes):
Message.log(
message_type=u"allmydata:web:common-render:bytes",
)
request.write(result)
request.finish()
elif isinstance(result, DecodedURL):
Message.log(
message_type=u"allmydata:web:common-render:DecodedURL",
)
_finish(redirectTo(str(result), request), render, request)
elif result is None:
Message.log(
message_type=u"allmydata:web:common-render:None",
)
request.finish()
elif result == NOT_DONE_YET:
Message.log(
message_type=u"allmydata:web:common-render:NOT_DONE_YET",
)
pass
else:
Message.log(
message_type=u"allmydata:web:common-render:unknown",
)
log.err("Request for {!r} handled by {!r} returned unusable {!r}".format(
request.uri,
fullyQualifiedName(render),
result,
))
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
_finish(b"Internal Server Error", render, request)
def _renderHTTP_exception(request, failure):
try:
text, code = humanize_failure(failure)
except:
log.msg("exception in humanize_failure")
log.msg("argument was %s" % (failure,))
log.err()
text = str(failure)
code = None
if code is not None:
return _renderHTTP_exception_simple(request, text, code)
accept = request.getHeader("accept")
if not accept:
accept = "*/*"
if "*/*" in accept or "text/*" in accept or "text/html" in accept:
request.setResponseCode(http.INTERNAL_SERVER_ERROR)
return template.renderElement(
request,
tags.html(
tags.head(
tags.title(u"Exception"),
),
tags.body(
FailureElement(failure),
),
),
)
# use plain text
traceback = failure.getTraceback()
return _renderHTTP_exception_simple(
request,
traceback,
http.INTERNAL_SERVER_ERROR,
)
def _renderHTTP_exception_simple(request, text, code):
request.setResponseCode(code)
request.setHeader("content-type", "text/plain;charset=utf-8")
if isinstance(text, unicode):
text = text.encode("utf-8")
request.setHeader("content-length", b"%d" % len(text))
return text
def handle_when_done(req, d):
when_done = get_arg(req, "when_done", None)
if when_done:
d.addCallback(lambda res: DecodedURL.from_text(when_done.decode("utf-8")))
return d
def url_for_string(req, url_string):
"""
Construct a universal URL using the given URL string.
:param IRequest req: The request being served. If ``redir_to`` is not
absolute then this is used to determine the net location of this
server and the resulting URL is made to point at it.
:param bytes url_string: A byte string giving a universal or absolute URL.
:return DecodedURL: An absolute URL based on this server's net location
and the given URL string.
"""
url = DecodedURL.from_text(url_string.decode("utf-8"))
if url.host == b"":
root = req.URLPath()
netloc = root.netloc.split(b":", 1)
if len(netloc) == 1:
host = netloc
port = None
else:
host = netloc[0]
port = int(netloc[1])
url = url.replace(
scheme=root.scheme.decode("ascii"),
host=host.decode("ascii"),
port=port,
)
return url

26
src/allmydata/web/common_py3.py
View File

@ -4,15 +4,7 @@ Common utilities that are available from Python 3.
Can eventually be merged back into allmydata.web.common.
"""
from future.utils import PY2
if PY2:
from nevow.inevow import IRequest as INevowRequest
else:
INevowRequest = None
from twisted.web import resource, http
from twisted.web.iweb import IRequest
from allmydata.util import abbreviate
@ -23,24 +15,20 @@ class WebError(Exception):
self.code = code
def get_arg(ctx_or_req, argname, default=None, multiple=False):
def get_arg(req, argname, default=None, multiple=False):
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
precedence. If multiple=True, this returns a tuple of arguments (possibly
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
"""
results = []
if PY2:
req = INevowRequest(ctx_or_req)
if argname in req.args:
results.extend(req.args[argname])
if req.fields and argname in req.fields:
results.append(req.fields[argname].value)
else:
req = IRequest(ctx_or_req)
if argname in req.args:
results.extend(req.args[argname])
if argname in req.args:
results.extend(req.args[argname])
if req.fields and argname in req.fields:
results.append(req.fields[argname].value)
if multiple:
return tuple(results)
if results:

27
src/allmydata/web/directory.py
View File

@ -58,6 +58,7 @@ from allmydata.web.common import (
SlotsSequenceElement,
exception_to_child,
render_exception,
handle_when_done,
)
from allmydata.web.filenode import ReplaceMeMixin, \
FileNodeHandler, PlaceHolderNodeHandler
@ -113,7 +114,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
# Rejecting URIs that contain empty path pieces (for example:
# "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was
# the old nevow behavior and it is encoded in the test suite;
# the old Nevow behavior and it is encoded in the test suite;
# we will follow suit.
for segment in req.prepath:
if not segment:
@ -206,6 +207,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
)
return make_handler_for(node, self.client, self.node, name)
@render_exception
def render_DELETE(self, req):
assert self.parentnode and self.name
d = self.parentnode.delete(self.name)
@ -310,13 +312,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
else:
raise WebError("POST to a directory with bad t=%s" % t)
when_done = get_arg(req, "when_done", None)
if when_done:
def done(res):
req.redirect(when_done)
return res
d.addCallback(done)
return d
return handle_when_done(req, d)
def _POST_mkdir(self, req):
name = get_arg(req, "name", "")
@ -402,9 +398,12 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
d.addBoth(_maybe_got_node)
# now we have a placeholder or a filenodehandler, and we can just
# delegate to it. We could return the resource back out of
# DirectoryNodeHandler.renderHTTP, and nevow would recurse into it,
# but the addCallback() that handles when_done= would break.
d.addCallback(lambda child: child.render(req))
# DirectoryNodeHandler.render_POST and it would get rendered but the
# addCallback() that handles when_done= would break.
def render_child(child):
req.dont_apply_extra_processing = True
return child.render(req)
d.addCallback(render_child)
return d
def _POST_uri(self, req):
@ -523,9 +522,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
d.addCallback(self._maybe_literal, CheckResultsRenderer)
return d
def _start_operation(self, monitor, renderer, ctx):
self._operations.add_monitor(ctx, monitor, renderer)
return self._operations.redirect_to(ctx)
def _start_operation(self, monitor, renderer, req):
self._operations.add_monitor(req, monitor, renderer)
return self._operations.redirect_to(req)
def _POST_start_deep_check(self, req):
# check this directory and everything reachable from it

46
src/allmydata/web/filenode.py
View File

@ -8,8 +8,6 @@ from twisted.web.resource import (
ErrorPage,
)
from nevow import url
from allmydata.interfaces import ExistingChildError
from allmydata.monitor import Monitor
from allmydata.immutable.upload import FileHandle
@ -34,8 +32,8 @@ from allmydata.web.common import (
render_exception,
should_create_intermediate_directories,
text_plain,
MyExceptionHandler,
WebError,
handle_when_done,
)
from allmydata.web.check_results import (
CheckResultsRenderer,
@ -150,10 +148,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
# placeholder.
raise WebError("POST to a file: bad t=%s" % t)
when_done = get_arg(req, "when_done", None)
if when_done:
d.addCallback(lambda res: when_done)
return d
return handle_when_done(req, d)
class FileNodeHandler(Resource, ReplaceMeMixin, object):
@ -315,10 +310,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
else:
raise WebError("POST to file: bad t=%s" % t)
when_done = get_arg(req, "when_done", None)
if when_done:
d.addCallback(lambda res: url.URL.fromString(when_done))
return d
return handle_when_done(req, d)
def _maybe_literal(self, res, Results_Class):
if res:
@ -485,24 +477,13 @@ class FileDownloader(Resource, object):
if req.method == "HEAD":
return ""
finished = []
def _request_finished(ign):
finished.append(True)
req.notifyFinish().addBoth(_request_finished)
d = self.filenode.read(req, first, size)
def _finished(ign):
if not finished:
req.finish()
def _error(f):
lp = log.msg("error during GET", facility="tahoe.webish", failure=f,
level=log.UNUSUAL, umid="xSiF3w")
if finished:
log.msg("but it's too late to tell them", parent=lp,
level=log.UNUSUAL, umid="j1xIbw")
return
req._tahoe_request_had_error = f # for HTTP-style logging
if f.check(defer.CancelledError):
# The HTTP connection was lost and we no longer have anywhere
# to send our result. Let this pass through.
return f
if req.startedWriting:
# The content-type is already set, and the response code has
# already been sent, so we can't provide a clean error
@ -513,15 +494,16 @@ class FileDownloader(Resource, object):
# error response be shorter than the intended results.
#
# We don't have a lot of options, unfortunately.
req.write("problem during download\n")
req.finish()
return b"problem during download\n"
else:
# We haven't written anything yet, so we can provide a
# sensible error message.
eh = MyExceptionHandler()
eh.renderHTTP_exception(req, f)
d.addCallbacks(_finished, _error)
return req.deferred
return f
d.addCallbacks(
lambda ignored: None,
_error,
)
return d
def _file_json_metadata(req, filenode, edge_metadata):

29
src/allmydata/web/operations.py
View File

@ -1,14 +1,15 @@
from future.utils import PY2
import time
if PY2:
from nevow import url
else:
# This module still needs porting to Python 3
url = None
from hyperlink import (
DecodedURL,
)
from twisted.web.template import (
renderer,
tags as T,
)
from twisted.python.urlpath import (
URLPath,
)
from twisted.python.failure import Failure
from twisted.internet import reactor, defer
from twisted.web import resource
@ -18,7 +19,6 @@ from twisted.application import service
from allmydata.web.common import (
WebError,
get_root,
get_arg,
boolean_of_arg,
exception_to_child,
@ -88,17 +88,14 @@ class OphandleTable(resource.Resource, service.Service):
"""
:param allmydata.webish.MyRequest req:
"""
ophandle = get_arg(req, "ophandle")
ophandle = get_arg(req, "ophandle").decode("utf-8")
assert ophandle
target = get_root(req) + "/operations/" + ophandle
here = DecodedURL.from_text(unicode(URLPath.fromRequest(req)))
target = here.click(u"/").child(u"operations", ophandle)
output = get_arg(req, "output")
if output:
target = target + "?output=%s" % output
# XXX: We have to use nevow.url here because nevow.appserver
# is unhappy with anything else; so this gets its own ticket.
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3314
return url.URL.fromString(target)
target = target.add(u"output", output.decode("utf-8"))
return target
@exception_to_child
def getChild(self, name, req):
@ -155,8 +152,6 @@ class ReloadMixin(object):
def refresh(self, req, tag):
if self.monitor.is_finished():
return ""
# dreid suggests ctx.tag(**dict([("http-equiv", "refresh")]))
# but I can't tell if he's joking or not
tag.attributes["http-equiv"] = "refresh"
tag.attributes["content"] = str(self.REFRESH_TIME)
return tag

2
src/allmydata/web/root.py
View File

@ -189,7 +189,7 @@ class FileHandler(resource.Resource, object):
return filenode.FileNodeDownloadHandler(self.client, node)
@render_exception
def render_GET(self, ctx):
def render_GET(self, req):
raise WebError("/file must be followed by a file-cap and a name",
http.NOT_FOUND)

6
src/allmydata/web/storage.py
View File

@ -1,3 +1,4 @@
from future.utils import PY2
import time, json
from twisted.python.filepath import FilePath
@ -317,4 +318,7 @@ class StorageStatus(MultiFormatResource):
"lease-checker": self._storage.lease_checker.get_state(),
"lease-checker-progress": self._storage.lease_checker.get_progress(),
}
return json.dumps(d, indent=1) + "\n"
result = json.dumps(d, indent=1) + "\n"
if PY2:
result = result.decode("utf-8")
return result.encode("utf-8")

8
src/allmydata/web/unlinked.py
View File

@ -1,5 +1,6 @@
import urllib
from twisted.web import http
from twisted.internet import defer
from twisted.python.filepath import FilePath
@ -10,7 +11,6 @@ from twisted.web.template import (
renderElement,
tags,
)
from nevow import url
from allmydata.immutable.upload import FileHandle
from allmydata.mutable.publish import MutableFileHandle
from allmydata.web.common import (
@ -21,6 +21,7 @@ from allmydata.web.common import (
get_format,
get_mutable_type,
render_exception,
url_for_string,
)
from allmydata.web import status
@ -66,7 +67,7 @@ def POSTUnlinkedCHK(req, client):
def _done(upload_results, redir_to):
if "%(uri)s" in redir_to:
redir_to = redir_to.replace("%(uri)s", urllib.quote(upload_results.get_uri()))
return url.URL.fromString(redir_to)
return url_for_string(req, redir_to)
d.addCallback(_done, when_done)
else:
# return the Upload Results page, which includes the URI
@ -160,7 +161,6 @@ def POSTUnlinkedCreateDirectory(req, client):
new_url = "uri/" + urllib.quote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
req.finish()
return ''
d.addCallback(_then_redir)
else:
@ -179,7 +179,6 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client):
new_url = "uri/" + urllib.quote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
req.finish()
return ''
d.addCallback(_then_redir)
else:
@ -198,7 +197,6 @@ def POSTUnlinkedCreateImmutableDirectory(req, client):
new_url = "uri/" + urllib.quote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
req.finish()
return ''
d.addCallback(_then_redir)
else:

209
src/allmydata/webish.py
View File

@ -1,43 +1,57 @@
import re, time
from functools import (
partial,
)
from cgi import (
FieldStorage,
)
from twisted.application import service, strports, internet
from twisted.web import http, static
from twisted.web import static
from twisted.web.http import (
parse_qs,
)
from twisted.web.server import (
Request,
Site,
)
from twisted.internet import defer
from twisted.internet.address import (
IPv4Address,
IPv6Address,
)
from nevow import appserver, inevow
from allmydata.util import log, fileutil
from allmydata.web import introweb, root
from allmydata.web.common import MyExceptionHandler
from allmydata.web.operations import OphandleTable
from .web.storage_plugins import (
StoragePlugins,
)
# we must override twisted.web.http.Request.requestReceived with a version
# that doesn't use cgi.parse_multipart() . Since we actually use Nevow, we
# override the nevow-specific subclass, nevow.appserver.NevowRequest . This
# is an exact copy of twisted.web.http.Request (from SVN HEAD on 10-Aug-2007)
# that modifies the way form arguments are parsed. Note that this sort of
# surgery may induce a dependency upon a particular version of twisted.web
class TahoeLAFSRequest(Request, object):
"""
``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
that are useful for Tahoe-LAFS.
parse_qs = http.parse_qs
class MyRequest(appserver.NevowRequest, object):
:ivar NoneType|FieldStorage fields: For POST requests, a structured
representation of the contents of the request body. For anything
else, ``None``.
"""
fields = None
_tahoe_request_had_error = None
def requestReceived(self, command, path, version):
"""Called by channel when all data has been received.
This method is not intended for users.
"""
self.content.seek(0,0)
Called by channel when all data has been received.
Override the base implementation to apply certain site-wide policies
and to provide less memory-intensive multipart/form-post handling for
large file uploads.
"""
self.content.seek(0)
self.args = {}
self.stack = []
self.setHeader("Referrer-Policy", "no-referrer")
self.method, self.uri = command, path
self.clientproto = version
@ -49,93 +63,36 @@ class MyRequest(appserver.NevowRequest, object):
self.path, argstring = x
self.args = parse_qs(argstring, 1)
# Adding security headers. These will be sent for *all* HTTP requests.
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"])
if self.method == 'POST':
# We use FieldStorage here because it performs better than
# cgi.parse_multipart(self.content, pdict) which is what
# twisted.web.http.Request uses.
self.fields = FieldStorage(
self.content,
{
name.lower(): value[-1]
for (name, value)
in self.requestHeaders.getAllRawHeaders()
},
environ={'REQUEST_METHOD': 'POST'})
self.content.seek(0)
# Argument processing.
self._tahoeLAFSSecurityPolicy()
## The original twisted.web.http.Request.requestReceived code parsed the
## content and added the form fields it found there to self.args . It
## did this with cgi.parse_multipart, which holds the arguments in RAM
## and is thus unsuitable for large file uploads. The Nevow subclass
## (nevow.appserver.NevowRequest) uses cgi.FieldStorage instead (putting
## the results in self.fields), which is much more memory-efficient.
## Since we know we're using Nevow, we can anticipate these arguments
## appearing in self.fields instead of self.args, and thus skip the
## parse-content-into-self.args step.
## args = self.args
## ctype = self.getHeader('content-type')
## if self.method == "POST" and ctype:
## mfd = 'multipart/form-data'
## key, pdict = cgi.parse_header(ctype)
## if key == 'application/x-www-form-urlencoded':
## args.update(parse_qs(self.content.read(), 1))
## elif key == mfd:
## try:
## args.update(cgi.parse_multipart(self.content, pdict))
## except KeyError, e:
## if e.args[0] == 'content-disposition':
## # Parse_multipart can't cope with missing
## # content-dispostion headers in multipart/form-data
## # parts, so we catch the exception and tell the client
## # it was a bad request.
## self.channel.transport.write(
## "HTTP/1.1 400 Bad Request\r\n\r\n")
## self.channel.transport.loseConnection()
## return
## raise
self.processing_started_timestamp = time.time()
self.process()
def _logger(self):
# we build up a log string that hides most of the cap, to preserve
# user privacy. We retain the query args so we can identify things
# like t=json. Then we send it to the flog. We make no attempt to
# match apache formatting. TODO: when we move to DSA dirnodes and
# shorter caps, consider exposing a few characters of the cap, or
# maybe a few characters of its hash.
x = self.uri.split("?", 1)
if len(x) == 1:
# no query args
path = self.uri
queryargs = ""
else:
path, queryargs = x
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these too.
if queryargs.startswith("uri="):
queryargs = "[uri=CENSORED]"
queryargs = "?" + queryargs
if path.startswith("/uri"):
path = "/uri/[CENSORED].."
elif path.startswith("/file"):
path = "/file/[CENSORED].."
elif path.startswith("/named"):
path = "/named/[CENSORED].."
uri = path + queryargs
error = ""
if self._tahoe_request_had_error:
error = " [ERROR]"
log.msg(
format=(
"web: %(clientip)s %(method)s %(uri)s %(code)s "
"%(length)s%(error)s"
),
clientip=_get_client_ip(self),
method=self.method,
uri=uri,
code=self.code,
length=(self.sentLength or "-"),
error=error,
facility="tahoe.webish",
level=log.OPERATIONAL,
)
def _tahoeLAFSSecurityPolicy(self):
"""
Set response properties related to Tahoe-LAFS-imposed security policy.
This will ensure that all HTTP requests received by the Tahoe-LAFS
HTTP server have this policy imposed, regardless of other
implementation details.
"""
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
self.responseHeaders.setRawHeaders("X-Frame-Options", ["DENY"])
# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
self.setHeader("Referrer-Policy", "no-referrer")
def _get_client_ip(request):
@ -150,6 +107,54 @@ def _get_client_ip(request):
return None
def _logFormatter(logDateTime, request):
# we build up a log string that hides most of the cap, to preserve
# user privacy. We retain the query args so we can identify things
# like t=json. Then we send it to the flog. We make no attempt to
# match apache formatting. TODO: when we move to DSA dirnodes and
# shorter caps, consider exposing a few characters of the cap, or
# maybe a few characters of its hash.
x = request.uri.split("?", 1)
if len(x) == 1:
# no query args
path = request.uri
queryargs = ""
else:
path, queryargs = x
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these too.
if queryargs.startswith("uri="):
queryargs = "uri=[CENSORED]"
queryargs = "?" + queryargs
if path.startswith("/uri/"):
path = "/uri/[CENSORED]"
elif path.startswith("/file/"):
path = "/file/[CENSORED]"
elif path.startswith("/named/"):
path = "/named/[CENSORED]"
uri = path + queryargs
template = "web: %(clientip)s %(method)s %(uri)s %(code)s %(length)s"
return template % dict(
clientip=_get_client_ip(request),
method=request.method,
uri=uri,
code=request.code,
length=(request.sentLength or "-"),
facility="tahoe.webish",
level=log.OPERATIONAL,
)
tahoe_lafs_site = partial(
Site,
requestFactory=TahoeLAFSRequest,
logFormatter=_logFormatter,
)
class WebishServer(service.MultiService):
name = "webish"
@ -175,15 +180,13 @@ class WebishServer(service.MultiService):
def buildServer(self, webport, nodeurl_path, staticdir):
self.webport = webport
self.site = site = appserver.NevowSite(self.root)
self.site.requestFactory = MyRequest
self.site.remember(MyExceptionHandler(), inevow.ICanHandleException)
self.site = tahoe_lafs_site(self.root)
self.staticdir = staticdir # so tests can check
if staticdir:
self.root.putChild("static", static.File(staticdir))
if re.search(r'^\d', webport):
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
s = strports.service(webport, site)
s = strports.service(webport, self.site)
s.setServiceParent(self)
self._scheme = None

20
static/tahoe.py
View File

@ -3,23 +3,19 @@
# Import this first to suppress deprecation warnings.
import allmydata
# nevow requires all these for its voodoo module import time adaptor registrations
from nevow import accessors, appserver, static, rend, url, util, query, i18n, flat
from nevow import guard, stan, testutil, context
from nevow.flat import flatmdom, flatstan, twist
from formless import webform, processors, annotate, iformless
from decimal import Decimal
from xml.dom import minidom
import allmydata.web
# junk to appease pyflakes's outrage
[
accessors, appserver, static, rend, url, util, query, i18n, flat, guard, stan, testutil,
context, flatmdom, flatstan, twist, webform, processors, annotate, iformless, Decimal,
minidom, allmydata,
]
# We import these things to give PyInstaller's dependency resolver some hints
# about what it needs to include. We don't use them otherwise _here_ but
# other parts of the codebase do. pyflakes points out that they are unused
# unless we use them. So ... use them.
Decimal
minidom
allmydata
from allmydata.scripts import runner
runner.run()
runner.run()