import time import json from twisted.web import http, server, resource, template from twisted.python import log from twisted.python.failure import Failure from zope.interface import Interface from nevow import loaders, appserver #from nevow.rend import Page from twisted.web.resource import Resource from nevow.inevow import IRequest from nevow.util import resource_filename from allmydata import blacklist from allmydata.interfaces import ExistingChildError, NoSuchChildError, \ FileTooLargeError, NotEnoughSharesError, NoSharesError, \ EmptyPathnameComponentError, MustBeDeepImmutableError, \ MustBeReadonlyError, MustNotBeUnknownRWError, SDMF_VERSION, MDMF_VERSION from allmydata.mutable.common import UnrecoverableFileError from allmydata.util import abbreviate from allmydata.util.hashutil import timing_safe_compare from allmydata.util.time_format import format_time, format_delta from allmydata.util.encodingutil import to_str, quote_output def get_filenode_metadata(filenode): metadata = {'mutable': filenode.is_mutable()} if metadata['mutable']: mutable_type = filenode.get_version() assert mutable_type in (SDMF_VERSION, MDMF_VERSION) if mutable_type == MDMF_VERSION: file_format = "MDMF" else: file_format = "SDMF" else: file_format = "CHK" metadata['format'] = file_format size = filenode.get_size() if size is not None: metadata['size'] = size return metadata class IOpHandleTable(Interface): pass def getxmlfile(name): return loaders.xmlfile(resource_filename('allmydata.web', '%s' % name)) def boolean_of_arg(arg): # TODO: "" if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"): raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST) return arg.lower() in ("true", "t", "1", "on") def parse_replace_arg(replace): if replace.lower() == "only-files": return replace try: return boolean_of_arg(replace) except WebError: raise WebError("invalid replace= argument: %r" % (replace,), http.BAD_REQUEST) def get_format(req, default="CHK"): arg = get_arg(req, "format", None) if not arg: if boolean_of_arg(get_arg(req, "mutable", "false")): return "SDMF" return default if arg.upper() == "CHK": return "CHK" elif arg.upper() == "SDMF": return "SDMF" elif arg.upper() == "MDMF": return "MDMF" else: raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg, http.BAD_REQUEST) def get_mutable_type(file_format): # accepts result of get_format() if file_format == "SDMF": return SDMF_VERSION elif file_format == "MDMF": return MDMF_VERSION else: # this is also used to identify which formats are mutable. Use # if get_mutable_type(file_format) is not None: # do_mutable() # else: # do_immutable() return None def parse_offset_arg(offset): # XXX: This will raise a ValueError when invoked on something that # is not an integer. Is that okay? Or do we want a better error # message? Since this call is going to be used by programmers and # their tools rather than users (through the wui), it is not # inconsistent to return that, I guess. if offset is not None: offset = int(offset) return offset def get_root(ctx_or_req): req = IRequest(ctx_or_req) # the addSlash=True gives us one extra (empty) segment depth = len(req.prepath) + len(req.postpath) - 1 link = "/".join([".."] * depth) return link def get_arg(ctx_or_req, argname, default=None, multiple=False): """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take precedence. If multiple=True, this returns a tuple of arguments (possibly empty), starting with all those in the query args. """ req = IRequest(ctx_or_req) results = [] if argname in req.args: results.extend(req.args[argname]) if req.fields and argname in req.fields: results.append(req.fields[argname].value) if multiple: return tuple(results) if results: return results[0] return default def convert_children_json(nodemaker, children_json): """I convert the JSON output of GET?t=json into the dict-of-nodes input to both dirnode.create_subdirectory() and client.create_directory(initial_children=). This is used by t=mkdir-with-children and t=mkdir-immutable""" children = {} if children_json: data = json.loads(children_json) for (namex, (ctype, propdict)) in data.iteritems(): namex = unicode(namex) writecap = to_str(propdict.get("rw_uri")) readcap = to_str(propdict.get("ro_uri")) metadata = propdict.get("metadata", {}) # name= argument is just for error reporting childnode = nodemaker.create_from_cap(writecap, readcap, name=namex) children[namex] = (childnode, metadata) return children def abbreviate_time(data): # 1.23s, 790ms, 132us if data is None: return "" s = float(data) if s >= 10: return abbreviate.abbreviate_time(data) if s >= 1.0: return "%.2fs" % s if s >= 0.01: return "%.0fms" % (1000*s) if s >= 0.001: return "%.1fms" % (1000*s) return "%.0fus" % (1000000*s) def compute_rate(bytes, seconds): if bytes is None: return None if seconds is None or seconds == 0: return None # negative values don't make sense here assert bytes > -1 assert seconds > 0 return 1.0 * bytes / seconds def abbreviate_rate(data): # 21.8kBps, 554.4kBps 4.37MBps if data is None: return "" r = float(data) if r > 1000000: return "%1.2fMBps" % (r/1000000) if r > 1000: return "%.1fkBps" % (r/1000) return "%.0fBps" % r def abbreviate_size(data): # 21.8kB, 554.4kB 4.37MB if data is None: return "" r = float(data) if r > 1000000000: return "%1.2fGB" % (r/1000000000) if r > 1000000: return "%1.2fMB" % (r/1000000) if r > 1000: return "%.1fkB" % (r/1000) return "%.0fB" % r def plural(sequence_or_length): if isinstance(sequence_or_length, int): length = sequence_or_length else: length = len(sequence_or_length) if length == 1: return "" return "s" def text_plain(text, ctx): req = IRequest(ctx) req.setHeader("content-type", "text/plain") req.setHeader("content-length", b"%d" % len(text)) return text def spaces_to_nbsp(text): return unicode(text).replace(u' ', u'\u00A0') def render_time_delta(time_1, time_2): return spaces_to_nbsp(format_delta(time_1, time_2)) def render_time(t): return spaces_to_nbsp(format_time(time.localtime(t))) def render_time_attr(t): return format_time(time.localtime(t)) class WebError(Exception): def __init__(self, text, code=http.BAD_REQUEST): self.text = text self.code = code # XXX: to make UnsupportedMethod return 501 NOT_IMPLEMENTED instead of 500 # Internal Server Error, we either need to do that ICanHandleException trick, # or make sure that childFactory returns a WebErrorResource (and never an # actual exception). The latter is growing increasingly annoying. def should_create_intermediate_directories(req): t = get_arg(req, "t", "").strip() return bool(req.method in ("PUT", "POST") and t not in ("delete", "rename", "rename-form", "check")) def humanize_failure(f): # return text, responsecode if f.check(EmptyPathnameComponentError): return ("The webapi does not allow empty pathname components, " "i.e. a double slash", http.BAD_REQUEST) if f.check(ExistingChildError): return ("There was already a child by that name, and you asked me " "to not replace it.", http.CONFLICT) if f.check(NoSuchChildError): quoted_name = quote_output(f.value.args[0], encoding="utf-8", quotemarks=False) return ("No such child: %s" % quoted_name, http.NOT_FOUND) if f.check(NotEnoughSharesError): t = ("NotEnoughSharesError: This indicates that some " "servers were unavailable, or that shares have been " "lost to server departure, hard drive failure, or disk " "corruption. You should perform a filecheck on " "this object to learn more.\n\nThe full error message is:\n" "%s") % str(f.value) return (t, http.GONE) if f.check(NoSharesError): t = ("NoSharesError: no shares could be found. " "Zero shares usually indicates a corrupt URI, or that " "no servers were connected, but it might also indicate " "severe corruption. You should perform a filecheck on " "this object to learn more.\n\nThe full error message is:\n" "%s") % str(f.value) return (t, http.GONE) if f.check(UnrecoverableFileError): t = ("UnrecoverableFileError: the directory (or mutable file) could " "not be retrieved, because there were insufficient good shares. " "This might indicate that no servers were connected, " "insufficient servers were connected, the URI was corrupt, or " "that shares have been lost due to server departure, hard drive " "failure, or disk corruption. You should perform a filecheck on " "this object to learn more.") return (t, http.GONE) if f.check(MustNotBeUnknownRWError): quoted_name = quote_output(f.value.args[1], encoding="utf-8") immutable = f.value.args[2] if immutable: t = ("MustNotBeUnknownRWError: an operation to add a child named " "%s to a directory was given an unknown cap in a write slot.\n" "If the cap is actually an immutable readcap, then using a " "webapi server that supports a later version of Tahoe may help.\n\n" "If you are using the webapi directly, then specifying an immutable " "readcap in the read slot (ro_uri) of the JSON PROPDICT, and " "omitting the write slot (rw_uri), would also work in this " "case.") % quoted_name else: t = ("MustNotBeUnknownRWError: an operation to add a child named " "%s to a directory was given an unknown cap in a write slot.\n" "Using a webapi server that supports a later version of Tahoe " "may help.\n\n" "If you are using the webapi directly, specifying a readcap in " "the read slot (ro_uri) of the JSON PROPDICT, as well as a " "writecap in the write slot if desired, would also work in this " "case.") % quoted_name return (t, http.BAD_REQUEST) if f.check(MustBeDeepImmutableError): quoted_name = quote_output(f.value.args[1], encoding="utf-8") t = ("MustBeDeepImmutableError: a cap passed to this operation for " "the child named %s, needed to be immutable but was not. Either " "the cap is being added to an immutable directory, or it was " "originally retrieved from an immutable directory as an unknown " "cap.") % quoted_name return (t, http.BAD_REQUEST) if f.check(MustBeReadonlyError): quoted_name = quote_output(f.value.args[1], encoding="utf-8") t = ("MustBeReadonlyError: a cap passed to this operation for " "the child named '%s', needed to be read-only but was not. " "The cap is being passed in a read slot (ro_uri), or was retrieved " "from a read slot as an unknown cap.") % quoted_name return (t, http.BAD_REQUEST) if f.check(blacklist.FileProhibited): t = "Access Prohibited: %s" % quote_output(f.value.reason, encoding="utf-8", quotemarks=False) return (t, http.FORBIDDEN) if f.check(WebError): return (f.value.text, f.value.code) if f.check(FileTooLargeError): return (f.getTraceback(), http.REQUEST_ENTITY_TOO_LARGE) return (str(f), None) class MyExceptionHandler(appserver.DefaultExceptionHandler, object): def simple(self, ctx, text, code=http.BAD_REQUEST): req = IRequest(ctx) req.setResponseCode(code) #req.responseHeaders.setRawHeaders("content-encoding", []) #req.responseHeaders.setRawHeaders("content-disposition", []) req.setHeader("content-type", "text/plain;charset=utf-8") if isinstance(text, unicode): text = text.encode("utf-8") req.setHeader("content-length", b"%d" % len(text)) req.write(text) # TODO: consider putting the requested URL here req.finishRequest(False) def renderHTTP_exception(self, ctx, f): try: text, code = humanize_failure(f) except: log.msg("exception in humanize_failure") log.msg("argument was %s" % (f,)) log.err() text, code = str(f), None if code is not None: return self.simple(ctx, text, code) if f.check(server.UnsupportedMethod): # twisted.web.server.Request.render() has support for transforming # this into an appropriate 501 NOT_IMPLEMENTED or 405 NOT_ALLOWED # return code, but nevow does not. req = IRequest(ctx) method = req.method return self.simple(ctx, "I don't know how to treat a %s request." % method, http.NOT_IMPLEMENTED) req = IRequest(ctx) accept = req.getHeader("accept") if not accept: accept = "*/*" if "*/*" in accept or "text/*" in accept or "text/html" in accept: super = appserver.DefaultExceptionHandler return super.renderHTTP_exception(self, ctx, f) # use plain text traceback = f.getTraceback() return self.simple(ctx, traceback, http.INTERNAL_SERVER_ERROR) class NeedOperationHandleError(WebError): pass class RenderMixin(object): def renderHTTP(self, ctx): request = IRequest(ctx) # if we were using regular twisted.web Resources (and the regular # twisted.web.server.Request object) then we could implement # render_PUT and render_GET. But Nevow's request handler # (NevowRequest.gotPageContext) goes directly to renderHTTP. Copy # some code from the Resource.render method that Nevow bypasses, to # do the same thing. m = getattr(self, 'render_' + request.method, None) if not m: raise server.UnsupportedMethod(getattr(self, 'allowedMethods', ())) return m(ctx) def render_OPTIONS(self, ctx): """ Handle HTTP OPTIONS request by adding appropriate headers. :param ctx: client transaction from which request is extracted :returns: str (empty) """ req = IRequest(ctx) req.setHeader("server", "Tahoe-LAFS gateway") methods = ', '.join([m[7:] for m in dir(self) if m.startswith('render_')]) req.setHeader("allow", methods) req.setHeader("public", methods) req.setHeader("compliance", "rfc=2068, rfc=2616") req.setHeader("content-length", 0) return "" class MultiFormatPage(Page): """ ```MultiFormatPage`` is a ``rend.Page`` that can be rendered in a number of different formats. Rendered format is controlled by a query argument (given by ``self.formatArgument``). Different resources may support different formats but ``json`` is a pretty common one. """ formatArgument = "t" formatDefault = None def renderHTTP(self, ctx): """ Dispatch to a renderer for a particular format, as selected by a query argument. A renderer for the format given by the query argument matching ``formatArgument`` will be selected and invoked. The default ``Page`` rendering behavior will be used if no format is selected (either by query arguments or by ``formatDefault``). :return: The result of the selected renderer. """ req = IRequest(ctx) t = get_arg(req, self.formatArgument, self.formatDefault) renderer = self._get_renderer(t) result = renderer(ctx) return result def _get_renderer(self, fmt): """ Get the renderer for the indicated format. :param bytes fmt: The format. If a method with a prefix of ``render_`` and a suffix of this format (upper-cased) is found, it will be used. :return: A callable which takes a Nevow context and renders a response. """ if fmt is None: return super(MultiFormatPage, self).renderHTTP try: renderer = getattr(self, "render_{}".format(fmt.upper())) except AttributeError: raise WebError( "Unknown {} value: {!r}".format(self.formatArgument, fmt), ) else: if renderer is None: return super(MultiFormatPage, self).renderHTTP return lambda ctx: renderer(IRequest(ctx)) class MultiFormatResource(resource.Resource, object): """ ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in a number of different formats. Rendered format is controlled by a query argument (given by ``self.formatArgument``). Different resources may support different formats but ``json`` is a pretty common one. ``html`` is the default format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" formatDefault = None def render(self, req): """ Dispatch to a renderer for a particular format, as selected by a query argument. A renderer for the format given by the query argument matching ``formatArgument`` will be selected and invoked. render_HTML will be used as a default if no format is selected (either by query arguments or by ``formatDefault``). :return: The result of the selected renderer. """ t = get_arg(req, self.formatArgument, self.formatDefault) renderer = self._get_renderer(t) return renderer(req) def _get_renderer(self, fmt): """ Get the renderer for the indicated format. :param str fmt: The format. If a method with a prefix of ``render_`` and a suffix of this format (upper-cased) is found, it will be used. :return: A callable which takes a twisted.web Request and renders a response. """ renderer = None if fmt is not None: try: renderer = getattr(self, "render_{}".format(fmt.upper())) except AttributeError: raise WebError( "Unknown {} value: {!r}".format(self.formatArgument, fmt), ) if renderer is None: renderer = self.render_HTML return renderer class SlotsSequenceElement(template.Element): """ ``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for twisted.web.template. Tags passed in to be templated will have two renderers available: ``item`` and ``tag``. """ def __init__(self, tag, seq): self.loader = template.TagLoader(tag) self.seq = seq @template.renderer def item(self, request, tag): """ A template renderer for each sequence item. ``tag`` will be cloned for each item in the sequence provided, and its slots filled from the sequence item. Each item must be dict-like enough for ``tag.fillSlots(**item)``. Each cloned tag will be siblings with no separator beween them. """ for item in self.seq: yield tag.clone(deep=False).fillSlots(**item) @template.renderer def empty(self, request, tag): """ A template renderer for empty sequences. This renderer will either return ``tag`` unmodified if the provided sequence has no items, or return the empty string if there are any items. """ if len(self.seq) > 0: return u'' else: return tag class TokenOnlyWebApi(resource.Resource, object): """ I provide a rend.Page implementation that only accepts POST calls, and only if they have a 'token=' arg with the correct authentication token (see :meth:`allmydata.client.Client.get_auth_token`). Callers must also provide the "t=" argument to indicate the return-value (the only valid value for this is "json") Subclasses should override 'post_json' which should process the API call and return a string which encodes a valid JSON object. This will only be called if the correct token is present and valid (during renderHTTP processing). """ def __init__(self, client): self.client = client def post_json(self, req): return NotImplemented def render(self, req): if req.method != 'POST': raise server.UnsupportedMethod(('POST',)) if req.args.get('token', False): raise WebError("Do not pass 'token' as URL argument", http.BAD_REQUEST) # not using get_arg() here because we *don't* want the token # argument to work if you passed it as a GET-style argument token = None if req.fields and 'token' in req.fields: token = req.fields['token'].value.strip() if not token: raise WebError("Missing token", http.UNAUTHORIZED) if not timing_safe_compare(token, self.client.get_auth_token()): raise WebError("Invalid token", http.UNAUTHORIZED) t = get_arg(req, "t", "").strip() if not t: raise WebError("Must provide 't=' argument") if t == u'json': try: return self.post_json(req) except WebError as e: req.setResponseCode(e.code) return json.dumps({"error": e.text}) except Exception as e: message, code = humanize_failure(Failure()) req.setResponseCode(500 if code is None else code) return json.dumps({"error": message}) else: raise WebError("'%s' invalid type for 't' arg" % (t,), http.BAD_REQUEST)