diff --git a/commandline.c b/commandline.c index b8e1e469..2e643b16 100644 --- a/commandline.c +++ b/commandline.c @@ -483,7 +483,7 @@ int app_echo(const struct cli_parsed *parsed, struct cli_context *context) DEBUGF("echo:argv[%d]=\"%s\"", i, arg); if (escapes) { unsigned char buf[strlen(arg)]; - size_t len = strn_fromprint(buf, sizeof buf, arg, '\0', NULL); + size_t len = strn_fromprint(buf, sizeof buf, arg, 0, '\0', NULL); cli_write(context, buf, len); } else cli_puts(context, arg); diff --git a/conf.h b/conf.h index 2d7b40fb..2a3fbb7d 100644 --- a/conf.h +++ b/conf.h @@ -233,8 +233,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "rhizome.h" -typedef char bool_t; - #define CONFIG_FILE_MAX_SIZE (32 * 1024) #define INTERFACE_NAME_STRLEN 40 diff --git a/conf_schema.h b/conf_schema.h index 5de82ff9..a44adc67 100644 --- a/conf_schema.h +++ b/conf_schema.h @@ -233,6 +233,7 @@ ATOM(bool_t, dnaresponses, 0, boolean,, "") ATOM(bool_t, dnahelper, 0, boolean,, "") ATOM(bool_t, queues, 0, boolean,, "") ATOM(bool_t, timing, 0, boolean,, "") +ATOM(bool_t, httpd, 0, boolean,, "") ATOM(bool_t, io, 0, boolean,, "") ATOM(bool_t, verbose_io, 0, boolean,, "") ATOM(bool_t, interactive_io, 0, boolean,, "") @@ -261,6 +262,7 @@ ATOM(bool_t, slipbytestream, 0, boolean,, "") ATOM(bool_t, packetconstruction, 0, boolean,, "") ATOM(bool_t, rhizome, 0, boolean,, "") ATOM(bool_t, rhizome_bind, 0, boolean,, "") +ATOM(bool_t, rhizome_httpd, 0, boolean,, "") ATOM(bool_t, rhizome_tx, 0, boolean,, "") ATOM(bool_t, rhizome_rx, 0, boolean,, "") ATOM(bool_t, rhizome_ads, 0, boolean,, "") @@ -375,6 +377,15 @@ STRUCT(rhizome_direct) SUB_STRUCT(peerlist, peer,) END_STRUCT +STRUCT(user) +STRING(50, password, "", str,, "Authentication password") +END_STRUCT + +ARRAY(userlist,) +KEY_STRING(25, str) +VALUE_SUB_STRUCT(user) +END_ARRAY(10) + STRUCT(rhizome_api_addfile) STRING(64, uri_path, "", absolute_path,, "URI path for HTTP add-file request") ATOM(struct in_addr, allow_host, hton_in_addr(INADDR_LOOPBACK), in_addr,, "IP address of host allowed to make HTTP add-file request") @@ -383,8 +394,13 @@ ATOM(sid_t, default_author, SID_ANY, sid,, "Author of ad ATOM(rhizome_bk_t, bundle_secret_key, RHIZOME_BK_NONE, rhizome_bk,, "Secret key of add-file bundle to try if sender not given") END_STRUCT +STRUCT(rhizome_api_restful) +SUB_STRUCT(userlist, users,) +END_STRUCT + STRUCT(rhizome_api) SUB_STRUCT(rhizome_api_addfile, addfile,) +SUB_STRUCT(rhizome_api_restful, restful,) END_STRUCT STRUCT(rhizome_http) diff --git a/constants.h b/constants.h index f99cc307..cc8b4ffb 100644 --- a/constants.h +++ b/constants.h @@ -220,5 +220,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define UNLOCK_CHALLENGE (0xF1) #define UNLOCK_RESPONSE (0xF2) +// should there be a types.h to hold this? +typedef char bool_t; + #endif // __SERVALD_CONSTANTS_H diff --git a/directory_service.c b/directory_service.c index c3727f87..6f09e32e 100644 --- a/directory_service.c +++ b/directory_service.c @@ -1,9 +1,11 @@ -#include "constants.h" -#include "mdp_client.h" +#ifdef HAVE_POLL_H #include +#endif #include #include +#include "constants.h" +#include "mdp_client.h" #include "str.h" struct item{ diff --git a/fakeradio.c b/fakeradio.c index b310fbe3..5727ad1b 100644 --- a/fakeradio.c +++ b/fakeradio.c @@ -4,7 +4,9 @@ #include #include #include +#ifdef HAVE_POLL_H #include +#endif #include #include #include diff --git a/fdqueue.c b/fdqueue.c index 1ef8aea4..83b65b3e 100644 --- a/fdqueue.c +++ b/fdqueue.c @@ -17,8 +17,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include -#include "serval.h" +#include "fdqueue.h" #include "conf.h" #include "str.h" #include "strbuf.h" diff --git a/fdqueue.h b/fdqueue.h new file mode 100644 index 00000000..11596cf3 --- /dev/null +++ b/fdqueue.h @@ -0,0 +1,92 @@ +/* +Serval DNA file descriptor queue +Copyright (C) 2012-2013 Serval Project, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVALDNA__FDQUEUE_H +#define __SERVALDNA__FDQUEUE_H + +#ifdef HAVE_POLL_H +#include +#endif +#include "os.h" +#include "log.h" + +struct profile_total { + struct profile_total *_next; + int _initialised; + const char *name; + time_ms_t max_time; + time_ms_t total_time; + time_ms_t child_time; + int calls; +}; + +struct call_stats{ + time_ms_t enter_time; + time_ms_t child_time; + struct profile_total *totals; + struct call_stats *prev; +}; + +struct sched_ent; + +typedef void (*ALARM_FUNCP) (struct sched_ent *alarm); + +struct sched_ent{ + struct sched_ent *_next; + struct sched_ent *_prev; + + ALARM_FUNCP function; + void *context; + struct pollfd poll; + // when we should first consider the alarm + time_ms_t alarm; + // the order we will prioritise the alarm + time_ms_t deadline; + struct profile_total *stats; + int _poll_index; +}; + +int is_scheduled(const struct sched_ent *alarm); +int _schedule(struct __sourceloc, struct sched_ent *alarm); +int _unschedule(struct __sourceloc, struct sched_ent *alarm); +int _watch(struct __sourceloc, struct sched_ent *alarm); +int _unwatch(struct __sourceloc, struct sched_ent *alarm); +#define schedule(alarm) _schedule(__WHENCE__, alarm) +#define unschedule(alarm) _unschedule(__WHENCE__, alarm) +#define watch(alarm) _watch(__WHENCE__, alarm) +#define unwatch(alarm) _unwatch(__WHENCE__, alarm) +int fd_poll(); + +/* function timing routines */ +int fd_clearstats(); +int fd_showstats(); +int fd_checkalarms(); +int fd_func_enter(struct __sourceloc, struct call_stats *this_call); +int fd_func_exit(struct __sourceloc, struct call_stats *this_call); +void dump_stack(int log_level); + +#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \ + struct call_stats _this_call={.totals=&_aggregate_stats}; \ + fd_func_enter(__HERE__, &_this_call); + +#define OUT() fd_func_exit(__HERE__, &_this_call) +#define RETURN(X) do { OUT(); return (X); } while (0); +#define RETURNNULL do { OUT(); return (NULL); } while (0); + +#endif // __SERVALDNA__FDQUEUE_H diff --git a/headerfiles.mk b/headerfiles.mk index 45528bd9..20fae731 100644 --- a/headerfiles.mk +++ b/headerfiles.mk @@ -18,6 +18,8 @@ HDRS= fifo.h \ crypto.h \ log.h \ net.h \ + fdqueue.h \ + http_server.h \ xprintf.h \ constants.h \ monitor-client.h \ diff --git a/http_server.c b/http_server.c new file mode 100644 index 00000000..87645cd8 --- /dev/null +++ b/http_server.c @@ -0,0 +1,1762 @@ +/* +Serval DNA - HTTP Server +Copyright (C) 2013 Serval Project, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include +#include +#include +#include "serval.h" +#include "conf.h" +#include "http_server.h" +#include "log.h" +#include "str.h" +#include "strbuf.h" +#include "strbuf_helpers.h" +#include "net.h" +#include "mem.h" + +#define BOUNDARY_STRING_MAXLEN 70 // legislated limit from RFC-1341 + +/* The (struct http_request).verb field points to one of these static strings, so that a simple + * equality test can be used, eg, (r->verb == HTTP_VERB_GET) instead of a strcmp(). + * + * @author Andrew Bettison + */ +const char HTTP_VERB_GET[] = "GET"; +const char HTTP_VERB_POST[] = "POST"; +const char HTTP_VERB_PUT[] = "PUT"; +const char HTTP_VERB_HEAD[] = "HEAD"; +const char HTTP_VERB_DELETE[] = "DELETE"; +const char HTTP_VERB_TRACE[] = "TRACE"; +const char HTTP_VERB_OPTIONS[] = "OPTIONS"; +const char HTTP_VERB_CONNECT[] = "CONNECT"; +const char HTTP_VERB_PATCH[] = "PATCH"; + +static struct { + const char *word; + size_t wordlen; +} http_verbs[] = { +#define VERB_ENTRY(NAME) { HTTP_VERB_##NAME, sizeof HTTP_VERB_##NAME - 1 } + VERB_ENTRY(GET), + VERB_ENTRY(POST), + VERB_ENTRY(PUT), + VERB_ENTRY(HEAD), + VERB_ENTRY(DELETE), + VERB_ENTRY(TRACE), + VERB_ENTRY(OPTIONS), + VERB_ENTRY(CONNECT), + VERB_ENTRY(PATCH) +#undef VERB_ENTRY +}; + +static struct profile_total http_server_stats = { + .name = "http_server_poll", +}; + +#define DEBUG_DUMP_PARSED(r) do { \ + if (config.debug.httpd) \ + DEBUGF("%s %s HTTP/%u.%u", r->verb ? r->verb : "NULL", alloca_str_toprint(r->path), r->version_major, r->version_minor);\ + } while (0) + +#define DEBUG_DUMP_PARSER(r) do { \ + if (config.debug.httpd) \ + DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \ + r->parsed - r->received, alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \ + r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor), \ + r->end - r->received, \ + r->request_content_remaining \ + ); \ + } while (0) + +static void http_server_poll(struct sched_ent *); +static int http_request_parse_verb(struct http_request *r); +static int http_request_parse_path(struct http_request *r); +static int http_request_parse_http_version(struct http_request *r); +static int http_request_start_parsing_headers(struct http_request *r); +static int http_request_parse_header(struct http_request *r); +static int http_request_start_body(struct http_request *r); +static int http_request_parse_body_form_data(struct http_request *r); +static void http_request_start_response(struct http_request *r); + +void http_request_init(struct http_request *r, int sockfd) +{ + assert(sockfd != -1); + r->request_header.content_length = CONTENT_LENGTH_UNKNOWN; + r->request_content_remaining = CONTENT_LENGTH_UNKNOWN; + r->alarm.stats = &http_server_stats; + r->alarm.function = http_server_poll; + if (r->idle_timeout == 0) + r->idle_timeout = 10000; // 10 seconds + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + r->alarm.poll.fd = sockfd; + r->alarm.poll.events = POLLIN; + r->phase = RECEIVE; + r->received = r->end = r->parsed = r->cursor = r->buffer; + r->parser = http_request_parse_verb; + watch(&r->alarm); + schedule(&r->alarm); +} + +void http_request_free_response_buffer(struct http_request *r) +{ + if (r->response_free_buffer) { + r->response_free_buffer(r->response_buffer); + r->response_free_buffer = NULL; + } + r->response_buffer = NULL; + r->response_buffer_size = 0; +} + +int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz) +{ + const char *const bufe = r->buffer + sizeof r->buffer; + assert(r->received < bufe); + size_t rbufsiz = bufe - r->received; + if (bufsiz <= rbufsiz) { + http_request_free_response_buffer(r); + r->response_buffer = (char *) r->received; + r->response_buffer_size = rbufsiz; + return 0; + } + if (bufsiz != r->response_buffer_size) { + http_request_free_response_buffer(r); + if ((r->response_buffer = emalloc(bufsiz)) == NULL) + return -1; + r->response_free_buffer = free; + r->response_buffer_size = bufsiz; + } + assert(r->response_buffer_size >= bufsiz); + assert(r->response_buffer != NULL); + return 0; +} + +void http_request_finalise(struct http_request *r) +{ + if (r->phase == DONE) + return; + assert(r->phase == RECEIVE || r->phase == TRANSMIT); + unschedule(&r->alarm); + unwatch(&r->alarm); + close(r->alarm.poll.fd); + r->alarm.poll.fd = -1; + if (r->finalise) + r->finalise(r); + r->finalise = NULL; + http_request_free_response_buffer(r); + r->phase = DONE; +} + +#define _SEP (1 << 0) +#define _BND (1 << 1) + +uint8_t http_ctype[256] = { + ['0'] = _BND, ['1'] = _BND, ['2'] = _BND, ['3'] = _BND, ['4'] = _BND, + ['5'] = _BND, ['6'] = _BND, ['7'] = _BND, ['8'] = _BND, ['9'] = _BND, + ['A'] = _BND, ['B'] = _BND, ['C'] = _BND, ['D'] = _BND, ['E'] = _BND, + ['F'] = _BND, ['G'] = _BND, ['H'] = _BND, ['I'] = _BND, ['J'] = _BND, + ['K'] = _BND, ['L'] = _BND, ['M'] = _BND, ['N'] = _BND, ['O'] = _BND, + ['P'] = _BND, ['Q'] = _BND, ['R'] = _BND, ['S'] = _BND, ['T'] = _BND, + ['U'] = _BND, ['V'] = _BND, ['W'] = _BND, ['X'] = _BND, ['Y'] = _BND, + ['Z'] = _BND, + ['a'] = _BND, ['b'] = _BND, ['c'] = _BND, ['d'] = _BND, ['e'] = _BND, + ['f'] = _BND, ['g'] = _BND, ['h'] = _BND, ['i'] = _BND, ['j'] = _BND, + ['k'] = _BND, ['l'] = _BND, ['m'] = _BND, ['n'] = _BND, ['o'] = _BND, + ['p'] = _BND, ['q'] = _BND, ['r'] = _BND, ['s'] = _BND, ['t'] = _BND, + ['u'] = _BND, ['v'] = _BND, ['w'] = _BND, ['x'] = _BND, ['y'] = _BND, + ['z'] = _BND, + ['+'] = _BND, ['-'] = _BND, ['.'] = _BND, ['/'] = _BND, [':'] = _BND, + ['_'] = _BND, + ['('] = _SEP | _BND, + [')'] = _SEP | _BND, + [','] = _SEP | _BND, + ['?'] = _SEP | _BND, + ['='] = _SEP | _BND, + [' '] = _SEP | _BND, + ['\t'] = _SEP, + ['<'] = _SEP, + ['>'] = _SEP, + ['@'] = _SEP, + [';'] = _SEP, + [':'] = _SEP, + ['\\'] = _SEP, + ['"'] = _SEP, + ['/'] = _SEP, + ['['] = _SEP, + [']'] = _SEP, + ['{'] = _SEP, + ['}'] = _SEP, +}; + +inline int is_http_char(char c) +{ + return c >= 0; +} + +inline int is_http_ctl(char c) +{ + return iscntrl(c); +} + +inline int is_http_separator(char c) +{ + return (http_ctype[(unsigned char) c] & _SEP) != 0; +} + +inline int is_http_boundary(char c) +{ + return (http_ctype[(unsigned char) c] & _BND) != 0; +} + +inline int is_http_token(char c) +{ + return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c); +} + +inline int is_valid_http_boundary_string(const char *s) +{ + if (s[0] == '\0') + return 0; + for (; *s; ++s) + if (!is_http_boundary(*s)) + return 0; + return s[-1] != ' '; +} + +struct substring { + const char *start; + const char *end; +}; + +#define alloca_substring_toprint(sub) alloca_toprint(-1, (sub).start, (sub).end - (sub).start) + +const struct substring substring_NULL = { NULL, NULL }; + +#if 0 +static int _matches(struct substring str, const char *text) +{ + return strlen(text) == str.end - str.start && memcmp(str.start, text, str.end - str.start) == 0; +} +#endif + +static const char * _reserve(struct http_request *r, struct substring str) +{ + char *reslim = r->buffer + sizeof r->buffer - 1024; // always leave this much unreserved space + assert(r->received <= reslim); + size_t len = str.end - str.start; + size_t siz = len + 1; + if (r->received + siz > reslim) { + r->response.result_code = 414; + return NULL; + } + if (r->received + siz > r->parsed) { + WARNF("Error during HTTP parsing, unparsed content %s would be overwritten by reserving %s", + alloca_toprint(30, r->parsed, r->end - r->parsed), + alloca_substring_toprint(str) + ); + r->response.result_code = 500; + return NULL; + } + char *ret = (char *) r->received; + if (ret != str.start) + memmove(ret, str.start, len); + ret[len] = '\0'; + r->received += siz; + assert(r->received <= r->parsed); + return ret; +} + +static const char * _reserve_str(struct http_request *r, const char *str) +{ + struct substring sub = { .start = str, .end = str + strlen(str) }; + return _reserve(r, sub); +} + +static inline int _end_of_content(struct http_request *r) +{ + return r->cursor == r->end && r->request_content_remaining == 0; +} + +static inline int _run_out(struct http_request *r) +{ + assert(r->cursor <= r->end); + return r->cursor == r->end; +} + +static inline int _buffer_full(struct http_request *r) +{ + const char *const bufend = r->buffer + sizeof r->buffer; + return r->parsed == r->received && (r->end == bufend || r->request_content_remaining == 0); +} + +static inline void _rewind(struct http_request *r) +{ + assert(r->parsed >= r->received); + r->cursor = r->parsed; +} + +static inline void _commit(struct http_request *r) +{ + assert(r->cursor <= r->end); + r->parsed = r->cursor; +} + +static inline void _skip_all(struct http_request *r) +{ + r->cursor = r->end; +} + +static inline int _skip_crlf(struct http_request *r) +{ + return !_run_out(r) && *r->cursor == '\r' && ++r->cursor && !_run_out(r) && *r->cursor == '\n' && ++r->cursor; +} + +static inline int _skip_to_crlf(struct http_request *r) +{ + for (; !_run_out(r); ++r->cursor) + if (r->cursor + 1 < r->end && r->cursor[0] == '\r' && r->cursor[1] == '\n') + return 1; + return 0; +} + +static inline void _rewind_optional_cr(struct http_request *r) +{ + if (r->cursor > r->parsed && r->cursor[-1] == '\r') + --r->cursor; +} + +static inline void _rewind_crlf(struct http_request *r) +{ + assert(r->cursor >= r->parsed + 2); + assert(r->cursor[-2] == '\r'); + assert(r->cursor[-1] == '\n'); + r->cursor -= 2; +} + +/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF + * as part of the end-of-line sequence and treats the CR as optional. This allows simple manual + * testing using telnet(1). + */ +static inline int _skip_eol(struct http_request *r) +{ + unsigned crcount = 0; + for (; !_run_out(r); ++r->cursor) { + switch (*r->cursor) { + case '\0': // ignore any leading NULs (telnet inserts them) + break; + case '\r': // ignore up to one leading CR + if (++crcount > 1) + return 0; + break; + case '\n': + ++r->cursor; + return 1; + default: + return 0; + } + } + return 0; +} + +/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF + * as part of the end-of-line sequence and treats the CR as optional. This allows simple manual + * testing using telnet(1). + */ +static int _skip_to_eol(struct http_request *r) +{ + const char *const start = r->cursor; + while (!_run_out(r) && *r->cursor != '\n') + ++r->cursor; + if (_run_out(r)) + return 0; + // consume preceding NULs (telnet inserts them) + while (r->cursor > start && r->cursor[-1] == '\0') + --r->cursor; + // consume a single preceding CR + if (r->cursor > start && r->cursor[-1] == '\r') + --r->cursor; + // consume any more preceding NULs + while (r->cursor > start && r->cursor[-1] == '\0') + --r->cursor; + return 1; +} + +static int _skip_literal(struct http_request *r, const char *literal) +{ + while (!_run_out(r) && *literal && *r->cursor == *literal) + ++literal, ++r->cursor; + return *literal == '\0'; +} + +static int _skip_literal_nocase(struct http_request *r, const char *literal) +{ + while (!_run_out(r) && *literal && toupper(*r->cursor) == toupper(*literal)) + ++literal, ++r->cursor; + return *literal == '\0'; +} + +static int _skip_optional_space(struct http_request *r) +{ + while (!_run_out(r) && (*r->cursor == ' ' || *r->cursor == '\t')) + ++r->cursor; + return 1; +} + +static inline int _skip_space(struct http_request *r) +{ + const char *const start = r->cursor; + _skip_optional_space(r); + return r->cursor > start; +} + +static size_t _skip_word_printable(struct http_request *r, struct substring *str) +{ + if (_run_out(r) || isspace(*r->cursor) || !isprint(*r->cursor)) + return 0; + const char *start = r->cursor; + for (++r->cursor; !_run_out(r) && !isspace(*r->cursor) && isprint(*r->cursor); ++r->cursor) + ; + if (_run_out(r)) + return 0; + assert(r->cursor > start); + assert(isspace(*r->cursor)); + if (str) { + str->start = start; + str->end = r->cursor; + } + return r->cursor - start; +} + +static size_t _skip_token(struct http_request *r, struct substring *str) +{ + if (_run_out(r) || !is_http_token(*r->cursor)) + return 0; + const char *start = r->cursor; + for (++r->cursor; !_run_out(r) && is_http_token(*r->cursor); ++r->cursor) + ; + if (_run_out(r)) + return 0; + assert(r->cursor > start); + assert(!is_http_token(*r->cursor)); + if (str) { + str->start = start; + str->end = r->cursor; + } + return r->cursor - start; +} + +static size_t _parse_token(struct http_request *r, char *dst, size_t dstsiz) +{ + struct substring str; + size_t len = _skip_token(r, &str); + if (len && dst) { + size_t cpy = len < dstsiz - 1 ? len : dstsiz - 1; + strncpy(dst, str.start, cpy)[cpy] = '\0'; + } + return len; +} + +static size_t _parse_quoted_string(struct http_request *r, char *dst, size_t dstsiz) +{ + assert(r->cursor <= r->end); + if (_run_out(r) || *r->cursor != '"') + return 0; + int slosh = 0; + size_t len = 0; + for (++r->cursor; !_run_out(r); ++r->cursor) { + if (!isprint(*r->cursor)) + return 0; + if (slosh) { + if (dst && len < dstsiz - 1) + dst[len] = *r->cursor; + ++len; + slosh = 0; + } else if (*r->cursor == '"') + break; + else if (*r->cursor == '\\') + slosh = 1; + else { + if (dst && len < dstsiz - 1) + dst[len] = *r->cursor; + ++len; + } + } + if (dst) + dst[len < dstsiz - 1 ? len : dstsiz - 1] = '\0'; + if (_run_out(r)) + return 0; + assert(*r->cursor == '"'); + ++r->cursor; + return len; +} + +static size_t _parse_token_or_quoted_string(struct http_request *r, char *dst, size_t dstsiz) +{ + assert(dstsiz > 0); + if (!_run_out(r) && *r->cursor == '"') + return _parse_quoted_string(r, dst, dstsiz); + return _parse_token(r, dst, dstsiz); +} + +static inline int _parse_http_size_t(struct http_request *r, http_size_t *szp) +{ + return !_run_out(r) && isdigit(*r->cursor) && str_to_uint64(r->cursor, 10, szp, &r->cursor); +} + +static inline int _parse_uint(struct http_request *r, unsigned int *uintp) +{ + return !_run_out(r) && isdigit(*r->cursor) && str_to_uint(r->cursor, 10, uintp, &r->cursor); +} + +static unsigned _parse_ranges(struct http_request *r, struct http_range *range, unsigned nrange) +{ + unsigned i = 0; + while (1) { + enum http_range_type type; + http_size_t first = 0, last = 0; + if (_skip_literal(r, "-")) { + if (!_parse_http_size_t(r, &last)) + return 0; + type = SUFFIX; + } + else if (_parse_http_size_t(r, &first) && _skip_literal(r, "-")) { + if (_parse_http_size_t(r, &last)) { + if (last < first) + return 0; + type = CLOSED; + } else + type = OPEN; + } else + return 0; + if (i < nrange) { + range[i].type = type; + range[i].first = first; + range[i].last = last; + } + ++i; + if (!_skip_literal(r, ",")) + break; + _skip_optional_space(r); + } + return i; +} + +static int _parse_quoted_rfc822_time(struct http_request *r, time_t *timep) +{ + char datestr[40]; + size_t n = _parse_quoted_string(r, datestr, sizeof datestr); + if (n == 0 || n >= sizeof datestr) + return 0; + // TODO: Move the following code into its own function in str.c + struct tm tm; + bzero(&tm, sizeof tm); + // TODO: Ensure this works in non-English locales, ie, "%a" still accepts "Mon", "Tue" etc. and + // "%b" still accepts "Jan", "Feb" etc. + // TODO: Support symbolic time zones, eg, "UT", "GMT", "UTC", "EST"... + const char *c = strptime(datestr, "%a, %d %b %Y %T ", &tm); + if ((c[0] == '-' || c[0] == '+') && isdigit(c[1]) && isdigit(c[2]) && isdigit(c[3]) && isdigit(c[4]) && c[5] == '\0') { + time_t zone = (c[0] == '-' ? -1 : 1) * ((c[1] - '0') * 600 + (c[2] - '0') * 60 + (c[3] - '0') * 10 + (c[4] - '0')); + const char *tz = getenv("TZ"); + if (tz) + tz = alloca_strdup(tz); + setenv("TZ", "", 1); + tzset(); + *timep = mktime(&tm) - zone; + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + tzset(); + return 1; + } + return 0; +} + +/* If parsing completes, then sets r->parser to the next parsing function and returns 0. If parsing + * cannot complete due to running out of data, returns 100 without changing r->parser, so this + * function will be called again once more data has been read. Returns a 4nn or 5nn HTTP result + * code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_verb(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + _rewind(r); + assert(r->cursor >= r->received); + assert(!_run_out(r)); + // Parse verb: GET, PUT, POST, etc. + assert(r->verb == NULL); + unsigned i; + for (i = 0; i < NELS(http_verbs); ++i) { + _rewind(r); + if (_skip_literal(r, http_verbs[i].word) && _skip_literal(r, " ")) { + r->verb = http_verbs[i].word; + break; + } + if (_run_out(r)) + return 100; // read more and try again + } + if (r->verb == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP request, invalid verb: %s", alloca_toprint(20, r->cursor, r->end - r->cursor)); + return 400; + } + _commit(r); + r->parser = http_request_parse_path; + return 0; +} + +/* If parsing completes, then sets r->parser to the next parsing function and returns 0. If parsing + * cannot complete due to running out of data, returns 100 without changing r->parser, so this + * function will be called again once more data has been read. Returns a 4nn or 5nn HTTP result + * code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_path(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + // Parse path: word immediately following verb, delimited by spaces. + assert(r->path == NULL); + struct substring path; + if (!(_skip_word_printable(r, &path) && _skip_literal(r, " "))) { + if (_run_out(r)) + return 100; // read more and try again + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request at path: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed)); + return 400; + } + _commit(r); + if ((r->path = _reserve(r, path)) == NULL) + return 0; // error + r->parser = http_request_parse_http_version; + return 0; +} + +/* If parsing completes, then sets r->parser to the next parsing function and returns 0. If parsing + * cannot complete due to running out of data, returns 100 without changing r->parser, so this + * function will be called again once more data has been read. Returns a 4nn or 5nn HTTP result + * code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_http_version(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + // Parse HTTP version: HTTP/m.n followed by CRLF. + assert(r->version_major == 0); + assert(r->version_minor == 0); + unsigned major, minor; + if (!( _skip_literal(r, "HTTP/") + && _parse_uint(r, &major) + && major > 0 && major < UINT8_MAX + && _skip_literal(r, ".") + && _parse_uint(r, &minor) + && minor < UINT8_MAX + && _skip_eol(r) + ) + ) { + if (_run_out(r)) + return 100; // read more and try again + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request at version: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed)); + return 400; + } + _commit(r); + r->version_major = major; + r->version_minor = minor; + r->parser = http_request_start_parsing_headers; + if (r->handle_first_line) + return r->handle_first_line(r); + return 0; // parsing complete +} + +/* Select the header parser. Returns 0 after setting the new parser function. Returns a 4nn or 5nn + * HTTP result code if the request cannot be handled (eg, unsupported HTTP version or invalid path). + * + * @author Andrew Bettison + */ +static int http_request_start_parsing_headers(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + assert(r->verb != NULL); + assert(r->path != NULL); + assert(r->version_major != 0); + if (r->version_major != 1) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP version: %u.%u", r->version_major, r->version_minor); + return 400; + } + r->parser = http_request_parse_header; + return 0; +} + +/* Parse one request header line. + * + * If the end of headers is parsed (blank line), then sets r->parser to the next parsing function + * and returns 0. If a single header line is successfully parsed, returns 0 after advancing + * r->parsed. If parsing cannot complete due to running out of data, returns 0 without changing + * r->parser, so this function will be called again once more data has been read. Returns a 4nn or + * 5nn HTTP result code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_header(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + _skip_to_eol(r); + const char *const eol = r->cursor; + _skip_eol(r); + if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)... + _commit(r); + if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + size_t unparsed = r->end - r->parsed; + if (unparsed > r->request_header.content_length) { + WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length)); + r->request_content_remaining = 0; + } + else + r->request_content_remaining = r->request_header.content_length - unparsed; + } + r->parser = http_request_start_body; + if (r->handle_headers) + return r->handle_headers(r); + return 0; + } + const char *const nextline = r->cursor; + _rewind(r); + const char *const sol = r->cursor; + if (_skip_literal_nocase(r, "Content-Length:")) { + _skip_optional_space(r); + http_size_t length; + if (_parse_http_size_t(r, &length) && _skip_optional_space(r) && r->cursor == eol) { + r->cursor = nextline; + _commit(r); + r->request_header.content_length = length; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Content-Length: %"PRIhttp_size_t, r->request_header.content_length); + return 0; + } + goto malformed; + } + _rewind(r); + if (_skip_literal_nocase(r, "Content-Type:")) { + _skip_optional_space(r); + struct substring type = substring_NULL; + struct substring subtype = substring_NULL; + char boundary[BOUNDARY_STRING_MAXLEN + 1]; + boundary[0] = '\0'; + if (_skip_token(r, &type) && _skip_literal(r, "/") && _skip_token(r, &subtype)) { + // Parse zero or more content-type parameters. + for (_skip_optional_space(r); r->cursor < eol && _skip_literal(r, ";"); _skip_optional_space(r)) { + _skip_optional_space(r); + const char *startparam = r->cursor; + if (_skip_literal(r, "boundary=")) { + size_t n = _parse_token_or_quoted_string(r, boundary, sizeof boundary); + if (n == 0 || n >= sizeof boundary || !is_valid_http_boundary_string(boundary)) + goto malformed; + continue; + } + // Silently ignore unrecognised parameters (eg, charset=) if they are well formed. + r->cursor = startparam; // partial rewind + if (_skip_token(r, NULL) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) + continue; + break; + } + if (r->cursor == eol) { + r->cursor = nextline; + _commit(r); + if ( (r->request_header.content_type = _reserve(r, type)) == NULL + || (r->request_header.content_subtype = _reserve(r, subtype)) == NULL + || (boundary[0] && (r->request_header.boundary = _reserve_str(r, boundary)) == NULL) + ) + return 0; // error + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Content-type: %s/%s%s%s", + r->request_header.content_type, + r->request_header.content_subtype, + r->request_header.boundary ? "; boundary=" : "", + r->request_header.boundary ? alloca_str_toprint(r->request_header.boundary) : "" + ); + return 0; + } + } + goto malformed; + } + _rewind(r); + if (_skip_literal_nocase(r, "Range:")) { + _skip_optional_space(r); + unsigned int n; + if ( _skip_literal(r, "bytes=") + && (n = _parse_ranges(r, r->request_header.content_ranges, NELS(r->request_header.content_ranges))) + && _skip_optional_space(r) + && (r->cursor == eol) + ) { + r->cursor = nextline; + _commit(r); + if (n > NELS(r->request_header.content_ranges)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("HTTP request Range header overflow (%u ranges in set, can only handle %u): %s", + n, NELS(r->request_header.content_ranges), alloca_toprint(-1, sol, eol - sol)); + // In this case ignore the Range: header -- respond with the entire resource. + r->request_header.content_range_count = 0; + } else { + r->request_header.content_range_count = n; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Range: bytes=%s", alloca_http_ranges(r->request_header.content_ranges)); + } + return 0; + } + goto malformed; + } + _rewind(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipped HTTP request header: %s", alloca_toprint(-1, sol, eol - sol)); + r->cursor = nextline; + _commit(r); + return 0; +malformed: + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP request header: %s", alloca_toprint(-1, sol, eol - sol)); + return 400; +} + +/* If parsing completes, then sets r->parser to the next parsing function and returns 0. If parsing + * cannot complete due to running out of data, returns 0 without changing r->parser, so this + * function will be called again once more data has been read. Returns a 4nn or 5nn HTTP result + * code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_start_body(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + assert(r->verb != NULL); + assert(r->path != NULL); + assert(r->version_major != 0); + assert(r->parsed <= r->end); + if (r->verb == HTTP_VERB_GET) { + // TODO: Implement HEAD requests (only send response header, not body) + if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb); + return 400; + } + if (r->request_header.content_type) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: Content-Type not allowed", r->verb); + return 400; + } + r->parser = NULL; + } + else if (r->verb == HTTP_VERB_POST) { + if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb); + return 411; + } + if (r->request_header.content_type == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: missing Content-Type header", r->verb); + return 400; + } + if ( strcmp(r->request_header.content_type, "multipart") == 0 + && strcmp(r->request_header.content_subtype, "form-data") == 0 + ) { + if (r->request_header.boundary == NULL || r->request_header.boundary[0] == '\0') { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: Content-Type %s/%s missing boundary parameter", + r->verb, r->request_header.content_type, r->request_header.content_subtype); + return 400; + } + r->parser = http_request_parse_body_form_data; + r->form_data_state = START; + } else { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP %s request: Content-Type %s/%s not supported", + r->verb, r->request_header.content_type, r->request_header.content_subtype); + return 415; + } + } + else { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP %s request", r->verb); + r->parser = NULL; + return 501; + } + if (_run_out(r)) + return 100; + return 0; +} + +/* Returns 1 if a MIME delimiter is skipped, 2 if a MIME close-delimiter is skipped. + */ +static int _skip_mime_boundary(struct http_request *r) +{ + if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.boundary)) + return 0; + if (_skip_literal(r, "--") && _skip_crlf(r)) + return 2; + if (_skip_crlf(r)) + return 1; + return 0; +} + +static int _parse_content_disposition(struct http_request *r, struct mime_content_disposition *cd) +{ + size_t n = _parse_token(r, cd->type, sizeof cd->type); + if (n == 0) + return 0; + if (n >= sizeof cd->type) { + WARNF("HTTP Content-Disposition type truncated: %s", alloca_str_toprint(cd->type)); + return 0; + } + while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) { + const char *start = r->cursor; + if (_skip_literal(r, "filename=")) { + size_t n = _parse_token_or_quoted_string(r, cd->filename, sizeof cd->filename); + if (n == 0) + return 0; + if (n >= sizeof cd->filename) { + WARNF("HTTP Content-Disposition filename truncated: %s", alloca_str_toprint(cd->filename)); + return 0; + } + continue; + } + r->cursor = start; + if (_skip_literal(r, "name=")) { + size_t n = _parse_token_or_quoted_string(r, cd->name, sizeof cd->name); + if (n == 0) + return 0; + if (n >= sizeof cd->name) { + WARNF("HTTP Content-Disposition name truncated: %s", alloca_str_toprint(cd->name)); + return 0; + } + continue; + } + r->cursor = start; + if (_skip_literal(r, "size=")) { + if (!_parse_http_size_t(r, &cd->size)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "creation-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->creation_date)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "modification-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->modification_date)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "read-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->read_date)) + goto malformed; + continue; + } + r->cursor = start; + struct substring param; + if (_skip_token(r, ¶m) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping HTTP Content-Disposition parameter: %s", alloca_substring_toprint(param)); + continue; + } +malformed: + WARNF("Malformed HTTP Content-Disposition: %s", alloca_toprint(50, r->cursor, r->end - r->cursor)); + return 0; + } + return 1; +} + +/* If parsing completes (ie, parsed to end of epilogue), then sets r->parser to NULL and returns 0, + * so this function will not be called again. If parsing cannot complete due to running out of + * data, returns 100, so this function will not be called again until more data has been read. + * Returns a 4nn or 5nn HTTP result code if parsing fails. Returns -1 if an unexpected error + * occurs. + * + * NOTE: No support for nested/mixed parts, as that would considerably complicate the parser. If + * the need arises in future, we will deal with it then. In the meantime, we will have something + * that meets our immediate needs for Rhizome Direct and a variety of use cases. + * + * @author Andrew Bettison + */ +static int http_request_parse_body_form_data(struct http_request *r) +{ + DEBUG_DUMP_PARSER(r); + int at_start = 0; + switch (r->form_data_state) { + case START: + if (config.debug.httpd) + DEBUGF("START"); + // The logic here allows for a missing initial CRLF before the first boundary line. + at_start = 1; + r->form_data_state = PREAMBLE; + // fall through + case PREAMBLE: { + if (config.debug.httpd) + DEBUGF("PREAMBLE"); + const char *start = r->parsed; + for (; at_start || _skip_to_crlf(r); at_start = 0) { + const char *end_preamble = r->cursor; + int b; + if ((b = _skip_mime_boundary(r))) { + assert(end_preamble >= r->parsed); + if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_preamble(%s length=%zu)", + alloca_toprint(50, r->parsed, end_preamble - r->parsed), end_preamble - r->parsed); + r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed); + } + _rewind_crlf(r); + _commit(r); + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_start()"); + r->form_data.handle_mime_part_start(r); + } + } else + r->form_data_state = EPILOGUE; + return 0; + } + } + if (_end_of_content(r)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s form data: missing first boundary", r->verb); + return 400; + } + _rewind_optional_cr(r); + _commit(r); + if (r->parsed > start && r->form_data.handle_mime_preamble) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_preamble(%s length=%zu)", + alloca_toprint(50, start, r->parsed - start), r->parsed - start); + r->form_data.handle_mime_preamble(r, start, r->parsed - start); + } + } + return 100; // need more data + case HEADER: { + if (config.debug.httpd) + DEBUGF("HEADER"); + // If not at a CRLF, then we are skipping through an over-long header that didn't + // fit into the buffer. Just discard bytes up to the next CRLF. + if (!_skip_crlf(r)) { + _skip_to_crlf(r); // advance to next CRLF or end of buffer + _rewind_optional_cr(r); // don't skip a CR at end of buffer (it might be part of a half-received CRLF) + assert(r->cursor > r->parsed); + if (r->debug_flag && *r->debug_flag) + DEBUGF("skipping %zu header bytes", r->cursor - r->parsed); + _commit(r); + return 0; + } + const char *sol = r->cursor; + // A blank line finishes the headers. The CRLF does not form part of the body. + if (_skip_crlf(r)) { + _commit(r); + r->form_data_state = BODY; + return 0; + } + if (_run_out(r)) + return 100; // read more and try again + r->cursor = sol; + // A mime boundary technically should not occur in the middle of the headers, but if it + // does, treat it as a zero-length body. + int b; + if ((b = _skip_mime_boundary(r))) { + _rewind_crlf(r); + _commit(r); + if (r->form_data.handle_mime_part_end) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_end()"); + r->form_data.handle_mime_part_end(r); + } + // A boundary in the middle of headers finishes the current part and starts a new part. + // An end boundary terminates the current part and starts the epilogue. + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_start()"); + r->form_data.handle_mime_part_start(r); + } + } + else + r->form_data_state = EPILOGUE; + return 0; + } + if (_run_out(r)) + return 100; // read more and try again + r->cursor = sol; + struct substring label; + if (_skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) { + size_t labellen = label.end - label.start; + char labelstr[labellen + 1]; + strncpy(labelstr, label.start, labellen)[labellen] = '\0'; + str_tolower_inplace(labelstr); + const char *value = r->cursor; + if (strcmp(labelstr, "content-disposition") == 0) { + struct mime_content_disposition cd; + bzero(&cd, sizeof cd); + if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) { + _rewind_crlf(r); + _commit(r); + if (r->form_data.handle_mime_content_disposition) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_content_disposition(%s)", alloca_mime_content_disposition(&cd)); + r->form_data.handle_mime_content_disposition(r, &cd); + } + return 0; + } + } + else if (_skip_to_crlf(r)) { + _commit(r); + if (r->form_data.handle_mime_header) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_header(%s, %s)", alloca_str_toprint(labelstr), alloca_toprint(-1, value, value - r->cursor)); + r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end + } + return 0; + } + } + r->cursor = sol; + if (_buffer_full(r)) { + // The line does not start with "Token:" and is too long to fit into the buffer. Start + // skipping it. + WARNF("Skipping unterminated HTTP MIME header %s", alloca_toprint(50, sol, r->end - sol)); + r->cursor = r->end; + _rewind_optional_cr(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("skipping %zu header bytes", r->cursor - r->parsed); + _commit(r); + return 0; + } + if (_run_out(r)) + return 100; // read more and try again + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(50, sol, r->end - sol)); + DEBUG_DUMP_PARSER(r); + } + return 400; + case BODY: + if (config.debug.httpd) + DEBUGF("BODY"); + const char *start = r->parsed; + while (_skip_to_crlf(r)) { + int b; + const char *end_body = r->cursor; + _skip_crlf(r); + if ((b = _skip_mime_boundary(r))) { + _rewind_crlf(r); + _commit(r); + if (end_body > start && r->form_data.handle_mime_body) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, end_body - start), end_body - start); + r->form_data.handle_mime_body(r, start, end_body - start); // excluding CRLF at end + } + if (r->form_data.handle_mime_part_end) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_end()"); + r->form_data.handle_mime_part_end(r); + } + r->form_data_state = EPILOGUE; + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_start()"); + r->form_data.handle_mime_part_start(r); + } + } + return 0; + } + } + if (_end_of_content(r)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s form data part: missing end boundary", r->verb); + return 400; + } + _rewind_optional_cr(r); + _commit(r); + if (r->parsed > start && r->form_data.handle_mime_body) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, r->parsed - start), r->parsed - start); + r->form_data.handle_mime_body(r, start, r->parsed - start); + } + return 100; // need more data + case EPILOGUE: + if (config.debug.httpd) + DEBUGF("EPILOGUE"); + r->cursor = r->end; + if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_epilogue(%s length=%zu)", + alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed); + r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed); + } + _commit(r); + assert(_run_out(r)); + if (_end_of_content(r)) + return 0; // done + return 100; // need more data + } + abort(); // not reached +} + +static ssize_t http_request_read(struct http_request *r, char *buf, size_t len) +{ + sigPipeFlag = 0; + ssize_t bytes = read_nonblock(r->alarm.poll.fd, buf, len); + if (bytes == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("HTTP socket read error, closing connection"); + http_request_finalise(r); + return -1; + } + if (sigPipeFlag) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Received SIGPIPE on HTTP socket read, closing connection"); + http_request_finalise(r); + return -1; + } + return bytes; +} + +static void http_request_receive(struct http_request *r) +{ + assert(r->phase == RECEIVE); + const char *const bufend = r->buffer + sizeof r->buffer; + assert(r->end <= bufend); + assert(r->parsed >= r->received); + assert(r->parsed <= r->end); + // If the end of content falls within the buffer, then there is no need to make any more room, + // just read up to the end of content. Otherwise, If buffer is running short on unused space, + // shift existing content in buffer down to make more room if possible. + size_t room = bufend - r->end; + if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && room > r->request_content_remaining) + room = r->request_content_remaining; + else { + size_t spare = r->parsed - r->received; + if (spare && (room < 128 || (room < 1024 && spare >= 32))) { + size_t unparsed = r->end - r->parsed; + memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst + r->parsed = r->received; + r->end = r->received + unparsed; + room = bufend - r->end; + if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && room > r->request_content_remaining) + room = r->request_content_remaining; + } + } + // If there is no more buffer space, fail the request. + if (room == 0) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Buffer size reached, reporting overflow"); + http_request_simple_response(r, 431, NULL); + return; + } + // Read up to the end of available buffer space or the end of content, whichever is first. Read + // as many bytes as possible into the unused buffer space. Any read error closes the connection + // without any response. + assert(room > 0); + if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN) + assert(room <= r->request_content_remaining); + ssize_t bytes = http_request_read(r, (char *)r->end, room); + if (bytes == -1) + return; + assert((size_t) bytes <= room); + // If no data was read, then just return to polling. Don't drop the connection on an empty read, + // because that drops connections when they shouldn't, including during testing. The inactivity + // timeout will drop inactive connections. + if (bytes == 0) + return; + r->end += (size_t) bytes; + if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN) + r->request_content_remaining -= (size_t) bytes; + // We got some data, so reset the inactivity timer and invoke the parsing state machine to process + // it. The state machine invokes the caller-supplied callback functions. + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + unschedule(&r->alarm); + schedule(&r->alarm); + // Parse the unparsed and received data. + while (r->phase == RECEIVE) { + int result; + _rewind(r); + DEBUG_DUMP_PARSER(r); + if (_end_of_content(r)) { + if (r->handle_content_end) + result = r->handle_content_end(r); + else { + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: no end-of-content function set"); + result = 500; + } + } else { + HTTP_REQUEST_PARSER oldparser = r->parser; + const char *oldparsed = r->parsed; + if (r->parser == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("No HTTP parser function set -- skipping %zu bytes", (size_t)(r->end - r->cursor)); + _skip_all(r); + _commit(r); + result = 0; + } else { + result = r->parser(r); + assert(r->parsed >= oldparsed); + } + if (r->phase != RECEIVE) + break; + if (result == 100) + return; // needs more data; poll again + if (result == 0 && r->parsed == oldparsed && r->parser == oldparser) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: parser function did not advance"); + DEBUG_DUMP_PARSER(r); + result = 500; + } + } + if (result >= 300) + r->response.result_code = result; + else if (result) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Internal failure parsing HTTP request: invalid result=%d", result); + r->response.result_code = 500; + } + if (r->response.result_code) + break; + if (result == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Unrecoverable error parsing HTTP request, closing connection"); + http_request_finalise(r); + return; + } + } + if (r->phase != RECEIVE) + return; + if (r->response.result_code == 0) { + WHY("No HTTP response set, using 500 Server Error"); + r->response.result_code = 500; + } + http_request_start_response(r); +} + +/* Write the current contents of the response buffer to the HTTP socket. When no more bytes can be + * written, return so that socket polling can continue. Once all bytes are sent, if there is a + * content generator function, invoke it to put more content in the response buffer, and write that + * content. + * + * @author Andrew Bettison + */ +static void http_request_send_response(struct http_request *r) +{ + assert(r->response_sent <= r->response_length); + while (r->response_sent < r->response_length) { + assert(r->response_buffer_sent <= r->response_buffer_length); + if (r->response_buffer_sent == r->response_buffer_length) { + if (r->response.content_generator) { + // Content generator must fill or partly fill response_buffer and set response_buffer_sent + // and response_buffer_length. May also malloc() a bigger buffer and set response_buffer to + // point to it. + r->response_buffer_sent = r->response_buffer_length = 0; + if (r->response.content_generator(r) == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Content generation error, closing connection"); + http_request_finalise(r); + return; + } + assert(r->response_buffer_sent <= r->response_buffer_length); + if (r->response_buffer_sent == r->response_buffer_length) { + WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)", + r->response_sent, r->response_length, r->response_length - r->response_sent); + http_request_finalise(r); + return; + } + } else { + WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes", + r->response_length, r->response_length - r->response_sent); + http_request_finalise(r); + return; + } + } + assert(r->response_buffer_sent < r->response_buffer_length); + size_t bytes = r->response_buffer_length - r->response_buffer_sent; + if (r->response_sent + bytes > r->response_length) { + WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating", + r->response_length, + r->response_sent + bytes - r->response_length); + bytes = r->response_length - r->response_sent; + } + sigPipeFlag = 0; + ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes); + if (written == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("HTTP socket write error, closing connection"); + http_request_finalise(r); + return; + } + if (sigPipeFlag) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Received SIGPIPE on HTTP socket write, closing connection"); + http_request_finalise(r); + return; + } + // If we wrote nothing, go back to polling. + if (written == 0) + return; + r->response_sent += (size_t) written; + r->response_buffer_sent += (size_t) written; + assert(r->response_sent <= r->response_length); + assert(r->response_buffer_sent <= r->response_buffer_length); + // Reset inactivity timer. + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + unschedule(&r->alarm); + schedule(&r->alarm); + // If we wrote less than we tried, then go back to polling. + if (written < (size_t) bytes) + return; + } + if (r->debug_flag && *r->debug_flag) + DEBUG("Done, closing connection"); + http_request_finalise(r); +} + +static void http_server_poll(struct sched_ent *alarm) +{ + struct http_request *r = (struct http_request *) alarm; + if (alarm->poll.revents == 0) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Timeout, closing connection"); + http_request_finalise(r); + } + else if (alarm->poll.revents & (POLLHUP | POLLERR)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Poll error (%s), closing connection", alloca_poll_events(alarm->poll.revents)); + http_request_finalise(r); + } + else { + if (r->phase == RECEIVE && (alarm->poll.revents & POLLIN)) + http_request_receive(r); // this could change the phase to TRANSMIT + if (r->phase == TRANSMIT && (alarm->poll.revents & POLLOUT)) + http_request_send_response(r); + } + // Any of the above calls could change the phase to DONE. + if (r->phase == DONE && r->free) + r->free(r); // after this, *r is no longer valid +} + +/* Copy the array of byte ranges, closing it (converting all ranges to CLOSED) using the supplied + * resource length. If a range is not satisfiable it is omitted from 'dst'. Returns the number of + * closed ranges written to 'dst'. + * + * @author Andrew Bettison + */ +unsigned http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t resource_length) +{ + unsigned i; + unsigned ndst = 0; + for (i = 0; i != nranges; ++i) { + http_size_t first = 0; + http_size_t last = resource_length - 1; + const struct http_range *range = &src[i]; + switch (range->type) { + case CLOSED: + last = range->last < resource_length ? range->last : resource_length - 1; + case OPEN: + first = range->first < resource_length ? range->first : resource_length; + break; + case SUFFIX: + first = range->last < resource_length ? resource_length - range->last : 0; + break; + default: + abort(); // not reached + } + if (first <= last) + dst[ndst++] = (struct http_range){ .type = CLOSED, .first=first, .last=last }; + } + return ndst; +} + +/* Return the total number of bytes represented by the given ranges which must all be CLOSED and + * valid. + * + * @author Andrew Bettison + */ +http_size_t http_range_bytes(const struct http_range *range, unsigned nranges) +{ + http_size_t bytes = 0; + unsigned i; + for (i = 0; i != nranges; ++i) { + assert(range[i].type == CLOSED); + assert(range[i].last >= range[i].first); + bytes += range[i].last - range[i].first + 1; + } + return bytes; +} + +/* Return appropriate message for HTTP response codes, both known and unknown. + */ +static const char *httpResultString(int response_code) +{ + switch (response_code) { + case 200: return "OK"; + case 201: return "Created"; + case 206: return "Partial Content"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 411: return "Length Required"; + case 414: return "Request-URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Requested Range Not Satisfiable"; + case 431: return "Request Header Fields Too Large"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + default: return (response_code <= 4) ? "Unknown status code" : "A suffusion of yellow"; + } +} + +static strbuf strbuf_append_quoted_string(strbuf sb, const char *str) +{ + strbuf_putc(sb, '"'); + for (; *str; ++str) { + if (*str == '"' || *str == '\\') + strbuf_putc(sb, '\\'); + strbuf_putc(sb, *str); + } + strbuf_putc(sb, '"'); + return sb; +} + +/* Render the HTTP response into the current response buffer. Return 1 if it fits, 0 if it does + * not. The buffer response_pointer may be NULL, in which case no response is rendered, but the + * content_length is still computed + * + * @author Andrew Bettison + */ +static int _render_response(struct http_request *r) +{ + struct http_response hr = r->response; + assert(hr.result_code != 0); + assert(hr.header.content_range_start <= hr.header.resource_length); + assert(hr.header.content_length <= hr.header.resource_length); + // To save page handlers having to decide between 200 (OK) and 206 (Partial Content), they can + // just send 200 and the content range fields, and this logic will detect if it should be 206. + if (hr.header.content_length > 0 && hr.header.content_length < hr.header.resource_length && hr.result_code == 200) + hr.result_code = 206; // Partial Content + const char *result_string = httpResultString(hr.result_code); + strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size); + if (hr.content == NULL && hr.content_generator == NULL) { + strbuf cb = strbuf_alloca(100 + strlen(result_string)); + strbuf_puts(cb, "

"); + strbuf_puts(cb, result_string); + strbuf_puts(cb, "

\r\n"); + hr.content = strbuf_str(cb); + hr.header.resource_length = hr.header.content_length = strbuf_len(cb); + hr.header.content_type = "text/html"; + hr.header.content_range_start = 0; + } + assert(hr.header.content_type != NULL); + assert(hr.header.content_type[0]); + strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", hr.result_code, result_string); + strbuf_sprintf(sb, "Content-Type: %s", hr.header.content_type); + if (hr.header.boundary) { + strbuf_puts(sb, "; boundary="); + if (strchr(hr.header.boundary, '"') || strchr(hr.header.boundary, '\\')) + strbuf_append_quoted_string(sb, hr.header.boundary); + else + strbuf_puts(sb, hr.header.boundary); + } + strbuf_puts(sb, "\r\n"); + if (hr.result_code == 206) { + // Must only use result code 206 (Partial Content) if the content is in fact less than the whole + // resource length. + assert(hr.header.content_length > 0); + assert(hr.header.content_length < hr.header.resource_length); + strbuf_sprintf(sb, + "Content-Range: bytes %"PRIhttp_size_t"-%"PRIhttp_size_t"/%"PRIhttp_size_t"\r\n", + hr.header.content_range_start, + hr.header.content_range_start + hr.header.content_length - 1, + hr.header.resource_length + ); + } + strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length); + strbuf_puts(sb, "\r\n"); + if (strbuf_overrun(sb)) + return 0; + r->response_length = strbuf_len(sb) + hr.header.content_length; + if (hr.content) { + if (r->response_buffer_size < r->response_length) + return 0; + bcopy(hr.content, strbuf_end(sb), hr.header.content_length); + r->response_buffer_length = r->response_length; + } else { + r->response_buffer_length = strbuf_len(sb); + } + r->response_buffer_sent = 0; + return 1; +} + +/* Returns with the length of the rendered response in r->response_length. If the rendered response + * did not fit into any available buffer, then returns with r->response_buffer == NULL, otherwise + * r->response_buffer points to the rendered response. + * + * @author Andrew Bettison + */ +static void http_request_render_response(struct http_request *r) +{ + // If there is no response buffer allocated yet, use the available part of the in-struct buffer. + http_request_set_response_bufsize(r, 1); + // Try rendering the response into the existing buffer. This will discover the length of the + // rendered headers, so after this step, whether or not the buffer was overrun, we know the total + // length of the response. + if (!_render_response(r)) { + // If the response did not fit into the existing buffer, then allocate a large buffer from the + // heap and try rendering again. + if (http_request_set_response_bufsize(r, r->response_length + 1) == -1) + WHY("Cannot render HTTP response, out of memory"); + else if (!_render_response(r)) + FATAL("Re-render of HTTP response overflowed buffer"); + } +} + +static size_t http_request_drain(struct http_request *r) +{ + assert(r->phase == RECEIVE); + char buf[8192]; + size_t drained = 0; + ssize_t bytes; + while ((bytes = http_request_read(r, buf, sizeof buf)) != -1 && bytes != 0) + drained += (size_t) bytes; + return drained; +} + +static void http_request_start_response(struct http_request *r) +{ + assert(r->phase == RECEIVE); + assert(r->response.result_code != 0); + if (r->response.content || r->response.content_generator) { + assert(r->response.header.content_type != NULL); + assert(r->response.header.content_type[0]); + } + // If HTTP responses are disabled (eg, for testing purposes) then skip all response construction + // and close the connection. + if (r->disable_tx_flag && *r->disable_tx_flag) { + INFO("HTTP transmit disabled, closing connection"); + http_request_finalise(r); + return; + } + // Drain the rest of the request that has not been received yet (eg, if sending an error response + // provoked while parsing the early part of a partially-received request). If a read error + // occurs, the connection is closed so the phase changes to DONE. + http_request_drain(r); + if (r->phase != RECEIVE) + return; + // If the response cannot be rendered, then render a 500 Server Error instead. If that fails, + // then just close the connection. + http_request_render_response(r); + if (r->response_buffer == NULL) { + WARN("Cannot render HTTP response, sending 500 Server Error instead"); + r->response.result_code = 500; + r->response.content = NULL; + http_request_render_response(r); + if (r->response_buffer == NULL) { + WHY("Cannot render HTTP 500 Server Error response, closing connection"); + http_request_finalise(r); + return; + } + } + r->response_sent = 0; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_buffer_length)); + r->phase = TRANSMIT; + r->alarm.poll.events = POLLOUT; + watch(&r->alarm); +} + +/* Start sending a static (pre-computed) response back to the client. The response's Content-Type + * is set by the 'mime_type' parameter (in the standard format "type/subtype"). The response's + * content is set from the 'body' and 'bytes' parameters, which need not point to persistent data, + * ie, the memory pointed to by 'body' is no longer referenced once this function returns. + * + * @author Andrew Bettison + */ +void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes) +{ + assert(r->phase == RECEIVE); + assert(result >= 100); + assert(result < 300); + assert(mime_type != NULL); + assert(mime_type[0]); + r->response.result_code = result; + r->response.header.content_type = mime_type; + r->response.header.content_range_start = 0; + r->response.header.content_length = r->response.header.resource_length = bytes; + r->response.content = body; + r->response.content_generator = NULL; + http_request_start_response(r); +} + +void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR generator) +{ + assert(r->phase == RECEIVE); + assert(result >= 100); + assert(result < 300); + assert(mime_type != NULL); + assert(mime_type[0]); + r->response.result_code = result; + r->response.header.content_type = mime_type; + r->response.content = NULL; + r->response.content_generator = generator; + http_request_start_response(r); +} + +/* Start sending a short response back to the client. The result code must be either a success + * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code. The 'body' argument + * may be a bare message which is enclosed in an HTML envelope to form the response content, so it + * may contain HTML markup. If the 'body' argument is NULL, then the response content is generated + * automatically from the result code. + * + * @author Andrew Bettison + */ +void http_request_simple_response(struct http_request *r, uint16_t result, const char *body) +{ + assert(r->phase == RECEIVE); + assert(result >= 200); + assert(result < 600); + strbuf h = NULL; + if (body) { + size_t html_len = strlen(body) + 40; + h = strbuf_alloca(html_len); + strbuf_sprintf(h, "

%03u %s

", result, body); + } + r->response.result_code = result; + r->response.header.content_type = "text/html"; + r->response.header.content_range_start = 0; + r->response.header.resource_length = r->response.header.content_length = h ? strbuf_len(h) : 0; + r->response.content = h ? strbuf_str(h) : NULL; + r->response.content_generator = NULL; + http_request_start_response(r); +} diff --git a/http_server.h b/http_server.h new file mode 100644 index 00000000..a647c30e --- /dev/null +++ b/http_server.h @@ -0,0 +1,156 @@ +/* +Serval DNA - HTTP Server API +Copyright (C) 2013 Serval Project, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVALDNA__HTTP_SERVER_H +#define __SERVALDNA__HTTP_SERVER_H + +#include +#include "constants.h" +#include "strbuf.h" +#include "fdqueue.h" + +/* Generic HTTP request handling. + * + * @author Andrew Bettison + */ + +extern const char HTTP_VERB_GET[]; +extern const char HTTP_VERB_POST[]; +extern const char HTTP_VERB_PUT[]; +extern const char HTTP_VERB_HEAD[]; +extern const char HTTP_VERB_DELETE[]; +extern const char HTTP_VERB_TRACE[]; +extern const char HTTP_VERB_OPTIONS[]; +extern const char HTTP_VERB_CONNECT[]; +extern const char HTTP_VERB_PATCH[]; + +typedef uint64_t http_size_t; +#define PRIhttp_size_t PRIu64 + +struct http_request; + +struct http_range { + enum http_range_type { NIL = 0, CLOSED, OPEN, SUFFIX } type; + http_size_t first; // only for CLOSED or OPEN + http_size_t last; // only for CLOSED or SUFFIX +}; + +unsigned http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length); +http_size_t http_range_bytes(const struct http_range *range, unsigned nranges); + +#define CONTENT_LENGTH_UNKNOWN UINT64_MAX + +struct http_request_headers { + http_size_t content_length; + const char *content_type; + const char *content_subtype; + const char *boundary; + unsigned short content_range_count; + struct http_range content_ranges[5]; +}; + +struct http_response_headers { + http_size_t content_length; + http_size_t content_range_start; // range_end = range_start + content_length - 1 + http_size_t resource_length; // size of entire resource + const char *content_type; // "type/subtype" + const char *boundary; +}; + +typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *); + +struct http_response { + uint16_t result_code; + struct http_response_headers header; + const char *content; + HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content +}; + +#define MIME_FILENAME_MAXLEN 127 + +struct mime_content_disposition { + char type[64]; + char name[64]; + char filename[MIME_FILENAME_MAXLEN + 1]; + http_size_t size; + time_t creation_date; + time_t modification_date; + time_t read_date; +}; + +struct http_mime_handler { + void (*handle_mime_preamble)(struct http_request *, const char *, size_t); + void (*handle_mime_part_start)(struct http_request *); + void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *); + void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t); + void (*handle_mime_body)(struct http_request *, const char *, size_t); + void (*handle_mime_part_end)(struct http_request *); + void (*handle_mime_epilogue)(struct http_request *, const char *, size_t); +}; + +struct http_request; + +void http_request_init(struct http_request *r, int sockfd); +void http_request_free_response_buffer(struct http_request *r); +int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz); +void http_request_finalise(struct http_request *r); +void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); +void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR); +void http_request_simple_response(struct http_request *r, uint16_t result, const char *body); + +typedef int (*HTTP_REQUEST_PARSER)(struct http_request *); + +struct http_request { + struct sched_ent alarm; // MUST BE FIRST ELEMENT + enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase; + bool_t *debug_flag; + bool_t *disable_tx_flag; + time_ms_t initiate_time; // time connection was initiated + time_ms_t idle_timeout; // disconnect if no bytes received for this long + struct sockaddr_in client_sockaddr_in; + HTTP_REQUEST_PARSER parser; // current parser function + HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed + HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed + HTTP_REQUEST_PARSER handle_content_end; // called after all content is received + enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state; + struct http_mime_handler form_data; // called to parse multipart/form-data body + void (*finalise)(struct http_request *); + void (*free)(void*); + const char *verb; // points to nul terminated static string, "GET", "PUT", etc. + const char *path; // points into buffer; nul terminated + uint8_t version_major; // m from from HTTP/m.n + uint8_t version_minor; // n from HTTP/m.n + struct http_request_headers request_header; + const char *received; // start of received data in buffer[] + const char *end; // end of received data in buffer[] + const char *parsed; // start of unparsed data in buffer[] + const char *cursor; // for parsing + http_size_t request_content_remaining; + struct http_response response; + http_size_t response_length; // total response bytes (header + content) + http_size_t response_sent; // for counting up to response_length + char *response_buffer; + size_t response_buffer_size; + size_t response_buffer_length; + size_t response_buffer_sent; + void (*response_free_buffer)(void*); + char buffer[8 * 1024]; +}; + +#endif // __SERVALDNA__HTTP_SERVER_H diff --git a/keyring.c b/keyring.c index f9ce8821..f9c60a6f 100644 --- a/keyring.c +++ b/keyring.c @@ -675,7 +675,7 @@ static int load_did_name(keypair *kp, const char *text) return WHY("duplicate DID"); const char *e = NULL; bzero(kp->private_key, kp->private_key_len); - strn_fromprint(kp->private_key, kp->private_key_len, t, '"', &e); + strn_fromprint(kp->private_key, kp->private_key_len, t, 0, '"', &e); if (*e != '"') return WHY("malformed DID quoted string"); t = e + 1; @@ -685,7 +685,7 @@ static int load_did_name(keypair *kp, const char *text) return WHY("duplicate Name"); const char *e = NULL; bzero(kp->public_key, kp->public_key_len); - strn_fromprint(kp->public_key, kp->public_key_len, t, '"', &e); + strn_fromprint(kp->public_key, kp->public_key_len, t, 0, '"', &e); if (*e != '"') return WHY("malformed Name quoted string"); t = e + 1; diff --git a/log.h b/log.h index bba3ec79..1d1b743f 100644 --- a/log.h +++ b/log.h @@ -153,8 +153,8 @@ struct strbuf; #define DEBUG(X) DEBUGF("%s", (X)) #define DEBUGF_perror(F,...) LOGF_perror(LOG_LEVEL_DEBUG, F, ##__VA_ARGS__) #define DEBUG_perror(X) DEBUGF_perror("%s", (X)) -#define D DEBUG("D") -#define T { if (config.debug.trace) DEBUG("T"); } +#define D (DEBUG("D"), 1) +#define T (config.debug.trace ? DEBUG("T") : 1) #define DEBUG_argv(X,ARGC,ARGV) logArgv(LOG_LEVEL_DEBUG, __WHENCE__, (X), (ARGC), (ARGV)) #define dump(X,A,N) logDump(LOG_LEVEL_DEBUG, __WHENCE__, (X), (const unsigned char *)(A), (size_t)(N)) diff --git a/meshms.c b/meshms.c index 6b5f1d8f..1f51ae26 100644 --- a/meshms.c +++ b/meshms.c @@ -49,7 +49,7 @@ struct ply_read{ // details of the current record uint64_t record_end_offset; uint16_t record_length; - int buffer_size; + size_t buffer_size; char type; // raw record data unsigned char *buffer; @@ -78,8 +78,7 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) alloca_tohex(keyring->contexts[cn]->identities[in] ->keypairs[kp]->private_key, crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES)); - int ret = rhizome_get_bundle_from_seed(m, seed); - if (ret<0) + if (rhizome_get_bundle_from_seed(m, seed) == -1) return -1; // always consider the content encrypted, we don't need to rely on the manifest itself. @@ -225,9 +224,9 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome if (rhizome_retrieve_manifest(bid, m)) return -1; int ret = rhizome_open_decrypt_read(m, NULL, &ply->read); - if (ret>0) + if (ret == 1) WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); - if (ret) + if (ret != 0) return ret; ply->read.offset = ply->read.length = m->fileLength; return 0; @@ -399,11 +398,11 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){ goto end; ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK); - if (ret<0) + if (ret == -1) goto end; if (ret==0){ - if (unpack_uint(ply.buffer, ply.record_length, &previous_ack)<0) + if (unpack_uint(ply.buffer, ply.record_length, &previous_ack) == -1) previous_ack=0; } if (config.debug.meshms) @@ -478,7 +477,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, bzero(&buff, sizeof(buff)); int ret = rhizome_open_decrypt_read(m, NULL, &read); - if (ret<0) + if (ret == -1) goto end; unsigned char version=0xFF; @@ -527,8 +526,9 @@ end: return 0; } -static int write_conversation(struct rhizome_write *write, struct conversations *conv){ - int len=0; +static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv) +{ + size_t len=0; if (!conv) return len; { @@ -541,14 +541,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations len+=pack_uint(&buffer[len], conv->read_offset); len+=pack_uint(&buffer[len], conv->their_size); int ret=rhizome_write_buffer(write, buffer, len); - if (ret<0) + if (ret == -1) return ret; }else{ len+=measure_packed_uint(conv->their_last_message); len+=measure_packed_uint(conv->read_offset); len+=measure_packed_uint(conv->their_size); } - DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %d", + DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %zu", alloca_tohex_sid_t(conv->them), conv->their_last_message, conv->read_offset, @@ -556,14 +556,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations len); } // write the two child nodes - int ret=write_conversation(write, conv->_left); - if (ret<0) + ssize_t ret = write_conversation(write, conv->_left); + if (ret == -1) return ret; - len+=ret; - ret=write_conversation(write, conv->_right); - if (ret<0) + len += (size_t) ret; + ret = write_conversation(write, conv->_right); + if (ret == -1) return ret; - len+=ret; + len += (size_t) ret; return len; } @@ -578,22 +578,22 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations * // TODO rebalance tree... // measure the final payload first - int len=write_conversation(NULL, conv); - if (len<0) + ssize_t len=write_conversation(NULL, conv); + if (len == -1) goto end; // then write it m->version++; rhizome_manifest_set_ll(m,"version",m->version); - m->fileLength = len+1; + m->fileLength = (size_t) len + 1; rhizome_manifest_set_ll(m,"filesize",m->fileLength); - if (rhizome_write_open_manifest(&write, m)) + if (rhizome_write_open_manifest(&write, m) == -1) goto end; unsigned char version=1; - if (rhizome_write_buffer(&write, &version, 1)<0) + if (rhizome_write_buffer(&write, &version, 1) == -1) goto end; - if (write_conversation(&write, conv)<0) + if (write_conversation(&write, conv) == -1) goto end; if (rhizome_finish_write(&write)) goto end; @@ -795,7 +795,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context // find their last ACK so we know if messages have been received int r = ply_find_next(&read_theirs, MESHMS_BLOCK_TYPE_ACK); if (r==0){ - if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack)<0) + if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack) == -1) their_last_ack=0; else their_ack_offset = read_theirs.record_end_offset; @@ -822,11 +822,11 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context // read their message list, and insert all messages that are included in the ack range if (conv->found_their_ply){ int ofs=unpack_uint(read_ours.buffer, read_ours.record_length, (uint64_t*)&read_theirs.read.offset); - if (ofs<0) + if (ofs == -1) break; uint64_t end_range; int x = unpack_uint(read_ours.buffer+ofs, read_ours.record_length - ofs, &end_range); - if (x<0) + if (x == -1) end_range=0; else end_range = read_theirs.read.offset - end_range; diff --git a/monitor-cli.c b/monitor-cli.c index cf10da1c..506ece60 100644 --- a/monitor-cli.c +++ b/monitor-cli.c @@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include +#ifdef HAVE_POLL_H #include +#endif #include #include "serval.h" diff --git a/overlay.c b/overlay.c index 6c70d943..d16b0ef9 100644 --- a/overlay.c +++ b/overlay.c @@ -133,12 +133,8 @@ schedule(&_sched_##X); } rhizome_cleanup(NULL); } - /* Rhizome http server needs to know which callback to attach - to client sockets, so provide it here, along with the name to - appear in time accounting statistics. */ - rhizome_http_server_start(rhizome_server_parse_http_request, - "rhizome_server_parse_http_request", - RHIZOME_HTTP_PORT,RHIZOME_HTTP_PORT_MAX); + // start the HTTP server if enabled + rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX); // start the dna helper if configured dna_helper_start(); diff --git a/performance_timing.c b/performance_timing.c index 7c989b18..cb188dc7 100644 --- a/performance_timing.c +++ b/performance_timing.c @@ -17,7 +17,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "serval.h" +#include "fdqueue.h" #include "conf.h" struct profile_total *stats_head=NULL; diff --git a/rhizome.h b/rhizome.h index fa64ee98..e856ff54 100644 --- a/rhizome.h +++ b/rhizome.h @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "sha2.h" #include "str.h" #include "strbuf.h" +#include "http_server.h" #include "nacl.h" #include @@ -269,7 +270,7 @@ int rhizome_str_is_bundle_crypt_key(const char *text); int rhizome_strn_is_file_hash(const char *text); int rhizome_str_is_file_hash(const char *text); -int http_header_complete(const char *buf, size_t len, size_t read_since_last_call); +int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call); typedef struct sqlite_retry_state { unsigned int limit; // do not retry once elapsed >= limit @@ -471,8 +472,8 @@ struct rhizome_write_buffer { struct rhizome_write_buffer *_next; int64_t offset; - int buffer_size; - int data_size; + size_t buffer_size; + size_t data_size; unsigned char data[0]; }; @@ -487,7 +488,7 @@ struct rhizome_write int64_t written_offset; int64_t file_length; struct rhizome_write_buffer *buffer_list; - int buffer_size; + size_t buffer_size; int crypt; unsigned char key[RHIZOME_CRYPT_KEY_BYTES]; @@ -502,7 +503,7 @@ struct rhizome_write struct rhizome_read_buffer{ uint64_t offset; unsigned char data[RHIZOME_CRYPT_PAGE_SIZE]; - int len; + size_t len; }; struct rhizome_read @@ -525,71 +526,27 @@ struct rhizome_read int64_t length; }; -typedef struct rhizome_http_request { - struct sched_ent alarm; - time_ms_t initiate_time; /* time connection was initiated */ - - struct sockaddr_in requestor; +/* Rhizome-specific HTTP request handling. + */ +typedef struct rhizome_http_request +{ + struct http_request http; // MUST BE FIRST ELEMENT - /* identify request from others being run. + /* Identify request from others being run. Monotonic counter feeds it. Only used for debugging when we write post-.log files for multi-part form requests. */ unsigned int uuid; - /* The HTTP request as currently received */ - int request_length; - int header_length; - char request[1024]; - - /* Nature of the request */ - int request_type; - /* All of the below are receiving data */ -#define RHIZOME_HTTP_REQUEST_RECEIVING -1 -#define RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART -2 - - // callback function to fill the response buffer - int (*generator)(struct rhizome_http_request *r); - - /* Local buffer of data to be sent. - If a RHIZOME_HTTP_REQUEST_FROMBUFFER, then the buffer is sent, and when empty - the request is closed. - Else emptying the buffer triggers a request to fetch more data. Only if no - more data is provided do we then close the request. */ - unsigned char *buffer; - int buffer_size; // size - int buffer_length; // number of bytes loaded into buffer - int buffer_offset; // where we are between [0,buffer_length) - struct rhizome_read read_state; - /* Path of request (used by POST multipart form requests where - the actual processing of the request does not occur while the - request headers are still available. */ - char path[1024]; - /* Boundary string for POST multipart form requests */ - char boundary_string[1024]; - int boundary_string_length; /* File currently being written to while decoding POST multipart form */ - FILE *field_file; + enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part; + int part_fd; + /* Which parts have been received in POST multipart form */ + bool_t received_manifest; + bool_t received_data; /* Name of data file supplied */ - char data_file_name[1024]; - /* Which fields have been seen in POST multipart form */ - int fields_seen; - /* The seen fields bitmap above shares values with the actual Rhizome Direct - state machine. The state numbers (and thus bitmap values for the various - fields) are listed here. - - To avoid confusion, we should not use single bit values for states that do - not correspond directly to a particular field. - Doesn't really matter what they are apart from not having exactly one bit set. - In fact, the only reason to not have exactly one bit set is so that we keep as - many bits available for field types as possible. - */ -#define RD_MIME_STATE_MANIFESTHEADERS (1<<0) -#define RD_MIME_STATE_DATAHEADERS (1<<1) -#define RD_MIME_STATE_INITIAL 0 -#define RD_MIME_STATE_PARTHEADERS 0xffff0000 -#define RD_MIME_STATE_BODY 0xffff0001 + char data_file_name[MIME_FILENAME_MAXLEN + 1]; /* The source specification data which are used in different ways by different request types */ @@ -607,15 +564,6 @@ typedef struct rhizome_http_request { } rhizome_http_request; -struct http_response { - unsigned int result_code; - const char * content_type; - uint64_t content_start; - uint64_t content_end; - uint64_t content_length; - const char * body; -}; - int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, uint64_t offset,int count,unsigned char *bytes, int type); @@ -629,9 +577,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r, int result, con int rhizome_server_http_response(rhizome_http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes); -int rhizome_http_server_start(int (*http_parse_func)(rhizome_http_request *), - const char *http_parse_func_description, - uint16_t port_low, uint16_t port_high); +int rhizome_http_server_start(uint16_t port_low, uint16_t port_high); int is_rhizome_enabled(); int is_rhizome_mdp_enabled(); @@ -654,12 +600,12 @@ typedef struct rhizome_direct_bundle_cursor { rhizome_bid_t bid_low; rhizome_bid_t bid_high; unsigned char *buffer; - int buffer_size; - int buffer_used; - int buffer_offset_bytes; + size_t buffer_size; + size_t buffer_used; + size_t buffer_offset_bytes; } rhizome_direct_bundle_cursor; -rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size); +rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size); void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r); int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r, unsigned char *pickled, @@ -723,7 +669,7 @@ rhizome_direct_sync_request *rhizome_direct_new_sync_request( void (*transport_specific_dispatch_function) (struct rhizome_direct_sync_request *), - int buffer_size,int interval, int mode, + size_t buffer_size, int interval, int mode, void *transport_specific_state); int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r); int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r); @@ -761,7 +707,7 @@ int rhizome_fetch_status_html(struct strbuf *b); int rhizome_fetch_has_queue_space(unsigned char log2_size); struct http_response_parts { - int code; + uint16_t code; char *reason; int64_t range_start; int64_t content_length; @@ -774,34 +720,34 @@ int unpack_http_response(char *response, struct http_response_parts *parts); int rhizome_exists(const rhizome_filehash_t *hashp); int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority); -int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size); -int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size); +int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size); +int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size); int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m); int rhizome_write_file(struct rhizome_write *write, const char *filename); int rhizome_fail_write(struct rhizome_write *write); int rhizome_finish_write(struct rhizome_write *write); int rhizome_import_file(rhizome_manifest *m, const char *filepath); -int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length); +int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length); int rhizome_stat_file(rhizome_manifest *m, const char *filepath); int rhizome_add_file(rhizome_manifest *m, const char *filepath); int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk); int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size); -int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len); +int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len); int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename); int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length); -int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, +int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, const unsigned char *key, const unsigned char *nonce); int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp); -int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length); -int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len); +int rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length); +int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len); int rhizome_read_close(struct rhizome_read *read); int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length); int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, - uint64_t fileOffset, unsigned char *buffer, int length); + uint64_t fileOffset, unsigned char *buffer, size_t length); int rhizome_cache_close(); int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp); diff --git a/rhizome_crypto.c b/rhizome_crypto.c index 7d9dd7e0..3cbb2a32 100644 --- a/rhizome_crypto.c +++ b/rhizome_crypto.c @@ -72,13 +72,13 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed) return -1; int ret=rhizome_retrieve_manifest(&key.Public, m); - if (ret<0) + if (ret == -1) return -1; m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID; m->cryptoSignPublic = key.Public; bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret); - if (ret>0) + if (ret == 1) rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); return ret; } @@ -590,22 +590,22 @@ static void add_nonce(unsigned char *nonce, int64_t value){ /* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries * for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE) */ -int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, +int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, const unsigned char *key, const unsigned char *nonce){ if (stream_offset<0) return WHY("Invalid stream offset"); int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1); - int offset=0; + size_t offset=0; unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES]; bcopy(nonce, block_nonce, sizeof(block_nonce)); add_nonce(block_nonce, nonce_offset); if (nonce_offset < stream_offset){ - int padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1); - int size = RHIZOME_CRYPT_PAGE_SIZE - padding; + size_t padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1); + size_t size = RHIZOME_CRYPT_PAGE_SIZE - padding; if (size>buffer_size) size=buffer_size; @@ -619,11 +619,11 @@ int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stre } while(offset < buffer_size){ - int size = buffer_size - offset; + size_t size = buffer_size - offset; if (size>RHIZOME_CRYPT_PAGE_SIZE) size=RHIZOME_CRYPT_PAGE_SIZE; - crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, size, block_nonce, key); + crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, (unsigned long long) size, block_nonce, key); add_nonce(block_nonce, RHIZOME_CRYPT_PAGE_SIZE); offset+=size; diff --git a/rhizome_direct.c b/rhizome_direct.c index 518b02d0..02d91833 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -135,7 +135,7 @@ rhizome_direct_sync_request *rhizome_direct_new_sync_request( void (*transport_specific_dispatch_function) (struct rhizome_direct_sync_request *), - int buffer_size,int interval, int mode, void *state) + size_t buffer_size, int interval, int mode, void *state) { assert(mode&3); @@ -260,8 +260,7 @@ int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r) multiple versions of a given bundle introduces only a slight complication. */ -rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response -(unsigned char *buffer,int size, int max_response_bytes) +rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *buffer,int size, int max_response_bytes) { if (size<10) return NULL; if (size>65536) return NULL; @@ -288,7 +287,7 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response } DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high,c->limit_size_high); - DEBUGF("c->buffer_size=%d",c->buffer_size); + DEBUGF("c->buffer_size=%zu",c->buffer_size); /* Get our list of BARs for the same cursor range */ int us_count=rhizome_direct_bundle_iterator_fill(c,-1); @@ -534,7 +533,7 @@ int app_rhizome_direct_sync(const struct cli_parsed *parsed, struct cli_context } } -rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size) +rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size) { rhizome_direct_bundle_cursor *r=calloc(sizeof(rhizome_direct_bundle_cursor),1); assert(r!=NULL); diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index ef6a9430..220fbcbd 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -28,658 +28,390 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include -int rhizome_direct_clear_temporary_files(rhizome_http_request *r) +static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_request *r, char *pathbuf, size_t bufsiz, const char *field) { - char filename[1024]; - char *fields[]={"manifest","data","unknown",NULL}; - int i; - - for(i=0;fields[i];i++) { - snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,fields[i]); - filename[1023]=0; - unlink(filename); + strbuf b = strbuf_local(pathbuf, bufsiz); + // TODO: use a temporary directory + strbuf_sprintf(b, "rhizomedirect.%d.%s", r->http.alarm.poll.fd, field); + if (strbuf_overrun(b)) { + WHYF("Rhizome Direct pathname overflow: %s", alloca_str_toprint(pathbuf)); + return -1; } return 0; } -int rhizome_direct_form_received(rhizome_http_request *r) +#define form_temporary_file_path(r,buf,field) _form_temporary_file_path(__WHENCE__, (r), (buf), sizeof(buf), (field)) + +static void rhizome_direct_clear_temporary_files(rhizome_http_request *r) { - /* Process completed form based on the set of fields seen */ - if (!strcmp(r->path,"/rhizome/import")) { - switch(r->fields_seen) { - case RD_MIME_STATE_MANIFESTHEADERS | RD_MIME_STATE_DATAHEADERS: { - /* Got a bundle to import */ - DEBUGF("Call bundle import for rhizomedata.%d.{data,file}", - r->alarm.poll.fd); - strbuf manifest_path = strbuf_alloca(50); - strbuf payload_path = strbuf_alloca(50); - strbuf_sprintf(manifest_path, "rhizomedirect.%d.manifest", r->alarm.poll.fd); - strbuf_sprintf(payload_path, "rhizomedirect.%d.data", r->alarm.poll.fd); - - int ret=0; - rhizome_manifest *m = rhizome_new_manifest(); - - if (!m) - ret=WHY("Out of manifests."); - else{ - ret=rhizome_bundle_import_files(m, strbuf_str(manifest_path), strbuf_str(payload_path)); - rhizome_manifest_free(m); - } - - rhizome_direct_clear_temporary_files(r); - /* report back to caller. - 200 = ok, which is probably appropriate for when we already had the bundle. - 201 = content created, which is probably appropriate for when we successfully - import a bundle (or if we already have it). - 403 = forbidden, which might be appropriate if we refuse to accept it, e.g., - the import fails due to malformed data etc. - (should probably also indicate if we have a newer version if possible) - */ - switch (ret) { - case 0: - return rhizome_server_simple_http_response(r, 201, "Bundle succesfully imported."); - case 2: - return rhizome_server_simple_http_response(r, 200, "Bundle already imported."); - } - return rhizome_server_simple_http_response(r, 500, "Server error: Rhizome import command failed."); - } - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - } - } else if (!strcmp(r->path,"/rhizome/enquiry")) { - int fd=-1; - char file[1024]; - switch(r->fields_seen) { - case RD_MIME_STATE_DATAHEADERS: - /* Read data buffer in, pass to rhizome direct for comparison with local - rhizome database, and send back responses. */ - snprintf(file,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,"data"); - fd=open(file,O_RDONLY); - if (fd == -1) { - WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(file)); - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't read a file"); - } - struct stat stat; - if (fstat(fd, &stat) == -1) { - WHYF_perror("stat(%d)", fd); - /* Clean up after ourselves */ - close(fd); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't stat a file"); - } - unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); - if (addr==MAP_FAILED) { - WHYF_perror("mmap(NULL, %"PRId64", PROT_READ, MAP_SHARED, %d, 0)", (int64_t) stat.st_size, fd); - /* Clean up after ourselves */ - close(fd); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't mmap() a file"); - } - /* Ask for a fill response. Regardless of the size of the set of BARs passed - to us, we will allow up to 64KB of response. */ - rhizome_direct_bundle_cursor - *c=rhizome_direct_get_fill_response(addr,stat.st_size,65536); - munmap(addr,stat.st_size); - close(fd); + const char *fields[] = { "manifest", "data" }; + int i; + for (i = 0; i != NELS(fields); ++i) { + char path[1024]; + if (form_temporary_file_path(r, path, fields[i]) != -1) + if (unlink(path) == -1 && errno != ENOENT) + WARNF_perror("unlink(%s)", alloca_str_toprint(path)); + } +} - if (c) - { - /* TODO: Write out_buffer as the body of the response. - We should be able to do this using the async framework fairly easily. - */ - - int bytes=c->buffer_offset_bytes+c->buffer_used; - r->buffer=malloc(bytes+1024); - r->buffer_size=bytes+1024; - r->buffer_offset=0; - assert(r->buffer); - - /* Write HTTP response header */ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code=200; - hr.content_type="binary/octet-stream"; - hr.content_length=bytes; - hr.body=NULL; - r->request_type=0; - rhizome_server_set_response(r,&hr); - assert(r->buffer_offset<1024); - - /* Now append body and send it back. */ - bcopy(c->buffer,&r->buffer[r->buffer_length],bytes); - r->buffer_length+=bytes; - r->buffer_offset=0; - - /* Clean up cursor after sending response */ - rhizome_direct_bundle_iterator_free(&c); - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return 0; - } - else - { - return rhizome_server_simple_http_response(r,500,"Could not get response to enquiry"); - } - - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return rhizome_server_simple_http_response(r, 404, "/rhizome/enquiry requires 'data' field"); - } - } - /* Allow servald to be configured to accept files without manifests via HTTP - from localhost, so that rhizome bundles can be created programatically. - There are probably still some security loop-holes here, which is part of - why we leave it disabled by default, but it will be sufficient for testing - possible uses, including integration with OpenDataKit. +int rhizome_direct_import_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (!r->received_manifest) { + http_request_simple_response(&r->http, 400, "Missing 'manifest' part"); + return 0; + } + if (!r->received_data) { + http_request_simple_response(&r->http, 400, "Missing 'data' part"); + return 0; + } + /* Got a bundle to import */ + char manifest_path[512]; + char payload_path[512]; + if ( form_temporary_file_path(r, manifest_path, "manifest") == -1 + || form_temporary_file_path(r, payload_path, "data") == -1 + ) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Call rhizome_bundle_import_files(%s, %s)", + alloca_str_toprint(manifest_path), + alloca_str_toprint(payload_path) + ); + int ret = 0; + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) + ret = WHY("Out of manifests"); + else { + ret = rhizome_bundle_import_files(m, manifest_path, payload_path); + rhizome_manifest_free(m); + } + rhizome_direct_clear_temporary_files(r); + /* report back to caller. + 200 = ok, which is probably appropriate for when we already had the bundle. + 201 = content created, which is probably appropriate for when we successfully + import a bundle (or if we already have it). + 403 = forbidden, which might be appropriate if we refuse to accept it, e.g., + the import fails due to malformed data etc. + (should probably also indicate if we have a newer version if possible) */ - else if (config.rhizome.api.addfile.uri_path[0] && strcmp(r->path, config.rhizome.api.addfile.uri_path) == 0) { - if (r->requestor.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr) { - DEBUGF("rhizome.api.addfile request received from %s, but is only allowed from %s", - inet_ntoa(r->requestor.sin_addr), - inet_ntoa(config.rhizome.api.addfile.allow_host) - ); + switch (ret) { + case 0: + http_request_simple_response(&r->http, 201, "Bundle succesfully imported"); + return 0; + case 2: + http_request_simple_response(&r->http, 200, "Bundle already imported"); + return 0; + } + http_request_simple_response(&r->http, 500, "Internal Error: Rhizome import failed"); + return 0; +} - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,404,"Not available from here."); +int rhizome_direct_enquiry_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (!r->received_data) { + http_request_simple_response(&r->http, 400, "Missing 'data' part"); + return 0; + } + char data_path[512]; + if (form_temporary_file_path(r, data_path, "data") == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Call rhizome_direct_fill_response(%s)", alloca_str_toprint(data_path)); + /* Read data buffer in, pass to rhizome direct for comparison with local + rhizome database, and send back responses. */ + int fd = open(data_path, O_RDONLY); + if (fd == -1) { + WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(data_path)); + /* Clean up after ourselves */ + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't read file"); + return 0; + } + struct stat stat; + if (fstat(fd, &stat) == -1) { + WHYF_perror("stat(%d)", fd); + /* Clean up after ourselves */ + close(fd); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't stat file"); + return 0; + } + unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (addr==MAP_FAILED) { + WHYF_perror("mmap(NULL,%"PRId64",PROT_READ,MAP_SHARED,%d,0)", (int64_t) stat.st_size, fd); + /* Clean up after ourselves */ + close(fd); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't mmap file"); + return 0; + } + /* Ask for a fill response. Regardless of the size of the set of BARs passed + to us, we will allow up to 64KB of response. */ + rhizome_direct_bundle_cursor *c = rhizome_direct_get_fill_response(addr, stat.st_size, 65536); + munmap(addr,stat.st_size); + close(fd); + if (c) { + size_t bytes = c->buffer_offset_bytes + c->buffer_used; + if (http_request_set_response_bufsize(&r->http, bytes) == -1) + http_request_simple_response(&r->http, 500, "Internal Error: Out of memory"); + else + http_request_response_static(&r->http, 200, "binary/octet-stream", (const char *)c->buffer, bytes); + rhizome_direct_bundle_iterator_free(&c); + } else + http_request_simple_response(&r->http, 500, "Internal Error: No response to enquiry"); + rhizome_direct_clear_temporary_files(r); + return 0; +} + +int rhizome_direct_addfile_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + // If given a file without a manifest, we should only accept if it we are configured to do so, and + // the connection is from localhost. Otherwise people could cause your servald to create + // arbitrary bundles, which would be bad. + if (!r->received_manifest) { + char payload_path[512]; + if (form_temporary_file_path(r, payload_path, "data") == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; } - - switch(r->fields_seen) { - case RD_MIME_STATE_DATAHEADERS: - /* We have been given a file without a manifest, we should only - accept if it we are configured to do so, and the connection is from - localhost. Otherwise people could cause your servald to create - arbitrary bundles, which would be bad. - */ - /* A bundle to import */ - DEBUGF("Call bundle import sans-manifest for rhizomedata.%d.{data,file}", - r->alarm.poll.fd); - - char filepath[1024]; - snprintf(filepath,1024,"rhizomedirect.%d.data",r->alarm.poll.fd); - - char manifestTemplate[1024]; - manifestTemplate[0] = '\0'; - if (config.rhizome.api.addfile.manifest_template_file[0]) { - strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate); - strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL); - if (access(manifestTemplate, R_OK) != 0) { - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file points to a file I could not read."); - } + if (config.debug.rhizome) + DEBUGF("Call rhizome_add_file(%s)", alloca_str_toprint(payload_path)); + char manifestTemplate[1024]; + manifestTemplate[0] = '\0'; + if (config.rhizome.api.addfile.manifest_template_file[0]) { + strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate); + strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL); + if (strbuf_overrun(b)) { + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Template path too long"); + return 0; } - - rhizome_manifest *m = rhizome_new_manifest(); - if (!m) - { - rhizome_server_simple_http_response(r,500,"No free manifest slots. Try again later."); - rhizome_direct_clear_temporary_files(r); - return WHY("Manifest struct could not be allocated -- not added to rhizome"); - } - - if (manifestTemplate[0]) - if (rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) { - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file can't be read as a manifest."); - } - - if (rhizome_stat_file(m, filepath)){ + if (access(manifestTemplate, R_OK) != 0) { + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Cannot read template"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Using manifest template %s", alloca_str_toprint(manifestTemplate)); + } + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) { + WHY("Manifest struct could not be allocated -- not added to rhizome"); + http_request_simple_response(&r->http, 500, "Internal Error: No free manifest slots"); + rhizome_direct_clear_temporary_files(r); + return 0; + } + if (manifestTemplate[0] && rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) { + WHY("Manifest template read failed"); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Malformed manifest template"); + return 0; + } + if (rhizome_stat_file(m, payload_path)) { + WHY("Payload file stat failed"); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Could not store file"); + return 0; + } + // If manifest template did not specify a service field, then by default it is "file". + if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) + rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); + sid_t *author = NULL; + if (!is_sid_t_any(config.rhizome.api.addfile.default_author)) + author = &config.rhizome.api.addfile.default_author; + rhizome_bk_t bsk = config.rhizome.api.addfile.bundle_secret_key; + if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)) { + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest"); + return 0; + } + m->payloadEncryption=0; + rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); + // import file contents + // TODO, stream file into database + if (m->fileLength) { + if (rhizome_add_file(m, payload_path)) { rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not store file"); + http_request_simple_response(&r->http, 500, "Internal Error: Could not store file"); + return 0; } - - if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); - - sid_t *author=NULL; - if (!is_sid_t_any(config.rhizome.api.addfile.default_author)) - author = &config.rhizome.api.addfile.default_author; - - rhizome_bk_t bsk; - memcpy(bsk.binary, config.rhizome.api.addfile.bundle_secret_key.binary, RHIZOME_BUNDLE_KEY_BYTES); - - if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)){ - rhizome_manifest_free(m); - m = NULL; - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not fill manifest default values"); - } - - m->payloadEncryption=0; - rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); - - // import file contents - // TODO, stream file into database - if (m->fileLength){ - if (rhizome_add_file(m, filepath)){ - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not store file"); - } - } - - rhizome_manifest *mout = NULL; - if (rhizome_manifest_finalise(m, &mout, 1)) { - if (mout && mout!=m) - rhizome_manifest_free(mout); - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500, - "Could not finalise manifest"); - } - - DEBUGF("Import sans-manifest appeared to succeed"); - - /* Respond with the manifest that was added. */ - rhizome_server_simple_http_response(r, 200, (char *)m->manifestdata); - - /* clean up after ourselves */ - if (mout && mout!=m) + } + rhizome_manifest *mout = NULL; + if (rhizome_manifest_finalise(m, &mout, 1)) { + if (mout && mout != m) rhizome_manifest_free(mout); rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); - + http_request_simple_response(&r->http, 500, "Internal Error: Could not finalise manifest"); return 0; - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return rhizome_server_simple_http_response(r, 400, "Rhizome create bundle from file API requires 'data' field"); } - } - - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - /* Report error */ - return rhizome_server_simple_http_response(r, 500, "Something went wrong. Probably a missing data or manifest part, or invalid combination of URI and data/manifest provision."); - -} - - -int rhizome_direct_process_mime_line(rhizome_http_request *r,char *buffer,int count) -{ - /* Check for boundary line at start of buffer. - Boundary line = CRLF + "--" + boundary_string + optional whitespace + CRLF - EXCEPT end of form boundary, which is: - CRLF + "--" + boundary_string + "--" + CRLF - - NOTE: We attach the "--" to boundary_string when setting things up so that - we don't have to keep manually checking for it here. - - NOTE: The parser eats the CRLF from the front, and attaches it to the end - of the previous line. This means we need to rewind 2 bytes from whatever - file we were writing to whenever we encounter a boundary line, at least - if those last two bytes were CRLF. That can be safely assumed if we - assume that the boundary string has been chosen to be a string never appearing - anywhere in the contents of the form. In practice, that is only "almost - certain" (according to the mathematical meaning of that phrase) if boundary - strings are randomly selected and are of sufficient length. - - NOTE: We are not supporting nested/mixed parts, as that would considerably - complicate the parser. If the need arises in future, we will deal with it - then. In the meantime, we will have something that meets our immediate - needs for Rhizome Direct and a variety of use cases. - */ - - /* Regardless of the state of the parser, the presence of boundary lines - is significant, so lets just check once, and remember the result. - Similarly check a few other conditions. */ - int boundaryLine=0; - if (!memcmp(buffer,r->boundary_string,r->boundary_string_length)) - boundaryLine=1; - - int endOfForm=0; - if (boundaryLine&& - buffer[r->boundary_string_length]=='-'&& - buffer[r->boundary_string_length+1]=='-') - endOfForm=1; - int blankLine=0; - if (!strcmp(buffer,"\r\n")) blankLine=1; - - switch(r->source_flags) { - case RD_MIME_STATE_INITIAL: - if (boundaryLine) r->source_flags=RD_MIME_STATE_PARTHEADERS; - break; - case RD_MIME_STATE_PARTHEADERS: - case RD_MIME_STATE_MANIFESTHEADERS: - case RD_MIME_STATE_DATAHEADERS: - if (blankLine) { - /* End of headers */ - if (r->source_flags==RD_MIME_STATE_PARTHEADERS) - { - /* Multiple content-disposition lines. This is very naughty. */ - rhizome_server_simple_http_response - (r, 400, "

Malformed multi-part form POST: Missing content-disposition lines in MIME encoded part.

\r\n"); - return -1; - } - - /* Prepare to write to file for field. - We may have multiple rhizome direct transactions running at the same - time on different TCP connections. So serialise using file descriptor. - We could use the boundary string or some other random thing, but using - the file descriptor places a reasonable upper limit on the clutter that - is possible, while still preventing collisions -- provided that we don't - close the file descriptor until we have completed processing the - request. */ - r->field_file=NULL; - char filename[1024]; - char *field="unknown"; - switch(r->source_flags) { - case RD_MIME_STATE_DATAHEADERS: field="data"; break; - case RD_MIME_STATE_MANIFESTHEADERS: field="manifest"; break; - } - snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,field); - filename[1023]=0; - r->field_file=fopen(filename,"w"); - if (!r->field_file) { - WHYF_perror("fopen(%s, \"w\")", alloca_str_toprint(filename)); - goto scram; - } - r->source_flags=RD_MIME_STATE_BODY; - } else { - char name[1024]; - char field[1024]; - if (sscanf(buffer, - "Content-Disposition: form-data; name=\"%[^\"]\";" - " filename=\"%[^\"]\"",field,name)==2) - { - if (r->source_flags!=RD_MIME_STATE_PARTHEADERS) - { - /* Multiple content-disposition lines. This is very naughty. */ - rhizome_server_simple_http_response - (r, 400, "

Malformed multi-part form POST: Multiple content-disposition lines in single MIME encoded part.

\r\n"); - return -1; - } - if (!strcasecmp(field,"manifest")) - r->source_flags=RD_MIME_STATE_MANIFESTHEADERS; - if (!strcasecmp(field,"data")) { - r->source_flags=RD_MIME_STATE_DATAHEADERS; - /* record file name of data field for HTTP manifest-less import */ - strncpy(r->data_file_name,name,1023); - r->data_file_name[1023]=0; - } - if (r->source_flags!=RD_MIME_STATE_PARTHEADERS) - r->fields_seen|=r->source_flags; - } - } - break; - case RD_MIME_STATE_BODY: - if (boundaryLine) { - r->source_flags=RD_MIME_STATE_PARTHEADERS; - /* We will have written an extra CRLF to the end of the file, so prune that off. */ - if (fflush(r->field_file) == EOF) { - WHYF_perror("fflush()"); - goto scram; - } - int fd = fileno(r->field_file); - off_t correct_size = ftell(r->field_file) - 2; - if (ftruncate(fd,correct_size) == -1) { - WHYF_perror("ftruncate()"); - goto scram; - } - if (fclose(r->field_file) == EOF) { - WHYF_perror("fclose()"); - r->field_file = NULL; - goto scram; - } - r->field_file = NULL; - } - else { - int written=fwrite(r->request,count,1,r->field_file); - if (written<1) - DEBUGF("Short write for multi-part form file -- %d bytes may be missing", - count); - } - break; - } - - if (endOfForm) { - /* End of form marker found. - Pass it to function that deals with what has been received, - and will also send response or close the http request if required. */ - - /* XXX Rewind last two bytes from file if open, and close file */ - - return rhizome_direct_form_received(r); - } - return 0; - -scram: - if (r->field_file) { - if (fclose(r->field_file) == EOF) - WARNF_perror("fclose()"); - } - rhizome_direct_clear_temporary_files(r); - rhizome_server_simple_http_response(r, 500, - "

Sorry, couldn't complete your request, reasonable as it was. Perhaps try again later.

\r\n"); - return -1; -} - -int rhizome_direct_process_post_multipart_bytes(rhizome_http_request *r,const char *bytes,int count) -{ - { - char logname[128]; - snprintf(logname,128,"post-%08x.log",r->uuid); - FILE *f=fopen(logname,"a"); - if (f) fwrite(bytes,count,1,f); - if (f) fclose(f); - } - - /* This function looks for multi-part form separators and descriptor lines, - and streams any "manifest" or "data" blocks to respectively named files. - - The challenge is that we might only get a partial boundary string passed - to us. So we need to remember the last KB or so of data and glue it to - the front of the current set of bytes. - - In multi-part form parsing we don't need r->request for anything, so if - we are not in a form part already, then we can stow the bytes there - for reexamination when more bytes arrive. - - Side effect will be that the entire boundary string and associated bits will - need to be <=1KB, the size of r->request. This seems quite reasonable. - - Example of such a block is: - - ------WebKitFormBoundaryEoJwSoSVW4qsrBZW - Content-Disposition: form-data; name="manifest"; filename="spleen" - Content-Type: application/octet-stream - */ - - int o; - - /* Split into lines and process each line separately using a - simple state machine. - Lines containing binary are truncated into arbitrarily length pieces, but - a newline will ALWAYS break the line. - */ - - for(o=0;orequest_length>0&&r->request[r->request_length-1]=='\r') - { newline=1; r->request_length--; } - if (r->request_length>1020) newline=2; - if (newline) { - /* Found end of line, so process it */ - if (newline==1) { - /* Put the real new line onto the end if it was present, so that - we don't go doing anything silly, like joining lines in files - that really were separated by CRLF, or similarly inserting CRLF - in the middle of slabs of bytes that were not CRLF terminated. - */ - r->request[r->request_length++]='\r'; - r->request[r->request_length++]='\n'; - } - r->request[r->request_length]=0; - if (rhizome_direct_process_mime_line(r,r->request,r->request_length)) - return -1; - r->request_length=0; - /* If a real new line was detected, then - don't include the \n as part of the next line. - But if it wasn't a real new line, then make sure we - don't loose the byte. */ - if (newline==1) continue; - } - - r->request[r->request_length++]=bytes[o]; - } - - r->source_count-=count; - if (r->source_count<=0) { - /* Got to end of multi-part form data */ - - /* If the form is still being processed, then flush things through */ - if (r->request_type<0) { - /* Flush out any remaining data */ - if (r->request_length) { - DEBUGF("Flushing last %d bytes",r->request_length); - r->request[r->request_length]=0; - rhizome_direct_process_mime_line(r,r->request,r->request_length); - } - return rhizome_direct_form_received(r); - } else { - /* Form has already been processed, so do nothing */ - } - } - return 0; -} - -struct http_request_parts { -}; - -int rhizome_direct_parse_http_request(rhizome_http_request *r) -{ - DEBUGF("uri=%s", alloca_str_toprint(config.rhizome.api.addfile.uri_path)); - - // Parse the HTTP request into verb, path, protocol, headers and content. - char *const request_end = r->request + r->request_length; - char *verb = r->request; - char *path = NULL; - char *proto = NULL; - size_t pathlen = 0; - char *headers = NULL; - int headerlen = 0; - char *content = NULL; - int contentlen = 0; - char *p; - if ((str_startswith(verb, "GET", (const char **)&p) || str_startswith(verb, "POST", (const char **)&p)) && isspace(*p)) { - *p++ = '\0'; - path = p; - while (p < request_end && !isspace(*p)) - ++p; - if (p < request_end) { - pathlen = p - path; - *p++ = '\0'; - proto = p; - if ( str_startswith(p, "HTTP/1.", (const char **)&p) - && (str_startswith(p, "0", (const char **)&p) || str_startswith(p, "1", (const char **)&p)) - && (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers)) - ) { - *p = '\0'; - char *eoh = str_str(headers, "\r\n\r\n", request_end - p); - if (eoh) { - content = eoh + 4; - headerlen = content - headers; - contentlen = request_end - content; - } - } - } - } - if (content == NULL) { - if (config.debug.rhizome_tx) - DEBUGF("Received malformed HTTP request %s", alloca_toprint(160, (const char *)r->request, r->request_length)); - return rhizome_server_simple_http_response(r, 400, "

Malformed request

\r\n"); - } - INFOF("RHIZOME HTTP SERVER, %s %s %s", verb, alloca_toprint(-1, path, pathlen), proto); - if (config.debug.rhizome_tx) - DEBUGF("headers %s", alloca_toprint(-1, headers, headerlen)); - if (strcmp(verb, "POST") == 0 - && ( strcmp(path, "/rhizome/import") == 0 - || strcmp(path, "/rhizome/enquiry") == 0 - || (config.rhizome.api.addfile.uri_path[0] && strcmp(path, config.rhizome.api.addfile.uri_path) == 0) - ) - ) { - const char *cl_str=str_str(headers,"Content-Length: ",headerlen); - const char *ct_str=str_str(headers,"Content-Type: multipart/form-data; boundary=",headerlen); - if (!cl_str) - return rhizome_server_simple_http_response(r,400,"

Missing Content-Length header

\r\n"); - if (!ct_str) - return rhizome_server_simple_http_response(r,400,"

Missing or unsupported Content-Type header

\r\n"); - /* ok, we have content-type and content-length, now make sure they are well formed. */ - int64_t content_length; - if (sscanf(cl_str,"Content-Length: %"PRId64,&content_length)!=1) - return rhizome_server_simple_http_response(r,400,"

Malformed Content-Length header

\r\n"); - char boundary_string[1024]; - int i; - ct_str+=strlen("Content-Type: multipart/form-data; boundary="); - for(i=0;i<1023&&*ct_str&&*ct_str!='\n'&&*ct_str!='\r';i++,ct_str++) - boundary_string[i]=*ct_str; - boundary_string[i] = '\0'; - if (i<4||i>128) - return rhizome_server_simple_http_response(r,400,"

Malformed Content-Type header

\r\n"); - - DEBUGF("content_length=%"PRId64", boundary_string=%s contentlen=%d", (int64_t) content_length, alloca_str_toprint(boundary_string), contentlen); - - /* Now start receiving and parsing multi-part data. If we already received some of the - post-header data, process that first. Tell the HTTP request that it has moved to multipart - form data parsing, and what the actual requested action is. - */ - - /* Remember boundary string and source path. - Put the preceeding -- on the front to make our life easier when - parsing the rest later. */ - strbuf bs = strbuf_local(r->boundary_string, sizeof r->boundary_string); - strbuf_puts(bs, "--"); - strbuf_puts(bs, boundary_string); - if (strbuf_overrun(bs)) - return rhizome_server_simple_http_response(r,500,"

Internal server error: Multipart boundary string too long

\r\n"); - strbuf ps = strbuf_local(r->path, sizeof r->path); - strbuf_puts(ps, path); - if (strbuf_overrun(ps)) - return rhizome_server_simple_http_response(r,500,"

Internal server error: Path too long

\r\n"); - r->boundary_string_length = strbuf_len(bs); - r->source_index = 0; - r->source_count = content_length; - r->request_type = RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART; - r->request_length = 0; - r->source_flags = 0; - - /* Find the end of the headers and start of any body bytes that we have read - so far. Copy the bytes to a separate buffer, because r->request and - r->request_length get used internally in the parser. - */ - if (contentlen) { - char buffer[contentlen]; - bcopy(content, buffer, contentlen); - rhizome_direct_process_post_multipart_bytes(r, buffer, contentlen); - } - - /* Handle the rest of the transfer asynchronously. */ + if (config.debug.rhizome) + DEBUGF("Import sans-manifest appeared to succeed"); + /* Respond with the manifest that was added. */ + http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes); + /* clean up after ourselves */ + if (mout && mout != m) + rhizome_manifest_free(mout); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); return 0; } else { - rhizome_server_simple_http_response(r, 404, "

Not found (OTHER)

\r\n"); + http_request_simple_response(&r->http, 501, "Not Implemented: Rhizome add with manifest"); + return 0; } - - /* Try sending data immediately. */ - rhizome_server_http_send_bytes(r); +} +void rhizome_direct_process_mime_start(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + assert(r->current_part == NONE); + assert(r->part_fd == -1); +} + +void rhizome_direct_process_mime_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (r->part_fd != -1) { + if (close(r->part_fd) == -1) { + WHYF_perror("close(%d)", r->part_fd); + http_request_simple_response(&r->http, 500, "Internal Error: Close temporary file failed"); + return; + } + r->part_fd = -1; + } + switch (r->current_part) { + case MANIFEST: + r->received_manifest = 1; + break; + case DATA: + r->received_data = 1; + break; + case NONE: + break; + } + r->current_part = NONE; +} + +void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (strcmp(cd->name, "data") == 0) { + r->current_part = DATA; + strncpy(r->data_file_name, cd->filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0'; + } + else if (strcmp(cd->name, "manifest") == 0) { + r->current_part = MANIFEST; + } else + return; + char path[512]; + if (form_temporary_file_path(r, path, cd->name) == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return; + } + if ((r->part_fd = open(path, O_WRONLY | O_CREAT, 0666)) == -1) { + WHYF_perror("open(%s,O_WRONLY|O_CREAT,0666)", alloca_str_toprint(path)); + http_request_simple_response(&r->http, 500, "Internal Error: Create temporary file failed"); + return; + } +} + +void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (r->part_fd != -1) { + if (write_all(r->part_fd, buf, len) == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed"); + return; + } + } +} + +int rhizome_direct_import(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_import_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; return 0; } +int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_enquiry_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; + return 0; +} + +/* Servald can be configured to accept files without manifests via HTTP from localhost, so that + * rhizome bundles can be created programatically. There are probably still some security + * loop-holes here, which is part of why we leave it disabled by default, but it will be sufficient + * for testing possible uses, including integration with OpenDataKit. + */ +int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + if ( r->http.client_sockaddr_in.sin_family != AF_INET + || r->http.client_sockaddr_in.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr + ) { + INFOF("rhizome.api.addfile request received from %s, but is only allowed from AF_INET %s", + alloca_sockaddr(&r->http.client_sockaddr_in, sizeof r->http.client_sockaddr_in), + alloca_in_addr(&config.rhizome.api.addfile.allow_host) + ); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 404, "

Not available from here

"); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_addfile_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; + return 0; +} + +int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder) +{ + if ( config.rhizome.api.addfile.uri_path[0] + && strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0 + ) + return rhizome_direct_addfile(r, remainder); + return 1; +} + static int receive_http_response(int sock, char *buffer, size_t buffer_len, struct http_response_parts *parts) { int len = 0; @@ -688,7 +420,7 @@ static int receive_http_response(int sock, char *buffer, size_t buffer_len, stru if ((count = read(sock, &buffer[len], buffer_len - len)) == -1) return WHYF_perror("read(%d, %p, %d)", sock, &buffer[len], (int)buffer_len - len); len += count; - } while (len < buffer_len && count != 0 && !http_header_complete(buffer, len, len)); + } while (len < buffer_len && count != 0 && !is_http_header_complete(buffer, len, len)); if (config.debug.rhizome_rx) DEBUGF("Received HTTP response %s", alloca_toprint(-1, buffer, len)); if (unpack_http_response(buffer, parts) == -1) @@ -702,7 +434,8 @@ static int receive_http_response(int sock, char *buffer, size_t buffer_len, stru DEBUGF("Invalid HTTP reply: missing Content-Length header"); return -1; } - DEBUGF("content_length=%"PRId64, parts->content_length); + if (config.debug.rhizome_rx) + DEBUGF("content_length=%"PRId64, parts->content_length); return len - (parts->content_start - buffer); } @@ -718,36 +451,38 @@ static int fill_buffer(int sock, unsigned char *buffer, int len, int buffer_size void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) { - DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high); + if (config.debug.rhizome_tx) + DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high); rhizome_direct_transport_state_http *state = r->transport_specific_state; sid_t zerosid = SID_ANY; int sock=socket(AF_INET, SOCK_STREAM, 0); if (sock==-1) { - WHY_perror("socket"); + WHY_perror("socket"); goto end; - } + } struct hostent *hostent; hostent = gethostbyname(state->host); if (!hostent) { - DEBUGF("could not resolve hostname"); + if (config.debug.rhizome_tx) + DEBUGF("could not resolve hostname"); goto end; } - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_port = htons(state->port); + struct sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(state->port); addr.sin_addr = *((struct in_addr *)hostent->h_addr); - bzero(&(addr.sin_zero),8); + bzero(&(addr.sin_zero),8); - if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) { - WHY_perror("connect"); + if (connect(sock, (struct sockaddr *)&addr, sizeof addr) == -1) { + WHYF_perror("connect(%s)", alloca_sockaddr(&addr, sizeof addr)); close(sock); goto end; } - + char boundary[20]; char buffer[8192]; @@ -785,7 +520,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) int len = strbuf_len(request); int sent=0; while(sentpullP) { /* Need to fetch manifest. Once we have the manifest, then we can - use our normal bundle fetch routines from rhizome_fetch.c + use our normal bundle fetch routines from rhizome_fetch.c Generate a request like: GET /rhizome/manifestbybar/ and add it to our list of HTTP fetch requests, then watch @@ -875,16 +612,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) Then as noted above, we can use that to pull the file down using existing routines. */ - DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i); + if (config.debug.rhizome_tx) + DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i); if (!rhizome_fetch_request_manifest_by_prefix(&addr, &zerosid, &actionlist[i+1], RHIZOME_BAR_PREFIX_BYTES)) { - /* Fetching the manifest, and then using it to see if we want to + /* Fetching the manifest, and then using it to see if we want to fetch the file for import is all handled asynchronously, so just wait for it to finish. */ while (rhizome_any_fetch_active() || rhizome_any_fetch_queued()) fd_poll(); } - + } else if (type==1&&r->pushP) { /* Form up the POST request to submit the appropriate bundle. */ @@ -897,11 +635,14 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } /* Get filehash and size from manifest if present */ - DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); - DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); - DEBUGF("file size = %"PRId64, m->fileLength); + if (config.debug.rhizome_tx) { + DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); + DEBUGF("file size = %"PRId64, m->fileLength); + } int64_t version = rhizome_manifest_get_ll(m, "version"); - DEBUGF("version = %"PRId64,version); + if (config.debug.rhizome_tx) + DEBUGF("version = %"PRId64,version); /* We now have everything we need to compose the POST request and send it. */ @@ -919,8 +660,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) "Content-Type: application/octet-stream\r\n" "\r\n"; /* Work out what the content length should be */ - DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", - m->manifest_all_bytes,m->manifest_bytes); + if (config.debug.rhizome_tx) + DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes); int content_length =strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */ +strlen(boundary) @@ -930,30 +671,28 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) +m->fileLength +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n"); - /* XXX For some reason the above is four bytes out, so fix that */ - content_length+=4; - int len=snprintf(buffer,8192,template,content_length,boundary); len+=snprintf(&buffer[len],8192-len,template2,boundary); memcpy(&buffer[len],m->manifestdata,m->manifest_all_bytes); len+=m->manifest_all_bytes; len+=snprintf(&buffer[len],8192-len,template3,boundary); - addr.sin_family = AF_INET; - addr.sin_port = htons(state->port); + addr.sin_family = AF_INET; + addr.sin_port = htons(state->port); addr.sin_addr = *((struct in_addr *)hostent->h_addr); - bzero(&(addr.sin_zero),8); - + bzero(&(addr.sin_zero),8); + sock=socket(AF_INET, SOCK_STREAM, 0); if (sock==-1) { - DEBUGF("could not open socket"); + if (config.debug.rhizome_tx) + DEBUGF("could not open socket"); goto closeit; - } - if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) - { + } + if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) { + if (config.debug.rhizome_tx) DEBUGF("Could not connect to remote"); - goto closeit; - } + goto closeit; + } int sent=0; /* Send buffer now */ @@ -968,12 +707,12 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) rhizome_filehash_t filehash; if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1) goto closeit; - + struct rhizome_read read; bzero(&read, sizeof read); if (rhizome_open_read(&read, &filehash)) goto closeit; - + int64_t read_ofs; for(read_ofs=0;read_ofsfileLength;){ unsigned char buffer[4096]; @@ -994,7 +733,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } write_ofs+=written; } - + read_ofs+=bytes_read; } rhizome_read_close(&read); @@ -1006,7 +745,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) int r=write(sock,&buffer[sent],len-sent); if (r>0) sent+=r; if (r<0) goto closeit; - } + } /* get response back. */ if (receive_http_response(sock, buffer, sizeof buffer, &parts) == -1) @@ -1023,7 +762,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } free(actionlist); - + /* now update cursor according to what range was covered in the response. We set our current position to just past the high limit of the returned cursor. @@ -1033,18 +772,20 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) end up in an infinite loop. We could also end up in a very long finite loop if the cursor doesn't advance far. A simple solution is to not adjust the cursor position, and simply re-attempt the sync until no actions result. - That will do for now. + That will do for now. */ #ifdef FANCY_CURSOR_POSITION_HANDLING rhizome_direct_bundle_cursor *c=rhizome_direct_bundle_iterator(10); assert(c!=NULL); - if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10)) - { + if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10)) { + if (config.debug.rhizome_tx) DEBUGF("Couldn't unpickle range. This should never happen. Assuming near and far cursor ranges match."); - } + } else { - DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high); - DEBUGF("c->buffer_size=%d",c->buffer_size); + if (config.debug.rhizome_tx) { + DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high); + DEBUGF("c->buffer_size=%d",c->buffer_size); + } r->cursor->size_low=c->limit_size_high; /* Set tail of BID to all high, as we assume the far end has returned all BIDs with the specified prefix. */ @@ -1055,7 +796,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) #endif end: - /* Warning: tail recursion when done this way. + /* Warning: tail recursion when done this way. Should be triggered by an asynchronous event. But this will do for now. */ rhizome_direct_continue_sync_request(r); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 20a51f26..7c8dc534 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -181,7 +181,7 @@ int rhizome_fetch_queue_bytes(){ return bytes; } -int rhizome_fetch_status_html(struct strbuf *b) +int rhizome_fetch_status_html(strbuf b) { int i,j; for(i=0;iprevious); slot->previous=NULL; }else{ - strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n", - slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength); + assert(slot->previous->fileLength >= slot->manifest->journalTail); + assert(slot->manifest->fileLength > 0); + strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n", + slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1); } } @@ -1483,7 +1485,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm) slot->alarm.deadline = slot->alarm.alarm + config.rhizome.idle_timeout; schedule(&slot->alarm); slot->request_len += bytes; - if (http_header_complete(slot->request, slot->request_len, bytes)) { + if (is_http_header_complete(slot->request, slot->request_len, bytes)) { if (config.debug.rhizome_rx) DEBUGF("Got HTTP reply: %s", alloca_toprint(160, slot->request, slot->request_len)); /* We have all the reply headers, so parse them, taking care of any following bytes of @@ -1495,9 +1497,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm) rhizome_fetch_switch_to_mdp(slot); return; } - if (parts.code != 200) { + if (parts.code != 200 && parts.code != 206) { if (config.debug.rhizome_rx) - DEBUGF("Failed HTTP request: rhizome server returned %d != 200 OK", parts.code); + DEBUGF("Failed HTTP request: rhizome server returned %03u", parts.code); rhizome_fetch_switch_to_mdp(slot); return; } @@ -1564,7 +1566,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm) This function takes a pointer to a buffer into which the entire HTTP response header has been read. The caller must have ensured that the buffer contains at least one consecutive pair of newlines '\n', optionally with carriage returns '\r' preceding and optionally interspersed with - nul characters '\0' (which can originate from telnet). The http_header_complete() function + nul characters '\0' (which can originate from telnet). The is_http_header_complete() function is useful for this. This returns pointers to within the supplied buffer, and may overwrite some characters in the buffer, for example to nul-terminate a string that was terminated by space ' ' or newline '\r' @@ -1576,7 +1578,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm) int unpack_http_response(char *response, struct http_response_parts *parts) { IN(); - parts->code = -1; + parts->code = 0; parts->reason = NULL; parts->range_start=0; parts->content_length = -1; diff --git a/rhizome_http.c b/rhizome_http.c index 3ae5c495..d08180eb 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -23,18 +23,73 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #ifdef HAVE_SYS_FILIO_H #include #endif +#include #include "serval.h" #include "overlay_address.h" #include "conf.h" #include "str.h" #include "rhizome.h" +#include "http_server.h" + #define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32 -struct sched_ent server_alarm; -struct profile_total server_stats; +struct http_handler{ + const char *path; + int (*parser)(rhizome_http_request *r, const char *remainder); +}; -struct profile_total connection_stats; +static int rhizome_status_page(rhizome_http_request *r, const char *remainder); +static int rhizome_file_page(rhizome_http_request *r, const char *remainder); +static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder); +static int interface_page(rhizome_http_request *r, const char *remainder); +static int neighbour_page(rhizome_http_request *r, const char *remainder); +static int fav_icon_header(rhizome_http_request *r, const char *remainder); +static int root_page(rhizome_http_request *r, const char *remainder); + +extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder); +extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder); +extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder); + +struct http_handler paths[]={ + {"/rhizome/status", rhizome_status_page}, + {"/rhizome/file/", rhizome_file_page}, + {"/rhizome/import", rhizome_direct_import}, + {"/rhizome/enquiry", rhizome_direct_enquiry}, + {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, + {"/rhizome/", rhizome_direct_dispatch}, + {"/interface/", interface_page}, + {"/neighbour/", neighbour_page}, + {"/favicon.ico", fav_icon_header}, + {"/", root_page}, +}; + +static int rhizome_dispatch(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + INFOF("RHIZOME HTTP SERVER, %s %s", r->http.verb, r->http.path); + r->http.response.content_generator = NULL; + unsigned i; + for (i = 0; i < NELS(paths); ++i) { + const char *remainder; + if (str_startswith(r->http.path, paths[i].path, &remainder)){ + int ret = paths[i].parser(r, remainder); + if (ret < 0) { + http_request_simple_response(&r->http, 500, NULL); + return 0; + } + if (ret == 0) + return 0; + } + } + http_request_simple_response(&r->http, 404, NULL); + return 0; +} + +struct sched_ent server_alarm; +struct profile_total server_stats = { + .name = "rhizome_server_poll", +}; /* HTTP server and client code for rhizome transfers and rhizome direct. @@ -47,9 +102,6 @@ static int rhizome_server_socket = -1; static int request_count=0; static time_ms_t rhizome_server_last_start_attempt = -1; -int (*rhizome_http_parse_func)(rhizome_http_request *)=NULL; -const char *rhizome_http_parse_func_description="(null)"; - // Format icon data using: // od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/ */,/g' unsigned char favicon_bytes[]={ @@ -88,9 +140,7 @@ int is_rhizome_http_server_running() Return 1 if the server is already started successfully. Return 2 if the server was not started because it is too soon since last failed attempt. */ -int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *), - const char *parse_func_desc, - uint16_t port_low, uint16_t port_high) +int rhizome_http_server_start(uint16_t port_low, uint16_t port_high) { if (rhizome_server_socket != -1) return 1; @@ -100,7 +150,7 @@ int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *), if (now < rhizome_server_last_start_attempt + 5000) return 2; rhizome_server_last_start_attempt = now; - if (config.debug.rhizome_tx) + if (config.debug.rhizome_httpd) DEBUGF("Starting rhizome HTTP server"); uint16_t port; @@ -162,15 +212,10 @@ success: else INFOF("HTTP SERVER (LIMITED SERVICE), START port=%"PRIu16" fd=%d", port, rhizome_server_socket); - /* Remember which function to call when handling client connections */ - rhizome_http_parse_func=parse_func; - rhizome_http_parse_func_description=parse_func_desc; - rhizome_http_server_port = port; /* Add Rhizome HTTPd server to list of file descriptors to watch */ server_alarm.function = rhizome_server_poll; - server_stats.name="rhizome_server_poll"; - server_alarm.stats=&server_stats; + server_alarm.stats = &server_stats; server_alarm.poll.fd = rhizome_server_socket; server_alarm.poll.events = POLLIN; watch(&server_alarm); @@ -178,90 +223,16 @@ success: } -void rhizome_client_poll(struct sched_ent *alarm) +static void rhizome_server_finalise_http_request(struct http_request *_r) { - rhizome_http_request *r = (rhizome_http_request *)alarm; - if (alarm->poll.revents == 0 || alarm->poll.revents & (POLLHUP | POLLERR)){ - if (config.debug.rhizome_tx) - DEBUGF("Closing connection due to timeout or error %d", alarm->poll.revents); - rhizome_server_free_http_request(r); - return; - } - - if (alarm->poll.revents & POLLIN){ - switch(r->request_type) - { - case RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART: - { - /* Reading multi-part form data. Read some bytes and proces them. */ - char buffer[16384]; - sigPipeFlag=0; - int bytes = read_nonblock(r->alarm.poll.fd, buffer, 16384); - /* If we got some data, see if we have found the end of the HTTP request */ - if (bytes > 0) { - // reset inactivity timer - r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - rhizome_direct_process_post_multipart_bytes(r,buffer,bytes); - } - /* We don't drop the connection on an empty read, because that results - in connections dropping when they shouldn't, including during testing. - The idle timeout should drop the connections instead. - */ - if (sigPipeFlag) { - if (config.debug.rhizome_tx) - DEBUG("Received SIGPIPE, closing connection"); - rhizome_server_free_http_request(r); - return; - } - } - break; - - case RHIZOME_HTTP_REQUEST_RECEIVING: - /* Keep reading until we have two CR/LFs in a row */ - r->request[r->request_length] = '\0'; - sigPipeFlag=0; - int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], sizeof r->request - r->request_length); - /* If we got some data, see if we have found the end of the HTTP request */ - if (bytes > 0) { - // reset inactivity timer - r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - r->request_length += bytes; - r->header_length = http_header_complete(r->request, r->request_length, bytes); - if (r->header_length){ - /* We have the request. Now parse it to see if we can respond to it */ - if (rhizome_http_parse_func!=NULL) - rhizome_http_parse_func(r); - } - } else { - if (config.debug.rhizome_tx) - DEBUG("Empty read, closing connection"); - rhizome_server_free_http_request(r); - return; - } - if (sigPipeFlag) { - if (config.debug.rhizome_tx) - DEBUG("Received SIGPIPE, closing connection"); - rhizome_server_free_http_request(r); - return; - } - break; - } - } - - if (alarm->poll.revents & POLLOUT){ - /* Socket already has request -- so just try to send some data. */ - rhizome_server_http_send_bytes(r); - } - return; + rhizome_http_request *r = (rhizome_http_request *) _r; + rhizome_read_close(&r->read_state); + request_count--; } -static unsigned int rhizome_http_request_uuid_counter=0; +static int rhizome_dispatch(struct http_request *); + +static unsigned int rhizome_http_request_uuid_counter = 0; void rhizome_server_poll(struct sched_ent *alarm) { @@ -269,7 +240,10 @@ void rhizome_server_poll(struct sched_ent *alarm) struct sockaddr addr; unsigned int addr_len = sizeof addr; int sock; - if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) != -1) { + if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) == -1) { + if (errno && errno != EAGAIN) + WARN_perror("accept"); + } else { struct sockaddr_in *peerip=NULL; if (addr.sa_family == AF_INET) { peerip = (struct sockaddr_in *)&addr; @@ -285,56 +259,34 @@ void rhizome_server_poll(struct sched_ent *alarm) addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data) ); } - rhizome_http_request *request = calloc(sizeof(rhizome_http_request), 1); + rhizome_http_request *request = emalloc_zero(sizeof(rhizome_http_request)); if (request == NULL) { - WHYF_perror("calloc(%u, 1)", (int)sizeof(rhizome_http_request)); - WHY("Cannot respond to request, out of memory"); + WHY("Cannot respond to HTTP request, out of memory"); close(sock); } else { request_count++; - request->uuid=rhizome_http_request_uuid_counter++; - if (peerip) request->requestor=*peerip; - else bzero(&request->requestor,sizeof(request->requestor)); - request->data_file_name[0]=0; - /* We are now trying to read the HTTP request */ - request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING; - request->alarm.function = rhizome_client_poll; - request->read_state.blob_fd=-1; - request->read_state.blob_rowid=-1; - connection_stats.name="rhizome_client_poll"; - request->alarm.stats=&connection_stats; - request->alarm.poll.fd=sock; - request->alarm.poll.events=POLLIN; - request->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT; - request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT; - // watch for the incoming http request - watch(&request->alarm); - // set an inactivity timeout to close the connection - schedule(&request->alarm); + request->uuid = rhizome_http_request_uuid_counter++; + request->data_file_name[0] = '\0'; + request->read_state.blob_fd = -1; + request->read_state.blob_rowid = -1; + if (peerip) + request->http.client_sockaddr_in = *peerip; + request->http.handle_headers = rhizome_dispatch; + request->http.debug_flag = &config.debug.rhizome_httpd; + request->http.disable_tx_flag = &config.debug.rhizome_nohttptx; + request->http.finalise = rhizome_server_finalise_http_request; + request->http.free = free; + request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT; + http_request_init(&request->http, sock); } } - if (errno && errno != EAGAIN) - WARN_perror("accept"); } if (alarm->poll.revents & (POLLHUP | POLLERR)) { INFO("Error on tcp listen socket"); } } -int rhizome_server_free_http_request(rhizome_http_request *r) -{ - unwatch(&r->alarm); - unschedule(&r->alarm); - close(r->alarm.poll.fd); - if (r->buffer) - free(r->buffer); - rhizome_read_close(&r->read_state); - free(r); - request_count--; - return 0; -} - -int http_header_complete(const char *buf, size_t len, size_t read_since_last_call) +int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call) { IN(); const char *bufend = buf + len; @@ -360,144 +312,158 @@ int http_header_complete(const char *buf, size_t len, size_t read_since_last_cal OUT(); } -static int neighbour_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int neighbour_page(rhizome_http_request *r, const char *remainder) { + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char buf[8*1024]; - strbuf b=strbuf_local(buf, sizeof buf); - + strbuf b = strbuf_local(buf, sizeof buf); sid_t neighbour_sid; if (str_to_sid_t(&neighbour_sid, remainder) == -1) - return -1; - + return 1; struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0); if (!neighbour) return 1; - strbuf_puts(b, ""); link_neighbour_status_html(b, neighbour); strbuf_puts(b, ""); if (strbuf_overrun(b)) return -1; - rhizome_server_simple_http_response(r, 200, buf); + http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); return 0; } -static int interface_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int interface_page(rhizome_http_request *r, const char *remainder) { + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char buf[8*1024]; strbuf b=strbuf_local(buf, sizeof buf); int index=atoi(remainder); if (index<0 || index>=OVERLAY_MAX_INTERFACES) return 1; - strbuf_puts(b, ""); interface_state_html(b, &overlay_interfaces[index]); strbuf_puts(b, ""); if (strbuf_overrun(b)) return -1; - - rhizome_server_simple_http_response(r, 200, buf); + http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); return 0; } -static int rhizome_status_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int rhizome_status_page(rhizome_http_request *r, const char *remainder) { if (!is_rhizome_http_enabled()) return 1; if (*remainder) return 1; - - char buf[32*1024]; - struct strbuf b; - strbuf_init(&b, buf, sizeof buf); - strbuf_puts(&b, ""); - strbuf_sprintf(&b, "%d HTTP requests
", request_count); - strbuf_sprintf(&b, "%d Bundles transferring via MDP
", rhizome_cache_count()); - rhizome_fetch_status_html(&b); - strbuf_puts(&b, ""); - if (strbuf_overrun(&b)) - return -1; - rhizome_server_simple_http_response(r, 200, buf); - return 0; -} - -static int rhizome_file_content(rhizome_http_request *r) -{ - int suggested_size=65536; - if (suggested_size > r->read_state.length - r->read_state.offset) - suggested_size = r->read_state.length - r->read_state.offset; - if (suggested_size<=0) + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); return 0; - - if (r->buffer_size < suggested_size){ - r->buffer_size = suggested_size; - if (r->buffer) - free(r->buffer); - r->buffer = malloc(r->buffer_size); } - - if (!r->buffer) + char buf[32*1024]; + strbuf b = strbuf_local(buf, sizeof buf); + strbuf_puts(b, ""); + strbuf_sprintf(b, "%d HTTP requests
", request_count); + strbuf_sprintf(b, "%d Bundles transferring via MDP
", rhizome_cache_count()); + rhizome_fetch_status_html(b); + strbuf_puts(b, ""); + if (strbuf_overrun(b)) return -1; - - r->buffer_length = rhizome_read(&r->read_state, r->buffer, r->buffer_size); + http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); return 0; } -static int rhizome_file_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int rhizome_file_content(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + assert(r->http.response_buffer_sent == 0); + assert(r->http.response_buffer_length == 0); + assert(r->read_state.offset < r->read_state.length); + uint64_t readlen = r->read_state.length - r->read_state.offset; + size_t suggested_size = 64 * 1024; + if (suggested_size > readlen) + suggested_size = readlen; + if (r->http.response_buffer_size < suggested_size) + http_request_set_response_bufsize(&r->http, suggested_size); + if (r->http.response_buffer == NULL) + http_request_set_response_bufsize(&r->http, 1); + if (r->http.response_buffer == NULL) + return -1; + ssize_t len = rhizome_read(&r->read_state, + (unsigned char *)r->http.response_buffer, + r->http.response_buffer_size); + if (len == -1) + return -1; + assert((size_t) len <= r->http.response_buffer_size); + r->http.response_buffer_length += (size_t) len; + return 0; +} + +static int rhizome_file_page(rhizome_http_request *r, const char *remainder) { /* Stream the specified payload */ if (!is_rhizome_http_enabled()) return 1; - - rhizome_filehash_t filehash; - if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) - return -1; - - bzero(&r->read_state, sizeof(r->read_state)); - - /* Refuse to honour HTTP request if required (used for debugging and - testing transition from HTTP to MDP) */ - if (rhizome_open_read(&r->read_state, &filehash)) - return 1; - - if (r->read_state.length==-1){ - if (rhizome_read(&r->read_state, NULL, 0)){ - rhizome_read_close(&r->read_state); - return 1; - } - } - - const char *range=str_str((char*)headers,"Range: bytes=",-1); - r->read_state.offset = r->source_index = 0; - - if (range){ - sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset); - if (0) - DEBUGF("Found range header %"PRId64,r->read_state.offset); - } - - if (r->read_state.length - r->read_state.offset<=0){ - rhizome_server_simple_http_response(r, 200, ""); + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); return 0; } - - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = 200; - hr.content_type = "application/binary"; - hr.content_start = r->read_state.offset; - hr.content_end = r->read_state.length; - hr.content_length = r->read_state.length; - hr.body = NULL; - r->generator = rhizome_file_content; - rhizome_server_set_response(r, &hr); + if (r->http.request_header.content_range_count > 1) { + // To support byte range sets, eg, Range: bytes=0-100,200-300,400- we would have + // to reply with a multipart/byteranges MIME content. + http_request_simple_response(&r->http, 501, "Not Implemented: Byte range sets"); + return 0; + } + rhizome_filehash_t filehash; + if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) + return 1; + bzero(&r->read_state, sizeof r->read_state); + int n = rhizome_open_read(&r->read_state, &filehash); + if (n == -1) { + http_request_simple_response(&r->http, 500, NULL); + return 0; + } + if (n != 0) + return 1; + if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) { + rhizome_read_close(&r->read_state); + return 1; + } + assert(r->read_state.length != -1); + r->http.response.header.resource_length = r->read_state.length; + if (r->http.request_header.content_range_count > 0) { + assert(r->http.request_header.content_range_count == 1); + struct http_range closed; + unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length); + if (n == 0 || http_range_bytes(&closed, 1) == 0) { + http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable + return 0; + } + r->http.response.header.content_range_start = closed.first; + r->http.response.header.content_length = closed.last - closed.first + 1; + r->read_state.offset = closed.first; + } else { + r->http.response.header.content_range_start = 0; + r->http.response.header.content_length = r->http.response.header.resource_length; + r->read_state.offset = 0; + } + http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content); return 0; } -static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder) { if (!is_rhizome_http_enabled()) return 1; + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } rhizome_bid_t prefix; const char *endp = NULL; unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp); @@ -505,27 +471,32 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde return 1; // not found rhizome_manifest *m = rhizome_new_manifest(); int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m); - if (ret==0) - rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes); + if (ret == -1) + http_request_simple_response(&r->http, 500, NULL); + else if (ret == 0) + http_request_response_static(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes); rhizome_manifest_free(m); - return ret; + return ret <= 0 ? 0 : 1; } -static int fav_icon_header(rhizome_http_request *r, const char *remainder, const char *headers) +static int fav_icon_header(rhizome_http_request *r, const char *remainder) { if (*remainder) return 1; - rhizome_server_http_response(r, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); + http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); return 0; } -static int root_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int root_page(rhizome_http_request *r, const char *remainder) { if (*remainder) return 1; - + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char temp[8192]; - strbuf b=strbuf_local(temp, sizeof(temp)); + strbuf b = strbuf_local(temp, sizeof temp); strbuf_sprintf(b, "" "

Hello, I'm %s*


" "Interfaces;
", @@ -533,250 +504,19 @@ static int root_page(rhizome_http_request *r, const char *remainder, const char int i; for (i=0;i%d: %s, TX: %d, RX: %d
", + strbuf_sprintf(b, "%d: %s, TX: %d, RX: %d
", i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count); } - strbuf_puts(b, "Neighbours;
"); link_neighbour_short_status_html(b, "/neighbour"); - if (is_rhizome_http_enabled()){ strbuf_puts(b, "Rhizome Status
"); } strbuf_puts(b, ""); - if (strbuf_overrun(b)) - return -1; - rhizome_server_simple_http_response(r, 200, temp); + if (strbuf_overrun(b)) { + WHY("HTTP Root page buffer overrun"); + http_request_simple_response(&r->http, 500, NULL); + } else + http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b)); return 0; } - -struct http_handler{ - const char *path; - int (*parser)(rhizome_http_request *r, const char *remainder, const char *headers); -}; - -struct http_handler paths[]={ - {"/rhizome/status", rhizome_status_page}, - {"/rhizome/file/", rhizome_file_page}, - {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, - {"/interface/", interface_page}, - {"/neighbour/", neighbour_page}, - {"/favicon.ico", fav_icon_header}, - {"/", root_page}, -}; - -int rhizome_direct_parse_http_request(rhizome_http_request *r); -int rhizome_server_parse_http_request(rhizome_http_request *r) -{ - // Start building up a response. - // Parse the HTTP "GET" line. - char *path = NULL; - char *headers = NULL; - if (str_startswith(r->request, "POST ", (const char **)&path)) { - return rhizome_direct_parse_http_request(r); - } else if (str_startswith(r->request, "GET ", (const char **)&path)) { - const char *p; - size_t header_length = 0; - size_t pathlen = 0; - // This loop is guaranteed to terminate before the end of the buffer, because we know that the - // buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block. - for (p = path; !isspace(*p); ++p) - ; - pathlen = p - path; - if ( str_startswith(p, " HTTP/1.", &p) - && (str_startswith(p, "0", &p) || str_startswith(p, "1", &p)) - && (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers)) - ){ - path[pathlen] = '\0'; - header_length = r->header_length - (headers - r->request); - headers[header_length] = '\0'; - }else - path = NULL; - } - - if (!path) { - if (config.debug.rhizome_tx) - DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length)); - rhizome_server_simple_http_response(r, 400, "

Malformed request

\r\n"); - return 0; - } - - char *id = NULL; - INFOF("RHIZOME HTTP SERVER, GET %s", path); - - int i; - r->generator=NULL; - - for (i=0;i

Internal Error

\r\n"); - if (ret>0) - rhizome_server_simple_http_response(r, 404, "

Not Found

\r\n"); - - /* Try sending data immediately. */ - rhizome_server_http_send_bytes(r); - - return 0; - } - } - - rhizome_server_simple_http_response(r, 404, "

Not Found

\r\n"); - return 0; -} - - -/* Return appropriate message for HTTP response codes, both known and unknown. */ -static const char *httpResultString(int response_code) { - switch (response_code) { - case 200: return "OK"; - case 201: return "Created"; - case 206: return "Partial Content"; - case 404: return "Not found"; - case 500: return "Internal server error"; - default: - if (response_code<=4) - return "Unknown status code"; - else - return "A suffusion of yellow"; - } -} - -static strbuf strbuf_build_http_response(strbuf sb, const struct http_response *h) -{ - strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", h->result_code, httpResultString(h->result_code)); - strbuf_sprintf(sb, "Content-type: %s\r\n", h->content_type); - if (h->content_end && h->content_length && (h->content_start!=0 || h->content_end!=h->content_length)) - strbuf_sprintf(sb, - "Content-range: bytes %"PRIu64"-%"PRIu64"/%"PRIu64"\r\n" - "Content-length: %"PRIu64"\r\n", - h->content_start, h->content_end, h->content_length, h->content_end - h->content_start); - else if (h->content_length) - strbuf_sprintf(sb, "Content-length: %"PRIu64"\r\n", h->content_length); - strbuf_puts(sb, "\r\n"); - return sb; -} - -int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h) -{ - r->request_type=0; - - if (config.debug.rhizome_nohttptx) - unwatch(&r->alarm); - else{ - /* Switching to writing, so update the call-back */ - r->alarm.poll.events=POLLOUT; - watch(&r->alarm); - } - - strbuf b = strbuf_local((char *) r->buffer, r->buffer_size); - strbuf_build_http_response(b, h); - if (r->buffer == NULL || strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) { - // Need a bigger buffer - if (r->buffer) - free(r->buffer); - r->buffer_size = strbuf_count(b) + 1; - if (h->body) - r->buffer_size += h->content_length; - r->buffer = malloc(r->buffer_size); - if (r->buffer == NULL) { - WHYF_perror("malloc(%u)", r->buffer_size); - r->buffer_size = 0; - return WHY("Cannot send response, out of memory"); - } - strbuf_init(b, (char *) r->buffer, r->buffer_size); - strbuf_build_http_response(b, h); - if (strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) - return WHYF("Bug! Cannot send response, buffer not big enough"); - } - r->buffer_length = strbuf_len(b); - if (h->body){ - bcopy(h->body, strbuf_end(b), h->content_length); - r->buffer_length+=h->content_length; - } - r->buffer_offset = 0; - if (config.debug.rhizome_tx) - DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->buffer, r->buffer_length)); - return 0; -} - -int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response) -{ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = result; - hr.content_type = "text/html"; - hr.content_length = strlen(response); - hr.body = response; - if (result==400) { - DEBUGF("Rejecting http request as malformed due to: %s", - response); - } - return rhizome_server_set_response(r, &hr); -} - -int rhizome_server_http_response(rhizome_http_request *r, int result, - const char *mime_type, const char *body, uint64_t bytes) -{ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = result; - hr.content_type = mime_type; - hr.content_length = bytes; - hr.body = body; - return rhizome_server_set_response(r, &hr); -} - -int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes) -{ - return rhizome_server_http_response(r, result, mime_type, NULL, bytes); -} - -/* - return codes: - 1: connection still open. - 0: connection finished. - <0: an error occurred. -*/ -int rhizome_server_http_send_bytes(rhizome_http_request *r) -{ - // Don't send anything if disabled for testing HTTP->MDP Rhizome failover - if (config.debug.rhizome_nohttptx) - return 1; - - // write one block of buffered data - if(r->buffer_offset < r->buffer_length){ - int bytes=r->buffer_length - r->buffer_offset; - bytes=write(r->alarm.poll.fd,&r->buffer[r->buffer_offset],bytes); - if (bytes<0){ - // stop writing when the tcp buffer is full - // TODO errors? - return 1; - } - r->buffer_offset+=bytes; - - // reset inactivity timer - r->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm+RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - - // allow other alarms to fire and wait for the next POLLOUT - return 1; - } - - r->buffer_offset=r->buffer_length=0; - - if (r->generator){ - r->generator(r); - } - - // once we've written the whole buffer, and nothing new has been generated, close the connection - if (!r->buffer_length){ - if (config.debug.rhizome_tx) - DEBUG("Closing connection, done"); - return rhizome_server_free_http_request(r); - } - return 1; -} diff --git a/rhizome_store.c b/rhizome_store.c index 5e761852..bbf6a51c 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -1,3 +1,4 @@ +#include #include "serval.h" #include "rhizome.h" #include "conf.h" @@ -71,7 +72,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path); write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664); - if (write->blob_fd<0) + if (write->blob_fd == -1) goto insert_row_fail; if (config.debug.externalblobs) @@ -111,7 +112,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex } if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){ - if (write->blob_fd>=0){ + if (write->blob_fd != -1){ if (config.debug.externalblobs) DEBUGF("Cancel write to fd %d", write->blob_fd); close(write->blob_fd); @@ -162,7 +163,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer // open database locks static int write_get_lock(struct rhizome_write *write_state){ - if (write_state->blob_fd>=0 || write_state->sql_blob) + if (write_state->blob_fd != -1 || write_state->sql_blob) return 0; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; @@ -186,14 +187,15 @@ static int write_get_lock(struct rhizome_write *write_state){ } // write data to disk -static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, int data_size){ +static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size) +{ if (data_size<=0) return 0; if (file_offset != write_state->written_offset) WARNF("Writing file data out of order! [%"PRId64",%"PRId64"]", file_offset, write_state->written_offset); - if (write_state->blob_fd>=0) { + if (write_state->blob_fd != -1) { int ofs=0; // keep trying until all of the data is written. lseek(write_state->blob_fd, file_offset, SEEK_SET); @@ -233,7 +235,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u // close database locks static int write_release_lock(struct rhizome_write *write_state){ int ret=0; - if (write_state->blob_fd>=0) + if (write_state->blob_fd != -1) return 0; if (write_state->sql_blob){ @@ -252,7 +254,8 @@ static int write_release_lock(struct rhizome_write *write_state){ // Write data buffers in any order, the data will be cached and streamed into the database in file order. // Though there is an upper bound on the amount of cached data -int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size){ +int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size) +{ if (offset + data_size > write_state->file_length) data_size = write_state->file_length - offset; @@ -260,12 +263,12 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi int ret=0; int should_write = 0; // if we are writing to a file, or already have the sql blob open, write as much as we can. - if (write_state->blob_fd>=0 || write_state->sql_blob){ + if (write_state->blob_fd != -1 || write_state->sql_blob){ should_write = 1; }else{ // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go. // (Not perfect if the range overlaps) - int64_t new_size = write_state->written_offset + write_state->buffer_size + data_size; + uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size; if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE) should_write = 1; } @@ -324,7 +327,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi if (!*ptr || offset < (*ptr)->offset){ // found the insert position in the list - int64_t size = data_size; + size_t size = data_size; // allow for buffers to overlap, we may need to split the incoming buffer into multiple pieces. if (*ptr && offset+size > (*ptr)->offset) @@ -348,7 +351,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi break; if (config.debug.rhizome) - DEBUGF("Caching block @%"PRId64", %"PRId64, offset, size); + DEBUGF("Caching block @%"PRId64", %zu", offset, size); struct rhizome_write_buffer *i = emalloc(size + sizeof(struct rhizome_write_buffer)); if (!i){ ret=-1; @@ -377,7 +380,8 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi return ret; } -int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size){ +int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size) +{ return rhizome_random_write(write_state, write_state->file_offset, buffer, data_size); } @@ -398,8 +402,8 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){ if (write->file_offset + size > write->file_length) size=write->file_length - write->file_offset; - int r = fread(buffer, 1, size, f); - if (r==-1){ + size_t r = fread(buffer, 1, size, f); + if (ferror(f)){ ret = WHY_perror("fread"); goto end; } @@ -417,7 +421,7 @@ end: int rhizome_fail_write(struct rhizome_write *write) { - if (write->blob_fd>=0){ + if (write->blob_fd != -1){ if (config.debug.externalblobs) DEBUGF("Closing and removing fd %d", write->blob_fd); close(write->blob_fd); @@ -572,12 +576,12 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath) } // store a whole payload from a single buffer -int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length) +int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length) { if (m->fileLength<=0) return 0; if (length!=m->fileLength) - return WHYF("Expected %"PRId64" bytes, got %d", m->fileLength, length); + return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length); /* Import the file first, checking the hash as we go */ struct rhizome_write write; @@ -723,62 +727,74 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp return 0; // file opened } +static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read *read_state, unsigned char *buffer, size_t bufsz) +{ + IN(); + if (read_state->blob_fd != -1) { + if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1) + RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset)); + if (bufsz == 0) + RETURN(0); + ssize_t rd = read(read_state->blob_fd, buffer, bufsz); + if (rd == -1) + RETURN(WHYF_perror("read(%d,%p,%zu)", read_state->blob_fd, buffer, bufsz)); + if (config.debug.externalblobs) + DEBUGF("Read %zu bytes from fd=%d @%"PRIx64, (size_t) rd, read_state->blob_fd, read_state->offset); + RETURN(rd); + } + if (read_state->blob_rowid == -1) + RETURN(WHY("file not open")); + sqlite3_blob *blob = NULL; + int ret; + do { + assert(blob == NULL); + ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob); + } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_open")); + if (ret != SQLITE_OK) { + assert(blob == NULL); + RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db))); + } + assert(blob != NULL); + if (read_state->length == -1) + read_state->length = sqlite3_blob_bytes(blob); + // A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out + // the length. + size_t bytes_read = 0; + if (buffer && bufsz && read_state->offset < read_state->length) { + bytes_read = read_state->length - read_state->offset; + if (bytes_read > bufsz) + bytes_read = bufsz; + assert(bytes_read > 0); + do { + ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset); + } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read")); + if (ret != SQLITE_OK) { + WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_db)); + sqlite3_blob_close(blob); + RETURN(-1); + } + } + sqlite3_blob_close(blob); + RETURN(bytes_read); + OUT(); +} + /* Read content from the store, hashing and decrypting as we go. Random access is supported, but hashing requires all payload contents to be read sequentially. */ // returns the number of bytes read -int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length) +ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, size_t buffer_length) { IN(); // hash check failed, just return an error if (read_state->invalid) RETURN(-1); - - int bytes_read = 0; - if (read_state->blob_fd >= 0) { - if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1) - RETURN(WHYF_perror("lseek(%d,%ld,SEEK_SET)", read_state->blob_fd, (long)read_state->offset)); - bytes_read = read(read_state->blob_fd, buffer, buffer_length); - if (bytes_read == -1) - RETURN(WHYF_perror("read(%d,%p,%ld)", read_state->blob_fd, buffer, (long)buffer_length)); - if (config.debug.externalblobs) - DEBUGF("Read %d bytes from fd %d @%"PRIx64, bytes_read, read_state->blob_fd, read_state->offset); - } else if (read_state->blob_rowid != -1) { - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - do{ - sqlite3_blob *blob = NULL; - int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob); - if (sqlite_code_busy(ret)) - goto again; - else if(ret!=SQLITE_OK) - RETURN(WHYF("sqlite3_blob_open failed: %s",sqlite3_errmsg(rhizome_db))); - if (read_state->length==-1) - read_state->length=sqlite3_blob_bytes(blob); - bytes_read = read_state->length - read_state->offset; - if (bytes_read>buffer_length) - bytes_read=buffer_length; - // allow the caller to do a dummy read, just to work out the length - if (!buffer) - bytes_read=0; - if (bytes_read>0){ - ret = sqlite3_blob_read(blob, buffer, bytes_read, read_state->offset); - if (sqlite_code_busy(ret)) - goto again; - else if(ret!=SQLITE_OK){ - WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db)); - sqlite3_blob_close(blob); - RETURN(-1); - } - } - sqlite3_blob_close(blob); - break; - again: - if (blob) sqlite3_blob_close(blob); - if (!sqlite_retry(&retry, "sqlite3_blob_open")) - RETURN(-1); - } while (1); - } else - RETURN(WHY("file not open")); - + + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + ssize_t n = rhizome_read_retry(&retry, read_state, buffer, buffer_length); + if (n == -1) + RETURN(-1); + size_t bytes_read = (size_t) n; + // hash the payload as we go, but only if we happen to read the payload data in order if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){ SHA512_Update(&read_state->sha512_context, buffer, bytes_read); @@ -810,9 +826,9 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf } /* Read len bytes from read->offset into data, using *buffer to cache any reads */ -int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len) +int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len) { - int bytes_copied=0; + size_t bytes_copied=0; while (len>0){ // make sure we only attempt to read data that actually exists @@ -822,7 +838,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer // if we can supply either the beginning or end of the data from cache, do that first. uint64_t ofs=read->offset - buffer->offset; if (ofs>=0 && ofs<=buffer->len){ - int size=len; + size_t size=len; if (size > buffer->len - ofs) size = buffer->len - ofs; if (size>0){ @@ -838,7 +854,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer ofs = (read->offset+len) - buffer->offset; if (ofs>0 && ofs<=buffer->len){ - int size=len; + size_t size=len; if (size > ofs) size = ofs; if (size>0){ @@ -854,10 +870,12 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer // remember the requested read offset so we can put it back ofs = read->offset; buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1); - buffer->len = rhizome_read(read, buffer->data, sizeof(buffer->data)); + ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data)); read->offset = ofs; - if (buffer->len<=0) - return buffer->len; + buffer->len = 0; + if (len == -1) + return -1; + buffer->len = (size_t) len; } return bytes_copied; } @@ -978,7 +996,7 @@ int rhizome_cache_count() } // read a block of data, caching meta data for reuse -int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length) +int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, size_t length) { // look for a cached entry struct cache_entry **ptr = find_entry_location(&root, bidp, version); @@ -1120,17 +1138,17 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write, unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE]; while(length>0){ - int size=sizeof(buffer); + size_t size=sizeof(buffer); if (size > length) size=length; - int r = rhizome_read(read, buffer, size); - if (r<0) + ssize_t r = rhizome_read(read, buffer, size); + if (r == -1) return r; - length -= r; + length -= (size_t) r; - if (rhizome_write_buffer(write, buffer, r)) + if (rhizome_write_buffer(write, buffer, (size_t) r)) return -1; } @@ -1194,12 +1212,12 @@ failure: return ret; } -int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len) +int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len) { struct rhizome_write write; bzero(&write, sizeof write); - int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, len); + int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, (uint64_t) len); if (ret) return -1; diff --git a/rhizome_sync.c b/rhizome_sync.c index ae6db73f..415b0944 100644 --- a/rhizome_sync.c +++ b/rhizome_sync.c @@ -439,7 +439,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t if (count){ mdp.out.payload_length = ob_position(b); - if (config.debug.rhizome) + if (config.debug.rhizome_ads) DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last); overlay_mdp_dispatch(&mdp,0,NULL,0); } @@ -449,7 +449,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t int rhizome_sync_announce() { + int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads); sync_send_response(NULL, 0, HEAD_FLAG, 5); + sqlite_set_tracefunc(oldfunc); return 0; } diff --git a/serval.h b/serval.h index c990ebc8..b6f40e62 100644 --- a/serval.h +++ b/serval.h @@ -1,7 +1,7 @@ /* -Serval Daemon +Serval DNA header file Copyright (C) 2010-2012 Paul Gardner-Stephen -Copyright (C) 2012 Serval Project Inc. +Copyright (C) 2012-2013 Serval Project, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -90,9 +90,6 @@ struct in_addr { #ifdef HAVE_SYS_TIME_H #include #endif -#ifdef HAVE_POLL_H -#include -#endif #ifdef HAVE_NETDB_H #include #endif @@ -109,6 +106,7 @@ struct in_addr { #include #include +#include "fdqueue.h" #include "cli.h" #include "constants.h" #include "mem.h" @@ -341,42 +339,6 @@ int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret); extern int sock; -struct profile_total { - struct profile_total *_next; - int _initialised; - const char *name; - time_ms_t max_time; - time_ms_t total_time; - time_ms_t child_time; - int calls; -}; - -struct call_stats{ - time_ms_t enter_time; - time_ms_t child_time; - struct profile_total *totals; - struct call_stats *prev; -}; - -struct sched_ent; - -typedef void (*ALARM_FUNCP) (struct sched_ent *alarm); - -struct sched_ent{ - struct sched_ent *_next; - struct sched_ent *_prev; - - ALARM_FUNCP function; - void *context; - struct pollfd poll; - // when we should first consider the alarm - time_ms_t alarm; - // the order we will prioritise the alarm - time_ms_t deadline; - struct profile_total *stats; - int _poll_index; -}; - struct limit_state{ // length of time for a burst time_ms_t burst_length; @@ -817,17 +779,6 @@ void sigIoHandler(int signal); int overlay_mdp_setup_sockets(); -int is_scheduled(const struct sched_ent *alarm); -int _schedule(struct __sourceloc, struct sched_ent *alarm); -int _unschedule(struct __sourceloc, struct sched_ent *alarm); -int _watch(struct __sourceloc, struct sched_ent *alarm); -int _unwatch(struct __sourceloc, struct sched_ent *alarm); -#define schedule(alarm) _schedule(__WHENCE__, alarm) -#define unschedule(alarm) _unschedule(__WHENCE__, alarm) -#define watch(alarm) _watch(__WHENCE__, alarm) -#define unwatch(alarm) _unwatch(__WHENCE__, alarm) -int fd_poll(); - void overlay_interface_discover(struct sched_ent *alarm); void overlay_packetradio_poll(struct sched_ent *alarm); int overlay_packetradio_setup_port(overlay_interface *interface); @@ -846,7 +797,6 @@ int overlay_queue_init(); void monitor_client_poll(struct sched_ent *alarm); void monitor_poll(struct sched_ent *alarm); -void rhizome_client_poll(struct sched_ent *alarm); void rhizome_fetch_poll(struct sched_ent *alarm); void rhizome_server_poll(struct sched_ent *alarm); @@ -858,22 +808,6 @@ time_ms_t limit_next_allowed(struct limit_state *state); int limit_is_allowed(struct limit_state *state); int limit_init(struct limit_state *state, int rate_micro_seconds); -/* function timing routines */ -int fd_clearstats(); -int fd_showstats(); -int fd_checkalarms(); -int fd_func_enter(struct __sourceloc, struct call_stats *this_call); -int fd_func_exit(struct __sourceloc, struct call_stats *this_call); -void dump_stack(int log_level); - -#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \ - struct call_stats _this_call={.totals=&_aggregate_stats}; \ - fd_func_enter(__HERE__, &_this_call); - -#define OUT() fd_func_exit(__HERE__, &_this_call) -#define RETURN(X) do { OUT(); return (X); } while (0); -#define RETURNNULL do { OUT(); return (NULL); } while (0); - int olsr_init_socket(void); int olsr_send(struct overlay_frame *frame); diff --git a/sourcefiles.mk b/sourcefiles.mk index 00ee579b..1b4516ac 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -14,6 +14,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)fdqueue.c \ $(SERVAL_BASE)fifo.c \ $(SERVAL_BASE)golay.c \ + $(SERVAL_BASE)http_server.c \ $(SERVAL_BASE)keyring.c \ $(SERVAL_BASE)log.c \ $(SERVAL_BASE)lsif.c \ diff --git a/str.c b/str.c index f1e7ef3d..182a7842 100644 --- a/str.c +++ b/str.c @@ -28,6 +28,7 @@ #include #include #include +#include const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; @@ -122,6 +123,14 @@ char *str_toupper_inplace(char *str) return str; } +char *str_tolower_inplace(char *str) +{ + register char *s; + for (s = str; *s; ++s) + *s = tolower(*s); + return str; +} + const char *strnchr(const char *s, size_t n, char c) { for (; n; --n, ++s) { @@ -227,17 +236,48 @@ char *str_str(char *haystack, const char *needle, int haystack_len) return NULL; } +int str_to_int(const char *str, int base, int *result, const char **afterp) +{ + if (isspace(*str)) + return 0; + const char *end = str; + errno = 0; + long value = strtol(str, (char**)&end, base); + if (afterp) + *afterp = end; + if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN || isdigit(*end) || (!afterp && *end)) + return 0; + if (result) + *result = value; + return 1; +} + +int str_to_uint(const char *str, int base, unsigned *result, const char **afterp) +{ + if (isspace(*str)) + return 0; + const char *end = str; + errno = 0; + unsigned long value = strtoul(str, (char**)&end, base); + if (afterp) + *afterp = end; + if (errno == ERANGE || end == str || value > UINT_MAX || isdigit(*end) || (!afterp && *end)) + return 0; + if (result) + *result = value; + return 1; +} + int str_to_int64(const char *str, int base, int64_t *result, const char **afterp) { if (isspace(*str)) return 0; const char *end = str; + errno = 0; long long value = strtoll(str, (char**)&end, base); - if (end == str) - return 0; if (afterp) *afterp = end; - else if (*end) + if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end)) return 0; if (result) *result = value; @@ -249,12 +289,11 @@ int str_to_uint64(const char *str, int base, uint64_t *result, const char **afte if (isspace(*str)) return 0; const char *end = str; + errno = 0; unsigned long long value = strtoull(str, (char**)&end, base); - if (end == str) - return 0; if (afterp) *afterp = end; - else if (*end) + if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end)) return 0; if (result) *result = value; @@ -295,8 +334,11 @@ int str_to_int64_scaled(const char *str, int base, int64_t *result, const char * { int64_t value; const char *end = str; - if (!str_to_int64(str, base, &value, &end)) + if (!str_to_int64(str, base, &value, &end)) { + if (afterp) + *afterp = end; return 0; + } value *= scale_factor(end, &end); if (afterp) *afterp = end; @@ -311,8 +353,11 @@ int str_to_uint64_scaled(const char *str, int base, uint64_t *result, const char { uint64_t value; const char *end = str; - if (!str_to_uint64(str, base, &value, &end)) + if (!str_to_uint64(str, base, &value, &end)) { + if (afterp) + *afterp = end; return 0; + } value *= scale_factor(end, &end); if (afterp) *afterp = end; @@ -347,8 +392,11 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft return 0; const char *end = str; unsigned long long value = strtoull(str, (char**)&end, 10) * precision; - if (end == str) + if (end == str) { + if (afterp) + *afterp = end; return 0; + } if (end[0] == '.' && isdigit(end[1])) { ++end; unsigned factor; @@ -407,11 +455,12 @@ size_t toprint_str_len(const char *srcStr, const char quotes[2]) return srcStr ? strbuf_count(strbuf_toprint_quoted(strbuf_local(NULL, 0), quotes, srcStr)) : 4; } -size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp) +size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp) { unsigned char *const odst = dst; - unsigned char *const edst = dst + dstlen; - while (*src && *src != endquote && dst < edst) { + unsigned char *const edst = dst + dstsiz; + const char *const esrc = srclen ? src + srclen : NULL; + while (src < esrc && *src && *src != endquote && dst < edst) { switch (*src) { case '\\': ++src; diff --git a/str.h b/str.h index 811ffc52..7f9827be 100644 --- a/str.h +++ b/str.h @@ -66,7 +66,9 @@ size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary); int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary); size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp); -#define alloca_tohex(buf,bytes) tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf)) +#define alloca_tohex(buf,bytes) tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf)) + +#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str)) __STR_INLINE int hexvalue(char c) { @@ -92,13 +94,15 @@ __STR_INLINE int hexvalue(char c) } int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value); + char *str_toupper_inplace(char *s); +char *str_tolower_inplace(char *s); char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]); char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]); size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2]); size_t toprint_str_len(const char *srcStr, const char quotes[2]); -size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp); +size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp); #define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), "``") + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), "``") #define alloca_str_toprint_quoted(str, quotes) toprint_str((char *)alloca(toprint_str_len((str), (quotes)) + 1), -1, (str), (quotes)) @@ -148,7 +152,7 @@ __STR_INLINE ssize_t str_rindex(const char *s, char c) * nul-terminated, but are held in a buffer which has an associated length. To avoid this function * running past the end of the buffer, the caller must ensure that the buffer contains a sub-string * that is not part of the sub-string being sought, eg, "\r\n\r\n" as detected by - * http_header_complete(). This guarantees that this function will return nonzero before running + * is_http_header_complete(). This guarantees that this function will return nonzero before running * past the end of the buffer. * * @author Andrew Bettison @@ -199,13 +203,16 @@ char *str_str(char *haystack, const char *needle, int haystack_len); /* Parse a string as an integer in ASCII radix notation in the given 'base' (eg, base=10 means * decimal). * - * Return 1 if a valid integer was parsed, storing the value in *result (unless result is NULL) and - * storing a pointer to the immediately succeeding character in *afterp (unless afterp is NULL, in - * which case returns 1 only if the immediately succeeding character is a nul '\0'). Returns 0 - * otherwise, leaving *result and *afterp unchanged. + * Returns 1 if a valid integer is parsed, storing the value in *result (unless result is NULL) and + * storing a pointer to the immediately succeeding character in *afterp. If afterp is NULL then + * returns 0 unless the immediately succeeding character is a NUL '\0'. If no integer is parsed or + * if the integer overflows (too many digits), then returns 0, leaving *result unchanged and setting + * setting *afterp to point to the character where parsing failed. * * @author Andrew Bettison */ +int str_to_int(const char *str, int base, int *result, const char **afterp); +int str_to_uint(const char *str, int base, unsigned *result, const char **afterp); int str_to_int64(const char *str, int base, int64_t *result, const char **afterp); int str_to_uint64(const char *str, int base, uint64_t *result, const char **afterp); diff --git a/strbuf_helpers.c b/strbuf_helpers.c index dd63d473..de8446d6 100644 --- a/strbuf_helpers.c +++ b/strbuf_helpers.c @@ -17,14 +17,16 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "strbuf_helpers.h" -#include #include #include #include #include #include +#include #include +#ifdef HAVE_POLL_H +#include +#endif #ifdef HAVE_NETINET_IN_H #include #endif @@ -32,6 +34,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #endif #include +#include "http_server.h" +#include "strbuf_helpers.h" static inline strbuf _toprint(strbuf sb, char c) { @@ -309,6 +313,16 @@ strbuf strbuf_append_socket_type(strbuf sb, int type) return sb; } +strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr) +{ + strbuf_sprintf(sb, " %u.%u.%u.%u", + ((unsigned char *) &addr->s_addr)[0], + ((unsigned char *) &addr->s_addr)[1], + ((unsigned char *) &addr->s_addr)[2], + ((unsigned char *) &addr->s_addr)[3]); + return sb; +} + strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen) { strbuf_append_socket_domain(sb, addr->sa_family); @@ -332,13 +346,9 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t break; case AF_INET: { const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr; - strbuf_sprintf(sb, " %u.%u.%u.%u:%u", - ((unsigned char *) &addr_in->sin_addr.s_addr)[0], - ((unsigned char *) &addr_in->sin_addr.s_addr)[1], - ((unsigned char *) &addr_in->sin_addr.s_addr)[2], - ((unsigned char *) &addr_in->sin_addr.s_addr)[3], - ntohs(addr_in->sin_port) - ); + strbuf_putc(sb, ' '); + strbuf_append_in_addr(sb, &addr_in->sin_addr); + strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port)); if (addrlen != sizeof(struct sockaddr_in)) strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in)); } @@ -391,3 +401,57 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt) strbuf_putc(sb, ']'); return sb; } + +strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels) +{ + unsigned i; + int first = 1; + for (i = 0; i != nels; ++i) { + const struct http_range *r = &ranges[i]; + switch (r->type) { + case NIL: break; + case CLOSED: + strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-%"PRIhttp_size_t, first ? "" : ",", r->first, r->last); + first = 0; + break; + case OPEN: + strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-", first ? "" : ",", r->first); + first = 0; + break; + case SUFFIX: + strbuf_sprintf(sb, "%s-%"PRIhttp_size_t, first ? "" : ",", r->last); + first = 0; + break; + } + } + return sb; +} + +strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd) +{ + strbuf_puts(sb, "type="); + strbuf_toprint_quoted(sb, "``", cd->type); + strbuf_puts(sb, " name="); + strbuf_toprint_quoted(sb, "``", cd->name); + strbuf_puts(sb, " filename="); + strbuf_toprint_quoted(sb, "``", cd->filename); + strbuf_puts(sb, " size="); + strbuf_sprintf(sb, "%"PRIhttp_size_t, cd->size); + struct tm tm; + strbuf_puts(sb, " creation_date="); + if (cd->creation_date) + strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->creation_date, &tm)); + else + strbuf_puts(sb, "0"); + strbuf_puts(sb, " modification_date="); + if (cd->modification_date) + strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->modification_date, &tm)); + else + strbuf_puts(sb, "0"); + strbuf_puts(sb, " read_date="); + if (cd->read_date) + strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->read_date, &tm)); + else + strbuf_puts(sb, "0"); + return sb; +} diff --git a/strbuf_helpers.h b/strbuf_helpers.h index ada181ae..289c7ddf 100644 --- a/strbuf_helpers.h +++ b/strbuf_helpers.h @@ -67,6 +67,7 @@ strbuf strbuf_path_join(strbuf sb, ...); * @author Andrew Bettison */ strbuf strbuf_append_poll_events(strbuf sb, short events); +#define alloca_poll_events(ev) strbuf_str(strbuf_append_poll_events(strbuf_alloca(200), (ev))) /* Append a nul-terminated string as a single-quoted shell word which, if * expanded in a shell command line, would evaluate to the original string. @@ -116,6 +117,14 @@ strbuf strbuf_append_socket_domain(strbuf sb, int domain); strbuf strbuf_append_socket_type(strbuf sb, int type); #define alloca_socket_type(type) strbuf_str(strbuf_append_socket_type(strbuf_alloca(15), type)) +/* Append a textual description of a struct in_addr (in network order) as IPv4 + * quartet "N.N.N.N". + * @author Andrew Bettison + */ +struct in_addr; +strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr); +#define alloca_in_addr(addr) strbuf_str(strbuf_append_in_addr(strbuf_alloca(16), (const struct in_addr *)(addr))) + /* Append a textual description of a struct sockaddr_in. * @author Andrew Bettison */ @@ -136,4 +145,18 @@ struct iovec; strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt); #define alloca_iovec(iov,cnt) strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt))) +/* Append a representation of a struct http_range[] array. + * @author Andrew Bettison + */ +struct http_range; +strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels); +#define alloca_http_ranges(ra) strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra))) + +/* Append a representation of a struct mime_content_disposition struct. + * @author Andrew Bettison + */ +struct mime_content_disposition; +strbuf strbuf_append_mime_content_disposition(strbuf, const struct mime_content_disposition *); +#define alloca_mime_content_disposition(cd) strbuf_str(strbuf_append_mime_content_disposition(strbuf_alloca(500), (cd))) + #endif //__STRBUF_HELPERS_H__ diff --git a/testdefs.sh b/testdefs.sh index 791c7fb3..2e083640 100644 --- a/testdefs.sh +++ b/testdefs.sh @@ -732,3 +732,52 @@ has_seen_instances() { instances_see_each_other() { foreach_instance "$@" has_seen_instances "$@" } + +# Setup function: +# - ensure that the given version of the curl(1) utility is available +# - remove all proxy settings +setup_curl() { + local minversion="${1?}" + local ver="$(curl --version | tr '\n' ' ')" + case "$ver" in + '') + fail "curl(1) command is not present" + ;; + curl\ *\ Protocols:*\ http\ *) + set -- $ver + tfw_cmp_version "$2" 7 + case $? in + 0|2) + unset http_proxy + unset HTTP_PROXY + unset HTTPS_PROXY + unset ALL_PROXY + return 0 + ;; + esac + fail "curl(1) version $2 is not adequate (expecting $minversion or higher)" + ;; + esac + fail "cannot parse output of curl --version: $ver" +} + +# Setup function: +# - ensure that version 1.2 or later of the jq(1) utility is available +setup_jq() { + local minversion="${1?}" + local ver="$(jq --version 2>&1)" + case "$ver" in + '') + fail "jq(1) command is not present" + ;; + jq\ version\ *) + set -- $ver + tfw_cmp_version "$3" "$minversion" + case $? in + 0|2) return 0;; + esac + fail "jq(1) version $3 is not adequate (need $minversion or higher)" + ;; + esac + fail "cannot parse output of jq --version: $ver" +} diff --git a/testframework.sh b/testframework.sh index 083f77b7..9c183ef4 100644 --- a/testframework.sh +++ b/testframework.sh @@ -840,6 +840,20 @@ tfw_quietly() { fi } +# Compare the two arguments as dotted ascii decimal version strings. +# Return 0 if they are equal, 1 if arg1 < arg2, 2 if arg1 > arg2 +tfw_cmp_version() { + local IFS=. + local i=0 a=($1) b=($2) + for (( i=0; i < ${#a[@]} || i < ${#b[@]}; ++i )); do + local ai="${a[i]:-0}" + local bi="${b[i]:-0}" + (( 10#$ai < 10#$bi )) && return 1 + (( 10#$ai > 10#$bi )) && return 2 + done + return 0 +} + # Append the contents of a file to the test case's stdout log. A normal 'cat' # to stdout would also do this, but tfw_cat echoes header and footer delimiter # lines around to content to help distinguish it, and also works even in a diff --git a/tests/framework b/tests/framework new file mode 100755 index 00000000..2343ceb2 --- /dev/null +++ b/tests/framework @@ -0,0 +1,47 @@ +#!/bin/bash + +# Tests for Serval rhizome operations. +# +# Copyright 2012 Serval Project, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +source "${0%/*}/../testframework.sh" + +shopt -s extglob + +test_tfw_cmp_version() { + execute --exit-status=1 tfw_cmp_version 1 2 + execute --exit-status=2 tfw_cmp_version 1.0.1 1.0.0 + execute --exit-status=1 tfw_cmp_version 1.0 1.1 + execute --exit-status=0 tfw_cmp_version 1 1 + execute --exit-status=1 tfw_cmp_version 2.1 2.2 + execute --exit-status=2 tfw_cmp_version 3.0.4.10 3.0.4.2 + execute --exit-status=1 tfw_cmp_version 4.08 4.08.01 + execute --exit-status=2 tfw_cmp_version 3.2.1.9.8144 3.2 + execute --exit-status=1 tfw_cmp_version 3.2 3.2.1.9.8144 + execute --exit-status=1 tfw_cmp_version 1.2 2.1 + execute --exit-status=2 tfw_cmp_version 2.1 1.2 + execute --exit-status=0 tfw_cmp_version 5.6.7 5.6.7 + execute --exit-status=0 tfw_cmp_version 1.01.1 1.1.1 + execute --exit-status=0 tfw_cmp_version 1.1.1 1.01.1 + execute --exit-status=0 tfw_cmp_version 1 1.0 + execute --exit-status=0 tfw_cmp_version 1.0 1 + execute --exit-status=0 tfw_cmp_version 1.0.2.0 1.0.2 + execute --exit-status=0 tfw_cmp_version 1..0 1.0 + execute --exit-status=0 tfw_cmp_version 1.0 1..0 +} + +runTests "$@" diff --git a/tests/rhizomechannels b/tests/rhizomechannels index 48c40a07..ee67bd7d 100755 --- a/tests/rhizomechannels +++ b/tests/rhizomechannels @@ -40,24 +40,13 @@ configure_servald_server() { set log.show_pid on \ set log.show_time on \ set debug.rhizome on \ + set debug.rhizome_httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on \ set server.respawn_on_crash off \ set mdp.iftype.wifi.tick_ms 500 } -setup_curl_7() { - case "$(curl --version | tr '\n' ' ')" in - curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;; - '') fail "curl(1) command is not present";; - *) fail "curl(1) version is not adequate (expecting 7 or higher)";; - esac - unset http_proxy - unset HTTP_PROXY - unset HTTPS_PROXY - unset ALL_PROXY -} - setup_common() { setup_servald assert_no_servald_processes diff --git a/tests/rhizomehttp b/tests/rhizomehttp new file mode 100755 index 00000000..8fa74c77 --- /dev/null +++ b/tests/rhizomehttp @@ -0,0 +1,140 @@ +#!/bin/bash + +# Tests for Serval rhizome operations. +# +# Copyright 2013 Serval Project, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +source "${0%/*}/../testframework.sh" +source "${0%/*}/../testdefs.sh" +source "${0%/*}/../testdefs_rhizome.sh" + +shopt -s extglob + +setup() { + setup_curl 7 + setup_jq 1.2 + setup_servald + set_instance +A + set_rhizome_config + executeOk_servald config \ + set rhizome.api.restful.users.harry.password potter \ + set rhizome.api.restful.users.ron.password weasley \ + set rhizome.api.restful.users.hermione.password grainger + create_single_identity + echo "$SIDA1" >sids + start_servald_instances +A + wait_until rhizome_http_server_started +A + get_rhizome_server_port PORTA +A +} + +set_rhizome_config() { + executeOk_servald config \ + set debug.rhizome on \ + set debug.verbose on \ + set log.console.level debug +} + +doc_AuthBasicMissing="Basic Authentication credentials are required" +test_AuthBasicMissing() { + execute --exit-status=67 curl \ + --silent --fail --show-error \ + --output http.output \ + --dump-header http.headers \ + "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" +} +teardown_AuthBasicMissing() { + tfw_cat http.headers http.output +} + +doc_AuthBasicWrong="Basic Authentication credentials must be correct" +test_AuthBasicWrong() { + execute --exit-status=67 curl \ + --silent --fail --show-error \ + --output http.output \ + --dump-header http.headers \ + --basic --user fred:nurks \ + "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" + executeOk curl \ + --silent --fail --show-error \ + --output http.output \ + --dump-header http.headers \ + --basic --user ron:weasley \ + "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" +} + +doc_RhizomeList="Fetch full Rhizome bundle list in JSON format" +setup_RhizomeList() { + for n in 1 2 3 4; do + create_file file$n ${n}k + executeOk_servald rhizome add file $SIDA file$n file$n.manifest + done +} +test_RhizomeList() { + executeOk curl \ + --silent --fail --show-error \ + --output http.output \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" +} + +doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format" +test_RhizomeListSinceJSON() { + : +} + +doc_RhizomeManifest="Fetch Rhizome bundle manifest" +test_RhizomeManifest() { + : +} + +doc_RhizomePayloadRaw="Fetch Rhizome raw payload" +test_RhizomePayloadRaw() { + : +} + +doc_RhizomePayloadDecrypted="Fetch Rhizome decrypted payload" +test_RhizomePayloadDecrypted() { + : +} + +doc_RhizomeInsert="Insert new Rhizome bundle" +test_RhizomeInsert() { + : +} + +doc_MeshmsListConversations="List MeshMS conversations" +test_MeshmsListConversations() { + : +} + +doc_MeshmsListMessages="List all MeshMS messages in a single conversation" +test_MeshmsListMessages() { + : +} + +doc_MeshmsListMessagesSince="List MeshMS messages in a single conversation since token" +test_MeshmsListMessagesSince() { + : +} + +doc_MeshmsSend="Send MeshMS message" +test_MeshmsSend() { + : +} + +runTests "$@" diff --git a/tests/rhizomeops b/tests/rhizomeops index 29f5edd0..c8af2b35 100755 --- a/tests/rhizomeops +++ b/tests/rhizomeops @@ -26,21 +26,24 @@ shopt -s extglob setup_rhizome() { set_instance +A - executeOk_servald config \ - set debug.rhizome on \ - set debug.verbose on \ - set log.console.level debug + set_rhizome_config create_single_identity + echo "$SIDA1" >sids set_instance +B + set_rhizome_config + create_identities 4 + echo "$SIDB1" >>sids + echo "$SIDB2" >>sids + echo "$SIDB3" >>sids + echo "$SIDB4" >>sids + assert [ $(sort sids | uniq | wc -l) -eq 5 ] +} + +set_rhizome_config() { executeOk_servald config \ set debug.rhizome on \ set debug.verbose on \ set log.console.level debug - create_identities 4 - assert [ $SIDB1 != $SIDA1 ] - assert [ $SIDB2 != $SIDA1 ] - assert [ $SIDB3 != $SIDA1 ] - assert [ $SIDB4 != $SIDA1 ] } doc_InitialEmptyList="Initial list is empty" diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol index 0f4663b3..2b81620e 100755 --- a/tests/rhizomeprotocol +++ b/tests/rhizomeprotocol @@ -41,24 +41,14 @@ configure_servald_server() { set log.console.show_pid on \ set log.console.show_time on \ set debug.rhizome on \ + set debug.httpd on \ + set debug.rhizome_httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on \ set server.respawn_on_crash off \ set mdp.iftype.wifi.tick_ms 500 } -setup_curl_7() { - case "$(curl --version | tr '\n' ' ')" in - curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;; - '') fail "curl(1) command is not present";; - *) fail "curl(1) version is not adequate (expecting 7 or higher)";; - esac - unset http_proxy - unset HTTP_PROXY - unset HTTPS_PROXY - unset ALL_PROXY -} - setup_common() { setup_servald assert_no_servald_processes @@ -228,6 +218,7 @@ start_radio_instance() { executeOk_servald config \ set debug.rhizome on \ set debug.rhizome_ads on \ + set debug.rhizome_httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on \ set debug.throttling on \ @@ -524,12 +515,13 @@ test_CorruptPayload() { wait_until grep -i "Stored file $FILEHASH" $LOGA } -doc_HttpFetchRange="Fetch a file range using HTTP GET." +doc_HttpFetchRange="Fetch a file range using HTTP GET" setup_HttpFetchRange() { - setup_curl_7 + setup_curl 7 setup_common set_instance +A - rhizome_add_file file1 + rhizome_add_file file1 100 + tail --bytes +33 file1 >file1.tail start_servald_instances +A wait_until rhizome_http_server_started +A get_rhizome_server_port PORTA +A @@ -542,14 +534,16 @@ test_HttpFetchRange() { --write-out '%{http_code}\n' \ --continue-at 32 \ "http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH" - tfw_cat http.headers http.output - assertGrep http.headers "Content-range:" - assertGrep http.headers "Content-length:" + tfw_cat -v http.headers http.output + assertGrep http.headers "^Content-Range: bytes 32-99/100 $" + assertGrep http.headers "^Content-Length: 68 $" + tfw_cat -v file1.tail http.output + assert cmp file1.tail http.output } doc_HttpImport="Import bundle using HTTP POST multi-part form." setup_HttpImport() { - setup_curl_7 + setup_curl 7 setup_common cat >README.WHYNOTSIPS <<'EOF' When we were looking at implementing secure calls for OpenBTS it was suggested @@ -599,7 +593,7 @@ test_HttpImport() { doc_HttpAddLocal="Add file locally using HTTP, returns manifest" setup_HttpAddLocal() { - setup_curl_7 + setup_curl 7 setup_common set_instance +A executeOk_servald config \ @@ -639,6 +633,7 @@ setup_direct() { set log.console.level debug \ set log.console.show_time on \ set debug.rhizome on \ + set debug.rhizome_httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on rhizome_add_file fileB1 2000 @@ -662,6 +657,8 @@ setup_DirectPush() { setup_common setup_direct setup_direct_peer + executeOk ls -l + tfw_cat --stdout } test_DirectPush() { set_instance +B diff --git a/tests/rhizomestress b/tests/rhizomestress index a99d7c6d..e5fa9757 100755 --- a/tests/rhizomestress +++ b/tests/rhizomestress @@ -40,6 +40,7 @@ configure_servald_server() { set log.file.show_pid on \ set log.file.show_time on \ set debug.rhizome off \ + set debug.rhizome_httpd off \ set debug.rhizome_tx off \ set debug.rhizome_rx off \ set server.respawn_on_crash off \ @@ -147,6 +148,7 @@ setup_StressRhizomeDirect() { executeOk_servald config \ set log.file.show_time on \ set debug.rhizome off \ + set debug.rhizome_httpd off \ set debug.rhizome_tx off \ set debug.rhizome_rx off \ set server.respawn_on_crash off \ diff --git a/vomp_console.c b/vomp_console.c index a3ee4c05..4db2ea06 100644 --- a/vomp_console.c +++ b/vomp_console.c @@ -21,8 +21,10 @@ #include #include #include -#include #include +#ifdef HAVE_POLL_H +#include +#endif #include "serval.h" #include "conf.h"