From fa21bec8804f404d7e67fb9bf295633caaeeaf7a Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Wed, 16 Oct 2013 18:25:58 +1030 Subject: [PATCH] Rewrite HTTP server --- conf.h | 2 - constants.h | 3 + headerfiles.mk | 1 + http_server.c | 1536 +++++++++++++++++++++++++++++++++++++++++ http_server.h | 156 +++++ overlay.c | 8 +- rhizome.h | 84 +-- rhizome_direct.c | 3 +- rhizome_direct_http.c | 1092 +++++++++++------------------ rhizome_http.c | 649 ++++++----------- rhizome_store.c | 121 ++-- rhizome_sync.c | 4 +- serval.h | 1 - sourcefiles.mk | 1 + strbuf_helpers.c | 27 + strbuf_helpers.h | 7 + 16 files changed, 2434 insertions(+), 1261 deletions(-) create mode 100644 http_server.c create mode 100644 http_server.h diff --git a/conf.h b/conf.h index 2d7b40fb..2a3fbb7d 100644 --- a/conf.h +++ b/conf.h @@ -233,8 +233,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "rhizome.h" -typedef char bool_t; - #define CONFIG_FILE_MAX_SIZE (32 * 1024) #define INTERFACE_NAME_STRLEN 40 diff --git a/constants.h b/constants.h index f99cc307..cc8b4ffb 100644 --- a/constants.h +++ b/constants.h @@ -220,5 +220,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define UNLOCK_CHALLENGE (0xF1) #define UNLOCK_RESPONSE (0xF2) +// should there be a types.h to hold this? +typedef char bool_t; + #endif // __SERVALD_CONSTANTS_H diff --git a/headerfiles.mk b/headerfiles.mk index 45528bd9..a5f7fca7 100644 --- a/headerfiles.mk +++ b/headerfiles.mk @@ -18,6 +18,7 @@ HDRS= fifo.h \ crypto.h \ log.h \ net.h \ + http_server.h \ xprintf.h \ constants.h \ monitor-client.h \ diff --git a/http_server.c b/http_server.c new file mode 100644 index 00000000..7a58282a --- /dev/null +++ b/http_server.c @@ -0,0 +1,1536 @@ +/* +Serval DNA - HTTP Server +Copyright (C) 2013 Serval Project, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION +#include +#include +#include +#include "serval.h" +#include "http_server.h" +#include "log.h" +#include "str.h" +#include "strbuf.h" +#include "strbuf_helpers.h" +#include "net.h" +#include "mem.h" + +#define BOUNDARY_STRING_MAXLEN 70 // legislated limit from RFC-1341 + +/* The (struct http_request).verb field points to one of these static strings, so that a simple + * equality test can be used, eg, (r->verb == HTTP_VERB_GET) instead of a strcmp(). + * + * @author Andrew Bettison + */ +const char HTTP_VERB_GET[] = "GET"; +const char HTTP_VERB_POST[] = "POST"; +const char HTTP_VERB_PUT[] = "PUT"; +const char HTTP_VERB_HEAD[] = "HEAD"; +const char HTTP_VERB_DELETE[] = "DELETE"; +const char HTTP_VERB_TRACE[] = "TRACE"; +const char HTTP_VERB_OPTIONS[] = "OPTIONS"; +const char HTTP_VERB_CONNECT[] = "CONNECT"; +const char HTTP_VERB_PATCH[] = "PATCH"; + +static struct { + const char *word; + size_t wordlen; +} http_verbs[] = { +#define VERB_ENTRY(NAME) { HTTP_VERB_##NAME, sizeof HTTP_VERB_##NAME - 1 } + VERB_ENTRY(GET), + VERB_ENTRY(POST), + VERB_ENTRY(PUT), + VERB_ENTRY(HEAD), + VERB_ENTRY(DELETE), + VERB_ENTRY(TRACE), + VERB_ENTRY(OPTIONS), + VERB_ENTRY(CONNECT), + VERB_ENTRY(PATCH) +#undef VERB_ENTRY +}; + +static struct profile_total http_server_stats = { + .name = "http_server_poll", +}; + +static void http_server_poll(struct sched_ent *); +static int http_request_parse_verb(struct http_request *r); +static int http_request_parse_path(struct http_request *r); +static int http_request_parse_http_version(struct http_request *r); +static int http_request_start_parsing_headers(struct http_request *r); +static int http_request_parse_header(struct http_request *r); +static int http_request_start_body(struct http_request *r); +static int http_request_parse_body_form_data(struct http_request *r); +static void http_request_start_response(struct http_request *r); + +void http_request_init(struct http_request *r, int sockfd) +{ + bzero(r, sizeof *r); + assert(sockfd != -1); + r->request_header.content_length = CONTENT_LENGTH_UNKNOWN; + r->request_content_remaining = CONTENT_LENGTH_UNKNOWN; + r->alarm.stats = &http_server_stats; + r->alarm.function = http_server_poll; + if (r->idle_timeout == 0) + r->idle_timeout = 10000; // 10 seconds + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + r->alarm.poll.fd = sockfd; + r->alarm.poll.events = POLLIN; + r->phase = RECEIVE; + r->received = r->end = r->parsed = r->cursor = r->buffer; + r->limit = r->buffer + sizeof r->buffer; + r->parser = http_request_parse_verb; + watch(&r->alarm); + schedule(&r->alarm); +} + +void http_request_free_response_buffer(struct http_request *r) +{ + if (r->response_free_buffer) { + r->response_free_buffer(r->response_buffer); + r->response_free_buffer = NULL; + } + r->response_buffer = NULL; + r->response_buffer_size = 0; +} + +int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz) +{ + const char *const bufe = r->buffer + sizeof r->buffer; + assert(r->received < bufe); + size_t rbufsiz = bufe - r->received; + if (bufsiz <= rbufsiz) { + http_request_free_response_buffer(r); + r->response_buffer = (char *) r->received; + r->response_buffer_size = rbufsiz; + return 0; + } + if (bufsiz != r->response_buffer_size) { + http_request_free_response_buffer(r); + if ((r->response_buffer = emalloc(bufsiz)) == NULL) + return -1; + r->response_free_buffer = free; + r->response_buffer_size = bufsiz; + } + assert(r->response_buffer_size >= bufsiz); + assert(r->response_buffer != NULL); + return 0; +} + +void http_request_finalise(struct http_request *r) +{ + if (r->phase == DONE) + return; + assert(r->phase == RECEIVE || r->phase == TRANSMIT); + unschedule(&r->alarm); + unwatch(&r->alarm); + close(r->alarm.poll.fd); + r->alarm.poll.fd = -1; + if (r->finalise) + r->finalise(r); + r->finalise = NULL; + http_request_free_response_buffer(r); + r->phase = DONE; +} + +#define _SEP (1 << 0) +#define _BND (1 << 1) + +uint8_t http_ctype[256] = { + ['0'] = _BND, ['1'] = _BND, ['2'] = _BND, ['3'] = _BND, ['4'] = _BND, + ['5'] = _BND, ['6'] = _BND, ['7'] = _BND, ['8'] = _BND, ['9'] = _BND, + ['A'] = _BND, ['B'] = _BND, ['C'] = _BND, ['D'] = _BND, ['E'] = _BND, + ['F'] = _BND, ['G'] = _BND, ['H'] = _BND, ['I'] = _BND, ['J'] = _BND, + ['K'] = _BND, ['L'] = _BND, ['M'] = _BND, ['N'] = _BND, ['O'] = _BND, + ['P'] = _BND, ['Q'] = _BND, ['R'] = _BND, ['S'] = _BND, ['T'] = _BND, + ['U'] = _BND, ['V'] = _BND, ['W'] = _BND, ['X'] = _BND, ['Y'] = _BND, + ['Z'] = _BND, + ['a'] = _BND, ['b'] = _BND, ['c'] = _BND, ['d'] = _BND, ['e'] = _BND, + ['f'] = _BND, ['g'] = _BND, ['h'] = _BND, ['i'] = _BND, ['j'] = _BND, + ['k'] = _BND, ['l'] = _BND, ['m'] = _BND, ['n'] = _BND, ['o'] = _BND, + ['p'] = _BND, ['q'] = _BND, ['r'] = _BND, ['s'] = _BND, ['t'] = _BND, + ['u'] = _BND, ['v'] = _BND, ['w'] = _BND, ['x'] = _BND, ['y'] = _BND, + ['z'] = _BND, + ['+'] = _BND, ['-'] = _BND, ['.'] = _BND, ['/'] = _BND, [':'] = _BND, + ['_'] = _BND, + ['('] = _SEP | _BND, + [')'] = _SEP | _BND, + [','] = _SEP | _BND, + ['?'] = _SEP | _BND, + ['='] = _SEP | _BND, + [' '] = _SEP | _BND, + ['\t'] = _SEP, + ['<'] = _SEP, + ['>'] = _SEP, + ['@'] = _SEP, + [';'] = _SEP, + [':'] = _SEP, + ['\\'] = _SEP, + ['"'] = _SEP, + ['/'] = _SEP, + ['['] = _SEP, + [']'] = _SEP, + ['{'] = _SEP, + ['}'] = _SEP, +}; + +inline int is_http_char(char c) +{ + return c >= 0; +} + +inline int is_http_ctl(char c) +{ + return iscntrl(c); +} + +inline int is_http_separator(char c) +{ + return (http_ctype[(unsigned char) c] & _SEP) != 0; +} + +inline int is_http_boundary(char c) +{ + return (http_ctype[(unsigned char) c] & _BND) != 0; +} + +inline int is_http_token(char c) +{ + return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c); +} + +inline int is_valid_http_boundary_string(const char *s) +{ + if (s[0] == '\0') + return 0; + for (; *s; ++s) + if (!is_http_boundary(*s)) + return 0; + return s[-1] != ' '; +} + +struct substring { + const char *start; + const char *end; +}; + +#define alloca_substring_toprint(sub) alloca_toprint(-1, (sub).start, (sub).end - (sub).start) + +const struct substring substring_NULL = { NULL, NULL }; + +#if 0 +static int _matches(struct substring str, const char *text) +{ + return strlen(text) == str.end - str.start && memcmp(str.start, text, str.end - str.start) == 0; +} +#endif + +static const char * _reserve(struct http_request *r, struct substring str) +{ + char *reslim = r->buffer + sizeof r->buffer - 1024; // always leave this much unreserved space + assert(r->received <= reslim); + size_t len = str.end - str.start; + size_t siz = len + 1; + if (r->received + siz > reslim) { + r->response.result_code = 414; + return NULL; + } + char *ret = (char *) r->received; + if (r->received + siz > r->parsed) { + WARNF("Error during HTTP parsing, unparsed content %s would be overwritten by reserving %s", + alloca_toprint(30, r->parsed, r->end - r->parsed), + alloca_substring_toprint(str) + ); + r->response.result_code = 500; + return NULL; + } + if (ret != str.start) + memmove(ret, str.start, len); + ret[len] = '\0'; + r->received += siz; + assert(r->received <= r->parsed); + return ret; +} + +static const char * _reserve_str(struct http_request *r, const char *str) +{ + struct substring sub = { .start = str, .end = str + strlen(str) }; + return _reserve(r, sub); +} + +static inline int _buffer_full(struct http_request *r) +{ + return r->parsed == r->received && r->end == r->limit; +} + +static inline int _run_out(struct http_request *r) +{ + assert(r->cursor <= r->end); + return r->cursor == r->end; +} + +static inline void _rewind(struct http_request *r) +{ + assert(r->parsed >= r->received); + r->cursor = r->parsed; +} + +static inline void _commit(struct http_request *r) +{ + assert(r->cursor <= r->end); + r->parsed = r->cursor; +} + +static inline int _skip_crlf(struct http_request *r) +{ + return !_run_out(r) && *r->cursor == '\r' && ++r->cursor && !_run_out(r) && *r->cursor == '\n' && ++r->cursor; +} + +static inline int _skip_to_crlf(struct http_request *r) +{ + const char *const start = r->cursor; + for (; !_run_out(r); ++r->cursor) + if (*r->cursor == '\n' && r->cursor > start + 1 && r->cursor[-2] == '\r') { + --r->cursor; + return 1; + } + return 0; +} + +static inline void _rewind_crlf(struct http_request *r) +{ + assert(r->cursor >= r->parsed + 2); + assert(r->cursor[-2] == '\r'); + assert(r->cursor[-1] == '\n'); + r->cursor -= 2; +} + +/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF + * as part of the end-of-line sequence and treats the CR as optional. This allows simple manual + * testing using telnet(1). + */ +static inline int _skip_eol(struct http_request *r) +{ + unsigned crcount = 0; + for (; !_run_out(r); ++r->cursor) { + switch (*r->cursor) { + case '\0': // ignore any leading NULs (telnet inserts them) + break; + case '\r': // ignore up to one leading CR + if (++crcount > 1) + return 0; + break; + case '\n': + ++r->cursor; + return 1; + default: + return 0; + } + } + return 0; +} + +/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF + * as part of the end-of-line sequence and treats the CR as optional. This allows simple manual + * testing using telnet(1). + */ +static int _skip_to_eol(struct http_request *r) +{ + const char *const start = r->cursor; + while (!_run_out(r) && *r->cursor != '\n') + ++r->cursor; + if (_run_out(r)) + return 0; + // consume preceding NULs (telnet inserts them) + while (r->cursor > start && r->cursor[-1] == '\0') + --r->cursor; + // consume a single preceding CR + if (r->cursor > start && r->cursor[-1] == '\r') + --r->cursor; + // consume any more preceding NULs + while (r->cursor > start && r->cursor[-1] == '\0') + --r->cursor; + return 1; +} + +static int _skip_literal(struct http_request *r, const char *literal) +{ + while (!_run_out(r) && *literal && *r->cursor == *literal) + ++literal, ++r->cursor; + return *literal == '\0'; +} + +static int _skip_literal_nocase(struct http_request *r, const char *literal) +{ + while (!_run_out(r) && *literal && toupper(*r->cursor) == toupper(*literal)) + ++literal, ++r->cursor; + return *literal == '\0'; +} + +static int _skip_optional_space(struct http_request *r) +{ + while (!_run_out(r) && (*r->cursor == ' ' || *r->cursor == '\t')) + ++r->cursor; + return 1; +} + +static inline int _skip_space(struct http_request *r) +{ + const char *const start = r->cursor; + _skip_optional_space(r); + return r->cursor > start; +} + +static size_t _skip_word_printable(struct http_request *r, struct substring *str) +{ + if (_run_out(r) || isspace(*r->cursor) || !isprint(*r->cursor)) + return 0; + const char *start = r->cursor; + for (++r->cursor; !_run_out(r) && !isspace(*r->cursor) && isprint(*r->cursor); ++r->cursor) + ; + if (_run_out(r)) + return 0; + assert(r->cursor > start); + assert(isspace(*r->cursor)); + if (str) { + str->start = start; + str->end = r->cursor; + } + return r->cursor - start; +} + +static size_t _skip_token(struct http_request *r, struct substring *str) +{ + if (_run_out(r) || !is_http_token(*r->cursor)) + return 0; + const char *start = r->cursor; + for (++r->cursor; !_run_out(r) && is_http_token(*r->cursor); ++r->cursor) + ; + if (_run_out(r)) + return 0; + assert(r->cursor > start); + assert(!is_http_token(*r->cursor)); + if (str) { + str->start = start; + str->end = r->cursor; + } + return r->cursor - start; +} + +static size_t _parse_token(struct http_request *r, char *dst, size_t dstsiz) +{ + struct substring str; + size_t len = _skip_token(r, &str); + if (len && dst) { + size_t cpy = len < dstsiz - 1 ? len : dstsiz - 1; + strncpy(dst, str.start, cpy)[cpy] = '\0'; + } + return len; +} + +static size_t _parse_quoted_string(struct http_request *r, char *dst, size_t dstsiz) +{ + assert(r->cursor <= r->end); + if (_run_out(r) || *r->cursor != '"') + return 0; + int slosh = 0; + size_t len = 0; + for (++r->cursor; !_run_out(r); ++r->cursor) { + if (!isprint(*r->cursor)) + return 0; + if (slosh) { + if (dst && len < dstsiz - 1) + dst[len] = *r->cursor; + ++len; + slosh = 0; + } else if (*r->cursor == '"') + break; + else if (*r->cursor == '\\') + slosh = 1; + else { + if (dst && len < dstsiz - 1) + dst[len] = *r->cursor; + ++len; + } + } + if (dst) + dst[len < dstsiz - 1 ? len : dstsiz - 1] = '\0'; + if (_run_out(r)) + return 0; + assert(*r->cursor == '"'); + ++r->cursor; + return len; +} + +static size_t _parse_token_or_quoted_string(struct http_request *r, char *dst, size_t dstsiz) +{ + assert(dstsiz > 0); + if (!_run_out(r) && *r->cursor == '"') + return _parse_quoted_string(r, dst, dstsiz); + return _parse_token(r, dst, dstsiz); +} + +static inline int _parse_http_size_t(struct http_request *r, http_size_t *szp) +{ + return !_run_out(r) && isdigit(*r->cursor) && str_to_uint64(r->cursor, 10, szp, &r->cursor); +} + +static inline int _parse_uint(struct http_request *r, unsigned int *uintp) +{ + return !_run_out(r) && isdigit(*r->cursor) && str_to_uint(r->cursor, 10, uintp, &r->cursor); +} + +static unsigned _parse_ranges(struct http_request *r, struct http_range *range, unsigned nrange) +{ + unsigned i; + for (i = 0; 1; ++i) { + enum http_range_type type; + http_size_t first = 0, last = 0; + if (_skip_literal(r, "-") && _parse_http_size_t(r, &last)) + type = SUFFIX; + else if (_parse_http_size_t(r, &first) && _skip_literal(r, "-")) + type = (_parse_http_size_t(r, &last)) ? CLOSED : OPEN; + else + return 0; + if (i < nrange) { + range[i].type = type; + range[i].first = first; + range[i].last = last; + } + if (!_skip_literal(r, ",")) + break; + _skip_optional_space(r); + } + return i; +} + +static int _parse_quoted_rfc822_time(struct http_request *r, time_t *timep) +{ + char datestr[40]; + size_t n = _parse_quoted_string(r, datestr, sizeof datestr); + if (n == 0 || n >= sizeof datestr) + return 0; + // TODO: Move the following code into its own function in str.c + struct tm tm; + bzero(&tm, sizeof tm); + // TODO: Ensure this works in non-English locales, ie, "%a" still accepts "Mon", "Tue" etc. and + // "%b" still accepts "Jan", "Feb" etc. + // TODO: Support symbolic time zones, eg, "UT", "GMT", "UTC", "EST"... + const char *c = strptime(datestr, "%a, %d %b %Y %T ", &tm); + if ((c[0] == '-' || c[0] == '+') && isdigit(c[1]) && isdigit(c[2]) && isdigit(c[3]) && isdigit(c[4]) && c[5] == '\0') { + time_t zone = (c[0] == '-' ? -1 : 1) * ((c[1] - '0') * 600 + (c[2] - '0') * 60 + (c[3] - '0') * 10 + (c[4] - '0')); + const char *tz = getenv("TZ"); + if (tz) + tz = alloca_strdup(tz); + setenv("TZ", "", 1); + tzset(); + *timep = mktime(&tm) - zone; + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + tzset(); + return 1; + } + return 0; +} + +/* Return 100 if more received data is needed. Returns 0 if parsing completes or fails. Returns a + * 4nn or 5nn HTTP result code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_verb(struct http_request *r) +{ + _rewind(r); + assert(r->cursor >= r->received); + assert(!_run_out(r)); + // Parse verb: GET, PUT, POST, etc. + assert(r->verb == NULL); + unsigned i; + for (i = 0; i < NELS(http_verbs); ++i) { + _rewind(r); + if (_skip_literal(r, http_verbs[i].word) && _skip_literal(r, " ")) { + r->verb = http_verbs[i].word; + break; + } + if (_run_out(r)) + return 100; // read more and try again + } + if (r->verb == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP request: invalid verb: %s", alloca_toprint(20, r->cursor, r->end - r->cursor)); + return 400; + } + _commit(r); + r->parser = http_request_parse_path; + return 0; +} + +/* Return 100 if more received data is needed. Returns 0 if parsing completes or fails. Returns a + * 4nn or 5nn HTTP result code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_path(struct http_request *r) +{ + // Parse path: word immediately following verb, delimited by spaces. + assert(r->path == NULL); + struct substring path; + if (!(_skip_word_printable(r, &path) && _skip_literal(r, " "))) { + if (_run_out(r)) + return 100; // read more and try again + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request at path: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed)); + return 400; + } + if ((r->path = _reserve(r, path)) == NULL) + return 0; // error + _commit(r); + r->parser = http_request_parse_http_version; + return 0; +} + +/* Return 100 if more received data is needed. Returns 0 if parsing completes or fails. Returns a + * 4nn or 5nn HTTP result code if parsing fails. Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_http_version(struct http_request *r) +{ + // Parse HTTP version: HTTP/m.n followed by CRLF. + assert(r->version_major == 0); + assert(r->version_minor == 0); + unsigned major, minor; + if (!( _skip_literal(r, "HTTP/") + && _parse_uint(r, &major) + && major > 0 && major < UINT8_MAX + && _skip_literal(r, ".") + && _parse_uint(r, &minor) + && minor > 0 && minor < UINT8_MAX + && _skip_eol(r) + ) + ) { + if (_run_out(r)) + return 100; // read more and try again + if (r->debug_flag && *r->debug_flag) + DEBUGF("HTTP %s malformed request: malformed version: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed)); + return 400; + } + _commit(r); + r->version_major = major; + r->version_minor = minor; + r->parser = http_request_start_parsing_headers; + if (r->handle_first_line) + return r->handle_first_line(r); + return 0; // parsing complete +} + +/* Select the header parser. Returns 0 after setting the new parser function. Returns a 4nn or 5nn + * HTTP result code if parsing fails. + * + * @author Andrew Bettison + */ +static int http_request_start_parsing_headers(struct http_request *r) +{ + assert(r->verb != NULL); + assert(r->path != NULL); + assert(r->version_major != 0); + assert(r->version_minor != 0); + if (r->version_major != 1) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP version: %u.%u", r->version_major, r->version_minor); + return 400; + } + r->parser = http_request_parse_header; + return 0; +} + +/* Parse one request header line. Returns 100 if more received data is needed. Returns 0 if there + * are no more headers or parsing fails. Returns a 4nn or 5nn HTTP result code if parsing fails. + * Returns -1 if an unexpected error occurs. + * + * @author Andrew Bettison + */ +static int http_request_parse_header(struct http_request *r) +{ + _skip_to_eol(r); + const char *const eol = r->cursor; + _skip_eol(r); + if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)... + _commit(r); + r->parser = http_request_start_body; + if (r->handle_headers) + return r->handle_headers(r); + return 0; + } + const char *const nextline = r->cursor; + _rewind(r); + const char *const sol = r->cursor; + if (_skip_literal_nocase(r, "Content-Length:")) { + _skip_optional_space(r); + http_size_t length; + if (_parse_http_size_t(r, &length) && _skip_optional_space(r) && r->cursor == eol) { + r->cursor = nextline; + _commit(r); + r->request_header.content_length = length; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Content-Length: %"PRIhttp_size_t, r->request_header.content_length); + return 0; + } + goto malformed; + } + _rewind(r); + if (_skip_literal_nocase(r, "Content-Type:")) { + _skip_optional_space(r); + struct substring type = substring_NULL; + struct substring subtype = substring_NULL; + char boundary[BOUNDARY_STRING_MAXLEN + 1]; + boundary[0] = '\0'; + if (_skip_token(r, &type) && _skip_literal(r, "/") && _skip_token(r, &subtype)) { + // Parse zero or more content-type parameters. + for (_skip_optional_space(r); r->cursor < eol && _skip_literal(r, ";"); _skip_optional_space(r)) { + _skip_optional_space(r); + const char *startparam = r->cursor; + if (_skip_literal(r, "boundary=")) { + size_t n = _parse_token_or_quoted_string(r, boundary, sizeof boundary); + if (n == 0 || n >= sizeof boundary || !is_valid_http_boundary_string(boundary)) + goto malformed; + continue; + } + // Silently ignore unrecognised parameters (eg, charset=) if they are well formed. + r->cursor = startparam; // partial rewind + if (_skip_token(r, NULL) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) + continue; + break; + } + if (r->cursor == eol) { + r->cursor = nextline; + _commit(r); + if ( (r->request_header.content_type = _reserve(r, type)) == NULL + || (r->request_header.content_subtype = _reserve(r, subtype)) == NULL + || (boundary[0] && (r->request_header.content_subtype = _reserve_str(r, boundary)) == NULL) + ) + return 0; // error + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Content-type: %s/%s%s%s", + r->request_header.content_type, + r->request_header.content_subtype, + r->request_header.boundary ? "; boundary=" : "", + r->request_header.boundary ? alloca_str_toprint(r->request_header.boundary) : "" + ); + return 0; + } + } + goto malformed; + } + _rewind(r); + if (_skip_literal_nocase(r, "Range:")) { + _skip_optional_space(r); + unsigned int n; + if ( _skip_literal(r, "bytes=") + && (n = _parse_ranges(r, r->request_header.content_ranges, NELS(r->request_header.content_ranges))) + && _skip_optional_space(r) + && (r->cursor == eol) + ) { + r->cursor = nextline; + _commit(r); + if (n > NELS(r->request_header.content_ranges)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("HTTP request Range header overflow (%u ranges in set, can only handle %u): %s", + n, NELS(r->request_header.content_ranges), alloca_toprint(-1, sol, eol - sol)); + // In this case ignore the Range: header -- respond with the entire resource. + r->request_header.content_range_count = 0; + } else { + r->request_header.content_range_count = n; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Range: bytes=%s", alloca_http_ranges(r->request_header.content_ranges)); + } + return 0; + } + goto malformed; + } + _rewind(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipped HTTP request header: %s", alloca_toprint(-1, sol, eol - sol)); + r->cursor = nextline; + _commit(r); + return 0; +malformed: + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP request header: %s", alloca_toprint(-1, sol, eol - sol)); + return 400; +} + +/* Select the header parser. Returns 0 after setting the new parser function. Returns a 4nn or 5nn + * HTTP result code if parsing fails. + * + * @author Andrew Bettison + */ +static int http_request_start_body(struct http_request *r) +{ + assert(r->verb != NULL); + assert(r->path != NULL); + assert(r->version_major != 0); + assert(r->version_minor != 0); + assert(r->parsed <= r->end); + if (r->verb == HTTP_VERB_GET) { + // TODO: Implement HEAD requests + if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: Content-Length not allowed", r->verb); + return 400; + } + if (r->request_header.content_type) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: Content-Type not allowed", r->verb); + return 400; + } + r->parser = NULL; + } + else if (r->verb == HTTP_VERB_POST) { + if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb); + return 400; + } + if (r->request_header.content_type == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: missing Content-Type header", r->verb); + return 400; + } + if ( strcmp(r->request_header.content_type, "multipart") == 0 + && strcmp(r->request_header.content_subtype, "form-data") == 0 + ) { + if (r->request_header.boundary == NULL || r->request_header.boundary[0] == '\0') { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s request: Content-Type %s/%s missing boundary parameter", + r->verb, r->request_header.content_type, r->request_header.content_subtype); + return 400; + } + r->parser = http_request_parse_body_form_data; + r->form_data_state = START; + } else { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP %s request: Content-Type %s/%s not supported", + r->verb, r->request_header.content_type, r->request_header.content_subtype); + return 415; + } + size_t unparsed = r->end - r->parsed; + if (unparsed > r->request_header.content_length) { + WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length)); + r->request_content_remaining = 0; + } + else + r->request_content_remaining = r->request_header.content_length - unparsed; + } + else { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unsupported HTTP %s request", r->verb); + r->parser = NULL; + return 501; + } + return 0; +} + +/* Returns 1 if a MIME delimiter is skipped, 2 if a MIME close-delimiter is skipped. + */ +static int _skip_mime_boundary(struct http_request *r) +{ + if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.boundary)) + return 0; + if (_skip_literal(r, "--") && _skip_optional_space(r) && _skip_crlf(r)) + return 2; + if (_skip_optional_space(r) && _skip_crlf(r)) + return 1; + return 0; +} + +static int _parse_content_disposition(struct http_request *r, struct mime_content_disposition *cd) +{ + size_t n = _parse_token(r, cd->type, sizeof cd->type); + if (n == 0) + return 0; + if (n >= sizeof cd->type) { + WARNF("HTTP Content-Disposition type truncated: %s", alloca_str_toprint(cd->type)); + return 0; + } + while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) { + const char *start = r->cursor; + if (_skip_literal(r, "filename=")) { + size_t n = _parse_token_or_quoted_string(r, cd->filename, sizeof cd->filename); + if (n == 0) + return 0; + if (n >= sizeof cd->filename) { + WARNF("HTTP Content-Disposition filename truncated: %s", alloca_str_toprint(cd->filename)); + return 0; + } + continue; + } + r->cursor = start; + if (_skip_literal(r, "name=")) { + size_t n = _parse_token_or_quoted_string(r, cd->name, sizeof cd->name); + if (n == 0) + return 0; + if (n >= sizeof cd->name) { + WARNF("HTTP Content-Disposition name truncated: %s", alloca_str_toprint(cd->name)); + return 0; + } + continue; + } + r->cursor = start; + if (_skip_literal(r, "size=")) { + if (!_parse_http_size_t(r, &cd->size)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "creation-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->creation_date)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "modification-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->modification_date)) + goto malformed; + continue; + } + r->cursor = start; + if (_skip_literal(r, "read-date=")) { + if (!_parse_quoted_rfc822_time(r, &cd->read_date)) + goto malformed; + continue; + } + r->cursor = start; + struct substring param; + if (_skip_token(r, ¶m) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping HTTP Content-Disposition parameter: %s", alloca_substring_toprint(param)); + continue; + } +malformed: + WARNF("Malformed HTTP Content-Disposition: %s", alloca_toprint(50, r->cursor, r->end - r->cursor)); + return 0; + } + return 1; +} + +/* Return zero if more received data is needed. The first call that completes parsing of the body + * returns 200. All subsequent calls return 100. Returns a 4nn or 5nn HTTP result code if parsing + * fails. + * + * NOTE: No support for nested/mixed parts, as that would considerably complicate the parser. If + * the need arises in future, we will deal with it then. In the meantime, we will have something + * that meets our immediate needs for Rhizome Direct and a variety of use cases. + * + * @author Andrew Bettison + */ +static int http_request_parse_body_form_data(struct http_request *r) +{ + int at_start = 0; + switch (r->form_data_state) { + case START: + // The logic here allows for a missing initial CRLF before the first boundary line. + at_start = 1; + r->form_data_state = PREAMBLE; + // fall through + case PREAMBLE: + while (!_run_out(r)) { + const char *end_preamble = r->cursor; + int b; + if ((_skip_crlf(r) || at_start) && (b = _skip_mime_boundary(r))) { + assert(end_preamble >= r->parsed); + if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) + r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed); + _rewind_crlf(r); + _commit(r); + r->form_data_state = EPILOGUE; + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) + r->form_data.handle_mime_part_start(r); + } + return 0; + } + _skip_to_crlf(r); + at_start = 0; + } + if (r->cursor > r->parsed && r->form_data.handle_mime_preamble) + r->form_data.handle_mime_preamble(r, r->parsed, r->parsed - r->cursor); + _commit(r); + return 0; + case HEADER: { + if (_skip_crlf(r) && _skip_crlf(r)) { + _commit(r); + r->form_data_state = BODY; + return 0; + } + if (_run_out(r)) + return 0; // read more and try again + _rewind(r); + int b; + if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) { + _rewind_crlf(r); + _commit(r); + if (r->form_data.handle_mime_part_end) + r->form_data.handle_mime_part_end(r); + r->form_data_state = EPILOGUE; + // Boundary in the middle of headers starts a new part. + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) + r->form_data.handle_mime_part_start(r); + } + return 0; + } + if (_run_out(r)) + return 0; // read more and try again + _rewind(r); + struct substring label; + if (_skip_crlf(r) && _skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) { + size_t labellen = label.end - label.start; + char labelstr[labellen + 1]; + strncpy(labelstr, label.start, labellen)[labellen] = '\0'; + str_tolower_inplace(labelstr); + const char *value = r->cursor; + if (strcmp(labelstr, "content-disposition") == 0) { + struct mime_content_disposition cd; + bzero(&cd, sizeof cd); + if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) { + if (r->form_data.handle_mime_content_disposition) + r->form_data.handle_mime_content_disposition(r, &cd); + _commit(r); + return 0; + } + } else if (_skip_to_crlf(r)) { + if (r->form_data.handle_mime_header) + r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end + _skip_crlf(r); + _commit(r); + return 0; + } + } + if (_run_out(r)) + return 0; // read more and try again + _rewind(r); + } + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed)); + return 400; + case BODY: + while (!_run_out(r)) { + int b; + const char *eol = r->cursor; + if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) { + _rewind_crlf(r); + _commit(r); + if (r->form_data.handle_mime_body) + r->form_data.handle_mime_body(r, r->parsed, r->parsed - eol); // excluding CRLF at end + if (r->form_data.handle_mime_part_end) + r->form_data.handle_mime_part_end(r); + r->form_data_state = EPILOGUE; + if (b == 1) { + r->form_data_state = HEADER; + if (r->form_data.handle_mime_part_start) + r->form_data.handle_mime_part_start(r); + } + return 0; + } + _skip_to_crlf(r); + } + if (r->cursor > r->parsed && r->form_data.handle_mime_body) + r->form_data.handle_mime_body(r, r->parsed, r->parsed - r->cursor); + _commit(r); + return 0; + case EPILOGUE: + r->cursor = r->end; + if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed) + r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed); + _commit(r); + return 0; + } + assert(0); // not reached +} + +static ssize_t http_request_read(struct http_request *r, char *buf, size_t len) +{ + sigPipeFlag = 0; + ssize_t bytes = read_nonblock(r->alarm.poll.fd, buf, len); + if (bytes == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("HTTP socket read error, closing connection"); + http_request_finalise(r); + return -1; + } + if (sigPipeFlag) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Received SIGPIPE on HTTP socket read, closing connection"); + http_request_finalise(r); + return -1; + } + return bytes; +} + +static void http_request_receive(struct http_request *r) +{ + assert(r->phase == RECEIVE); + r->limit = r->buffer + sizeof r->buffer; + assert(r->end < r->limit); + size_t room = r->limit - r->end; + assert(r->parsed >= r->received); + assert(r->parsed <= r->end); + // If buffer is running short on unused space, shift existing content in buffer down to make more + // room if possible. + if ( (room < 128 || (room < 1024 && r->parsed - r->received >= 32)) + && (r->request_content_remaining == CONTENT_LENGTH_UNKNOWN || room < r->request_content_remaining) + ) { + size_t unparsed = r->end - r->parsed; + memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst + r->parsed = r->received; + r->end = r->received + unparsed; + room = r->limit - r->end; + } + // If there is no more buffer space, fail the request. + if (room == 0) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Buffer size reached, reporting overflow"); + http_request_simple_response(r, 431, NULL); + return; + } + // Read up to the end of available buffer space or the end of content, whichever is first. + size_t read_size = room; + if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && read_size > r->request_content_remaining) { + r->limit = r->end + r->request_content_remaining; + read_size = r->request_content_remaining; + } + if (read_size != 0) { + // Read as many bytes as possible into the unused buffer space. Any read error + // closes the connection without any response. + ssize_t bytes = http_request_read(r, (char *)r->end, read_size); + if (bytes == -1) + return; + // If no data was read, then just return to polling. Don't drop the connection on an empty read, + // because that drops connections when they shouldn't, including during testing. The inactivity + // timeout will drop the connections instead. + if (bytes == 0) + return; + r->end += (size_t) bytes; + // We got some data, so reset the inactivity timer and invoke the parsing state machine to process + // it. The state machine invokes the caller-supplied callback functions. + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + unschedule(&r->alarm); + schedule(&r->alarm); + } + // Parse the unparsed and received data. + while (r->phase == RECEIVE) { + int result; + if (_run_out(r) && r->request_content_remaining == 0) { + if (r->handle_content_end) + result = r->handle_content_end(r); + else { + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: no end-of-content function set"); + result = 500; + } + } else { + const char *oldparsed = r->parsed; + if (r->parser == NULL) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: no parser function set"); + result = 500; + } else { + _rewind(r); + result = r->parser(r); + assert(r->parsed >= oldparsed); + } + if (result == 100) { + if (_run_out(r)) + return; // needs more data; poll again + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: parser function returned 100 but not run out"); + result = 500; + } + if (result == 0 && r->parsed == oldparsed) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Internal failure parsing HTTP request: parser function did not advance"); + result = 500; + } + } + if (result >= 300) + r->response.result_code = result; + else if (result) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Internal failure parsing HTTP request: invalid result=%d", result); + r->response.result_code = 500; + } + if (r->response.result_code) { + http_request_start_response(r); + return; + } + if (result == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Unrecoverable error parsing HTTP request, closing connection"); + http_request_finalise(r); + return; + } + } + if (r->phase != RECEIVE) + return; + if (r->response.result_code == 0) { + WHY("No HTTP response set, using 500 Server Error"); + r->response.result_code = 500; + } + http_request_start_response(r); +} + +/* Write the current contents of the response buffer to the HTTP socket. When no more bytes can be + * written, return so that socket polling can continue. Once all bytes are sent, if there is a + * content generator function, invoke it to put more content in the response buffer, and write that + * content. + * + * @author Andrew Bettison + */ +static void http_request_send_response(struct http_request *r) +{ + assert(r->response_sent <= r->response_length); + while (r->response_sent < r->response_length) { + assert(r->response_buffer_sent <= r->response_buffer_length); + if (r->response_buffer_sent == r->response_buffer_length) { + if (r->response.content_generator) { + // Content generator must fill response_buffer[] and set response_buffer_length. + r->response_buffer_sent = r->response_buffer_length = 0; + if (r->response.content_generator(r) == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Content generation error, closing connection"); + http_request_finalise(r); + return; + } + assert(r->response_buffer_sent <= r->response_buffer_length); + if (r->response_buffer_sent == r->response_buffer_length) { + WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)", + r->response_sent, r->response_length, r->response_length - r->response_sent); + http_request_finalise(r); + return; + } + } else { + WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes", + r->response_length, r->response_length - r->response_sent); + http_request_finalise(r); + return; + } + } + assert(r->response_buffer_sent < r->response_buffer_length); + size_t bytes = r->response_buffer_length - r->response_buffer_sent; + if (r->response_sent + bytes > r->response_length) { + WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating", + r->response_length, + r->response_sent + bytes - r->response_length); + bytes = r->response_length - r->response_sent; + } + sigPipeFlag = 0; + ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes); + if (written == -1) { + if (r->debug_flag && *r->debug_flag) + DEBUG("HTTP socket write error, closing connection"); + http_request_finalise(r); + return; + } + if (sigPipeFlag) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Received SIGPIPE on HTTP socket write, closing connection"); + http_request_finalise(r); + return; + } + // If we wrote nothing, go back to polling. + if (written == 0) + return; + r->response_sent += (size_t) written; + r->response_buffer_sent += (size_t) written; + assert(r->response_sent <= r->response_length); + assert(r->response_buffer_sent <= r->response_buffer_length); + // Reset inactivity timer. + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + unschedule(&r->alarm); + schedule(&r->alarm); + // If we wrote less than we tried, then go back to polling. + if (written < (size_t) bytes) + return; + } + if (r->debug_flag && *r->debug_flag) + DEBUG("Done, closing connection"); + http_request_finalise(r); +} + +static void http_server_poll(struct sched_ent *alarm) +{ + struct http_request *r = (struct http_request *) alarm; + if (alarm->poll.revents == 0) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Timeout, closing connection"); + http_request_finalise(r); + } + else if (alarm->poll.revents & (POLLHUP | POLLERR)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Poll error (%s), closing connection", alloca_poll_events(alarm->poll.revents)); + http_request_finalise(r); + } + else { + if (r->phase == RECEIVE && (alarm->poll.revents & POLLIN)) + http_request_receive(r); // this could change the phase to TRANSMIT + if (r->phase == TRANSMIT && (alarm->poll.revents & POLLOUT)) + http_request_send_response(r); + } + // Any of the above calls could change the phase to DONE. + if (r->phase == DONE && r->free) + r->free(r); // after this, *r is no longer valid +} + +/* Count the number of bytes in a given set of ranges, given the length of the content to which the + * ranges will be applied. + * + * @author Andrew Bettison + */ +http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t resource_length) +{ + return http_range_close(NULL, range, nranges, resource_length); +} + +/* Copy the array of byte ranges, closing it (converting all ranges to CLOSED) using the supplied + * resource length. + * + * @author Andrew Bettison + */ +http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t resource_length) +{ + http_size_t bytes = 0; + unsigned i; + for (i = 0; i != nranges; ++i) { + http_size_t first = 0; + http_size_t last = resource_length; + const struct http_range *range = &src[i]; + switch (range->type) { + case CLOSED: + last = range->last < resource_length ? range->last : resource_length; + case OPEN: + first = range->first < resource_length ? range->first : resource_length; + break; + case SUFFIX: + first = range->last < resource_length ? resource_length - range->last : 0; + break; + default: + abort(); // not reached + } + assert(first <= last); + if (dst) + *dst++ = (struct http_range){ .type = CLOSED, .first=first, .last=last }; + bytes += last - first; + } + return bytes; +} + +/* Return appropriate message for HTTP response codes, both known and unknown. + */ +static const char *httpResultString(int response_code) +{ + switch (response_code) { + case 200: return "OK"; + case 201: return "Created"; + case 206: return "Partial Content"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 414: return "Request-URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Requested Range Not Satisfiable"; + case 431: return "Request Header Fields Too Large"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + default: return (response_code <= 4) ? "Unknown status code" : "A suffusion of yellow"; + } +} + +static strbuf strbuf_append_quoted_string(strbuf sb, const char *str) +{ + strbuf_putc(sb, '"'); + for (; *str; ++str) { + if (*str == '"' || *str == '\\') + strbuf_putc(sb, '\\'); + strbuf_putc(sb, *str); + } + strbuf_putc(sb, '"'); + return sb; +} + +/* Render the HTTP response into the current response buffer. Return 1 if it fits, 0 if it does + * not. The buffer response_pointer may be NULL, in which case no response is rendered, but the + * content_length is still computed + * + * @author Andrew Bettison + */ +static int _render_response(struct http_request *r) +{ + strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size); + struct http_response hr = r->response; + assert(hr.result_code != 0); + const char *result_string = httpResultString(hr.result_code); + if (hr.content == NULL) { + strbuf cb = strbuf_alloca(100 + strlen(result_string)); + strbuf_puts(cb, "

"); + strbuf_puts(cb, result_string); + strbuf_puts(cb, "

\r\n"); + hr.content = strbuf_str(cb); + hr.header.resource_length = hr.header.content_length = strbuf_len(cb); + hr.header.content_type = "text/html"; + hr.header.content_range_start = 0; + } + assert(hr.header.content_type != NULL); + strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", hr.result_code, result_string); + strbuf_sprintf(sb, "Content-Type: %s", hr.header.content_type); + if (hr.header.boundary) { + strbuf_puts(sb, "; boundary="); + if (strchr(hr.header.boundary, '"') || strchr(hr.header.boundary, '\\')) + strbuf_append_quoted_string(sb, hr.header.boundary); + else + strbuf_puts(sb, hr.header.boundary); + } + strbuf_puts(sb, "\r\n"); + assert(hr.header.content_range_start <= hr.header.resource_length); + assert(hr.header.content_length <= hr.header.resource_length); + if (hr.header.content_range_start && hr.header.content_length) + strbuf_sprintf(sb, + "Content-Range: bytes %"PRIhttp_size_t"-%"PRIhttp_size_t"/%"PRIhttp_size_t"\r\n", + hr.header.content_range_start, + hr.header.content_range_start + hr.header.content_length - 1, + hr.header.resource_length + ); + strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length); + strbuf_puts(sb, "\r\n"); + r->response_length = strbuf_len(sb) + hr.header.content_length; + if (strbuf_overrun(sb) || (r->response_buffer_size < r->response_length)) + return 0; + bcopy(hr.content, strbuf_end(sb), hr.header.content_length); + r->response_buffer_sent = 0; + r->response_buffer_length = r->response_length; + return 1; +} + +/* Returns with the length of the rendered response in r->response_length. If the rendered response + * did not fit into any available buffer, then returns with r->response_buffer == NULL, otherwise + * r->response_buffer points to the rendered response. + * + * @author Andrew Bettison + */ +static void http_request_render_response(struct http_request *r) +{ + // If there is no response buffer allocated yet, use the available part of the in-struct buffer. + http_request_set_response_bufsize(r, 1); + // Try rendering the response into the existing buffer. This will discover the length of the + // rendered headers, so after this step, whether or not the buffer was overrun, we know the total + // length of the response. + if (!_render_response(r)) { + // If the response did not fit into the existing buffer, then allocate a large buffer from the + // heap and try rendering again. + if (http_request_set_response_bufsize(r, r->response_length + 1) == -1) + WHY("Cannot render HTTP response, out of memory"); + else if (!_render_response(r)) + FATAL("Re-render of HTTP response overflowed buffer"); + } +} + +static void http_request_start_response(struct http_request *r) +{ + // If HTTP responses are disabled (eg, for testing purposes) then skip all response construction + // and close the connection. + if (r->disable_tx_flag && *r->disable_tx_flag) { + INFO("HTTP transmit disabled, closing connection"); + http_request_finalise(r); + return; + } + // Drain the rest of the request that has not been received yet (eg, if sending an error response + // provoked while parsing the early part of a partially-received request). If a read error + // occurs, the connection is closed. + ssize_t bytes; + while ((bytes = http_request_read(r, (char *) r->received, (r->buffer + sizeof r->buffer) - r->received)) > 0) + ; + if (bytes == -1) + return; + // If the response cannot be rendered, then render a 500 Server Error instead. If that fails, + // then just close the connection. + http_request_render_response(r); + if (r->response_buffer == NULL) { + WARN("Cannot render HTTP response, sending 500 Server Error instead"); + r->response.result_code = 500; + r->response.content = NULL; + http_request_render_response(r); + if (r->response_buffer == NULL) { + WHY("Cannot render HTTP 500 Server Error response, closing connection"); + http_request_finalise(r); + return; + } + } + r->response_sent = 0; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_length)); + r->phase = TRANSMIT; + r->alarm.poll.events = POLLOUT; + watch(&r->alarm); +} + +void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes) +{ + assert(result != 0); + r->response.result_code = result; + r->response.header.content_type = mime_type; + r->response.header.content_range_start = 0; + r->response.header.content_length = r->response.header.resource_length = bytes; + r->response.content = body; + r->response.content_generator = NULL; + http_request_start_response(r); +} + +void http_request_simple_response(struct http_request *r, uint16_t result, const char *body) +{ + strbuf h = NULL; + if (body) { + size_t html_len = strlen(body) + 40; + char html[html_len]; + h = strbuf_local(html, html_len); + strbuf_sprintf(h, "

%s

", body); + } + r->response.result_code = result; + r->response.header.content_type = body ? "text/html" : NULL; + r->response.header.content_range_start = 0; + r->response.header.resource_length = r->response.header.content_length = h ? strbuf_len(h) : 0; + r->response.content = h ? strbuf_str(h) : NULL; + r->response.content_generator = NULL; + http_request_start_response(r); +} + +void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes) +{ + r->response.result_code = result; + r->response.content = NULL; + r->response.content_generator = NULL; + http_request_start_response(r); +} diff --git a/http_server.h b/http_server.h new file mode 100644 index 00000000..7b851dbd --- /dev/null +++ b/http_server.h @@ -0,0 +1,156 @@ +/* +Serval DNA - HTTP Server API +Copyright (C) 2013 Serval Project, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVALDNA__HTTP_SERVER_H +#define __SERVALDNA__HTTP_SERVER_H + +#include +#include "constants.h" +#include "strbuf.h" + +/* Generic HTTP request handling. + * + * @author Andrew Bettison + */ + +extern const char HTTP_VERB_GET[]; +extern const char HTTP_VERB_POST[]; +extern const char HTTP_VERB_PUT[]; +extern const char HTTP_VERB_HEAD[]; +extern const char HTTP_VERB_DELETE[]; +extern const char HTTP_VERB_TRACE[]; +extern const char HTTP_VERB_OPTIONS[]; +extern const char HTTP_VERB_CONNECT[]; +extern const char HTTP_VERB_PATCH[]; + +typedef uint64_t http_size_t; +#define PRIhttp_size_t PRIu64 + +struct http_request; + +struct http_range { + enum http_range_type { NIL = 0, CLOSED, OPEN, SUFFIX } type; + http_size_t first; // only for CLOSED or OPEN + http_size_t last; // only for CLOSED or SUFFIX +}; + +http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length); +http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t content_length); + +#define CONTENT_LENGTH_UNKNOWN UINT64_MAX + +struct http_request_headers { + http_size_t content_length; + const char *content_type; + const char *content_subtype; + const char *boundary; + unsigned short content_range_count; + struct http_range content_ranges[5]; +}; + +struct http_response_headers { + http_size_t content_length; + http_size_t content_range_start; // range_end = range_start + content_length - 1 + http_size_t resource_length; // size of entire resource + const char *content_type; // "type/subtype" + const char *boundary; +}; + +struct http_response { + uint16_t result_code; + struct http_response_headers header; + const char *content; + int (*content_generator)(struct http_request *); // callback to produce more content +}; + +#define MIME_FILENAME_MAXLEN 127 + +struct mime_content_disposition { + char type[64]; + char name[64]; + char filename[MIME_FILENAME_MAXLEN + 1]; + http_size_t size; + time_t creation_date; + time_t modification_date; + time_t read_date; +}; + +struct http_mime_handler { + void (*handle_mime_preamble)(struct http_request *, const char *, size_t); + void (*handle_mime_part_start)(struct http_request *); + void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *); + void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t); + void (*handle_mime_body)(struct http_request *, const char *, size_t); + void (*handle_mime_part_end)(struct http_request *); + void (*handle_mime_epilogue)(struct http_request *, const char *, size_t); +}; + +struct http_request; + +void http_request_init(struct http_request *r, int sockfd); +void http_request_free_response_buffer(struct http_request *r); +int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz); +void http_request_finalise(struct http_request *r); +void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); +void http_request_simple_response(struct http_request *r, uint16_t result, const char *body); +void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes); + +#ifdef __SERVALDNA__HTTP_SERVER_IMPLEMENTATION + +struct http_request { + struct sched_ent alarm; // MUST BE FIRST ELEMENT + enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase; + bool_t *debug_flag; + bool_t *disable_tx_flag; + time_ms_t initiate_time; // time connection was initiated + time_ms_t idle_timeout; // disconnect if no bytes received for this long + struct sockaddr_in client_in_addr; + int (*parser)(struct http_request *); // current parser function + int (*handle_first_line)(struct http_request *); // called after first line is parsed + int (*handle_headers)(struct http_request *); // called after all headers are parsed + int (*handle_content_end)(struct http_request *); // called after all content is received + enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state; + struct http_mime_handler form_data; // called to parse multipart/form-data body + void (*finalise)(struct http_request *); + void (*free)(void*); + const char *verb; // points to nul terminated static string, "GET", "PUT", etc. + const char *path; // points into buffer; nul terminated + uint8_t version_major; // m from from HTTP/m.n + uint8_t version_minor; // n from HTTP/m.n + struct http_request_headers request_header; + const char *received; // start of received data in buffer[] + const char *end; // end of received data in buffer[] + const char *limit; // end of content in buffer[] + const char *parsed; // start of unparsed data in buffer[] + const char *cursor; // for parsing + http_size_t request_content_remaining; + struct http_response response; + http_size_t response_length; // total response bytes (header + content) + http_size_t response_sent; // for counting up to response_length + char *response_buffer; + size_t response_buffer_size; + size_t response_buffer_length; + size_t response_buffer_sent; + void (*response_free_buffer)(void*); + char buffer[8 * 1024]; +}; + +#endif // __SERVALDNA__HTTP_SERVER_IMPLEMENTATION + +#endif // __SERVALDNA__HTTP_SERVER_H diff --git a/overlay.c b/overlay.c index 6c70d943..d16b0ef9 100644 --- a/overlay.c +++ b/overlay.c @@ -133,12 +133,8 @@ schedule(&_sched_##X); } rhizome_cleanup(NULL); } - /* Rhizome http server needs to know which callback to attach - to client sockets, so provide it here, along with the name to - appear in time accounting statistics. */ - rhizome_http_server_start(rhizome_server_parse_http_request, - "rhizome_server_parse_http_request", - RHIZOME_HTTP_PORT,RHIZOME_HTTP_PORT_MAX); + // start the HTTP server if enabled + rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX); // start the dna helper if configured dna_helper_start(); diff --git a/rhizome.h b/rhizome.h index 9b73ff68..54201f44 100644 --- a/rhizome.h +++ b/rhizome.h @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "sha2.h" #include "str.h" #include "strbuf.h" +#include "http_server.h" #include "nacl.h" #include @@ -525,71 +526,27 @@ struct rhizome_read int64_t length; }; -typedef struct rhizome_http_request { - struct sched_ent alarm; - time_ms_t initiate_time; /* time connection was initiated */ - - struct sockaddr_in requestor; +/* Rhizome-specific HTTP request handling. + */ +typedef struct rhizome_http_request +{ + struct http_request http; // MUST BE FIRST ELEMENT - /* identify request from others being run. + /* Identify request from others being run. Monotonic counter feeds it. Only used for debugging when we write post-.log files for multi-part form requests. */ unsigned int uuid; - /* The HTTP request as currently received */ - int request_length; - int header_length; - char request[1024]; - - /* Nature of the request */ - int request_type; - /* All of the below are receiving data */ -#define RHIZOME_HTTP_REQUEST_RECEIVING -1 -#define RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART -2 - - // callback function to fill the response buffer - int (*generator)(struct rhizome_http_request *r); - - /* Local buffer of data to be sent. - If a RHIZOME_HTTP_REQUEST_FROMBUFFER, then the buffer is sent, and when empty - the request is closed. - Else emptying the buffer triggers a request to fetch more data. Only if no - more data is provided do we then close the request. */ - unsigned char *buffer; - int buffer_size; // size - int buffer_length; // number of bytes loaded into buffer - int buffer_offset; // where we are between [0,buffer_length) - struct rhizome_read read_state; - /* Path of request (used by POST multipart form requests where - the actual processing of the request does not occur while the - request headers are still available. */ - char path[1024]; - /* Boundary string for POST multipart form requests */ - char boundary_string[1024]; - int boundary_string_length; /* File currently being written to while decoding POST multipart form */ - FILE *field_file; + enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part; + int part_fd; + /* Which parts have been received in POST multipart form */ + bool_t received_manifest; + bool_t received_data; /* Name of data file supplied */ - char data_file_name[1024]; - /* Which fields have been seen in POST multipart form */ - int fields_seen; - /* The seen fields bitmap above shares values with the actual Rhizome Direct - state machine. The state numbers (and thus bitmap values for the various - fields) are listed here. - - To avoid confusion, we should not use single bit values for states that do - not correspond directly to a particular field. - Doesn't really matter what they are apart from not having exactly one bit set. - In fact, the only reason to not have exactly one bit set is so that we keep as - many bits available for field types as possible. - */ -#define RD_MIME_STATE_MANIFESTHEADERS (1<<0) -#define RD_MIME_STATE_DATAHEADERS (1<<1) -#define RD_MIME_STATE_INITIAL 0 -#define RD_MIME_STATE_PARTHEADERS 0xffff0000 -#define RD_MIME_STATE_BODY 0xffff0001 + char data_file_name[MIME_FILENAME_MAXLEN + 1]; /* The source specification data which are used in different ways by different request types */ @@ -607,15 +564,6 @@ typedef struct rhizome_http_request { } rhizome_http_request; -struct http_response { - unsigned int result_code; - const char * content_type; - uint64_t content_start; - uint64_t content_end; - uint64_t content_length; - const char * body; -}; - int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, uint64_t offset,int count,unsigned char *bytes, int type); @@ -629,9 +577,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r, int result, con int rhizome_server_http_response(rhizome_http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes); -int rhizome_http_server_start(int (*http_parse_func)(rhizome_http_request *), - const char *http_parse_func_description, - uint16_t port_low, uint16_t port_high); +int rhizome_http_server_start(uint16_t port_low, uint16_t port_high); int is_rhizome_enabled(); int is_rhizome_mdp_enabled(); @@ -787,7 +733,7 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath); int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk); int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size); -int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len); +int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len); int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename); int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length); diff --git a/rhizome_direct.c b/rhizome_direct.c index 518b02d0..b4ce9631 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -260,8 +260,7 @@ int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r) multiple versions of a given bundle introduces only a slight complication. */ -rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response -(unsigned char *buffer,int size, int max_response_bytes) +rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *buffer,int size, int max_response_bytes) { if (size<10) return NULL; if (size>65536) return NULL; diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index c8b075cc..b8d6125a 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -28,658 +28,390 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include -int rhizome_direct_clear_temporary_files(rhizome_http_request *r) +static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_request *r, char *pathbuf, size_t bufsiz, const char *field) { - char filename[1024]; - char *fields[]={"manifest","data","unknown",NULL}; - int i; - - for(i=0;fields[i];i++) { - snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,fields[i]); - filename[1023]=0; - unlink(filename); + strbuf b = strbuf_local(pathbuf, bufsiz); + // TODO: use a temporary directory + strbuf_sprintf(b, "rhizomedirect.%d.%s", r->http.alarm.poll.fd, field); + if (strbuf_overrun(b)) { + WHYF("Rhizome Direct pathname overflow: %s", alloca_str_toprint(pathbuf)); + return -1; } return 0; } -int rhizome_direct_form_received(rhizome_http_request *r) +#define form_temporary_file_path(r,buf,field) _form_temporary_file_path(__WHENCE__, (r), (buf), sizeof(buf), (field)) + +static void rhizome_direct_clear_temporary_files(rhizome_http_request *r) { - /* Process completed form based on the set of fields seen */ - if (!strcmp(r->path,"/rhizome/import")) { - switch(r->fields_seen) { - case RD_MIME_STATE_MANIFESTHEADERS | RD_MIME_STATE_DATAHEADERS: { - /* Got a bundle to import */ - DEBUGF("Call bundle import for rhizomedata.%d.{data,file}", - r->alarm.poll.fd); - strbuf manifest_path = strbuf_alloca(50); - strbuf payload_path = strbuf_alloca(50); - strbuf_sprintf(manifest_path, "rhizomedirect.%d.manifest", r->alarm.poll.fd); - strbuf_sprintf(payload_path, "rhizomedirect.%d.data", r->alarm.poll.fd); - - int ret=0; - rhizome_manifest *m = rhizome_new_manifest(); - - if (!m) - ret=WHY("Out of manifests."); - else{ - ret=rhizome_bundle_import_files(m, strbuf_str(manifest_path), strbuf_str(payload_path)); - rhizome_manifest_free(m); - } - - rhizome_direct_clear_temporary_files(r); - /* report back to caller. - 200 = ok, which is probably appropriate for when we already had the bundle. - 201 = content created, which is probably appropriate for when we successfully - import a bundle (or if we already have it). - 403 = forbidden, which might be appropriate if we refuse to accept it, e.g., - the import fails due to malformed data etc. - (should probably also indicate if we have a newer version if possible) - */ - switch (ret) { - case 0: - return rhizome_server_simple_http_response(r, 201, "Bundle succesfully imported."); - case 2: - return rhizome_server_simple_http_response(r, 200, "Bundle already imported."); - } - return rhizome_server_simple_http_response(r, 500, "Server error: Rhizome import command failed."); - } - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - } - } else if (!strcmp(r->path,"/rhizome/enquiry")) { - int fd=-1; - char file[1024]; - switch(r->fields_seen) { - case RD_MIME_STATE_DATAHEADERS: - /* Read data buffer in, pass to rhizome direct for comparison with local - rhizome database, and send back responses. */ - snprintf(file,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,"data"); - fd=open(file,O_RDONLY); - if (fd == -1) { - WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(file)); - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't read a file"); - } - struct stat stat; - if (fstat(fd, &stat) == -1) { - WHYF_perror("stat(%d)", fd); - /* Clean up after ourselves */ - close(fd); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't stat a file"); - } - unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); - if (addr==MAP_FAILED) { - WHYF_perror("mmap(NULL, %"PRId64", PROT_READ, MAP_SHARED, %d, 0)", (int64_t) stat.st_size, fd); - /* Clean up after ourselves */ - close(fd); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Couldn't mmap() a file"); - } - /* Ask for a fill response. Regardless of the size of the set of BARs passed - to us, we will allow up to 64KB of response. */ - rhizome_direct_bundle_cursor - *c=rhizome_direct_get_fill_response(addr,stat.st_size,65536); - munmap(addr,stat.st_size); - close(fd); + const char *fields[] = { "manifest", "data" }; + int i; + for (i = 0; i != NELS(fields); ++i) { + char path[1024]; + if (form_temporary_file_path(r, path, fields[i]) != -1) + if (unlink(path) == -1 && errno != ENOENT) + WARNF_perror("unlink(%s)", alloca_str_toprint(path)); + } +} - if (c) - { - /* TODO: Write out_buffer as the body of the response. - We should be able to do this using the async framework fairly easily. - */ - - int bytes=c->buffer_offset_bytes+c->buffer_used; - r->buffer=malloc(bytes+1024); - r->buffer_size=bytes+1024; - r->buffer_offset=0; - assert(r->buffer); - - /* Write HTTP response header */ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code=200; - hr.content_type="binary/octet-stream"; - hr.content_length=bytes; - hr.body=NULL; - r->request_type=0; - rhizome_server_set_response(r,&hr); - assert(r->buffer_offset<1024); - - /* Now append body and send it back. */ - bcopy(c->buffer,&r->buffer[r->buffer_length],bytes); - r->buffer_length+=bytes; - r->buffer_offset=0; - - /* Clean up cursor after sending response */ - rhizome_direct_bundle_iterator_free(&c); - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return 0; - } - else - { - return rhizome_server_simple_http_response(r,500,"Could not get response to enquiry"); - } - - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return rhizome_server_simple_http_response(r, 404, "/rhizome/enquiry requires 'data' field"); - } - } - /* Allow servald to be configured to accept files without manifests via HTTP - from localhost, so that rhizome bundles can be created programatically. - There are probably still some security loop-holes here, which is part of - why we leave it disabled by default, but it will be sufficient for testing - possible uses, including integration with OpenDataKit. +int rhizome_direct_import_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (!r->received_manifest) { + http_request_simple_response(&r->http, 400, "Missing 'manifest' part"); + return 0; + } + if (!r->received_data) { + http_request_simple_response(&r->http, 400, "Missing 'data' part"); + return 0; + } + /* Got a bundle to import */ + char manifest_path[512]; + char payload_path[512]; + if ( form_temporary_file_path(r, manifest_path, "manifest") == -1 + || form_temporary_file_path(r, payload_path, "data") == -1 + ) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Call rhizome_bundle_import_files(%s, %s)", + alloca_str_toprint(manifest_path), + alloca_str_toprint(payload_path) + ); + int ret = 0; + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) + ret = WHY("Out of manifests"); + else { + ret = rhizome_bundle_import_files(m, manifest_path, payload_path); + rhizome_manifest_free(m); + } + rhizome_direct_clear_temporary_files(r); + /* report back to caller. + 200 = ok, which is probably appropriate for when we already had the bundle. + 201 = content created, which is probably appropriate for when we successfully + import a bundle (or if we already have it). + 403 = forbidden, which might be appropriate if we refuse to accept it, e.g., + the import fails due to malformed data etc. + (should probably also indicate if we have a newer version if possible) */ - else if (config.rhizome.api.addfile.uri_path[0] && strcmp(r->path, config.rhizome.api.addfile.uri_path) == 0) { - if (r->requestor.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr) { - DEBUGF("rhizome.api.addfile request received from %s, but is only allowed from %s", - inet_ntoa(r->requestor.sin_addr), - inet_ntoa(config.rhizome.api.addfile.allow_host) - ); + switch (ret) { + case 0: + http_request_simple_response(&r->http, 201, "Bundle succesfully imported"); + return 0; + case 2: + http_request_simple_response(&r->http, 200, "Bundle already imported"); + return 0; + } + http_request_simple_response(&r->http, 500, "Internal Error: Rhizome import failed"); + return 0; +} - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,404,"Not available from here."); +int rhizome_direct_enquiry_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (!r->received_data) { + http_request_simple_response(&r->http, 400, "Missing 'data' part"); + return 0; + } + char data_path[512]; + if (form_temporary_file_path(r, data_path, "data") == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Call rhizome_direct_fill_response(%s)", alloca_str_toprint(data_path)); + /* Read data buffer in, pass to rhizome direct for comparison with local + rhizome database, and send back responses. */ + int fd = open(data_path, O_RDONLY); + if (fd == -1) { + WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(data_path)); + /* Clean up after ourselves */ + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't read file"); + return 0; + } + struct stat stat; + if (fstat(fd, &stat) == -1) { + WHYF_perror("stat(%d)", fd); + /* Clean up after ourselves */ + close(fd); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't stat file"); + return 0; + } + unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (addr==MAP_FAILED) { + WHYF_perror("mmap(NULL,%"PRId64",PROT_READ,MAP_SHARED,%d,0)", (int64_t) stat.st_size, fd); + /* Clean up after ourselves */ + close(fd); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Couldn't mmap file"); + return 0; + } + /* Ask for a fill response. Regardless of the size of the set of BARs passed + to us, we will allow up to 64KB of response. */ + rhizome_direct_bundle_cursor *c = rhizome_direct_get_fill_response(addr, stat.st_size, 65536); + munmap(addr,stat.st_size); + close(fd); + if (c) { + size_t bytes = c->buffer_offset_bytes + c->buffer_used; + if (http_request_set_response_bufsize(&r->http, bytes) == -1) + http_request_simple_response(&r->http, 500, "Internal Error: Out of memory"); + else + http_request_response(&r->http, 200, "binary/octet-stream", (const char *)c->buffer, bytes); + rhizome_direct_bundle_iterator_free(&c); + } else + http_request_simple_response(&r->http, 500, "Internal Error: No response to enquiry"); + rhizome_direct_clear_temporary_files(r); + return 0; +} + +int rhizome_direct_addfile_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + // If given a file without a manifest, we should only accept if it we are configured to do so, and + // the connection is from localhost. Otherwise people could cause your servald to create + // arbitrary bundles, which would be bad. + if (!r->received_manifest) { + char payload_path[512]; + if (form_temporary_file_path(r, payload_path, "data") == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return 0; } - - switch(r->fields_seen) { - case RD_MIME_STATE_DATAHEADERS: - /* We have been given a file without a manifest, we should only - accept if it we are configured to do so, and the connection is from - localhost. Otherwise people could cause your servald to create - arbitrary bundles, which would be bad. - */ - /* A bundle to import */ - DEBUGF("Call bundle import sans-manifest for rhizomedata.%d.{data,file}", - r->alarm.poll.fd); - - char filepath[1024]; - snprintf(filepath,1024,"rhizomedirect.%d.data",r->alarm.poll.fd); - - char manifestTemplate[1024]; - manifestTemplate[0] = '\0'; - if (config.rhizome.api.addfile.manifest_template_file[0]) { - strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate); - strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL); - if (access(manifestTemplate, R_OK) != 0) { - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file points to a file I could not read."); - } + if (config.debug.rhizome) + DEBUGF("Call rhizome_add_file(%s)", alloca_str_toprint(payload_path)); + char manifestTemplate[1024]; + manifestTemplate[0] = '\0'; + if (config.rhizome.api.addfile.manifest_template_file[0]) { + strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate); + strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL); + if (strbuf_overrun(b)) { + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Template path too long"); + return 0; } - - rhizome_manifest *m = rhizome_new_manifest(); - if (!m) - { - rhizome_server_simple_http_response(r,500,"No free manifest slots. Try again later."); - rhizome_direct_clear_temporary_files(r); - return WHY("Manifest struct could not be allocated -- not added to rhizome"); - } - - if (manifestTemplate[0]) - if (rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) { - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file can't be read as a manifest."); - } - - if (rhizome_stat_file(m, filepath)){ + if (access(manifestTemplate, R_OK) != 0) { + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Cannot read template"); + return 0; + } + if (config.debug.rhizome) + DEBUGF("Using manifest template %s", alloca_str_toprint(manifestTemplate)); + } + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) { + WHY("Manifest struct could not be allocated -- not added to rhizome"); + http_request_simple_response(&r->http, 500, "Internal Error: No free manifest slots"); + rhizome_direct_clear_temporary_files(r); + return 0; + } + if (manifestTemplate[0] && rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) { + WHY("Manifest template read failed"); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Malformed manifest template"); + return 0; + } + if (rhizome_stat_file(m, payload_path)) { + WHY("Payload file stat failed"); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Could not store file"); + return 0; + } + // If manifest template did not specify a service field, then by default it is "file". + if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) + rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); + sid_t *author = NULL; + if (!is_sid_t_any(config.rhizome.api.addfile.default_author)) + author = &config.rhizome.api.addfile.default_author; + rhizome_bk_t bsk = config.rhizome.api.addfile.bundle_secret_key; + if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)) { + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest"); + return 0; + } + m->payloadEncryption=0; + rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); + // import file contents + // TODO, stream file into database + if (m->fileLength) { + if (rhizome_add_file(m, payload_path)) { rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not store file"); + http_request_simple_response(&r->http, 500, "Internal Error: Could not store file"); + return 0; } - - if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); - - sid_t *author=NULL; - if (!is_sid_t_any(config.rhizome.api.addfile.default_author)) - author = &config.rhizome.api.addfile.default_author; - - rhizome_bk_t bsk; - memcpy(bsk.binary, config.rhizome.api.addfile.bundle_secret_key.binary, RHIZOME_BUNDLE_KEY_BYTES); - - if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)){ - rhizome_manifest_free(m); - m = NULL; - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not fill manifest default values"); - } - - m->payloadEncryption=0; - rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); - - // import file contents - // TODO, stream file into database - if (m->fileLength){ - if (rhizome_add_file(m, filepath)){ - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500,"Could not store file"); - } - } - - rhizome_manifest *mout = NULL; - if (rhizome_manifest_finalise(m, &mout, 1)) { - if (mout && mout!=m) - rhizome_manifest_free(mout); - rhizome_manifest_free(m); - rhizome_direct_clear_temporary_files(r); - return rhizome_server_simple_http_response(r,500, - "Could not finalise manifest"); - } - - DEBUGF("Import sans-manifest appeared to succeed"); - - /* Respond with the manifest that was added. */ - rhizome_server_simple_http_response(r, 200, (char *)m->manifestdata); - - /* clean up after ourselves */ - if (mout && mout!=m) + } + rhizome_manifest *mout = NULL; + if (rhizome_manifest_finalise(m, &mout, 1)) { + if (mout && mout != m) rhizome_manifest_free(mout); rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); - + http_request_simple_response(&r->http, 500, "Internal Error: Could not finalise manifest"); return 0; - break; - default: - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - - return rhizome_server_simple_http_response(r, 400, "Rhizome create bundle from file API requires 'data' field"); } - } - - /* Clean up after ourselves */ - rhizome_direct_clear_temporary_files(r); - /* Report error */ - return rhizome_server_simple_http_response(r, 500, "Something went wrong. Probably a missing data or manifest part, or invalid combination of URI and data/manifest provision."); - -} - - -int rhizome_direct_process_mime_line(rhizome_http_request *r,char *buffer,int count) -{ - /* Check for boundary line at start of buffer. - Boundary line = CRLF + "--" + boundary_string + optional whitespace + CRLF - EXCEPT end of form boundary, which is: - CRLF + "--" + boundary_string + "--" + CRLF - - NOTE: We attach the "--" to boundary_string when setting things up so that - we don't have to keep manually checking for it here. - - NOTE: The parser eats the CRLF from the front, and attaches it to the end - of the previous line. This means we need to rewind 2 bytes from whatever - file we were writing to whenever we encounter a boundary line, at least - if those last two bytes were CRLF. That can be safely assumed if we - assume that the boundary string has been chosen to be a string never appearing - anywhere in the contents of the form. In practice, that is only "almost - certain" (according to the mathematical meaning of that phrase) if boundary - strings are randomly selected and are of sufficient length. - - NOTE: We are not supporting nested/mixed parts, as that would considerably - complicate the parser. If the need arises in future, we will deal with it - then. In the meantime, we will have something that meets our immediate - needs for Rhizome Direct and a variety of use cases. - */ - - /* Regardless of the state of the parser, the presence of boundary lines - is significant, so lets just check once, and remember the result. - Similarly check a few other conditions. */ - int boundaryLine=0; - if (!memcmp(buffer,r->boundary_string,r->boundary_string_length)) - boundaryLine=1; - - int endOfForm=0; - if (boundaryLine&& - buffer[r->boundary_string_length]=='-'&& - buffer[r->boundary_string_length+1]=='-') - endOfForm=1; - int blankLine=0; - if (!strcmp(buffer,"\r\n")) blankLine=1; - - switch(r->source_flags) { - case RD_MIME_STATE_INITIAL: - if (boundaryLine) r->source_flags=RD_MIME_STATE_PARTHEADERS; - break; - case RD_MIME_STATE_PARTHEADERS: - case RD_MIME_STATE_MANIFESTHEADERS: - case RD_MIME_STATE_DATAHEADERS: - if (blankLine) { - /* End of headers */ - if (r->source_flags==RD_MIME_STATE_PARTHEADERS) - { - /* Multiple content-disposition lines. This is very naughty. */ - rhizome_server_simple_http_response - (r, 400, "

Malformed multi-part form POST: Missing content-disposition lines in MIME encoded part.

\r\n"); - return -1; - } - - /* Prepare to write to file for field. - We may have multiple rhizome direct transactions running at the same - time on different TCP connections. So serialise using file descriptor. - We could use the boundary string or some other random thing, but using - the file descriptor places a reasonable upper limit on the clutter that - is possible, while still preventing collisions -- provided that we don't - close the file descriptor until we have completed processing the - request. */ - r->field_file=NULL; - char filename[1024]; - char *field="unknown"; - switch(r->source_flags) { - case RD_MIME_STATE_DATAHEADERS: field="data"; break; - case RD_MIME_STATE_MANIFESTHEADERS: field="manifest"; break; - } - snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,field); - filename[1023]=0; - r->field_file=fopen(filename,"w"); - if (!r->field_file) { - WHYF_perror("fopen(%s, \"w\")", alloca_str_toprint(filename)); - goto scram; - } - r->source_flags=RD_MIME_STATE_BODY; - } else { - char name[1024]; - char field[1024]; - if (sscanf(buffer, - "Content-Disposition: form-data; name=\"%[^\"]\";" - " filename=\"%[^\"]\"",field,name)==2) - { - if (r->source_flags!=RD_MIME_STATE_PARTHEADERS) - { - /* Multiple content-disposition lines. This is very naughty. */ - rhizome_server_simple_http_response - (r, 400, "

Malformed multi-part form POST: Multiple content-disposition lines in single MIME encoded part.

\r\n"); - return -1; - } - if (!strcasecmp(field,"manifest")) - r->source_flags=RD_MIME_STATE_MANIFESTHEADERS; - if (!strcasecmp(field,"data")) { - r->source_flags=RD_MIME_STATE_DATAHEADERS; - /* record file name of data field for HTTP manifest-less import */ - strncpy(r->data_file_name,name,1023); - r->data_file_name[1023]=0; - } - if (r->source_flags!=RD_MIME_STATE_PARTHEADERS) - r->fields_seen|=r->source_flags; - } - } - break; - case RD_MIME_STATE_BODY: - if (boundaryLine) { - r->source_flags=RD_MIME_STATE_PARTHEADERS; - /* We will have written an extra CRLF to the end of the file, so prune that off. */ - if (fflush(r->field_file) == EOF) { - WHYF_perror("fflush()"); - goto scram; - } - int fd = fileno(r->field_file); - off_t correct_size = ftell(r->field_file) - 2; - if (ftruncate(fd,correct_size) == -1) { - WHYF_perror("ftruncate()"); - goto scram; - } - if (fclose(r->field_file) == EOF) { - WHYF_perror("fclose()"); - r->field_file = NULL; - goto scram; - } - r->field_file = NULL; - } - else { - int written=fwrite(r->request,count,1,r->field_file); - if (written<1) - DEBUGF("Short write for multi-part form file -- %d bytes may be missing", - count); - } - break; - } - - if (endOfForm) { - /* End of form marker found. - Pass it to function that deals with what has been received, - and will also send response or close the http request if required. */ - - /* XXX Rewind last two bytes from file if open, and close file */ - - return rhizome_direct_form_received(r); - } - return 0; - -scram: - if (r->field_file) { - if (fclose(r->field_file) == EOF) - WARNF_perror("fclose()"); - } - rhizome_direct_clear_temporary_files(r); - rhizome_server_simple_http_response(r, 500, - "

Sorry, couldn't complete your request, reasonable as it was. Perhaps try again later.

\r\n"); - return -1; -} - -int rhizome_direct_process_post_multipart_bytes(rhizome_http_request *r,const char *bytes,int count) -{ - { - char logname[128]; - snprintf(logname,128,"post-%08x.log",r->uuid); - FILE *f=fopen(logname,"a"); - if (f) fwrite(bytes,count,1,f); - if (f) fclose(f); - } - - /* This function looks for multi-part form separators and descriptor lines, - and streams any "manifest" or "data" blocks to respectively named files. - - The challenge is that we might only get a partial boundary string passed - to us. So we need to remember the last KB or so of data and glue it to - the front of the current set of bytes. - - In multi-part form parsing we don't need r->request for anything, so if - we are not in a form part already, then we can stow the bytes there - for reexamination when more bytes arrive. - - Side effect will be that the entire boundary string and associated bits will - need to be <=1KB, the size of r->request. This seems quite reasonable. - - Example of such a block is: - - ------WebKitFormBoundaryEoJwSoSVW4qsrBZW - Content-Disposition: form-data; name="manifest"; filename="spleen" - Content-Type: application/octet-stream - */ - - int o; - - /* Split into lines and process each line separately using a - simple state machine. - Lines containing binary are truncated into arbitrarily length pieces, but - a newline will ALWAYS break the line. - */ - - for(o=0;orequest_length>0&&r->request[r->request_length-1]=='\r') - { newline=1; r->request_length--; } - if (r->request_length>1020) newline=2; - if (newline) { - /* Found end of line, so process it */ - if (newline==1) { - /* Put the real new line onto the end if it was present, so that - we don't go doing anything silly, like joining lines in files - that really were separated by CRLF, or similarly inserting CRLF - in the middle of slabs of bytes that were not CRLF terminated. - */ - r->request[r->request_length++]='\r'; - r->request[r->request_length++]='\n'; - } - r->request[r->request_length]=0; - if (rhizome_direct_process_mime_line(r,r->request,r->request_length)) - return -1; - r->request_length=0; - /* If a real new line was detected, then - don't include the \n as part of the next line. - But if it wasn't a real new line, then make sure we - don't loose the byte. */ - if (newline==1) continue; - } - - r->request[r->request_length++]=bytes[o]; - } - - r->source_count-=count; - if (r->source_count<=0) { - /* Got to end of multi-part form data */ - - /* If the form is still being processed, then flush things through */ - if (r->request_type<0) { - /* Flush out any remaining data */ - if (r->request_length) { - DEBUGF("Flushing last %d bytes",r->request_length); - r->request[r->request_length]=0; - rhizome_direct_process_mime_line(r,r->request,r->request_length); - } - return rhizome_direct_form_received(r); - } else { - /* Form has already been processed, so do nothing */ - } - } - return 0; -} - -struct http_request_parts { -}; - -int rhizome_direct_parse_http_request(rhizome_http_request *r) -{ - DEBUGF("uri=%s", alloca_str_toprint(config.rhizome.api.addfile.uri_path)); - - // Parse the HTTP request into verb, path, protocol, headers and content. - char *const request_end = r->request + r->request_length; - char *verb = r->request; - char *path = NULL; - char *proto = NULL; - size_t pathlen = 0; - char *headers = NULL; - int headerlen = 0; - char *content = NULL; - int contentlen = 0; - char *p; - if ((str_startswith(verb, "GET", (const char **)&p) || str_startswith(verb, "POST", (const char **)&p)) && isspace(*p)) { - *p++ = '\0'; - path = p; - while (p < request_end && !isspace(*p)) - ++p; - if (p < request_end) { - pathlen = p - path; - *p++ = '\0'; - proto = p; - if ( str_startswith(p, "HTTP/1.", (const char **)&p) - && (str_startswith(p, "0", (const char **)&p) || str_startswith(p, "1", (const char **)&p)) - && (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers)) - ) { - *p = '\0'; - char *eoh = str_str(headers, "\r\n\r\n", request_end - p); - if (eoh) { - content = eoh + 4; - headerlen = content - headers; - contentlen = request_end - content; - } - } - } - } - if (content == NULL) { - if (config.debug.rhizome_httpd) - DEBUGF("Received malformed HTTP request %s", alloca_toprint(160, (const char *)r->request, r->request_length)); - return rhizome_server_simple_http_response(r, 400, "

Malformed request

\r\n"); - } - INFOF("RHIZOME HTTP SERVER, %s %s %s", verb, alloca_toprint(-1, path, pathlen), proto); - if (config.debug.rhizome_httpd) - DEBUGF("headers %s", alloca_toprint(-1, headers, headerlen)); - if (strcmp(verb, "POST") == 0 - && ( strcmp(path, "/rhizome/import") == 0 - || strcmp(path, "/rhizome/enquiry") == 0 - || (config.rhizome.api.addfile.uri_path[0] && strcmp(path, config.rhizome.api.addfile.uri_path) == 0) - ) - ) { - const char *cl_str=str_str(headers,"Content-Length: ",headerlen); - const char *ct_str=str_str(headers,"Content-Type: multipart/form-data; boundary=",headerlen); - if (!cl_str) - return rhizome_server_simple_http_response(r,400,"

Missing Content-Length header

\r\n"); - if (!ct_str) - return rhizome_server_simple_http_response(r,400,"

Missing or unsupported Content-Type header

\r\n"); - /* ok, we have content-type and content-length, now make sure they are well formed. */ - int64_t content_length; - if (sscanf(cl_str,"Content-Length: %"PRId64,&content_length)!=1) - return rhizome_server_simple_http_response(r,400,"

Malformed Content-Length header

\r\n"); - char boundary_string[1024]; - int i; - ct_str+=strlen("Content-Type: multipart/form-data; boundary="); - for(i=0;i<1023&&*ct_str&&*ct_str!='\n'&&*ct_str!='\r';i++,ct_str++) - boundary_string[i]=*ct_str; - boundary_string[i] = '\0'; - if (i<4||i>128) - return rhizome_server_simple_http_response(r,400,"

Malformed Content-Type header

\r\n"); - - DEBUGF("content_length=%"PRId64", boundary_string=%s contentlen=%d", (int64_t) content_length, alloca_str_toprint(boundary_string), contentlen); - - /* Now start receiving and parsing multi-part data. If we already received some of the - post-header data, process that first. Tell the HTTP request that it has moved to multipart - form data parsing, and what the actual requested action is. - */ - - /* Remember boundary string and source path. - Put the preceeding -- on the front to make our life easier when - parsing the rest later. */ - strbuf bs = strbuf_local(r->boundary_string, sizeof r->boundary_string); - strbuf_puts(bs, "--"); - strbuf_puts(bs, boundary_string); - if (strbuf_overrun(bs)) - return rhizome_server_simple_http_response(r,500,"

Internal server error: Multipart boundary string too long

\r\n"); - strbuf ps = strbuf_local(r->path, sizeof r->path); - strbuf_puts(ps, path); - if (strbuf_overrun(ps)) - return rhizome_server_simple_http_response(r,500,"

Internal server error: Path too long

\r\n"); - r->boundary_string_length = strbuf_len(bs); - r->source_index = 0; - r->source_count = content_length; - r->request_type = RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART; - r->request_length = 0; - r->source_flags = 0; - - /* Find the end of the headers and start of any body bytes that we have read - so far. Copy the bytes to a separate buffer, because r->request and - r->request_length get used internally in the parser. - */ - if (contentlen) { - char buffer[contentlen]; - bcopy(content, buffer, contentlen); - rhizome_direct_process_post_multipart_bytes(r, buffer, contentlen); - } - - /* Handle the rest of the transfer asynchronously. */ + if (config.debug.rhizome) + DEBUGF("Import sans-manifest appeared to succeed"); + /* Respond with the manifest that was added. */ + http_request_response(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes); + /* clean up after ourselves */ + if (mout && mout != m) + rhizome_manifest_free(mout); + rhizome_manifest_free(m); + rhizome_direct_clear_temporary_files(r); return 0; } else { - rhizome_server_simple_http_response(r, 404, "

Not found (OTHER)

\r\n"); + http_request_simple_response(&r->http, 501, "Not Implemented: Rhizome add with manifest"); + return 0; } - - /* Try sending data immediately. */ - rhizome_server_http_send_bytes(r); +} +void rhizome_direct_process_mime_start(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + assert(r->current_part == NONE); + assert(r->part_fd == -1); +} + +void rhizome_direct_process_mime_end(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (r->part_fd != -1) { + if (close(r->part_fd) == -1) { + WHYF_perror("close(%d)", r->part_fd); + http_request_simple_response(&r->http, 500, "Internal Error: Close temporary file failed"); + return; + } + r->part_fd = -1; + } + switch (r->current_part) { + case MANIFEST: + r->received_manifest = 1; + break; + case DATA: + r->received_data = 1; + break; + case NONE: + break; + } +} + +void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (strcmp(cd->name, "data") == 0) { + r->current_part = DATA; + strncpy(r->data_file_name, cd->filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0'; + } + else if (strcmp(cd->name, "manifest") == 0) { + r->current_part = MANIFEST; + } else + return; + char path[512]; + if (form_temporary_file_path(r, path, cd->name) == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); + return; + } + if ((r->part_fd = open(path, O_WRONLY | O_CREAT, 0666)) == -1) { + WHYF_perror("open(%s,O_WRONLY|O_CREAT,0666)", alloca_str_toprint(path)); + http_request_simple_response(&r->http, 500, "Internal Error: Create temporary file failed"); + return; + } +} + +void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + if (r->part_fd != -1) { + if (write_all(r->part_fd, buf, len) == -1) { + http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed"); + return; + } + } +} + +int rhizome_direct_import(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_import_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; return 0; } +int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_enquiry_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; + return 0; +} + +/* Servald can be configured to accept files without manifests via HTTP from localhost, so that + * rhizome bundles can be created programatically. There are probably still some security + * loop-holes here, which is part of why we leave it disabled by default, but it will be sufficient + * for testing possible uses, including integration with OpenDataKit. + */ +int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_POST) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + if (cmp_sockaddr((struct sockaddr *)&r->http.client_in_addr, sizeof r->http.client_in_addr, + (struct sockaddr *)&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host + ) != 0 + ) { + INFOF("rhizome.api.addfile request received from %s, but is only allowed from %s", + alloca_sockaddr(&r->http.client_in_addr, sizeof r->http.client_in_addr), + alloca_sockaddr(&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host) + ); + rhizome_direct_clear_temporary_files(r); + http_request_simple_response(&r->http, 404, "

Not available from here

"); + return 0; + } + r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; + r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; + r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; + r->http.handle_content_end = rhizome_direct_addfile_end; + r->current_part = NONE; + r->part_fd = -1; + r->data_file_name[0] = '\0'; + return 0; +} + +int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder) +{ + if ( config.rhizome.api.addfile.uri_path[0] + && strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0 + ) + return rhizome_direct_addfile(r, remainder); + return 1; +} + static int receive_http_response(int sock, char *buffer, size_t buffer_len, struct http_response_parts *parts) { int len = 0; @@ -702,7 +434,8 @@ static int receive_http_response(int sock, char *buffer, size_t buffer_len, stru DEBUGF("Invalid HTTP reply: missing Content-Length header"); return -1; } - DEBUGF("content_length=%"PRId64, parts->content_length); + if (config.debug.rhizome_rx) + DEBUGF("content_length=%"PRId64, parts->content_length); return len - (parts->content_start - buffer); } @@ -718,36 +451,38 @@ static int fill_buffer(int sock, unsigned char *buffer, int len, int buffer_size void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) { - DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high); + if (config.debug.rhizome_tx) + DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high); rhizome_direct_transport_state_http *state = r->transport_specific_state; sid_t zerosid = SID_ANY; int sock=socket(AF_INET, SOCK_STREAM, 0); if (sock==-1) { - WHY_perror("socket"); + WHY_perror("socket"); goto end; - } + } struct hostent *hostent; hostent = gethostbyname(state->host); if (!hostent) { - DEBUGF("could not resolve hostname"); + if (config.debug.rhizome_tx) + DEBUGF("could not resolve hostname"); goto end; } - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_port = htons(state->port); + struct sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(state->port); addr.sin_addr = *((struct in_addr *)hostent->h_addr); - bzero(&(addr.sin_zero),8); + bzero(&(addr.sin_zero),8); - if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) { - WHY_perror("connect"); + if (connect(sock, (struct sockaddr *)&addr, sizeof addr) == -1) { + WHYF_perror("connect(%s)", alloca_sockaddr(&addr, sizeof addr)); close(sock); goto end; } - + char boundary[20]; char buffer[8192]; @@ -785,7 +520,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) int len = strbuf_len(request); int sent=0; while(sentpullP) { /* Need to fetch manifest. Once we have the manifest, then we can - use our normal bundle fetch routines from rhizome_fetch.c + use our normal bundle fetch routines from rhizome_fetch.c Generate a request like: GET /rhizome/manifestbybar/ and add it to our list of HTTP fetch requests, then watch @@ -875,16 +612,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) Then as noted above, we can use that to pull the file down using existing routines. */ - DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i); + if (config.debug.rhizome_tx) + DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i); if (!rhizome_fetch_request_manifest_by_prefix(&addr, &zerosid, &actionlist[i+1], RHIZOME_BAR_PREFIX_BYTES)) { - /* Fetching the manifest, and then using it to see if we want to + /* Fetching the manifest, and then using it to see if we want to fetch the file for import is all handled asynchronously, so just wait for it to finish. */ while (rhizome_any_fetch_active() || rhizome_any_fetch_queued()) fd_poll(); } - + } else if (type==1&&r->pushP) { /* Form up the POST request to submit the appropriate bundle. */ @@ -897,11 +635,14 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } /* Get filehash and size from manifest if present */ - DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); - DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); - DEBUGF("file size = %"PRId64, m->fileLength); + if (config.debug.rhizome_tx) { + DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); + DEBUGF("file size = %"PRId64, m->fileLength); + } int64_t version = rhizome_manifest_get_ll(m, "version"); - DEBUGF("version = %"PRId64,version); + if (config.debug.rhizome_tx) + DEBUGF("version = %"PRId64,version); /* We now have everything we need to compose the POST request and send it. */ @@ -919,8 +660,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) "Content-Type: application/octet-stream\r\n" "\r\n"; /* Work out what the content length should be */ - DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", - m->manifest_all_bytes,m->manifest_bytes); + if (config.debug.rhizome_tx) + DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes); int content_length =strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */ +strlen(boundary) @@ -939,21 +680,22 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) len+=m->manifest_all_bytes; len+=snprintf(&buffer[len],8192-len,template3,boundary); - addr.sin_family = AF_INET; - addr.sin_port = htons(state->port); + addr.sin_family = AF_INET; + addr.sin_port = htons(state->port); addr.sin_addr = *((struct in_addr *)hostent->h_addr); - bzero(&(addr.sin_zero),8); - + bzero(&(addr.sin_zero),8); + sock=socket(AF_INET, SOCK_STREAM, 0); if (sock==-1) { - DEBUGF("could not open socket"); + if (config.debug.rhizome_tx) + DEBUGF("could not open socket"); goto closeit; - } - if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) - { + } + if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) { + if (config.debug.rhizome_tx) DEBUGF("Could not connect to remote"); - goto closeit; - } + goto closeit; + } int sent=0; /* Send buffer now */ @@ -968,12 +710,12 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) rhizome_filehash_t filehash; if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1) goto closeit; - + struct rhizome_read read; bzero(&read, sizeof read); if (rhizome_open_read(&read, &filehash)) goto closeit; - + int64_t read_ofs; for(read_ofs=0;read_ofsfileLength;){ unsigned char buffer[4096]; @@ -994,7 +736,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } write_ofs+=written; } - + read_ofs+=bytes_read; } rhizome_read_close(&read); @@ -1006,7 +748,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) int r=write(sock,&buffer[sent],len-sent); if (r>0) sent+=r; if (r<0) goto closeit; - } + } /* get response back. */ if (receive_http_response(sock, buffer, sizeof buffer, &parts) == -1) @@ -1023,7 +765,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) } free(actionlist); - + /* now update cursor according to what range was covered in the response. We set our current position to just past the high limit of the returned cursor. @@ -1033,18 +775,20 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) end up in an infinite loop. We could also end up in a very long finite loop if the cursor doesn't advance far. A simple solution is to not adjust the cursor position, and simply re-attempt the sync until no actions result. - That will do for now. + That will do for now. */ #ifdef FANCY_CURSOR_POSITION_HANDLING rhizome_direct_bundle_cursor *c=rhizome_direct_bundle_iterator(10); assert(c!=NULL); - if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10)) - { + if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10)) { + if (config.debug.rhizome_tx) DEBUGF("Couldn't unpickle range. This should never happen. Assuming near and far cursor ranges match."); - } + } else { - DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high); - DEBUGF("c->buffer_size=%d",c->buffer_size); + if (config.debug.rhizome_tx) { + DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high); + DEBUGF("c->buffer_size=%d",c->buffer_size); + } r->cursor->size_low=c->limit_size_high; /* Set tail of BID to all high, as we assume the far end has returned all BIDs with the specified prefix. */ @@ -1055,7 +799,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) #endif end: - /* Warning: tail recursion when done this way. + /* Warning: tail recursion when done this way. Should be triggered by an asynchronous event. But this will do for now. */ rhizome_direct_continue_sync_request(r); diff --git a/rhizome_http.c b/rhizome_http.c index fe9b848d..418e653f 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -23,18 +23,74 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #ifdef HAVE_SYS_FILIO_H #include #endif +#include +#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION #include "serval.h" #include "overlay_address.h" #include "conf.h" #include "str.h" #include "rhizome.h" +#include "http_server.h" + #define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32 -struct sched_ent server_alarm; -struct profile_total server_stats; +struct http_handler{ + const char *path; + int (*parser)(rhizome_http_request *r, const char *remainder); +}; -struct profile_total connection_stats; +static int rhizome_status_page(rhizome_http_request *r, const char *remainder); +static int rhizome_file_page(rhizome_http_request *r, const char *remainder); +static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder); +static int interface_page(rhizome_http_request *r, const char *remainder); +static int neighbour_page(rhizome_http_request *r, const char *remainder); +static int fav_icon_header(rhizome_http_request *r, const char *remainder); +static int root_page(rhizome_http_request *r, const char *remainder); + +extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder); +extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder); +extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder); + +struct http_handler paths[]={ + {"/rhizome/status", rhizome_status_page}, + {"/rhizome/file/", rhizome_file_page}, + {"/rhizome/import", rhizome_direct_import}, + {"/rhizome/enquiry", rhizome_direct_enquiry}, + {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, + {"/rhizome/", rhizome_direct_dispatch}, + {"/interface/", interface_page}, + {"/neighbour/", neighbour_page}, + {"/favicon.ico", fav_icon_header}, + {"/", root_page}, +}; + +static int rhizome_dispatch(struct http_request *hr) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + INFOF("RHIZOME HTTP SERVER, %s %s", r->http.verb, r->http.path); + r->http.response.content_generator = NULL; + unsigned i; + for (i = 0; i < NELS(paths); ++i) { + const char *remainder; + if (str_startswith(r->http.path, paths[i].path, &remainder)){ + int ret = paths[i].parser(r, remainder); + if (ret < 0) { + http_request_simple_response(&r->http, 500, NULL); + return 0; + } + if (ret == 0) + return 0; + } + } + http_request_simple_response(&r->http, 404, NULL); + return 0; +} + +struct sched_ent server_alarm; +struct profile_total server_stats = { + .name = "rhizome_server_poll", +}; /* HTTP server and client code for rhizome transfers and rhizome direct. @@ -47,9 +103,6 @@ static int rhizome_server_socket = -1; static int request_count=0; static time_ms_t rhizome_server_last_start_attempt = -1; -int (*rhizome_http_parse_func)(rhizome_http_request *)=NULL; -const char *rhizome_http_parse_func_description="(null)"; - // Format icon data using: // od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/ */,/g' unsigned char favicon_bytes[]={ @@ -88,9 +141,7 @@ int is_rhizome_http_server_running() Return 1 if the server is already started successfully. Return 2 if the server was not started because it is too soon since last failed attempt. */ -int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *), - const char *parse_func_desc, - uint16_t port_low, uint16_t port_high) +int rhizome_http_server_start(uint16_t port_low, uint16_t port_high) { if (rhizome_server_socket != -1) return 1; @@ -162,15 +213,10 @@ success: else INFOF("HTTP SERVER (LIMITED SERVICE), START port=%"PRIu16" fd=%d", port, rhizome_server_socket); - /* Remember which function to call when handling client connections */ - rhizome_http_parse_func=parse_func; - rhizome_http_parse_func_description=parse_func_desc; - rhizome_http_server_port = port; /* Add Rhizome HTTPd server to list of file descriptors to watch */ server_alarm.function = rhizome_server_poll; - server_stats.name="rhizome_server_poll"; - server_alarm.stats=&server_stats; + server_alarm.stats = &server_stats; server_alarm.poll.fd = rhizome_server_socket; server_alarm.poll.events = POLLIN; watch(&server_alarm); @@ -178,90 +224,16 @@ success: } -void rhizome_client_poll(struct sched_ent *alarm) +static void rhizome_server_finalise_http_request(struct http_request *_r) { - rhizome_http_request *r = (rhizome_http_request *)alarm; - if (alarm->poll.revents == 0 || alarm->poll.revents & (POLLHUP | POLLERR)){ - if (config.debug.rhizome_tx) - DEBUGF("Closing connection due to timeout or error %d", alarm->poll.revents); - rhizome_server_free_http_request(r); - return; - } - - if (alarm->poll.revents & POLLIN){ - switch(r->request_type) - { - case RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART: - { - /* Reading multi-part form data. Read some bytes and proces them. */ - char buffer[16384]; - sigPipeFlag=0; - int bytes = read_nonblock(r->alarm.poll.fd, buffer, 16384); - /* If we got some data, see if we have found the end of the HTTP request */ - if (bytes > 0) { - // reset inactivity timer - r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - rhizome_direct_process_post_multipart_bytes(r,buffer,bytes); - } - /* We don't drop the connection on an empty read, because that results - in connections dropping when they shouldn't, including during testing. - The idle timeout should drop the connections instead. - */ - if (sigPipeFlag) { - if (config.debug.rhizome_tx) - DEBUG("Received SIGPIPE, closing connection"); - rhizome_server_free_http_request(r); - return; - } - } - break; - - case RHIZOME_HTTP_REQUEST_RECEIVING: - /* Keep reading until we have two CR/LFs in a row */ - r->request[r->request_length] = '\0'; - sigPipeFlag=0; - int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], sizeof r->request - r->request_length); - /* If we got some data, see if we have found the end of the HTTP request */ - if (bytes > 0) { - // reset inactivity timer - r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - r->request_length += bytes; - r->header_length = is_http_header_complete(r->request, r->request_length, bytes); - if (r->header_length){ - /* We have the request. Now parse it to see if we can respond to it */ - if (rhizome_http_parse_func!=NULL) - rhizome_http_parse_func(r); - } - } else { - if (config.debug.rhizome_tx) - DEBUG("Empty read, closing connection"); - rhizome_server_free_http_request(r); - return; - } - if (sigPipeFlag) { - if (config.debug.rhizome_httpd) - DEBUG("Received SIGPIPE, closing HTTP connection"); - rhizome_server_free_http_request(r); - return; - } - break; - } - } - - if (alarm->poll.revents & POLLOUT){ - /* Socket already has request -- so just try to send some data. */ - rhizome_server_http_send_bytes(r); - } - return; + rhizome_http_request *r = (rhizome_http_request *) _r; + rhizome_read_close(&r->read_state); + request_count--; } -static unsigned int rhizome_http_request_uuid_counter=0; +static int rhizome_dispatch(struct http_request *); + +static unsigned int rhizome_http_request_uuid_counter = 0; void rhizome_server_poll(struct sched_ent *alarm) { @@ -269,7 +241,10 @@ void rhizome_server_poll(struct sched_ent *alarm) struct sockaddr addr; unsigned int addr_len = sizeof addr; int sock; - if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) != -1) { + if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) == -1) { + if (errno && errno != EAGAIN) + WARN_perror("accept"); + } else { struct sockaddr_in *peerip=NULL; if (addr.sa_family == AF_INET) { peerip = (struct sockaddr_in *)&addr; @@ -285,55 +260,32 @@ void rhizome_server_poll(struct sched_ent *alarm) addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data) ); } - rhizome_http_request *request = calloc(sizeof(rhizome_http_request), 1); + rhizome_http_request *request = emalloc_zero(sizeof(rhizome_http_request)); if (request == NULL) { - WHYF_perror("calloc(%u, 1)", (int)sizeof(rhizome_http_request)); - WHY("Cannot respond to request, out of memory"); + WHY("Cannot respond to HTTP request, out of memory"); close(sock); } else { request_count++; - request->uuid=rhizome_http_request_uuid_counter++; - if (peerip) request->requestor=*peerip; - else bzero(&request->requestor,sizeof(request->requestor)); - request->data_file_name[0]=0; - /* We are now trying to read the HTTP request */ - request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING; - request->alarm.function = rhizome_client_poll; - request->read_state.blob_fd=-1; - request->read_state.blob_rowid=-1; - connection_stats.name="rhizome_client_poll"; - request->alarm.stats=&connection_stats; - request->alarm.poll.fd=sock; - request->alarm.poll.events=POLLIN; - request->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT; - request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT; - // watch for the incoming http request - watch(&request->alarm); - // set an inactivity timeout to close the connection - schedule(&request->alarm); + request->uuid = rhizome_http_request_uuid_counter++; + request->data_file_name[0] = '\0'; + request->read_state.blob_fd = -1; + request->read_state.blob_rowid = -1; + if (peerip) + request->http.client_in_addr = *peerip; + request->http.handle_headers = rhizome_dispatch; + request->http.debug_flag = &config.debug.rhizome_httpd; + request->http.finalise = rhizome_server_finalise_http_request; + request->http.free = free; + request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT; + http_request_init(&request->http, sock); } } - if (errno && errno != EAGAIN) - WARN_perror("accept"); } if (alarm->poll.revents & (POLLHUP | POLLERR)) { INFO("Error on tcp listen socket"); } } -int rhizome_server_free_http_request(rhizome_http_request *r) -{ - unwatch(&r->alarm); - unschedule(&r->alarm); - close(r->alarm.poll.fd); - if (r->buffer) - free(r->buffer); - rhizome_read_close(&r->read_state); - free(r); - request_count--; - return 0; -} - int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call) { IN(); @@ -360,53 +312,59 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_ OUT(); } -static int neighbour_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int neighbour_page(rhizome_http_request *r, const char *remainder) { + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char buf[8*1024]; - strbuf b=strbuf_local(buf, sizeof buf); - + strbuf b = strbuf_local(buf, sizeof buf); sid_t neighbour_sid; if (str_to_sid_t(&neighbour_sid, remainder) == -1) - return -1; - + return 1; struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0); if (!neighbour) return 1; - strbuf_puts(b, ""); link_neighbour_status_html(b, neighbour); strbuf_puts(b, ""); if (strbuf_overrun(b)) return -1; - rhizome_server_simple_http_response(r, 200, buf); + http_request_simple_response(&r->http, 200, buf); return 0; } -static int interface_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int interface_page(rhizome_http_request *r, const char *remainder) { + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char buf[8*1024]; strbuf b=strbuf_local(buf, sizeof buf); int index=atoi(remainder); if (index<0 || index>=OVERLAY_MAX_INTERFACES) return 1; - strbuf_puts(b, ""); interface_state_html(b, &overlay_interfaces[index]); strbuf_puts(b, ""); if (strbuf_overrun(b)) return -1; - - rhizome_server_simple_http_response(r, 200, buf); + http_request_simple_response(&r->http, 200, buf); return 0; } -static int rhizome_status_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int rhizome_status_page(rhizome_http_request *r, const char *remainder) { if (!is_rhizome_http_enabled()) return 1; if (*remainder) return 1; - + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char buf[32*1024]; struct strbuf b; strbuf_init(&b, buf, sizeof buf); @@ -417,87 +375,100 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder, c strbuf_puts(&b, ""); if (strbuf_overrun(&b)) return -1; - rhizome_server_simple_http_response(r, 200, buf); + http_request_simple_response(&r->http, 200, buf); return 0; } -static int rhizome_file_content(rhizome_http_request *r) +static int rhizome_file_content(struct http_request *hr) { - int suggested_size=65536; - if (suggested_size > r->read_state.length - r->read_state.offset) - suggested_size = r->read_state.length - r->read_state.offset; - if (suggested_size<=0) + rhizome_http_request *r = (rhizome_http_request *) hr; + assert(r->http.response_length < r->http.response_buffer_size); + assert(r->read_state.offset <= r->read_state.length); + uint64_t readlen = r->read_state.length - r->read_state.offset; + if (readlen == 0) return 0; - - if (r->buffer_size < suggested_size){ - r->buffer_size = suggested_size; - if (r->buffer) - free(r->buffer); - r->buffer = malloc(r->buffer_size); - } - - if (!r->buffer) + size_t suggested_size = 64 * 1024; + if (suggested_size > readlen) + suggested_size = readlen; + if (r->http.response_buffer_size < suggested_size) + http_request_set_response_bufsize(&r->http, suggested_size); + if (r->http.response_buffer == NULL) + http_request_set_response_bufsize(&r->http, 1); + if (r->http.response_buffer == NULL) return -1; - - r->buffer_length = rhizome_read(&r->read_state, r->buffer, r->buffer_size); + size_t space = r->http.response_buffer_size - r->http.response_length; + int len = rhizome_read(&r->read_state, + (unsigned char *)r->http.response_buffer + r->http.response_length, + space); + if (len == -1) + return -1; + assert(len <= space); + r->http.response_length += len; return 0; } -static int rhizome_file_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int rhizome_file_page(rhizome_http_request *r, const char *remainder) { /* Stream the specified payload */ if (!is_rhizome_http_enabled()) return 1; - - rhizome_filehash_t filehash; - if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) - return -1; - - bzero(&r->read_state, sizeof(r->read_state)); - - /* Refuse to honour HTTP request if required (used for debugging and - testing transition from HTTP to MDP) */ - if (rhizome_open_read(&r->read_state, &filehash)) - return 1; - - if (r->read_state.length==-1){ - if (rhizome_read(&r->read_state, NULL, 0)){ - rhizome_read_close(&r->read_state); - return 1; - } - } - - const char *range=str_str((char*)headers,"Range: bytes=",-1); - r->read_state.offset = r->source_index = 0; - - if (range){ - sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset); - if (0) - DEBUGF("Found range header %"PRId64,r->read_state.offset); - } - - if (r->read_state.length - r->read_state.offset<=0){ - rhizome_server_simple_http_response(r, 200, ""); + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); return 0; } - - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = 200; - hr.content_type = "application/binary"; - hr.content_start = r->read_state.offset; - hr.content_end = r->read_state.length; - hr.content_length = r->read_state.length; - hr.body = NULL; - r->generator = rhizome_file_content; - rhizome_server_set_response(r, &hr); + if (r->http.request_header.content_range_count > 1) { + // To support byte range sets, eg, Range: bytes=0-100,200-300,400- we would have + // to reply with a multipart/byteranges MIME content. + http_request_simple_response(&r->http, 501, "Not Implemented: Byte range sets"); + return 0; + } + rhizome_filehash_t filehash; + if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) + return 1; + bzero(&r->read_state, sizeof r->read_state); + int n = rhizome_open_read(&r->read_state, &filehash); + if (n == -1) { + http_request_simple_response(&r->http, 500, NULL); + return 0; + } + if (n != 0) + return 1; + if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) { + rhizome_read_close(&r->read_state); + return 1; + } + assert(r->read_state.length != -1); + int result_code = 200; + struct http_range closed = (struct http_range){ .first = 0, .last = r->read_state.length }; + if (r->http.request_header.content_range_count > 0) { + if (http_range_bytes(r->http.request_header.content_ranges, + r->http.request_header.content_range_count, + r->read_state.length + ) == 0 + ) { + http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable + return 0; + } + result_code = 206; // Partial Content + http_range_close(&closed, &r->http.request_header.content_ranges[0], 1, r->read_state.length); + } + r->http.response.header.content_range_start = closed.first; + r->http.response.header.resource_length = closed.last; + r->http.response.header.content_length = closed.last - closed.first; + r->read_state.offset = closed.first; + r->http.response.content_generator = rhizome_file_content; + http_request_response(&r->http, result_code, "application/binary", NULL, 0); return 0; } -static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder) { if (!is_rhizome_http_enabled()) return 1; + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } rhizome_bid_t prefix; const char *endp = NULL; unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp); @@ -505,27 +476,32 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde return 1; // not found rhizome_manifest *m = rhizome_new_manifest(); int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m); - if (ret==0) - rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes); + if (ret == -1) + http_request_simple_response(&r->http, 500, NULL); + else if (ret == 0) + http_request_response(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes); rhizome_manifest_free(m); - return ret; + return ret <= 0 ? 0 : 1; } -static int fav_icon_header(rhizome_http_request *r, const char *remainder, const char *headers) +static int fav_icon_header(rhizome_http_request *r, const char *remainder) { if (*remainder) return 1; - rhizome_server_http_response(r, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); + http_request_response(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); return 0; } -static int root_page(rhizome_http_request *r, const char *remainder, const char *headers) +static int root_page(rhizome_http_request *r, const char *remainder) { if (*remainder) return 1; - + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } char temp[8192]; - strbuf b=strbuf_local(temp, sizeof(temp)); + strbuf b = strbuf_local(temp, sizeof temp); strbuf_sprintf(b, "" "

Hello, I'm %s*


" "Interfaces;
", @@ -533,250 +509,19 @@ static int root_page(rhizome_http_request *r, const char *remainder, const char int i; for (i=0;i%d: %s, TX: %d, RX: %d
", + strbuf_sprintf(b, "%d: %s, TX: %d, RX: %d
", i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count); } - strbuf_puts(b, "Neighbours;
"); link_neighbour_short_status_html(b, "/neighbour"); - if (is_rhizome_http_enabled()){ strbuf_puts(b, "Rhizome Status
"); } strbuf_puts(b, ""); - if (strbuf_overrun(b)) - return -1; - rhizome_server_simple_http_response(r, 200, temp); + if (strbuf_overrun(b)) { + WHY("HTTP Root page buffer overrun"); + http_request_simple_response(&r->http, 500, NULL); + } else + http_request_simple_response(&r->http, 200, temp); return 0; } - -struct http_handler{ - const char *path; - int (*parser)(rhizome_http_request *r, const char *remainder, const char *headers); -}; - -struct http_handler paths[]={ - {"/rhizome/status", rhizome_status_page}, - {"/rhizome/file/", rhizome_file_page}, - {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, - {"/interface/", interface_page}, - {"/neighbour/", neighbour_page}, - {"/favicon.ico", fav_icon_header}, - {"/", root_page}, -}; - -int rhizome_direct_parse_http_request(rhizome_http_request *r); -int rhizome_server_parse_http_request(rhizome_http_request *r) -{ - // Start building up a response. - // Parse the HTTP "GET" line. - char *path = NULL; - char *headers = NULL; - if (str_startswith(r->request, "POST ", (const char **)&path)) { - return rhizome_direct_parse_http_request(r); - } else if (str_startswith(r->request, "GET ", (const char **)&path)) { - const char *p; - size_t header_length = 0; - size_t pathlen = 0; - // This loop is guaranteed to terminate before the end of the buffer, because we know that the - // buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block. - for (p = path; !isspace(*p); ++p) - ; - pathlen = p - path; - if ( str_startswith(p, " HTTP/1.", &p) - && (str_startswith(p, "0", &p) || str_startswith(p, "1", &p)) - && (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers)) - ){ - path[pathlen] = '\0'; - header_length = r->header_length - (headers - r->request); - headers[header_length] = '\0'; - }else - path = NULL; - } - - if (!path) { - if (config.debug.rhizome_httpd) - DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length)); - rhizome_server_simple_http_response(r, 400, "

Malformed request

\r\n"); - return 0; - } - - char *id = NULL; - INFOF("RHIZOME HTTP SERVER, GET %s", path); - - int i; - r->generator=NULL; - - for (i=0;i

Internal Error

\r\n"); - if (ret>0) - rhizome_server_simple_http_response(r, 404, "

Not Found

\r\n"); - - /* Try sending data immediately. */ - rhizome_server_http_send_bytes(r); - - return 0; - } - } - - rhizome_server_simple_http_response(r, 404, "

Not Found

\r\n"); - return 0; -} - - -/* Return appropriate message for HTTP response codes, both known and unknown. */ -static const char *httpResultString(int response_code) { - switch (response_code) { - case 200: return "OK"; - case 201: return "Created"; - case 206: return "Partial Content"; - case 404: return "Not found"; - case 500: return "Internal server error"; - default: - if (response_code<=4) - return "Unknown status code"; - else - return "A suffusion of yellow"; - } -} - -static strbuf strbuf_build_http_response(strbuf sb, const struct http_response *h) -{ - strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", h->result_code, httpResultString(h->result_code)); - strbuf_sprintf(sb, "Content-type: %s\r\n", h->content_type); - if (h->content_end && h->content_length && (h->content_start!=0 || h->content_end!=h->content_length)) - strbuf_sprintf(sb, - "Content-range: bytes %"PRIu64"-%"PRIu64"/%"PRIu64"\r\n" - "Content-length: %"PRIu64"\r\n", - h->content_start, h->content_end, h->content_length, h->content_end - h->content_start); - else if (h->content_length) - strbuf_sprintf(sb, "Content-length: %"PRIu64"\r\n", h->content_length); - strbuf_puts(sb, "\r\n"); - return sb; -} - -int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h) -{ - r->request_type=0; - - if (config.debug.rhizome_nohttptx) - unwatch(&r->alarm); - else{ - /* Switching to writing, so update the call-back */ - r->alarm.poll.events=POLLOUT; - watch(&r->alarm); - } - - strbuf b = strbuf_local((char *) r->buffer, r->buffer_size); - strbuf_build_http_response(b, h); - if (r->buffer == NULL || strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) { - // Need a bigger buffer - if (r->buffer) - free(r->buffer); - r->buffer_size = strbuf_count(b) + 1; - if (h->body) - r->buffer_size += h->content_length; - r->buffer = malloc(r->buffer_size); - if (r->buffer == NULL) { - WHYF_perror("malloc(%u)", r->buffer_size); - r->buffer_size = 0; - return WHY("Cannot send response, out of memory"); - } - strbuf_init(b, (char *) r->buffer, r->buffer_size); - strbuf_build_http_response(b, h); - if (strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) - return WHYF("Bug! Cannot send response, buffer not big enough"); - } - r->buffer_length = strbuf_len(b); - if (h->body){ - bcopy(h->body, strbuf_end(b), h->content_length); - r->buffer_length+=h->content_length; - } - r->buffer_offset = 0; - if (config.debug.rhizome_httpd) - DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->buffer, r->buffer_length)); - return 0; -} - -int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response) -{ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = result; - hr.content_type = "text/html"; - hr.content_length = strlen(response); - hr.body = response; - if (result==400) { - DEBUGF("Rejecting http request as malformed due to: %s", - response); - } - return rhizome_server_set_response(r, &hr); -} - -int rhizome_server_http_response(rhizome_http_request *r, int result, - const char *mime_type, const char *body, uint64_t bytes) -{ - struct http_response hr; - bzero(&hr, sizeof hr); - hr.result_code = result; - hr.content_type = mime_type; - hr.content_length = bytes; - hr.body = body; - return rhizome_server_set_response(r, &hr); -} - -int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes) -{ - return rhizome_server_http_response(r, result, mime_type, NULL, bytes); -} - -/* - return codes: - 1: connection still open. - 0: connection finished. - <0: an error occurred. -*/ -int rhizome_server_http_send_bytes(rhizome_http_request *r) -{ - // Don't send anything if disabled for testing HTTP->MDP Rhizome failover - if (config.debug.rhizome_nohttptx) - return 1; - - // write one block of buffered data - if(r->buffer_offset < r->buffer_length){ - int bytes=r->buffer_length - r->buffer_offset; - bytes=write(r->alarm.poll.fd,&r->buffer[r->buffer_offset],bytes); - if (bytes<0){ - // stop writing when the tcp buffer is full - // TODO errors? - return 1; - } - r->buffer_offset+=bytes; - - // reset inactivity timer - r->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT; - r->alarm.deadline = r->alarm.alarm+RHIZOME_IDLE_TIMEOUT; - unschedule(&r->alarm); - schedule(&r->alarm); - - // allow other alarms to fire and wait for the next POLLOUT - return 1; - } - - r->buffer_offset=r->buffer_length=0; - - if (r->generator){ - r->generator(r); - } - - // once we've written the whole buffer, and nothing new has been generated, close the connection - if (!r->buffer_length){ - if (config.debug.rhizome_httpd) - DEBUG("Closing connection, done"); - return rhizome_server_free_http_request(r); - } - return 1; -} diff --git a/rhizome_store.c b/rhizome_store.c index 5e761852..b9d8fa51 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -1,3 +1,4 @@ +#include #include "serval.h" #include "rhizome.h" #include "conf.h" @@ -71,7 +72,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path); write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664); - if (write->blob_fd<0) + if (write->blob_fd == -1) goto insert_row_fail; if (config.debug.externalblobs) @@ -111,7 +112,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex } if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){ - if (write->blob_fd>=0){ + if (write->blob_fd != -1){ if (config.debug.externalblobs) DEBUGF("Cancel write to fd %d", write->blob_fd); close(write->blob_fd); @@ -162,7 +163,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer // open database locks static int write_get_lock(struct rhizome_write *write_state){ - if (write_state->blob_fd>=0 || write_state->sql_blob) + if (write_state->blob_fd != -1 || write_state->sql_blob) return 0; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; @@ -193,7 +194,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u if (file_offset != write_state->written_offset) WARNF("Writing file data out of order! [%"PRId64",%"PRId64"]", file_offset, write_state->written_offset); - if (write_state->blob_fd>=0) { + if (write_state->blob_fd != -1) { int ofs=0; // keep trying until all of the data is written. lseek(write_state->blob_fd, file_offset, SEEK_SET); @@ -233,7 +234,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u // close database locks static int write_release_lock(struct rhizome_write *write_state){ int ret=0; - if (write_state->blob_fd>=0) + if (write_state->blob_fd != -1) return 0; if (write_state->sql_blob){ @@ -260,7 +261,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi int ret=0; int should_write = 0; // if we are writing to a file, or already have the sql blob open, write as much as we can. - if (write_state->blob_fd>=0 || write_state->sql_blob){ + if (write_state->blob_fd != -1 || write_state->sql_blob){ should_write = 1; }else{ // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go. @@ -417,7 +418,7 @@ end: int rhizome_fail_write(struct rhizome_write *write) { - if (write->blob_fd>=0){ + if (write->blob_fd != -1){ if (config.debug.externalblobs) DEBUGF("Closing and removing fd %d", write->blob_fd); close(write->blob_fd); @@ -723,6 +724,58 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp return 0; // file opened } +static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read *read_state, unsigned char *buffer, size_t bufsz) +{ + IN(); + if (read_state->blob_fd != -1) { + if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1) + RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset)); + if (bufsz == 0) + RETURN(0); + ssize_t rd = read(read_state->blob_fd, buffer, bufsz); + if (rd == -1) + RETURN(WHYF_perror("read(%d,%p,%zu)", read_state->blob_fd, buffer, bufsz)); + if (config.debug.externalblobs) + DEBUGF("Read %zu bytes from fd=%d @%"PRIx64, (size_t) rd, read_state->blob_fd, read_state->offset); + RETURN(rd); + } + if (read_state->blob_rowid == -1) + RETURN(WHY("file not open")); + sqlite3_blob *blob = NULL; + int ret; + do { + assert(blob == NULL); + ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob); + } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_open")); + if (ret != SQLITE_OK) { + assert(blob == NULL); + RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db))); + } + assert(blob != NULL); + if (read_state->length == -1) + read_state->length = sqlite3_blob_bytes(blob); + // A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out + // the length. + size_t bytes_read = 0; + if (buffer && bufsz && read_state->offset < read_state->length) { + bytes_read = read_state->length - read_state->offset; + if (bytes_read > bufsz) + bytes_read = bufsz; + assert(bytes_read > 0); + do { + ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset); + } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read")); + if (ret != SQLITE_OK) { + WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_db)); + sqlite3_blob_close(blob); + RETURN(-1); + } + } + sqlite3_blob_close(blob); + RETURN(bytes_read); + OUT(); +} + /* Read content from the store, hashing and decrypting as we go. Random access is supported, but hashing requires all payload contents to be read sequentially. */ // returns the number of bytes read @@ -732,53 +785,13 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf // hash check failed, just return an error if (read_state->invalid) RETURN(-1); - - int bytes_read = 0; - if (read_state->blob_fd >= 0) { - if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1) - RETURN(WHYF_perror("lseek(%d,%ld,SEEK_SET)", read_state->blob_fd, (long)read_state->offset)); - bytes_read = read(read_state->blob_fd, buffer, buffer_length); - if (bytes_read == -1) - RETURN(WHYF_perror("read(%d,%p,%ld)", read_state->blob_fd, buffer, (long)buffer_length)); - if (config.debug.externalblobs) - DEBUGF("Read %d bytes from fd %d @%"PRIx64, bytes_read, read_state->blob_fd, read_state->offset); - } else if (read_state->blob_rowid != -1) { - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - do{ - sqlite3_blob *blob = NULL; - int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob); - if (sqlite_code_busy(ret)) - goto again; - else if(ret!=SQLITE_OK) - RETURN(WHYF("sqlite3_blob_open failed: %s",sqlite3_errmsg(rhizome_db))); - if (read_state->length==-1) - read_state->length=sqlite3_blob_bytes(blob); - bytes_read = read_state->length - read_state->offset; - if (bytes_read>buffer_length) - bytes_read=buffer_length; - // allow the caller to do a dummy read, just to work out the length - if (!buffer) - bytes_read=0; - if (bytes_read>0){ - ret = sqlite3_blob_read(blob, buffer, bytes_read, read_state->offset); - if (sqlite_code_busy(ret)) - goto again; - else if(ret!=SQLITE_OK){ - WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db)); - sqlite3_blob_close(blob); - RETURN(-1); - } - } - sqlite3_blob_close(blob); - break; - again: - if (blob) sqlite3_blob_close(blob); - if (!sqlite_retry(&retry, "sqlite3_blob_open")) - RETURN(-1); - } while (1); - } else - RETURN(WHY("file not open")); - + + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + ssize_t n = rhizome_read_retry(&retry, read_state, buffer, buffer_length); + if (n == -1) + RETURN(-1); + size_t bytes_read = (size_t) n; + // hash the payload as we go, but only if we happen to read the payload data in order if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){ SHA512_Update(&read_state->sha512_context, buffer, bytes_read); diff --git a/rhizome_sync.c b/rhizome_sync.c index ae6db73f..415b0944 100644 --- a/rhizome_sync.c +++ b/rhizome_sync.c @@ -439,7 +439,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t if (count){ mdp.out.payload_length = ob_position(b); - if (config.debug.rhizome) + if (config.debug.rhizome_ads) DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last); overlay_mdp_dispatch(&mdp,0,NULL,0); } @@ -449,7 +449,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t int rhizome_sync_announce() { + int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads); sync_send_response(NULL, 0, HEAD_FLAG, 5); + sqlite_set_tracefunc(oldfunc); return 0; } diff --git a/serval.h b/serval.h index c990ebc8..8f23d4b4 100644 --- a/serval.h +++ b/serval.h @@ -846,7 +846,6 @@ int overlay_queue_init(); void monitor_client_poll(struct sched_ent *alarm); void monitor_poll(struct sched_ent *alarm); -void rhizome_client_poll(struct sched_ent *alarm); void rhizome_fetch_poll(struct sched_ent *alarm); void rhizome_server_poll(struct sched_ent *alarm); diff --git a/sourcefiles.mk b/sourcefiles.mk index 00ee579b..1b4516ac 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -14,6 +14,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)fdqueue.c \ $(SERVAL_BASE)fifo.c \ $(SERVAL_BASE)golay.c \ + $(SERVAL_BASE)http_server.c \ $(SERVAL_BASE)keyring.c \ $(SERVAL_BASE)log.c \ $(SERVAL_BASE)lsif.c \ diff --git a/strbuf_helpers.c b/strbuf_helpers.c index dd63d473..8ec48516 100644 --- a/strbuf_helpers.c +++ b/strbuf_helpers.c @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include +#include #include #ifdef HAVE_NETINET_IN_H #include @@ -32,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #endif #include +#include "http_server.h" static inline strbuf _toprint(strbuf sb, char c) { @@ -391,3 +393,28 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt) strbuf_putc(sb, ']'); return sb; } + +strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels) +{ + unsigned i; + int first = 1; + for (i = 0; i != nels; ++i) { + const struct http_range *r = &ranges[i]; + switch (r->type) { + case NIL: break; + case CLOSED: + strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-%"PRIhttp_size_t, first ? "" : ",", r->first, r->last); + first = 0; + break; + case OPEN: + strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-", first ? "" : ",", r->first); + first = 0; + break; + case SUFFIX: + strbuf_sprintf(sb, "%s-%"PRIhttp_size_t, first ? "" : ",", r->last); + first = 0; + break; + } + } + return sb; +} diff --git a/strbuf_helpers.h b/strbuf_helpers.h index ada181ae..cb545983 100644 --- a/strbuf_helpers.h +++ b/strbuf_helpers.h @@ -136,4 +136,11 @@ struct iovec; strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt); #define alloca_iovec(iov,cnt) strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt))) +/* Append a representation of a struct http_range[] array. + * @author Andrew Bettison + */ +struct http_range; +strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels); +#define alloca_http_ranges(ra) strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra))) + #endif //__STRBUF_HELPERS_H__