diff --git a/commandline.c b/commandline.c index 04ddd772..c6db0e7d 100644 --- a/commandline.c +++ b/commandline.c @@ -1952,8 +1952,7 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex return WHYF("Invalid verb ? r->verb : "NULL", alloca_str_toprint(r->path), r->version_major, r->version_minor);\ } while (0) #define DEBUG_DUMP_PARSER(r) do { \ - if (config.debug.httpd) \ + if (config.debug.http_server) \ DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \ (int)(r->parsed - r->received), alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \ (int)(r->cursor - r->received), alloca_toprint(50, r->cursor, r->end - r->cursor), \ @@ -1208,14 +1213,14 @@ static int http_request_parse_body_form_data(struct http_request *r) int at_start = 0; switch (r->form_data_state) { case START: - if (config.debug.httpd) + if (config.debug.http_server) DEBUGF("START"); // The logic here allows for a missing initial CRLF before the first boundary line. at_start = 1; r->form_data_state = PREAMBLE; // fall through case PREAMBLE: { - if (config.debug.httpd) + if (config.debug.http_server) DEBUGF("PREAMBLE"); char *start = r->parsed; for (; at_start || _skip_to_crlf(r); at_start = 0) { @@ -1241,7 +1246,7 @@ static int http_request_parse_body_form_data(struct http_request *r) } return 100; // need more data case HEADER: { - if (config.debug.httpd) + if (config.debug.http_server) DEBUGF("HEADER"); // If not at a CRLF, then we are skipping through an over-long header that didn't // fit into the buffer. Just discard bytes up to the next CRLF. @@ -1365,7 +1370,7 @@ static int http_request_parse_body_form_data(struct http_request *r) } return 400; case BODY: - if (config.debug.httpd) + if (config.debug.http_server) DEBUGF("BODY"); char *start = r->parsed; while (_skip_to_crlf(r)) { @@ -1396,7 +1401,7 @@ static int http_request_parse_body_form_data(struct http_request *r) _INVOKE_HANDLER_BUF_LEN(handle_mime_body, start, r->parsed); return 100; // need more data case EPILOGUE: - if (config.debug.httpd) + if (config.debug.http_server) DEBUGF("EPILOGUE"); r->cursor = r->end; assert(r->cursor >= r->parsed); @@ -1785,6 +1790,50 @@ static const char *httpResultString(int response_code) } } +static strbuf strbuf_status_body(strbuf sb, struct http_response *hr, const char *message) +{ + if ( hr->header.content_type == CONTENT_TYPE_TEXT + || (hr->header.content_type && strcmp(hr->header.content_type, CONTENT_TYPE_TEXT) == 0) + ) { + hr->header.content_type = CONTENT_TYPE_TEXT; + strbuf_sprintf(sb, "%03u %s", hr->result_code, message); + if (hr->result_extra_label) { + strbuf_puts(sb, "\r\n"); + strbuf_puts(sb, hr->result_extra_label); + strbuf_puts(sb, "="); + strbuf_json_atom_as_text(sb, &hr->result_extra_value); + } + strbuf_puts(sb, "\r\n"); + } + else if ( hr->header.content_type == CONTENT_TYPE_JSON + || (hr->header.content_type && strcmp(hr->header.content_type, CONTENT_TYPE_JSON) == 0) + ) { + hr->header.content_type = CONTENT_TYPE_JSON; + strbuf_sprintf(sb, "{\n \"http_status_code\": %u,\n \"http_status_message\": ", hr->result_code); + strbuf_json_string(sb, message); + if (hr->result_extra_label) { + strbuf_puts(sb, ",\n "); + strbuf_json_string(sb, hr->result_extra_label); + strbuf_puts(sb, ": "); + strbuf_json_atom_as_html(sb, &hr->result_extra_value); + } + strbuf_puts(sb, "\n}"); + } + else { + hr->header.content_type = CONTENT_TYPE_HTML; + strbuf_sprintf(sb, "\n

%03u %s

", hr->result_code, message); + if (hr->result_extra_label) { + strbuf_puts(sb, "\n
"); + strbuf_html_escape(sb, hr->result_extra_label, strlen(hr->result_extra_label)); + strbuf_puts(sb, "
"); + strbuf_json_atom_as_html(sb, &hr->result_extra_value); + strbuf_puts(sb, "
"); + } + strbuf_puts(sb, "\n"); + } + return sb; +} + /* Render the HTTP response into the current response buffer. Return 1 if it fits, 0 if it does * not. The buffer response_pointer may be NULL, in which case no response is rendered, but the * content_length is still computed @@ -1833,16 +1882,16 @@ static int _render_response(struct http_request *r) } } else { // If no content is supplied at all, then render a standard, short body based solely on result - // code. + // code, consistent with the response Content-Type if already set (HTML if not set). assert(hr.header.content_length == CONTENT_LENGTH_UNKNOWN); assert(hr.header.resource_length == CONTENT_LENGTH_UNKNOWN); assert(hr.header.content_range_start == 0); assert(hr.result_code != 206); - strbuf cb = strbuf_alloca(100 + strlen(result_string)); - strbuf_sprintf(cb, "

%03u %s

", hr.result_code, result_string); + strbuf cb; + STRBUF_ALLOCA_FIT(cb, 40 + strlen(result_string), (strbuf_status_body(cb, &hr, result_string))); hr.content = strbuf_str(cb); - hr.header.content_type = "text/html"; - hr.header.resource_length = hr.header.content_length = strbuf_len(cb); + hr.header.content_length = strbuf_len(cb); + hr.header.resource_length = hr.header.content_length; hr.header.content_range_start = 0; } assert(hr.header.content_type != NULL); @@ -2045,25 +2094,21 @@ void http_request_response_generated(struct http_request *r, int result, const c } /* Start sending a short response back to the client. The result code must be either a success - * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code. The 'body' argument - * may be a bare message which is enclosed in an HTML envelope to form the response content, so it - * may contain HTML markup. If the 'body' argument is NULL, then the response content is generated - * automatically from the result code. + * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code. The 'message' + * argument may be a bare message which is enclosed in an HTML envelope to form the response + * content, so it may contain HTML markup. If the 'message' argument is NULL, then the response + * content is generated automatically from the result code. * * @author Andrew Bettison */ -void http_request_simple_response(struct http_request *r, uint16_t result, const char *body) +void http_request_simple_response(struct http_request *r, uint16_t result, const char *message) { assert(r->phase == RECEIVE); - strbuf h = NULL; - if (body) { - size_t html_len = strlen(body) + 40; - h = strbuf_alloca(html_len); - strbuf_sprintf(h, "

%03u %s

", result, body); - } r->response.result_code = result; - r->response.header.content_type = "text/html"; r->response.header.content_range_start = 0; + strbuf h = NULL; + if (message) + STRBUF_ALLOCA_FIT(h, 40 + strlen(message), (strbuf_status_body(h, &r->response, message))); if (h) { r->response.header.resource_length = r->response.header.content_length = strbuf_len(h); r->response.content = strbuf_str(h); @@ -2071,3 +2116,28 @@ void http_request_simple_response(struct http_request *r, uint16_t result, const r->response.content_generator = NULL; http_request_start_response(r); } + +int generate_http_content_from_strbuf_chunks( + struct http_request *r, + char *buf, + size_t bufsz, + struct http_content_generator_result *result, + HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER *chunker +) +{ + assert(bufsz > 0); + strbuf b = strbuf_local((char *)buf, bufsz); + int ret; + while ((ret = chunker(r, b)) != -1) { + if (strbuf_overrun(b)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b)); + result->need = strbuf_count(b) + 1 - result->generated; + break; + } + result->generated = strbuf_len(b); + if (ret == 0) + break; + } + return ret; +} diff --git a/http_server.h b/http_server.h index e88d7034..5456258b 100644 --- a/http_server.h +++ b/http_server.h @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include "constants.h" #include "strbuf.h" +#include "strbuf_helpers.h" #include "fdqueue.h" /* Generic HTTP request handling. @@ -56,6 +57,11 @@ http_size_t http_range_bytes(const struct http_range *range, unsigned nranges); #define CONTENT_LENGTH_UNKNOWN UINT64_MAX +extern const char CONTENT_TYPE_TEXT[]; +extern const char CONTENT_TYPE_HTML[]; +extern const char CONTENT_TYPE_JSON[]; +extern const char CONTENT_TYPE_BLOB[]; + struct mime_content_type { char type[64]; char subtype[64]; @@ -63,7 +69,6 @@ struct mime_content_type { char charset[31]; }; - struct http_client_authorization { enum http_authorization_scheme { NOAUTH = 0, BASIC } scheme; union { @@ -105,6 +110,8 @@ typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, siz struct http_response { uint16_t result_code; + const char *result_extra_label; + struct json_atom result_extra_value; struct http_response_headers header; const char *content; HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content @@ -152,6 +159,9 @@ void http_request_response_static(struct http_request *r, int result, const char void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *); void http_request_simple_response(struct http_request *r, uint16_t result, const char *body); +typedef int (HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER)(struct http_request *, strbuf); +int generate_http_content_from_strbuf_chunks(struct http_request *, char *, size_t, struct http_content_generator_result *, HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER *); + typedef int HTTP_REQUEST_PARSER(struct http_request *); typedef void HTTP_RENDERER(struct http_request *, strbuf); diff --git a/httpd.c b/httpd.c new file mode 100644 index 00000000..8f4ddeeb --- /dev/null +++ b/httpd.c @@ -0,0 +1,507 @@ +/* +Serval DNA HTTP external interface +Copyright (C) 2014 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "serval.h" +#include "conf.h" +#include "httpd.h" +#include "overlay_address.h" +#include "overlay_interface.h" + +#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32 + +static HTTP_HANDLER root_page; +static HTTP_HANDLER fav_icon_header; +static HTTP_HANDLER interface_page; +static HTTP_HANDLER neighbour_page; + +HTTP_HANDLER restful_rhizome_bundlelist_json; +HTTP_HANDLER restful_rhizome_newsince; +HTTP_HANDLER restful_rhizome_insert; +HTTP_HANDLER restful_rhizome_; +HTTP_HANDLER restful_meshms_; + +HTTP_HANDLER rhizome_status_page; +HTTP_HANDLER rhizome_file_page; +HTTP_HANDLER manifest_by_prefix_page; + +HTTP_HANDLER rhizome_direct_import; +HTTP_HANDLER rhizome_direct_enquiry; +HTTP_HANDLER rhizome_direct_dispatch; + +struct http_handler { + const char *path; + HTTP_HANDLER *parser; +}; + +struct http_handler paths[]={ + {"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json}, + {"/restful/rhizome/newsince/", restful_rhizome_newsince}, + {"/restful/rhizome/insert", restful_rhizome_insert}, + {"/restful/rhizome/", restful_rhizome_}, + {"/restful/meshms/", restful_meshms_}, + {"/rhizome/status", rhizome_status_page}, + {"/rhizome/file/", rhizome_file_page}, + {"/rhizome/import", rhizome_direct_import}, + {"/rhizome/enquiry", rhizome_direct_enquiry}, + {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, + {"/rhizome/", rhizome_direct_dispatch}, + {"/interface/", interface_page}, + {"/neighbour/", neighbour_page}, + {"/favicon.ico", fav_icon_header}, + {"/", root_page}, +}; + +static int httpd_dispatch(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + INFOF("HTTP SERVER, %s %s", r->http.verb, r->http.path); + r->http.response.content_generator = NULL; + unsigned i; + for (i = 0; i < NELS(paths); ++i) { + const char *remainder; + if (str_startswith(r->http.path, paths[i].path, &remainder)){ + int result = paths[i].parser(r, remainder); + if (result == -1 || (result >= 200 && result < 600)) + return result; + if (result == 1) + return 0; + if (result) + return WHYF("dispatch function for %s returned invalid result %d", paths[i].path, result); + } + } + return 404; +} + +void httpd_server_poll(struct sched_ent *); +struct sched_ent server_alarm; +struct profile_total server_stats = { + .name = "httpd_server_poll", +}; + +uint16_t httpd_server_port = 0; +unsigned int httpd_request_count = 0; + +static int httpd_server_socket = -1; +static time_ms_t httpd_server_last_start_attempt = -1; + +// Format icon data using: +// od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/ */,/g' +unsigned char favicon_bytes[]={ +0,0,1,0,1,0,16,16,16,0,0,0,0,0,40,1 +,0,0,22,0,0,0,40,0,0,0,16,0,0,0,32,0 +,0,0,1,0,4,0,0,0,0,0,128,0,0,0,0,0 +,0,0,0,0,0,0,16,0,0,0,0,0,0,0,104,158 +,168,0,163,233,247,0,104,161,118,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,17 +,17,17,17,18,34,17,17,18,34,17,17,18,34,17,17,2 +,34,17,17,18,34,17,16,18,34,1,17,17,1,17,1,17 +,1,16,1,16,17,17,17,17,1,17,16,16,17,17,17,17 +,1,17,18,34,17,17,17,16,17,17,2,34,17,17,17,16 +,17,16,18,34,17,17,17,16,17,1,17,1,17,17,17,18 +,34,17,17,16,17,17,17,18,34,17,17,18,34,17,17,18 +,34,17,17,18,34,17,17,16,17,17,17,18,34,17,17,16 +,17,17,17,17,17,0,17,1,17,17,17,17,17,17,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +int favicon_len=318; + +int is_httpd_server_running() +{ + return httpd_server_socket != -1; +} + +/* Start the Rhizome HTTP server by creating a socket, binding it to an available port, and + marking it as passive. If called repeatedly and frequently, this function will only try to start + the server after a certain time has elapsed since the last attempt. + Return -1 if an error occurs (message logged). + Return 0 if the server was started. + Return 1 if the server is already started successfully. + Return 2 if the server was not started because it is too soon since last failed attempt. + */ +int httpd_server_start(uint16_t port_low, uint16_t port_high) +{ + if (httpd_server_socket != -1) + return 1; + + /* Only try to start http server every five seconds. */ + time_ms_t now = gettime_ms(); + if (now < httpd_server_last_start_attempt + 5000) + return 2; + httpd_server_last_start_attempt = now; + if (config.debug.httpd) + DEBUGF("Starting HTTP server"); + + uint16_t port; + for (port = port_low; port <= port_high; ++port) { + /* Create a new socket, reusable and non-blocking. */ + if (httpd_server_socket == -1) { + httpd_server_socket = socket(AF_INET,SOCK_STREAM,0); + if (httpd_server_socket == -1) { + WHY_perror("socket"); + goto error; + } + int on=1; + if (setsockopt(httpd_server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) == -1) { + WHY_perror("setsockopt(REUSEADDR)"); + goto error; + } + if (ioctl(httpd_server_socket, FIONBIO, (char *)&on) == -1) { + WHY_perror("ioctl(FIONBIO)"); + goto error; + } + } + /* Bind it to the next port we want to try. */ + struct sockaddr_in address; + bzero((char *) &address, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(port); + if (bind(httpd_server_socket, (struct sockaddr *) &address, sizeof(address)) == -1) { + if (errno != EADDRINUSE) { + WHY_perror("bind"); + goto error; + } + } else { + /* We bound to a port. The battle is half won. Now we have to successfully listen on that + port, which could also fail with EADDRINUSE, in which case we have to scrap the socket and + create a new one, because once bound, a socket stays bound. + */ + if (listen(httpd_server_socket, 20) != -1) + goto success; + if (errno != EADDRINUSE) { + WHY_perror("listen"); + goto error; + } + close(httpd_server_socket); + httpd_server_socket = -1; + } + } + WHYF("No ports available in range %u to %u", HTTPD_PORT, HTTPD_PORT_MAX); +error: + if (httpd_server_socket != -1) { + close(httpd_server_socket); + httpd_server_socket = -1; + } + return WHY("Failed to start HTTP server"); + +success: + INFOF("HTTP SERVER START port=%"PRIu16" fd=%d services=RESTful%s%s", + port, + httpd_server_socket, + config.rhizome.http.enable ? ",Rhizome" : "", + config.rhizome.api.addfile.uri_path[0] ? ",RhizomeDirect" : "" + ); + httpd_server_port = port; + /* Add Rhizome HTTPd server to list of file descriptors to watch */ + server_alarm.function = httpd_server_poll; + server_alarm.stats = &server_stats; + server_alarm.poll.fd = httpd_server_socket; + server_alarm.poll.events = POLLIN; + watch(&server_alarm); + return 0; +} + +static void httpd_server_finalise_http_request(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + if (r->manifest) { + rhizome_manifest_free(r->manifest); + r->manifest = NULL; + } + if (r->finalise_union) { + r->finalise_union(r); + r->finalise_union = NULL; + } + if (httpd_request_count) + --httpd_request_count; +} + +static int httpd_dispatch(struct http_request *); + +static unsigned int http_request_uuid_counter = 0; + +void httpd_server_poll(struct sched_ent *alarm) +{ + if (alarm->poll.revents & (POLLIN | POLLOUT)) { + struct sockaddr addr; + unsigned int addr_len = sizeof addr; + int sock; + if ((sock = accept(httpd_server_socket, &addr, &addr_len)) == -1) { + if (errno && errno != EAGAIN) + WARN_perror("accept"); + } else { + struct sockaddr_in *peerip=NULL; + if (addr.sa_family == AF_INET) { + peerip = (struct sockaddr_in *)&addr; // network order + INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u", + addr_len, peerip->sin_family, peerip->sin_port, + ((unsigned char*)&peerip->sin_addr.s_addr)[0], + ((unsigned char*)&peerip->sin_addr.s_addr)[1], + ((unsigned char*)&peerip->sin_addr.s_addr)[2], + ((unsigned char*)&peerip->sin_addr.s_addr)[3] + ); + } else { + INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u data=%s", + addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data) + ); + } + httpd_request *request = emalloc_zero(sizeof(httpd_request)); + if (request == NULL) { + WHY("Cannot respond to HTTP request, out of memory"); + close(sock); + } else { + ++httpd_request_count; + request->uuid = http_request_uuid_counter++; + if (peerip) + request->http.client_sockaddr_in = *peerip; + request->http.handle_headers = httpd_dispatch; + request->http.debug_flag = &config.debug.httpd; + request->http.disable_tx_flag = &config.debug.nohttptx; + request->http.finalise = httpd_server_finalise_http_request; + request->http.free = free; + request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT; + http_request_init(&request->http, sock); + } + } + } + if (alarm->poll.revents & (POLLHUP | POLLERR)) { + INFO("Error on tcp listen socket"); + } +} + +int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call) +{ + IN(); + const char *bufend = buf + len; + const char *p = buf; + size_t tail = read_since_last_call + 4; + if (tail < len) + p = bufend - tail; + int count = 0; + for (; p != bufend; ++p) { + switch (*p) { + case '\n': + if (++count==2) + RETURN(p - buf); + case '\r': // ignore CR + case '\0': // ignore NUL (telnet inserts them) + break; + default: + count = 0; + break; + } + } + RETURN(0); + OUT(); +} + +static int is_from_loopback(const struct http_request *r) +{ + return r->client_sockaddr_in.sin_family == AF_INET + && ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET; +} + +/* Return 1 if the given authorization credentials are acceptable. + * Return 0 if not. + */ +static int is_authorized(const struct http_client_authorization *auth) +{ + if (auth->scheme != BASIC) + return 0; + unsigned i; + for (i = 0; i != config.rhizome.api.restful.users.ac; ++i) { + if ( strcmp(config.rhizome.api.restful.users.av[i].key, auth->credentials.basic.user) == 0 + && strcmp(config.rhizome.api.restful.users.av[i].value.password, auth->credentials.basic.password) == 0 + ) + return 1; + } + return 0; +} + +int authorize(struct http_request *r) +{ + if (!is_from_loopback(r)) + return 403; + if (!is_authorized(&r->request_header.authorization)) { + r->response.header.www_authenticate.scheme = BASIC; + r->response.header.www_authenticate.realm = "Serval Rhizome"; + return 401; + } + return 0; +} + +int accumulate_text(httpd_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len) +{ + if (len) { + size_t newlen = *textlenp + len; + if (newlen > textsiz) { + if (config.debug.httpd) + DEBUGF("Form part \"%s\" too long, %zu bytes overflows maximum %zu by %zu", + partname, newlen, textsiz, (size_t)(newlen - textsiz) + ); + strbuf msg = strbuf_alloca(100); + strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname); + http_request_simple_response(&r->http, 403, strbuf_str(msg)); + return 0; + } + memcpy(textbuf + *textlenp, buf, len); + *textlenp = newlen; + } + return 1; +} + +int form_buf_malloc_init(struct form_buf_malloc *f, size_t size_limit) +{ + assert(f->buffer == NULL); + assert(f->buffer_alloc_size == 0); + assert(f->length == 0); + f->size_limit = size_limit; + return 0; +} + +int form_buf_malloc_accumulate(httpd_request *r, const char *partname, struct form_buf_malloc *f, const char *buf, size_t len) +{ + if (len == 0) + return 0; + size_t newlen = f->length + len; + if (newlen > f->size_limit) { + if (config.debug.httpd) + DEBUGF("form part \"%s\" overflow, %zu bytes exceeds limit %zu by %zu", + partname, newlen, f->size_limit, (size_t)(newlen - f->size_limit) + ); + strbuf msg = strbuf_alloca(100); + strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname); + http_request_simple_response(&r->http, 403, strbuf_str(msg)); + return 403; + } + if (newlen > f->buffer_alloc_size) { + if ((f->buffer = erealloc(f->buffer, newlen)) == NULL) { + http_request_simple_response(&r->http, 500, NULL); + return 500; + } + f->buffer_alloc_size = newlen; + } + memcpy(f->buffer + f->length, buf, len); + f->length = newlen; + return 0; +} + +void form_buf_malloc_release(struct form_buf_malloc *f) +{ + if (f->buffer) { + free(f->buffer); + f->buffer = NULL; + } + f->buffer_alloc_size = 0; + f->length = 0; + f->size_limit = 0; +} + +int http_response_form_part(httpd_request *r, const char *what, const char *partname, const char *text, size_t textlen) +{ + if (config.debug.httpd) + DEBUGF("%s \"%s\" form part %s", what, partname, text ? alloca_toprint(-1, text, textlen) : ""); + strbuf msg = strbuf_alloca(100); + strbuf_sprintf(msg, "%s \"%s\" form part", what, partname); + http_request_simple_response(&r->http, 403, strbuf_str(msg)); + return 403; +} + +static int root_page(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + if (r->http.verb != HTTP_VERB_GET) + return 405; + char temp[8192]; + strbuf b = strbuf_local(temp, sizeof temp); + strbuf_sprintf(b, "" + "

Hello, I'm %s*


" + "Interfaces;
", + alloca_tohex_sid_t_trunc(my_subscriber->sid, 16)); + int i; + for (i=0;i%d: %s, TX: %d, RX: %d
", + i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count); + } + strbuf_puts(b, "Neighbours;
"); + link_neighbour_short_status_html(b, "/neighbour"); + if (is_rhizome_http_enabled()){ + strbuf_puts(b, "Rhizome Status
"); + } + strbuf_puts(b, ""); + if (strbuf_overrun(b)) { + WHY("HTTP Root page buffer overrun"); + return 500; + } + http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, temp, strbuf_len(b)); + return 1; +} + +static int fav_icon_header(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); + return 1; +} + +static int neighbour_page(httpd_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_GET) + return 405; + char buf[8*1024]; + strbuf b = strbuf_local(buf, sizeof buf); + sid_t neighbour_sid; + if (str_to_sid_t(&neighbour_sid, remainder) == -1) + return 404; + struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0); + if (!neighbour) + return 404; + strbuf_puts(b, ""); + link_neighbour_status_html(b, neighbour); + strbuf_puts(b, ""); + if (strbuf_overrun(b)) + return -1; + http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, buf, strbuf_len(b)); + return 1; +} + +static int interface_page(httpd_request *r, const char *remainder) +{ + if (r->http.verb != HTTP_VERB_GET) + return 405; + char buf[8*1024]; + strbuf b=strbuf_local(buf, sizeof buf); + int index=atoi(remainder); + if (index<0 || index>=OVERLAY_MAX_INTERFACES) + return 404; + strbuf_puts(b, ""); + interface_state_html(b, &overlay_interfaces[index]); + strbuf_puts(b, ""); + if (strbuf_overrun(b)) + return -1; + http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, buf, strbuf_len(b)); + return 1; +} diff --git a/httpd.h b/httpd.h new file mode 100644 index 00000000..d8f4b26a --- /dev/null +++ b/httpd.h @@ -0,0 +1,214 @@ +/* +Serval DNA Rhizome HTTP interface +Copyright (C) 2013-2014 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVAL_DNA__HTTPD_H +#define __SERVAL_DNA__HTTPD_H + +#include "rhizome.h" +#include "meshms.h" +#include "http_server.h" + +int is_httpd_server_running(); + +#define HTTPD_PORT 4110 +#define HTTPD_PORT_MAX 4150 + +extern uint16_t httpd_server_port; +extern unsigned int httpd_request_count; + +enum list_phase { LIST_HEADER = 0, LIST_ROWS, LIST_END, LIST_DONE }; + +struct form_buf_malloc { + char *buffer; + size_t size_limit; // == 0 means no limit + size_t buffer_alloc_size; + size_t length; +}; + +struct httpd_request; + +int form_buf_malloc_init(struct form_buf_malloc *, size_t size_limit); +int form_buf_malloc_accumulate(struct httpd_request *, const char *partname, struct form_buf_malloc *, const char *, size_t); +void form_buf_malloc_release(struct form_buf_malloc *); + +typedef struct httpd_request +{ + struct http_request http; // MUST BE FIRST ELEMENT + + /* Identify request from others being run. Monotonic counter feeds it. Only + * used for debugging when we write post-.log files for multi-part form + * requests. + */ + unsigned int uuid; + + /* For requests/responses that pertain to a single manifest. + */ + rhizome_manifest *manifest; + + /* For requests/responses that contain one or two SIDs. + */ + sid_t sid1; + sid_t sid2; + + /* For requests/responses that contain a Rhizome Bundle ID. + */ + rhizome_bid_t bid; + + /* For requests/responses that contain a 64-bit unsigned integer (eg, SQLite ROWID, byte offset). + */ + uint64_t ui64; + + /* Finaliser for union contents (below). + */ + void (*finalise_union)(struct httpd_request *); + + /* Mutually exclusive response arguments. + */ + union { + + /* For receiving Rhizome Direct import request + */ + struct { + // Which part is currently being received + const char *current_part; + // Temporary file currently current part is being written to + int part_fd; + // Which parts have already been received + bool_t received_manifest; + bool_t received_data; + // Name of data file supplied in part's Content-Disposition header, filename + // parameter (if any) + char data_file_name[MIME_FILENAME_MAXLEN + 1]; + } + direct_import; + + /* For receiving RESTful Rhizome insert request + */ + struct { + // Which part is currently being received + const char *current_part; + // Which parts have already been received + bool_t received_author; + bool_t received_secret; + bool_t received_manifest; + bool_t received_payload; + // For storing the "bundle-author" hex SID as we receive it + char author_hex[SID_STRLEN]; + size_t author_hex_len; + sid_t author; + // For storing the "bundle-secret" hex as we receive it + char secret_hex[RHIZOME_BUNDLE_KEY_STRLEN]; + size_t secret_hex_len; + rhizome_bk_t bundle_secret; + // The "force-new" parameter + char force_new_text[5]; // enough for "false" + size_t force_new_text_len; + bool_t force_new; + // For storing the manifest text (malloc/realloc) as we receive it + struct form_buf_malloc manifest; + // For receiving the payload + enum rhizome_payload_status payload_status; + uint64_t payload_size; + struct rhizome_write write; + } + insert; + + /* For responses that send part or all of a payload. + */ + struct rhizome_read read_state; + + /* For responses that list manifests. + */ + struct { + enum list_phase phase; + uint64_t rowid_highest; + size_t rowcount; + time_ms_t end_time; + struct rhizome_list_cursor cursor; + } + rhlist; + + /* For responses that list MeshMS conversations. + */ + struct { + enum list_phase phase; + size_t rowcount; + struct meshms_conversations *conv; + struct meshms_conversation_iterator iter; + } + mclist; + + /* For responses that list MeshMS messages in a single conversation. + */ + struct { + enum meshms_which_ply token_which_ply; + uint64_t token_offset; + enum meshms_which_ply latest_which_ply; + uint64_t latest_offset; + time_ms_t end_time; + uint64_t highest_ack_offset; + enum list_phase phase; + size_t rowcount; + struct meshms_message_iterator iter; + int finished; + } + msglist; + + /* For responses that send a MeshMS message. + */ + struct { + // Which part is currently being received + const char *current_part; + // Which parts have already been received + bool_t received_message; + // The text of the message to send + struct form_buf_malloc message; + } + sendmsg; + + } u; + +} httpd_request; + +int httpd_server_start(uint16_t port_low, uint16_t port_high); + +typedef int HTTP_HANDLER(httpd_request *r, const char *remainder); + +int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call); +int authorize(struct http_request *r); +int http_response_form_part(httpd_request *r, const char *what, const char *partname, const char *text, size_t textlen); +int accumulate_text(httpd_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len); + +int rhizome_response_content_init_filehash(httpd_request *r, const rhizome_filehash_t *hash); +int rhizome_response_content_init_payload(httpd_request *r, rhizome_manifest *); +HTTP_CONTENT_GENERATOR rhizome_payload_content; + +struct http_response_parts { + uint16_t code; + char *reason; + uint64_t range_start; + uint64_t content_length; + char *content_start; +}; + +#define HTTP_RESPONSE_CONTENT_LENGTH_UNSET UINT64_MAX + +int unpack_http_response(char *response, struct http_response_parts *parts); + +#endif // __SERVAL_DNA__HTTPD_H diff --git a/meshms.c b/meshms.c index df667575..12f2f29f 100644 --- a/meshms.c +++ b/meshms.c @@ -17,9 +17,11 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#define __MESHMS_INLINE #include #include "serval.h" #include "rhizome.h" +#include "meshms.h" #include "log.h" #include "conf.h" #include "crypto.h" @@ -28,72 +30,26 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "dataformats.h" #define MESHMS_BLOCK_TYPE_ACK 0x01 -#define MESHMS_BLOCK_TYPE_MESSAGE 0x02 -#define MESHMS_BLOCK_TYPE_BID_REFERENCE 0x03 +#define MESHMS_BLOCK_TYPE_MESSAGE 0x02 // NUL-terminated UTF8 string -// the manifest details for one half of a conversation -struct ply{ - rhizome_bid_t bundle_id; - uint64_t version; - uint64_t tail; - uint64_t size; -}; - -struct conversations{ - // binary tree - struct conversations *_left; - struct conversations *_right; - - // who are we talking to? - sid_t them; - - char found_my_ply; - struct ply my_ply; - - char found_their_ply; - struct ply their_ply; - - // what is the offset of their last message - uint64_t their_last_message; - // what is the last message we marked as read - uint64_t read_offset; - // our cached value for the last known size of their ply - uint64_t their_size; -}; - -// cursor state for reading one half of a conversation -struct ply_read{ - // rhizome payload - struct rhizome_read read; - // block buffer - struct rhizome_read_buffer buff; - - // details of the current record - uint64_t record_end_offset; - uint16_t record_length; - size_t buffer_size; - char type; - // raw record data - unsigned char *buffer; -}; - -static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv); - -static void free_conversations(struct conversations *conv){ - if (!conv) - return; - free_conversations(conv->_left); - free_conversations(conv->_right); - free(conv); +void meshms_free_conversations(struct meshms_conversations *conv) +{ + if (conv) { + if (conv->_parent && conv != conv->_parent->_left && conv != conv->_parent->_right) + WHYF("deformed MeshMS conversation tree"); + meshms_free_conversations(conv->_left); + meshms_free_conversations(conv->_right); + free(conv); + } } -static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) +static enum meshms_status get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) { /* Find our private key */ unsigned cn=0, in=0, kp=0; if (!keyring_find_sid(keyring,&cn,&in,&kp,my_sidp)) - return WHYF("SID was not found in keyring: %s", alloca_tohex_sid_t(*my_sidp)); - + return MESHMS_STATUS_SID_LOCKED; + char seed[1024]; snprintf(seed, sizeof(seed), "incorrection%sconcentrativeness", @@ -101,7 +57,7 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) ->keypairs[kp]->private_key, crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES)); if (rhizome_get_bundle_from_seed(m, seed) == -1) - return -1; + return MESHMS_STATUS_ERROR; // always consider the content encrypted, we don't need to rely on the manifest itself. rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED); @@ -121,35 +77,38 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) ); } } else { - if (strcmp(m->service, RHIZOME_SERVICE_FILE) != 0) - return WHYF("Invalid manifest, service=%s but should be %s", m->service, RHIZOME_SERVICE_MESHMS2); + if (strcmp(m->service, RHIZOME_SERVICE_FILE) != 0) { + WARNF("Invalid conversations manifest, service=%s but should be %s", m->service, RHIZOME_SERVICE_FILE); + return MESHMS_STATUS_PROTOCOL_FAULT; + } } - return 0; + return MESHMS_STATUS_OK; } -static struct conversations *add_conv(struct conversations **conv, const sid_t *them) +static struct meshms_conversations *add_conv(struct meshms_conversations **conv, const sid_t *them) { - struct conversations **ptr = conv; - while(*ptr){ + struct meshms_conversations **ptr = conv; + struct meshms_conversations *parent = NULL; + while (*ptr) { int cmp = cmp_sid_t(&(*ptr)->them, them); if (cmp == 0) break; + parent = *ptr; if (cmp < 0) ptr = &(*ptr)->_left; else ptr = &(*ptr)->_right; } - if (!*ptr){ - *ptr = emalloc_zero(sizeof(struct conversations)); - if (*ptr) - (*ptr)->them = *them; + if (*ptr == NULL && (*ptr = emalloc_zero(sizeof(struct meshms_conversations))) != NULL) { + (*ptr)->_parent = parent; + (*ptr)->them = *them; } return *ptr; } // find matching conversations // if their_sid == my_sid, return all conversations with any recipient -static int get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv) +static enum meshms_status get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite3_stmt *statement = sqlite_prepare_bind(&retry, @@ -164,7 +123,7 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si END ); if (!statement) - return -1; + return MESHMS_STATUS_ERROR; if (config.debug.meshms) { const char *my_sid_hex = alloca_tohex_sid_t(*my_sid); const char *their_sid_hex = alloca_tohex_sid_t(*(their_sid ? their_sid : my_sid)); @@ -197,10 +156,10 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si continue; } } - struct conversations *ptr = add_conv(conv, &their_sid); + struct meshms_conversations *ptr = add_conv(conv, &their_sid); if (!ptr) break; - struct ply *p; + struct meshms_ply *p; if (them==sender){ ptr->found_their_ply=1; p=&ptr->their_ply; @@ -214,22 +173,24 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si p->size = size; } sqlite3_finalize(statement); - return 0; + return MESHMS_STATUS_OK; } -static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid) +static enum meshms_status find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv) { - struct conversations *conv=NULL; - if (meshms_conversations_list(my_sid, their_sid, &conv)) - return NULL; - if (!conv){ - conv = emalloc_zero(sizeof(struct conversations)); - conv->them = *their_sid; + enum meshms_status status; + if (meshms_failed(status = meshms_conversations_list(my_sid, their_sid, conv))) + return status; + if (*conv == NULL) { + if ((*conv = (struct meshms_conversations *) emalloc_zero(sizeof(struct meshms_conversations))) == NULL) + return MESHMS_STATUS_ERROR; + (*conv)->them = *their_sid; + status = MESHMS_STATUS_UPDATED; } - return conv; + return status; } -static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m) +static int create_ply(const sid_t *my_sid, struct meshms_conversations *conv, rhizome_manifest *m) { if (config.debug.meshms) DEBUGF("Creating ply for my_sid=%s them=%s", @@ -249,199 +210,221 @@ static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_m return 0; } -static int append_footer(unsigned char *buffer, char type, int payload_len) +static int append_footer(unsigned char *buffer, char type, size_t message_len) { - payload_len = (payload_len << 4) | (type&0xF); - write_uint16(buffer, payload_len); + assert(message_len <= MESHMS_MESSAGE_MAX_LEN); + message_len = (message_len << 4) | (type&0xF); + write_uint16(buffer, message_len); return 2; } -static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome_manifest *m) +static enum meshms_status ply_read_open(struct meshms_ply_read *ply, const rhizome_bid_t *bid, rhizome_manifest *m) { if (config.debug.meshms) DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid)); - if (rhizome_retrieve_manifest(bid, m)) - return -1; + switch (rhizome_retrieve_manifest(bid, m)) { + case 0: + break; + case -1: + return MESHMS_STATUS_ERROR; + default: // bundle not found + return MESHMS_STATUS_PROTOCOL_FAULT; + } enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &ply->read); - if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) + if (config.debug.meshms) + DEBUGF("pstatus=%d", pstatus); + if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) { WARNF("Payload was not found for manifest %s, %"PRIu64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); + return MESHMS_STATUS_PROTOCOL_FAULT; + } + if (pstatus == RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL) + return MESHMS_STATUS_SID_LOCKED; if (pstatus != RHIZOME_PAYLOAD_STATUS_STORED && pstatus != RHIZOME_PAYLOAD_STATUS_EMPTY) - return -1; + return MESHMS_STATUS_ERROR; assert(m->filesize != RHIZOME_SIZE_UNSET); ply->read.offset = ply->read.length = m->filesize; - return 0; + return MESHMS_STATUS_OK; } -static void ply_read_close(struct ply_read *ply) +static void ply_read_close(struct meshms_ply_read *ply) { - if (ply->buffer){ - free(ply->buffer); - ply->buffer=NULL; + if (ply->record){ + free(ply->record); + ply->record=NULL; } - ply->buffer_size=0; + ply->record_size=0; ply->buff.len=0; rhizome_read_close(&ply->read); } // read the next record from the ply (backwards) -// returns 1 on EOF, -1 on failure -static int ply_read_next(struct ply_read *ply) +// returns MESHMS_STATUS_UPDATED if the read advances to a new record, MESHMS_STATUS_OK if at the +// end of records +static enum meshms_status ply_read_prev(struct meshms_ply_read *ply) { ply->record_end_offset = ply->read.offset; unsigned char footer[2]; if (ply->read.offset <= sizeof footer) { if (config.debug.meshms) - DEBUGF("EOF"); - return 1; + DEBUG("EOF"); + return MESHMS_STATUS_OK; } ply->read.offset -= sizeof footer; - ssize_t read; - read = rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer); - if (read == -1) - return WHYF("rhizome_read_buffered() failed"); - if ((size_t) read != sizeof footer) - return WHYF("Expected %zu bytes read, got %zu", (size_t) sizeof footer, (size_t) read); + ssize_t read = rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer); + if (read == -1) { + WHYF("rhizome_read_buffered() failed"); + return MESHMS_STATUS_ERROR; + } + if ((size_t) read != sizeof footer) { + WHYF("Expected %zu bytes read, got %zu", (size_t) sizeof footer, (size_t) read); + return MESHMS_STATUS_PROTOCOL_FAULT; + } // (rhizome_read automatically advances the offset by the number of bytes read) ply->record_length=read_uint16(footer); ply->type = ply->record_length & 0xF; ply->record_length = ply->record_length>>4; - if (config.debug.meshms) DEBUGF("Found record %d, length %d @%"PRId64, ply->type, ply->record_length, ply->record_end_offset); - // need to allow for advancing the tail and cutting a message in half. if (ply->record_length + sizeof footer > ply->read.offset){ if (config.debug.meshms) DEBUGF("EOF"); - return 1; + return MESHMS_STATUS_OK; } - ply->read.offset -= ply->record_length + sizeof(footer); uint64_t record_start = ply->read.offset; - - if (ply->buffer_size < ply->record_length){ - ply->buffer_size = ply->record_length; - unsigned char *b=realloc(ply->buffer, ply->buffer_size); + if (ply->record_size < ply->record_length){ + ply->record_size = ply->record_length; + unsigned char *b = erealloc(ply->record, ply->record_size); if (!b) - return WHY("realloc() failed"); - ply->buffer = b; + return MESHMS_STATUS_ERROR; + ply->record = b; } - - read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length); - if (read == -1) + read = rhizome_read_buffered(&ply->read, &ply->buff, ply->record, ply->record_length); + if (read == -1) { return WHYF("rhizome_read_buffered() failed"); - if ((size_t) read != ply->record_length) - return WHYF("Expected %u bytes read, got %zu", ply->record_length, (size_t) read); - + return MESHMS_STATUS_ERROR; + } + if ((size_t) read != ply->record_length) { + WHYF("Expected %u bytes read, got %zu", ply->record_length, (size_t) read); + return MESHMS_STATUS_PROTOCOL_FAULT; + } ply->read.offset = record_start; - return 0; + return MESHMS_STATUS_UPDATED; } // keep reading past messages until you find this type. -static int ply_find_next(struct ply_read *ply, char type){ - while(1){ - int ret = ply_read_next(ply); - if (ret || ply->type==type) - return ret; - } +static enum meshms_status ply_find_prev(struct meshms_ply_read *ply, char type) +{ + enum meshms_status status; + while ((status = ply_read_prev(ply)) == MESHMS_STATUS_UPDATED && ply->type != type) + ; + return status; } -static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, unsigned char *buffer, int len) +static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshms_conversations *conv, unsigned char *buffer, int len) { - int ret=-1; + enum meshms_status status = MESHMS_STATUS_ERROR; rhizome_manifest *mout = NULL; rhizome_manifest *m = rhizome_new_manifest(); if (!m) goto end; - if (conv->found_my_ply){ - if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m)) - goto end; + switch (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m)) { + case 0: + break; + case -1: + status = MESHMS_STATUS_ERROR; + goto end; + default: + status = MESHMS_STATUS_PROTOCOL_FAULT; + goto end; + } rhizome_authenticate_author(m); - if (!m->haveSecret || m->authorship != AUTHOR_AUTHENTIC) - goto end; - }else{ - if (create_ply(my_sid, conv, m)) + if (!m->haveSecret || m->authorship != AUTHOR_AUTHENTIC) { + status = MESHMS_STATUS_PROTOCOL_FAULT; goto end; + } + } else if (create_ply(my_sid, conv, m) == -1) { + status = MESHMS_STATUS_ERROR; + goto end; } assert(m->haveSecret); assert(m->authorship == AUTHOR_AUTHENTIC); - enum rhizome_payload_status pstatus = rhizome_append_journal_buffer(m, 0, buffer, len); - if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW) + if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW) { + status = MESHMS_STATUS_ERROR; goto end; - - enum rhizome_bundle_status status = rhizome_manifest_finalise(m, &mout, 1); - switch (status) { + } + enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1); + if (config.debug.meshms) + DEBUGF("bstatus=%d", bstatus); + switch (bstatus) { case RHIZOME_BUNDLE_STATUS_ERROR: - // error is already logged + // error has already been logged + status = MESHMS_STATUS_ERROR; break; case RHIZOME_BUNDLE_STATUS_NEW: - ret = 0; + status = MESHMS_STATUS_UPDATED; break; case RHIZOME_BUNDLE_STATUS_SAME: case RHIZOME_BUNDLE_STATUS_DUPLICATE: case RHIZOME_BUNDLE_STATUS_OLD: + status = MESHMS_STATUS_PROTOCOL_FAULT; WHYF("MeshMS ply manifest (version=%"PRIu64") gazumped by Rhizome store (version=%"PRIu64")", m->version, mout->version); break; case RHIZOME_BUNDLE_STATUS_INCONSISTENT: + status = MESHMS_STATUS_PROTOCOL_FAULT; WHYF("MeshMS ply manifest not consistent with payload"); break; case RHIZOME_BUNDLE_STATUS_FAKE: + status = MESHMS_STATUS_PROTOCOL_FAULT; WHYF("MeshMS ply manifest is not signed"); break; case RHIZOME_BUNDLE_STATUS_INVALID: + status = MESHMS_STATUS_PROTOCOL_FAULT; WHYF("MeshMS ply manifest is invalid"); break; default: - FATALF("status=%d", status); + FATALF("bstatus=%d", bstatus); } end: if (mout && mout!=m) rhizome_manifest_free(mout); if (m) rhizome_manifest_free(m); - return ret; + return status; } // update if any conversations are unread or need to be acked. -// return -1 for failure, 1 if the conversation index needs to be saved. -static int update_conversation(const sid_t *my_sid, struct conversations *conv) +// return MESHMS_STATUS_UPDATED if the conversation index needs to be saved. +static enum meshms_status update_conversation(const sid_t *my_sid, struct meshms_conversations *conv) { if (config.debug.meshms) DEBUG("Checking if conversation needs to be acked"); // Nothing to be done if they have never sent us anything if (!conv->found_their_ply) - return 0; + return MESHMS_STATUS_OK; rhizome_manifest *m_ours = NULL; rhizome_manifest *m_theirs = rhizome_new_manifest(); if (!m_theirs) - return -1; + return MESHMS_STATUS_ERROR; - struct ply_read ply; + struct meshms_ply_read ply; bzero(&ply, sizeof(ply)); - int ret=-1; - + enum meshms_status status = MESHMS_STATUS_ERROR; if (config.debug.meshms) DEBUG("Locating their last message"); - - if (ply_read_open(&ply, &conv->their_ply.bundle_id, m_theirs)) + if (meshms_failed(status = ply_read_open(&ply, &conv->their_ply.bundle_id, m_theirs))) goto end; - - ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_MESSAGE); - if (ret!=0){ - // no messages indicates that we didn't do anthing - if (ret>0) - ret=0; + if (meshms_failed(status = ply_find_prev(&ply, MESHMS_BLOCK_TYPE_MESSAGE))) goto end; - } - if (conv->their_last_message == ply.record_end_offset){ // nothing has changed since last time - ret=0; + status = MESHMS_STATUS_OK; goto end; } @@ -458,18 +441,18 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv) DEBUG("Locating our previous ack"); m_ours = rhizome_new_manifest(); - if (!m_ours) + if (!m_ours) { + status = MESHMS_STATUS_ERROR; goto end; - if (ply_read_open(&ply, &conv->my_ply.bundle_id, m_ours)) + } + if (meshms_failed(status = ply_read_open(&ply, &conv->my_ply.bundle_id, m_ours))) goto end; - - ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK); - if (ret == -1) + if (meshms_failed(status = ply_find_prev(&ply, MESHMS_BLOCK_TYPE_ACK))) goto end; - - if (ret==0){ - if (unpack_uint(ply.buffer, ply.record_length, &previous_ack) == -1) + if (status == MESHMS_STATUS_UPDATED) { + if (unpack_uint(ply.record, ply.record_length, &previous_ack) == -1) previous_ack=0; + status = MESHMS_STATUS_OK; } if (config.debug.meshms) DEBUGF("Previous ack is %"PRId64, previous_ack); @@ -477,96 +460,105 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv) }else{ if (config.debug.meshms) DEBUGF("No outgoing ply"); + status = MESHMS_STATUS_PROTOCOL_FAULT; } - if (previous_ack >= conv->their_last_message){ // their last message has already been acked - ret=1; + status = MESHMS_STATUS_UPDATED; goto end; } - + // append an ack for their message if (config.debug.meshms) DEBUGF("Creating ACK for %"PRId64" - %"PRId64, previous_ack, conv->their_last_message); - unsigned char buffer[24]; int ofs=0; ofs+=pack_uint(&buffer[ofs], conv->their_last_message); if (previous_ack) ofs+=pack_uint(&buffer[ofs], conv->their_last_message - previous_ack); ofs+=append_footer(buffer+ofs, MESHMS_BLOCK_TYPE_ACK, ofs); - ret = append_meshms_buffer(my_sid, conv, buffer, ofs); - + status = append_meshms_buffer(my_sid, conv, buffer, ofs); + if (config.debug.meshms) + DEBUGF("status=%d", status); end: ply_read_close(&ply); if (m_ours) rhizome_manifest_free(m_ours); if (m_theirs) rhizome_manifest_free(m_theirs); - // if it's all good, remember the size of their ply at the time we examined it. - if (ret>=0) + if (!meshms_failed(status)) conv->their_size = conv->their_ply.size; - - return ret; + return status; } -// update conversations, and return 1 if the conversation index should be saved -static int update_conversations(const sid_t *my_sid, struct conversations *conv) +// update conversations, and return MESHMS_STATUS_UPDATED if the conversation index should be saved +static enum meshms_status update_conversations(const sid_t *my_sid, struct meshms_conversations *conv) { - if (!conv) - return 0; - int ret = 0; - if (update_conversations(my_sid, conv->_left)) - ret=1; - - if (conv->their_size != conv->their_ply.size){ - if (update_conversation(my_sid, conv)>0) - ret=1; + enum meshms_status rstatus = MESHMS_STATUS_OK; + if (conv) { + enum meshms_status status; + if (meshms_failed(status = update_conversations(my_sid, conv->_left))) + return status; + if (status == MESHMS_STATUS_UPDATED) + rstatus = MESHMS_STATUS_UPDATED; + if (conv->their_size != conv->their_ply.size) { + if (meshms_failed(status = update_conversation(my_sid, conv))) + return status; + if (status == MESHMS_STATUS_UPDATED) + rstatus = MESHMS_STATUS_UPDATED; + } + if (meshms_failed(status = update_conversations(my_sid, conv->_right))) + return status; + if (status == MESHMS_STATUS_UPDATED) + rstatus = MESHMS_STATUS_UPDATED; } - - if (update_conversations(my_sid, conv->_right)) - ret=1; - - return ret; + return rstatus; } // read our cached conversation list from our rhizome payload // if we can't load the existing data correctly, just ignore it. -static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct conversations **conv) +static enum meshms_status read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct meshms_conversations **conv) { if (m->haveSecret==NEW_BUNDLE_ID) - return 0; - + return MESHMS_STATUS_OK; + struct rhizome_read read; bzero(&read, sizeof(read)); struct rhizome_read_buffer buff; bzero(&buff, sizeof(buff)); - - int ret = -1; + + enum meshms_status status = MESHMS_STATUS_ERROR; enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &read); - if (pstatus != RHIZOME_PAYLOAD_STATUS_STORED) + if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) { + WARNF("Payload was not found for manifest %s, %"PRIu64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); + goto fault; + } + if (pstatus != RHIZOME_PAYLOAD_STATUS_STORED && pstatus != RHIZOME_PAYLOAD_STATUS_EMPTY) goto end; - + unsigned char version=0xFF; ssize_t r = rhizome_read_buffered(&read, &buff, &version, 1); if (r == -1) goto end; if (version != 1) { WARNF("Expected version 1 (got 0x%02x)", version); - goto end; + goto fault; } - while (1) { sid_t sid; r = rhizome_read_buffered(&read, &buff, sid.binary, sizeof sid.binary); + if (r == 0) { + status = MESHMS_STATUS_OK; + goto end; + } if (r != sizeof sid.binary) break; if (config.debug.meshms) DEBUGF("Reading existing conversation for %s", alloca_tohex_sid_t(sid)); if (their_sid && cmp_sid_t(&sid, their_sid) != 0) continue; - struct conversations *ptr = add_conv(conv, &sid); + struct meshms_conversations *ptr = add_conv(conv, &sid); if (!ptr) goto end; unsigned char details[8*3]; @@ -589,13 +581,14 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, ofs += unpacked; read.offset += ofs - bytes; } - ret = 0; +fault: + status = MESHMS_STATUS_PROTOCOL_FAULT; end: rhizome_read_close(&read); - return ret; + return status; } -static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv) +static ssize_t write_conversation(struct rhizome_write *write, struct meshms_conversations *conv) { size_t len=0; if (!conv) @@ -637,13 +630,13 @@ static ssize_t write_conversation(struct rhizome_write *write, struct conversati return len; } -static int write_known_conversations(rhizome_manifest *m, struct conversations *conv) +static enum meshms_status write_known_conversations(rhizome_manifest *m, struct meshms_conversations *conv) { rhizome_manifest *mout=NULL; struct rhizome_write write; bzero(&write, sizeof(write)); - int ret=-1; + enum meshms_status status = MESHMS_STATUS_ERROR; // TODO rebalance tree... @@ -669,91 +662,287 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations * goto end; rhizome_manifest_set_filehash(m, &write.id); } - enum rhizome_bundle_status status = rhizome_manifest_finalise(m, &mout, 1); - switch (status) { + enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1); + switch (bstatus) { case RHIZOME_BUNDLE_STATUS_ERROR: // error is already logged break; case RHIZOME_BUNDLE_STATUS_NEW: - ret = 0; + status = MESHMS_STATUS_OK; break; case RHIZOME_BUNDLE_STATUS_SAME: case RHIZOME_BUNDLE_STATUS_DUPLICATE: case RHIZOME_BUNDLE_STATUS_OLD: WHYF("MeshMS conversation manifest (version=%"PRIu64") gazumped by Rhizome store (version=%"PRIu64")", m->version, mout->version); + status = MESHMS_STATUS_PROTOCOL_FAULT; break; case RHIZOME_BUNDLE_STATUS_INCONSISTENT: WHY("MeshMS conversation manifest not consistent with payload"); + status = MESHMS_STATUS_PROTOCOL_FAULT; break; case RHIZOME_BUNDLE_STATUS_FAKE: WHY("MeshMS conversation manifest is not signed"); + status = MESHMS_STATUS_PROTOCOL_FAULT; break; case RHIZOME_BUNDLE_STATUS_INVALID: WHY("MeshMS conversation manifest is invalid"); + status = MESHMS_STATUS_PROTOCOL_FAULT; break; default: - FATALF("status=%d", status); + FATALF("bstatus=%d", bstatus); } end: - if (ret) + if (meshms_failed(status)) rhizome_fail_write(&write); if (mout && m!=mout) rhizome_manifest_free(mout); - return ret; + return status; } // read information about existing conversations from a rhizome payload -static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv) +enum meshms_status meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv) { - int ret=-1; + enum meshms_status status = MESHMS_STATUS_ERROR; rhizome_manifest *m = rhizome_new_manifest(); if (!m) goto end; - if (get_my_conversation_bundle(my_sid, m)) + if (meshms_failed(status = get_my_conversation_bundle(my_sid, m))) goto end; - // read conversations payload - if (read_known_conversations(m, their_sid, conv)) + if (meshms_failed(status = read_known_conversations(m, their_sid, conv))) goto end; - - if (get_database_conversations(my_sid, their_sid, conv)) + if (meshms_failed(status = get_database_conversations(my_sid, their_sid, conv))) goto end; - - if (update_conversations(my_sid, *conv) && !their_sid){ - if (write_known_conversations(m, *conv)) - goto end; - } - ret=0; - + if ((status = update_conversations(my_sid, *conv)) == MESHMS_STATUS_UPDATED && their_sid == NULL) + status = write_known_conversations(m, *conv); end: rhizome_manifest_free(m); - return ret; + if (config.debug.meshms) + DEBUGF("status=%d", status); + return status; } -// recursively traverse the conversation tree in sorted order and output the details of each conversation -static int output_conversations(struct cli_context *context, struct conversations *conv, - int output, int offset, int count){ - if (!conv) - return 0; - - int traverse_count = output_conversations(context, conv->_left, output, offset, count); - if (count <0 || output + traverse_count < offset + count){ - if (output + traverse_count >= offset){ - cli_put_long(context, output + traverse_count, ":"); - cli_put_hexvalue(context, conv->them.binary, sizeof(conv->them), ":"); - cli_put_string(context, conv->read_offset < conv->their_last_message ? "unread":"", ":"); - cli_put_long(context, conv->their_last_message, ":"); - cli_put_long(context, conv->read_offset, "\n"); - } - traverse_count++; +/* Start traversing the given conversation binary tree in infix order. + * + * @author Andrew Bettison + */ +void meshms_conversation_iterator_start(struct meshms_conversation_iterator *it, struct meshms_conversations *conv) +{ + assert(conv->_parent == NULL); // can only iterate over whole tree + it->current = conv; + // infix traversal; descend to the leftmost leaf and start there + while (it->current->_left) + it->current = it->current->_left; +} + +/* Advance to the next conversation in the tree. + * + * @author Andrew Bettison + */ +void meshms_conversation_iterator_advance(struct meshms_conversation_iterator *it) +{ + assert(it->current != NULL); // do not call on a finished iterator + if (it->current->_right) { + it->current = it->current->_right; + while (it->current->_left) + it->current = it->current->_left; } - traverse_count += output_conversations(context, conv->_right, output + traverse_count, offset, count); - return traverse_count; + else { + while (1) { + struct meshms_conversations *conv = it->current; + it->current = it->current->_parent; + if (it->current == NULL || conv == it->current->_left) + break; + assert(conv == it->current->_right); + } + } +} + +enum meshms_status meshms_message_iterator_open(struct meshms_message_iterator *iter, const sid_t *me, const sid_t *them) +{ + enum meshms_status status; + bzero(iter, sizeof *iter); + if (meshms_failed(status = find_or_create_conv(me, them, &iter->_conv))) + goto fail; + assert(iter->_conv != NULL); + iter->_my_sid = *me; + iter->my_sid = &iter->_my_sid; + iter->their_sid = &iter->_conv->them; + iter->my_ply_bid = &iter->_conv->my_ply.bundle_id; + iter->their_ply_bid = &iter->_conv->their_ply.bundle_id; + iter->read_offset = iter->_conv->read_offset; + // If I have never sent a message (or acked any of theirs), there are no messages in the thread. + if (iter->_conv->found_my_ply) { + if ((iter->_my_manifest = rhizome_new_manifest()) == NULL) + goto error; + if (meshms_failed(status = ply_read_open(&iter->_my_reader, &iter->_conv->my_ply.bundle_id, iter->_my_manifest))) + goto fail; + if (iter->_conv->found_their_ply) { + if ((iter->_their_manifest = rhizome_new_manifest()) == NULL) + goto error; + if (meshms_failed(status = ply_read_open(&iter->_their_reader, &iter->_conv->their_ply.bundle_id, iter->_their_manifest))) + goto fail; + // Find their latest ACK so we know which of my messages have been delivered. + if (meshms_failed(status = ply_find_prev(&iter->_their_reader, MESHMS_BLOCK_TYPE_ACK))) + goto fail; + if (status == MESHMS_STATUS_UPDATED) { + if (unpack_uint(iter->_their_reader.record, iter->_their_reader.record_length, &iter->latest_ack_my_offset) == -1) + iter->latest_ack_my_offset = 0; + else + iter->latest_ack_offset = iter->_their_reader.record_end_offset; + if (config.debug.meshms) + DEBUGF("Found their last ack @%"PRId64, iter->latest_ack_my_offset); + } + // Re-seek to end of their ply. + iter->_their_reader.read.offset = iter->_their_reader.read.length; + } + } else { + if (config.debug.meshms) + DEBUGF("Did not find sender's ply; no messages in thread"); + } + iter->_in_ack = 0; + return MESHMS_STATUS_OK; +error: + status = MESHMS_STATUS_ERROR; +fail: + meshms_message_iterator_close(iter); + return status; +} + +int meshms_message_iterator_is_open(const struct meshms_message_iterator *iter) +{ + return iter->_conv != NULL; +} + +void meshms_message_iterator_close(struct meshms_message_iterator *iter) +{ + if (iter->_my_manifest) { + ply_read_close(&iter->_my_reader); + rhizome_manifest_free(iter->_my_manifest); + iter->_my_manifest = NULL; + } + if (iter->_their_manifest){ + ply_read_close(&iter->_their_reader); + rhizome_manifest_free(iter->_their_manifest); + iter->_their_manifest = NULL; + } + meshms_free_conversations(iter->_conv); + iter->_conv = NULL; +} + +enum meshms_status meshms_message_iterator_prev(struct meshms_message_iterator *iter) +{ + assert(iter->_conv != NULL); + if (iter->_conv->found_my_ply) { + assert(iter->_my_manifest != NULL); + if (iter->_conv->found_their_ply) + assert(iter->_their_manifest != NULL); + } + enum meshms_status status = MESHMS_STATUS_UPDATED; + while (status == MESHMS_STATUS_UPDATED) { + if (iter->_in_ack) { + if (config.debug.meshms) + DEBUGF("Reading other log from %"PRId64", to %"PRId64, iter->_their_reader.read.offset, iter->_end_range); + if (meshms_failed(status = ply_read_prev(&iter->_their_reader))) + break; + iter->which_ply = THEIR_PLY; + if (status == MESHMS_STATUS_UPDATED && iter->_their_reader.read.offset >= iter->_end_range) { + switch (iter->_their_reader.type) { + case MESHMS_BLOCK_TYPE_ACK: + iter->type = ACK_RECEIVED; + iter->offset = iter->_their_reader.record_end_offset; + iter->text = NULL; + iter->text_length = 0; + if (unpack_uint(iter->_their_reader.record, iter->_their_reader.record_length, &iter->ack_offset) == -1) + iter->ack_offset = 0; + return status; + case MESHMS_BLOCK_TYPE_MESSAGE: + iter->type = MESSAGE_RECEIVED; + iter->offset = iter->_their_reader.record_end_offset; + iter->text = (const char *)iter->_their_reader.record; + iter->text_length = iter->_their_reader.record_length; + if ( iter->_their_reader.record_length != 0 + && iter->_their_reader.record[iter->_their_reader.record_length - 1] == '\0' + ) { + iter->read = iter->_their_reader.record_end_offset <= iter->_conv->read_offset; + return status; + } + WARN("Malformed MeshMS2 ply journal, missing NUL terminator"); + return MESHMS_STATUS_PROTOCOL_FAULT; + } + continue; + } + iter->_in_ack = 0; + status = MESHMS_STATUS_UPDATED; + } + else if ((status = ply_read_prev(&iter->_my_reader)) == MESHMS_STATUS_UPDATED) { + if (config.debug.meshms) + DEBUGF("Offset %"PRId64", type %d, read_offset %"PRId64, iter->_my_reader.read.offset, iter->_my_reader.type, iter->read_offset); + iter->which_ply = MY_PLY; + switch (iter->_my_reader.type) { + case MESHMS_BLOCK_TYPE_ACK: + // Read the received messages up to the ack'ed offset + if (iter->_conv->found_their_ply) { + int ofs = unpack_uint(iter->_my_reader.record, iter->_my_reader.record_length, (uint64_t*)&iter->_their_reader.read.offset); + if (ofs == -1) { + WHYF("Malformed ACK"); + return MESHMS_STATUS_PROTOCOL_FAULT; + } + uint64_t end_range; + int x = unpack_uint(iter->_my_reader.record + ofs, iter->_my_reader.record_length - ofs, &end_range); + if (x == -1) + iter->_end_range = 0; + else + iter->_end_range = iter->_their_reader.read.offset - end_range; + // TODO tail + // just in case we don't have the full bundle anymore + if (iter->_their_reader.read.offset > iter->_their_reader.read.length) + iter->_their_reader.read.offset = iter->_their_reader.read.length; + iter->_in_ack = 1; + } + break; + case MESHMS_BLOCK_TYPE_MESSAGE: + iter->type = MESSAGE_SENT; + iter->offset = iter->_my_reader.record_end_offset; + iter->text = (const char *)iter->_my_reader.record; + iter->text_length = iter->_my_reader.record_length; + iter->delivered = iter->latest_ack_my_offset && iter->_my_reader.record_end_offset <= iter->latest_ack_my_offset; + return status; + } + } + } + return status; +} + +enum meshms_status meshms_send_message(const sid_t *sender, const sid_t *recipient, const char *message, size_t message_len) +{ + assert(message_len != 0); + if (message_len > MESHMS_MESSAGE_MAX_LEN) { + WHY("message too long"); + return MESHMS_STATUS_ERROR; + } + struct meshms_conversations *conv = NULL; + enum meshms_status status; + if (!meshms_failed(status = find_or_create_conv(sender, recipient, &conv))) { + assert(conv != NULL); + // construct a message payload + // TODO, new format here. + unsigned char buffer[message_len + 4]; + strncpy((char*)buffer, message, message_len); + // ensure message is NUL terminated + if (message[message_len - 1] != '\0') + buffer[message_len++] = '\0'; + message_len += append_footer(buffer + message_len, MESHMS_BLOCK_TYPE_MESSAGE, message_len); + status = append_meshms_buffer(sender, conv, buffer, message_len); + } + meshms_free_conversations(conv); + return status; } // output the list of existing conversations for a given local identity -int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context *context){ +int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context *context) +{ const char *sidhex, *offset_str, *count_str; if (cli_arg(parsed, "sid", &sidhex, str_is_subscriber_id, "") == -1 || cli_arg(parsed, "offset", &offset_str, NULL, "0")==-1 @@ -775,20 +964,36 @@ int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context return -1; } - struct conversations *conv=NULL; - if (meshms_conversations_list(&sid, NULL, &conv)){ + struct meshms_conversations *conv=NULL; + enum meshms_status status; + if (meshms_failed(status = meshms_conversations_list(&sid, NULL, &conv))) { keyring_free(keyring); - return -1; - } + return status; + } const char *names[]={ "_id","recipient","read", "last_message", "read_offset" }; cli_columns(context, 5, names); - int rows = output_conversations(context, conv, 0, offset, count); + int rows = 0; + if (conv) { + struct meshms_conversation_iterator it; + for (meshms_conversation_iterator_start(&it, conv); + it.current && (count < 0 || rows < offset + count); + meshms_conversation_iterator_advance(&it), ++rows + ) { + if (rows >= offset) { + cli_put_long(context, rows, ":"); + cli_put_hexvalue(context, it.current->them.binary, sizeof(it.current->them), ":"); + cli_put_string(context, it.current->read_offset < it.current->their_last_message ? "unread":"", ":"); + cli_put_long(context, it.current->their_last_message, ":"); + cli_put_long(context, it.current->read_offset, "\n"); + } + } + } cli_row_count(context, rows); - free_conversations(conv); + meshms_free_conversations(conv); keyring_free(keyring); return 0; } @@ -815,23 +1020,10 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context return WHY("invalid sender SID"); if (str_to_sid_t(&their_sid, their_sidhex) == -1) return WHY("invalid recipient SID"); - struct conversations *conv = find_or_create_conv(&my_sid, &their_sid); - if (!conv) { - keyring_free(keyring); - return -1; - } - // construct a message payload - int message_len = strlen(message)+1; - - // TODO, new format here. - unsigned char buffer[message_len+3]; - strcpy((char*)buffer, message); // message - message_len+=append_footer(buffer+message_len, MESHMS_BLOCK_TYPE_MESSAGE, message_len); - int ret = append_meshms_buffer(&my_sid, conv, buffer, message_len); - - free_conversations(conv); + // include terminating NUL + enum meshms_status status = meshms_send_message(&my_sid, &their_sid, message, strlen(message) + 1); keyring_free(keyring); - return ret; + return meshms_failed(status) ? status : 0; } int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context) @@ -840,7 +1032,6 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1 || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1) return -1; - if (create_serval_instance_dir() == -1) return -1; if (!(keyring = keyring_open_instance_cli(parsed))) @@ -848,7 +1039,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context if (rhizome_opendb() == -1){ keyring_free(keyring); return -1; - } + } sid_t my_sid, their_sid; if (str_to_sid_t(&my_sid, my_sidhex) == -1){ keyring_free(keyring); @@ -857,147 +1048,65 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context if (str_to_sid_t(&their_sid, their_sidhex) == -1){ keyring_free(keyring); return WHY("invalid recipient SID"); - } - struct conversations *conv=find_or_create_conv(&my_sid, &their_sid); - if (!conv){ - keyring_free(keyring); - return -1; } - int ret=-1; - + struct meshms_message_iterator iter; + enum meshms_status status; + if (meshms_failed(status = meshms_message_iterator_open(&iter, &my_sid, &their_sid))) { + keyring_free(keyring); + return status; + } const char *names[]={ "_id","offset","type","message" }; - cli_columns(context, 4, names); - - rhizome_manifest *m_ours=NULL, *m_theirs=NULL; - struct ply_read read_ours, read_theirs; - - // if we've never sent a message, (or acked theirs), there is nothing to show - if (!conv->found_my_ply){ - ret=0; - cli_row_count(context, 0); - if (config.debug.meshms) - DEBUGF("Did not find my ply"); - goto end; - } - - // start reading messages from both ply's in reverse order - bzero(&read_ours, sizeof(read_ours)); - bzero(&read_theirs, sizeof(read_theirs)); - - m_ours = rhizome_new_manifest(); - if (!m_ours) - goto end; - if (ply_read_open(&read_ours, &conv->my_ply.bundle_id, m_ours)) - goto end; - - uint64_t their_last_ack=0; - uint64_t their_ack_offset=0; - int64_t unread_mark=conv->read_offset; - - if (conv->found_their_ply){ - m_theirs = rhizome_new_manifest(); - if (!m_theirs) - goto end; - if (ply_read_open(&read_theirs, &conv->their_ply.bundle_id, m_theirs)) - goto end; - - // find their last ACK so we know if messages have been received - int r = ply_find_next(&read_theirs, MESHMS_BLOCK_TYPE_ACK); - if (r==0){ - if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack) == -1) - their_last_ack=0; - else - their_ack_offset = read_theirs.record_end_offset; - if (config.debug.meshms) - DEBUGF("Found their last ack @%"PRId64, their_last_ack); - } - } - - int id=0; - while(ply_read_next(&read_ours)==0){ - if (config.debug.meshms) - DEBUGF("Offset %"PRId64", type %d, read_offset %"PRId64, read_ours.read.offset, read_ours.type, conv->read_offset); - - if (their_last_ack && their_last_ack >= read_ours.record_end_offset){ - cli_put_long(context, id++, ":"); - cli_put_long(context, their_ack_offset, ":"); - cli_put_string(context, "ACK", ":"); - cli_put_string(context, "delivered", "\n"); - their_last_ack = 0; - } - - switch(read_ours.type){ - case MESHMS_BLOCK_TYPE_ACK: - // read their message list, and insert all messages that are included in the ack range - if (conv->found_their_ply){ - int ofs=unpack_uint(read_ours.buffer, read_ours.record_length, (uint64_t*)&read_theirs.read.offset); - if (ofs == -1) - break; - uint64_t end_range; - int x = unpack_uint(read_ours.buffer+ofs, read_ours.record_length - ofs, &end_range); - if (x == -1) - end_range=0; - else - end_range = read_theirs.read.offset - end_range; - - // TODO tail - // just incase we don't have the full bundle anymore - if (read_theirs.read.offset > read_theirs.read.length) - read_theirs.read.offset = read_theirs.read.length; - - if (config.debug.meshms) - DEBUGF("Reading other log from %"PRId64", to %"PRId64, read_theirs.read.offset, end_range); - while(ply_find_next(&read_theirs, MESHMS_BLOCK_TYPE_MESSAGE)==0){ - if (read_theirs.read.offset < end_range) - break; - - if (unread_mark >= (int64_t)read_theirs.record_end_offset){ - cli_put_long(context, id++, ":"); - cli_put_long(context, unread_mark, ":"); - cli_put_string(context, "MARK", ":"); - cli_put_string(context, "read", "\n"); - unread_mark = -1; - } - - cli_put_long(context, id++, ":"); - cli_put_long(context, read_theirs.record_end_offset, ":"); - cli_put_string(context, "<", ":"); - cli_put_string(context, (char *)read_theirs.buffer, "\n"); - } + bool_t marked_delivered = 0; + bool_t marked_read = 0; + int id = 0; + while ((status = meshms_message_iterator_prev(&iter)) == MESHMS_STATUS_UPDATED) { + switch (iter.type) { + case MESSAGE_SENT: + if (iter.delivered && !marked_delivered){ + cli_put_long(context, id++, ":"); + cli_put_long(context, iter.latest_ack_offset, ":"); + cli_put_string(context, "ACK", ":"); + cli_put_string(context, "delivered", "\n"); + marked_delivered = 1; } - break; - case MESHMS_BLOCK_TYPE_MESSAGE: // TODO new message format here cli_put_long(context, id++, ":"); - cli_put_long(context, read_ours.record_end_offset, ":"); + cli_put_long(context, iter.offset, ":"); cli_put_string(context, ">", ":"); - cli_put_string(context, (char *)read_ours.buffer, "\n"); + cli_put_string(context, iter.text, "\n"); + break; + case ACK_RECEIVED: + break; + case MESSAGE_RECEIVED: + if (iter.read && !marked_read) { + cli_put_long(context, id++, ":"); + cli_put_long(context, iter.read_offset, ":"); + cli_put_string(context, "MARK", ":"); + cli_put_string(context, "read", "\n"); + marked_read = 1; + } + // TODO new message format here + cli_put_long(context, id++, ":"); + cli_put_long(context, iter.offset, ":"); + cli_put_string(context, "<", ":"); + cli_put_string(context, iter.text, "\n"); break; } } - - cli_row_count(context, id); - ret=0; - -end: - if (m_ours){ - rhizome_manifest_free(m_ours); - ply_read_close(&read_ours); - } - if (m_theirs){ - rhizome_manifest_free(m_theirs); - ply_read_close(&read_theirs); - } - free_conversations(conv); + if (!meshms_failed(status)) + cli_row_count(context, id); + meshms_message_iterator_close(&iter); keyring_free(keyring); - return ret; + return status; } -static int mark_read(struct conversations *conv, const sid_t *their_sid, const char *offset_str){ - int ret=0; +// Returns the number of read markers moved. +static unsigned mark_read(struct meshms_conversations *conv, const sid_t *their_sid, const char *offset_str) +{ + unsigned ret=0; if (conv){ int cmp = their_sid ? cmp_sid_t(&conv->them, their_sid) : 0; if (!their_sid || cmp<0){ @@ -1035,7 +1144,7 @@ int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *UN || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, NULL) == -1 || cli_arg(parsed, "offset", &offset_str, NULL, NULL)==-1) return -1; - + if (create_serval_instance_dir() == -1) return -1; if (!(keyring = keyring_open_instance_cli(parsed))) @@ -1043,44 +1152,39 @@ int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *UN if (rhizome_opendb() == -1){ keyring_free(keyring); return -1; - } + } sid_t my_sid, their_sid; fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary)); if (their_sidhex) fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary)); - - int ret=-1; - struct conversations *conv=NULL; + + enum meshms_status status = MESHMS_STATUS_ERROR; + struct meshms_conversations *conv = NULL; rhizome_manifest *m = rhizome_new_manifest(); if (!m) goto end; - if (get_my_conversation_bundle(&my_sid, m)) + if (meshms_failed(status = get_my_conversation_bundle(&my_sid, m))) goto end; - // read all conversations, so we can write them again - if (read_known_conversations(m, NULL, &conv)) + if (meshms_failed(status = read_known_conversations(m, NULL, &conv))) goto end; - // read the full list of conversations from the database too - if (get_database_conversations(&my_sid, NULL, &conv)) + if (meshms_failed(status = get_database_conversations(&my_sid, NULL, &conv))) goto end; - // check if any incoming conversations need to be acked or have new messages and update the read offset - int changed = update_conversations(&my_sid, conv); + int changed = 0; + if (meshms_failed(status = update_conversations(&my_sid, conv))) + goto end; + if (status == MESHMS_STATUS_UPDATED) + changed = 1; if (mark_read(conv, their_sidhex?&their_sid:NULL, offset_str)) changed =1; - if (changed){ - // save the conversation list - if (write_known_conversations(m, conv)) - goto end; - } - - ret=0; - + if (changed) + status = write_known_conversations(m, conv); end: if (m) rhizome_manifest_free(m); - free_conversations(conv); + meshms_free_conversations(conv); keyring_free(keyring); - return ret; + return status; } diff --git a/meshms.h b/meshms.h new file mode 100644 index 00000000..c4d78f09 --- /dev/null +++ b/meshms.h @@ -0,0 +1,184 @@ +/* +Serval DNA MeshMS +Copyright (C) 2014 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVAL_DNA__MESHMS_H +#define __SERVAL_DNA__MESHMS_H + +#ifndef __MESHMS_INLINE +# if __GNUC__ && !__GNUC_STDC_INLINE__ +# define __MESHMS_INLINE extern inline +# else +# define __MESHMS_INLINE inline +# endif +#endif + +#include "serval.h" +#include "rhizome.h" + +#define MESHMS_MESSAGE_MAX_LEN 4095 + +/* The result of a MeshMS operation. Negative indicates failure, zero or + * positive success. + */ +enum meshms_status { + MESHMS_STATUS_ERROR = -1, // unexpected error (underlying failure) + MESHMS_STATUS_OK = 0, // operation succeeded, no bundle changed + MESHMS_STATUS_UPDATED = 1, // operation succeeded, bundle updated + MESHMS_STATUS_SID_LOCKED = 2, // cannot decode or send messages for that SID + MESHMS_STATUS_PROTOCOL_FAULT = 3, // missing or faulty ply bundle +}; + +__MESHMS_INLINE int meshms_failed(enum meshms_status status) { + return status != MESHMS_STATUS_OK && status != MESHMS_STATUS_UPDATED; +} + +// the manifest details for one half of a conversation +struct meshms_ply { + rhizome_bid_t bundle_id; + uint64_t version; + uint64_t tail; + uint64_t size; +}; + +struct meshms_conversations { + // binary tree + struct meshms_conversations *_left; + struct meshms_conversations *_right; + // keeping a pointer to parent node here means the traversal iterator does not need a stack, so + // there is no fixed limit on the tree depth + struct meshms_conversations *_parent; + + // who are we talking to? + sid_t them; + + char found_my_ply; + struct meshms_ply my_ply; + + char found_their_ply; + struct meshms_ply their_ply; + + // what is the offset of their last message + uint64_t their_last_message; + // what is the last message we marked as read + uint64_t read_offset; + // our cached value for the last known size of their ply + uint64_t their_size; +}; + +// cursor state for reading one half of a conversation +struct meshms_ply_read { + // rhizome payload + struct rhizome_read read; + // block buffer + struct rhizome_read_buffer buff; + // details of the current record + uint64_t record_end_offset; + uint16_t record_length; + size_t record_size; + char type; + // raw record data + unsigned char *record; +}; + +/* Fetch the list of all MeshMS conversations into a binary tree whose nodes + * are all allocated by malloc(3). + */ +enum meshms_status meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv); +void meshms_free_conversations(struct meshms_conversations *conv); + +/* For iterating over a binary tree of all MeshMS conversations, as created by + * meshms_conversations_list(). + * + * struct meshms_conversation_iterator it; + * meshms_conversation_iterator_start(&it, conv); + * while (it.current) { + * ... + * meshms_conversation_iterator_advance(&it); + * } + */ +struct meshms_conversation_iterator { + struct meshms_conversations *current; +}; +void meshms_conversation_iterator_start(struct meshms_conversation_iterator *, struct meshms_conversations *); +void meshms_conversation_iterator_advance(struct meshms_conversation_iterator *); + +/* For iterating through the messages in a single MeshMS conversation; both + * plys threaded (interleaved) in the order as seen by the sender. The + * meshms_message_iterator_prev() function returns MESHMS_STATUS_UPDATED if it + * advances the iterator to a message, or MESHMS_STATUS_OK if there are no more + * messages. Any other return value indicates failure. + * + * struct meshms_message_iterator it; + * enum meshms_status status; + * if (meshms_failed(status = meshms_message_iterator_open(&it, &sender_sid, &recip_sid))) + * return -1; + * while ((status = meshms_message_iterator_prev(&it)) == MESHMS_STATUS_UPDATED) { + * ... + * } + * meshms_message_iterator_close(&it); + * if (meshms_failed(status)) + * return -1; + * ... + */ +struct meshms_message_iterator { + // Public fields that remain fixed for the life of the iterator: + const sid_t *my_sid; + const sid_t *their_sid; + const rhizome_bid_t *my_ply_bid; + const rhizome_bid_t *their_ply_bid; + uint64_t latest_ack_offset; // offset in remote (their) ply of most recent ACK + uint64_t latest_ack_my_offset; // offset in my ply of most recent message ACKed by them + uint64_t read_offset; // offset in remote (their) ply of most recent message read by me + // The following public fields change per message: + enum meshms_which_ply { MY_PLY, THEIR_PLY } which_ply; + enum { MESSAGE_SENT, MESSAGE_RECEIVED, ACK_RECEIVED } type; + // For MESSAGE_SENT 'offset' is the byte position within the local ply + // (mine). For MESSAGE_RECEIVED and ACK_RECEIVED, it is the byte position + // within the remote ply (theirs). + uint64_t offset; + const char *text; // text of UTF8 message (NUL terminated) + size_t text_length; // excluding terminating NUL + union { + bool_t delivered; // for MESSAGE_SENT + bool_t read; // for MESSAGE_RECEIVED + uint64_t ack_offset; // for ACK_RECEIVED + }; + // Private implementation -- could change, so don't use them. + sid_t _my_sid; + struct meshms_conversations *_conv; + rhizome_manifest *_my_manifest; + rhizome_manifest *_their_manifest; + struct meshms_ply_read _my_reader; + struct meshms_ply_read _their_reader; + uint64_t _end_range; + bool_t _in_ack; +}; +enum meshms_status meshms_message_iterator_open(struct meshms_message_iterator *, const sid_t *me, const sid_t *them); +int meshms_message_iterator_is_open(const struct meshms_message_iterator *); +void meshms_message_iterator_close(struct meshms_message_iterator *); +enum meshms_status meshms_message_iterator_prev(struct meshms_message_iterator *); + +/* Append a message ('message_len' bytes of UTF8 at 'message') to the sender's + * ply in the conversation between 'sender' and 'recipient'. If no + * conversation (ply bundle) exists, then create it. Returns 0 on success, -1 + * on error (already logged). + */ +enum meshms_status meshms_send_message(const sid_t *sender, const sid_t *recipient, const char *message, size_t message_len); + +#endif // __SERVAL_DNA__MESHMS_H diff --git a/meshms_restful.c b/meshms_restful.c new file mode 100644 index 00000000..7dbd7bdb --- /dev/null +++ b/meshms_restful.c @@ -0,0 +1,558 @@ +/* +Serval DNA MeshMS HTTP RESTful interface +Copyright (C) 2013,2014 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "conf.h" +#include "serval.h" +#include "httpd.h" +#include "strbuf_helpers.h" + +static void finalise_union_meshms_conversationlist(httpd_request *r) +{ + meshms_free_conversations(r->u.mclist.conv); + r->u.mclist.conv = NULL; +} + +static void finalise_union_meshms_messagelist(httpd_request *r) +{ + meshms_message_iterator_close(&r->u.msglist.iter); +} + +static void finalise_union_meshms_sendmessage(httpd_request *r) +{ + form_buf_malloc_release(&r->u.sendmsg.message); +} + +#define MESHMS_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(rhizome_bid_t) + sizeof(uint64_t))) +#define alloca_meshms_token(bid, offset) meshms_token_to_str(alloca(MESHMS_TOKEN_STRLEN + 1), (bid), (offset)) + +static char *meshms_token_to_str(char *buf, const rhizome_bid_t *bid, uint64_t offset) +{ + struct iovec iov[2]; + iov[0].iov_base = (void *) bid->binary; + iov[0].iov_len = sizeof bid->binary; + iov[1].iov_base = &offset; + iov[1].iov_len = sizeof offset; + size_t n = base64url_encodev(buf, iov, 2); + assert(n == MESHMS_TOKEN_STRLEN); + buf[n] = '\0'; + return buf; +} + +static int strn_to_meshms_token(const char *str, rhizome_bid_t *bidp, uint64_t *offsetp, const char **afterp) +{ + unsigned char token[sizeof bidp->binary + sizeof *offsetp]; + if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token) + return 0; + memcpy(bidp->binary, token, sizeof bidp->binary); + memcpy(offsetp, token + sizeof bidp->binary, sizeof *offsetp); + return 1; +} + +static int http_request_meshms_response(struct httpd_request *r, uint16_t result, const char *message, enum meshms_status status) +{ + r->http.response.result_extra_label = "meshms_status_code"; + r->http.response.result_extra_value.type = JSON_INTEGER; + r->http.response.result_extra_value.u.integer = status; + switch (status) { + case MESHMS_STATUS_OK: + if (!result) + result = 200; + if (!message) + message = "OK"; + break; + case MESHMS_STATUS_UPDATED: + if (!result) + result = 201; + if (!message) + message = "Updated"; + break; + case MESHMS_STATUS_SID_LOCKED: + if (!result) + result = 403; + if (!message) + message = "Identity unknown"; + break; + case MESHMS_STATUS_PROTOCOL_FAULT: + if (!result) + result = 403; + if (!message) + message = "MeshMS protocol fault"; + break; + case MESHMS_STATUS_ERROR: + if (!result) + result = 500; + break; + default: + WHYF("Invalid MeshMS status code %d", status); + result = 500; + break; + } + assert(result != 0); + http_request_simple_response(&r->http, result, message); + return result; +} + +static HTTP_HANDLER restful_meshms_conversationlist_json; +static HTTP_HANDLER restful_meshms_messagelist_json; +static HTTP_HANDLER restful_meshms_newsince_messagelist_json; +static HTTP_HANDLER restful_meshms_sendmessage; + +int restful_meshms_(httpd_request *r, const char *remainder) +{ + r->http.response.header.content_type = CONTENT_TYPE_JSON; + if (!is_rhizome_http_enabled()) + return 403; + const char *verb = HTTP_VERB_GET; + HTTP_HANDLER *handler = NULL; + const char *end; + if (strn_to_sid_t(&r->sid1, remainder, &end) != -1) { + remainder = end; + if (strcmp(remainder, "/conversationlist.json") == 0) { + handler = restful_meshms_conversationlist_json; + remainder = ""; + } else if (*remainder == '/' && strn_to_sid_t(&r->sid2, remainder + 1, &end) != -1) { + remainder = end; + if (strcmp(remainder, "/messagelist.json") == 0) { + handler = restful_meshms_messagelist_json; + remainder = ""; + } + else if ( str_startswith(remainder, "/newsince/", &end) + && strn_to_meshms_token(end, &r->bid, &r->ui64, &end) + && strcmp(end, "/messagelist.json") == 0 + ) { + handler = restful_meshms_newsince_messagelist_json; + remainder = ""; + } + else if (strcmp(remainder, "/sendmessage") == 0) { + handler = restful_meshms_sendmessage; + verb = HTTP_VERB_POST; + remainder = ""; + } + } + } + if (handler == NULL) + return 404; + if (r->http.verb != verb) + return 405; + int ret = authorize(&r->http); + if (ret) + return ret; + ret = handler(r, remainder); + return ret; +} + +static HTTP_CONTENT_GENERATOR restful_meshms_conversationlist_json_content; + +static int restful_meshms_conversationlist_json(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_meshms_conversationlist; + r->u.mclist.phase = LIST_HEADER; + r->u.mclist.rowcount = 0; + r->u.mclist.conv = NULL; + enum meshms_status status; + if (meshms_failed(status = meshms_conversations_list(&r->sid1, NULL, &r->u.mclist.conv))) + return http_request_meshms_response(r, 0, NULL, status); + meshms_conversation_iterator_start(&r->u.mclist.iter, r->u.mclist.conv); + http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_conversationlist_json_content); + return 1; +} + +static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_conversationlist_json_content_chunk; + +static int restful_meshms_conversationlist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) +{ + return generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_meshms_conversationlist_json_content_chunk); +} + +static int restful_meshms_conversationlist_json_content_chunk(struct http_request *hr, strbuf b) +{ + httpd_request *r = (httpd_request *) hr; + // The "my_sid" and "their_sid" per-conversation fields allow the same JSON structure to be used + // in a future, non-SID-specific request, eg, to list all conversations for all currently open + // identities. + const char *headers[] = { + "_id", + "my_sid", + "their_sid", + "read", + "last_message", + "read_offset" + }; + switch (r->u.mclist.phase) { + case LIST_HEADER: + strbuf_puts(b, "{\n\"header\":["); + unsigned i; + for (i = 0; i != NELS(headers); ++i) { + if (i) + strbuf_putc(b, ','); + strbuf_json_string(b, headers[i]); + } + strbuf_puts(b, "],\n\"rows\":["); + if (!strbuf_overrun(b)) + r->u.mclist.phase = LIST_ROWS; + return 1; + case LIST_ROWS: + if (r->u.mclist.iter.current == NULL) { + r->u.mclist.phase = LIST_END; + // fall through... + } else { + if (r->u.mclist.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + strbuf_sprintf(b, "%u", r->u.mclist.rowcount); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->sid1.binary, sizeof r->sid1.binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.mclist.iter.current->them.binary, sizeof r->u.mclist.iter.current->them.binary); + strbuf_putc(b, ','); + strbuf_json_boolean(b, r->u.mclist.iter.current->read_offset >= r->u.mclist.iter.current->their_last_message); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.mclist.iter.current->their_last_message); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.mclist.iter.current->read_offset); + strbuf_puts(b, "]"); + if (!strbuf_overrun(b)) { + meshms_conversation_iterator_advance(&r->u.mclist.iter); + ++r->u.mclist.rowcount; + } + return 1; + } + // fall through... + case LIST_END: + strbuf_puts(b, "\n]\n}\n"); + if (!strbuf_overrun(b)) + r->u.mclist.phase = LIST_DONE; + // fall through... + case LIST_DONE: + return 0; + } + abort(); + return 0; +} + +static HTTP_CONTENT_GENERATOR restful_meshms_messagelist_json_content; + +static enum meshms_status reopen_meshms_message_iterator(httpd_request *r) +{ + if (!meshms_message_iterator_is_open(&r->u.msglist.iter)) { + enum meshms_status status; + if ( meshms_failed(status = meshms_message_iterator_open(&r->u.msglist.iter, &r->sid1, &r->sid2)) + || meshms_failed(status = meshms_message_iterator_prev(&r->u.msglist.iter)) + ) + return status; + r->u.msglist.finished = status != MESHMS_STATUS_UPDATED; + if (!r->u.msglist.finished) { + r->u.msglist.latest_which_ply = r->u.msglist.iter.which_ply; + r->u.msglist.latest_offset = r->u.msglist.iter.offset; + } + } + return MESHMS_STATUS_OK; +} + +static int restful_meshms_messagelist_json(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_meshms_messagelist; + r->u.msglist.rowcount = 0; + r->u.msglist.phase = LIST_HEADER; + r->u.msglist.token_offset = 0; + r->u.msglist.end_time = 0; + enum meshms_status status; + if (meshms_failed(status = reopen_meshms_message_iterator(r))) + return http_request_meshms_response(r, 0, NULL, status); + http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_messagelist_json_content); + return 1; +} + +static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_meshms_messagelist; + r->u.msglist.rowcount = 0; + r->u.msglist.phase = LIST_HEADER; + enum meshms_status status; + if (meshms_failed(status = reopen_meshms_message_iterator(r))) + return http_request_meshms_response(r, 0, NULL, status); + if (cmp_rhizome_bid_t(&r->bid, r->u.msglist.iter.my_ply_bid) == 0) + r->u.msglist.token_which_ply = MY_PLY; + else if (cmp_rhizome_bid_t(&r->bid, r->u.msglist.iter.their_ply_bid) == 0) + r->u.msglist.token_which_ply = THEIR_PLY; + else { + http_request_simple_response(&r->http, 404, "Unmatched token"); + return 404; + } + r->u.msglist.token_offset = r->ui64; + r->u.msglist.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000; + http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_messagelist_json_content); + return 1; +} + +static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_messagelist_json_content_chunk; + +static int restful_meshms_messagelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) +{ + httpd_request *r = (httpd_request *) hr; + if (meshms_failed(reopen_meshms_message_iterator(r))) + return -1; + return generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_meshms_messagelist_json_content_chunk); +} + +static int restful_meshms_messagelist_json_content_chunk(struct http_request *hr, strbuf b) +{ + httpd_request *r = (httpd_request *) hr; + // Include "my_sid" and "their_sid" per-message, so that the same JSON structure can be used by a + // future, non-SID-specific request (eg, to get all messages for all currently open identities). + const char *headers[] = { + "type", + "my_sid", + "their_sid", + "offset", + "token", + "text", + "delivered", + "read", + "ack_offset" + }; + switch (r->u.msglist.phase) { + case LIST_HEADER: + strbuf_puts(b, "{\n"); + if (!r->u.msglist.end_time) { + strbuf_sprintf(b, "\"read_offset\":%"PRIu64",\n\"latest_ack_offset\":%"PRIu64",\n", + r->u.msglist.iter.read_offset, + r->u.msglist.iter.latest_ack_my_offset + ); + } + strbuf_puts(b, "\"header\":["); + unsigned i; + for (i = 0; i != NELS(headers); ++i) { + if (i) + strbuf_putc(b, ','); + strbuf_json_string(b, headers[i]); + } + strbuf_puts(b, "],\n\"rows\":["); + if (!strbuf_overrun(b)) + r->u.msglist.phase = r->u.msglist.finished ? LIST_END : LIST_ROWS; + return 1; + case LIST_ROWS: + { + if ( r->u.msglist.finished + || (r->u.msglist.token_which_ply == r->u.msglist.iter.which_ply && r->u.msglist.iter.offset <= r->u.msglist.token_offset) + ) { + time_ms_t now; + if (r->u.msglist.end_time && (now = gettime_ms()) < r->u.msglist.end_time) { + r->u.msglist.token_which_ply = r->u.msglist.latest_which_ply; + r->u.msglist.token_offset = r->u.msglist.latest_offset; + meshms_message_iterator_close(&r->u.msglist.iter); + time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms; + if (wake_at > r->u.msglist.end_time) + wake_at = r->u.msglist.end_time; + http_request_pause_response(&r->http, wake_at); + return 0; + } + } else { + switch (r->u.msglist.iter.type) { + case MESSAGE_SENT: + if (r->u.msglist.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + strbuf_json_string(b, ">"); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset); + strbuf_putc(b, ','); + strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->my_ply.bundle_id, r->u.msglist.iter.offset)); + strbuf_putc(b, ','); + strbuf_json_string(b, r->u.msglist.iter.text); + strbuf_putc(b, ','); + strbuf_json_boolean(b, r->u.msglist.iter.delivered); + strbuf_putc(b, ','); + strbuf_json_boolean(b, 0); + strbuf_putc(b, ','); + strbuf_json_null(b); + strbuf_puts(b, "]"); + break; + case MESSAGE_RECEIVED: + if (r->u.msglist.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + strbuf_json_string(b, "<"); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset); + strbuf_putc(b, ','); + strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->their_ply.bundle_id, r->u.msglist.iter.offset)); + strbuf_putc(b, ','); + strbuf_json_string(b, r->u.msglist.iter.text); + strbuf_putc(b, ','); + strbuf_json_boolean(b, 1); + strbuf_putc(b, ','); + strbuf_json_boolean(b, r->u.msglist.iter.read); + strbuf_putc(b, ','); + strbuf_json_null(b); + strbuf_puts(b, "]"); + break; + case ACK_RECEIVED: + // Don't send old (irrelevant) ACKs. + if (r->u.msglist.iter.ack_offset > r->u.msglist.highest_ack_offset) { + if (r->u.msglist.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + strbuf_json_string(b, "ACK"); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset); + strbuf_putc(b, ','); + strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->their_ply.bundle_id, r->u.msglist.iter.offset)); + strbuf_putc(b, ','); + strbuf_json_string(b, r->u.msglist.iter.text); + strbuf_putc(b, ','); + strbuf_json_boolean(b, 1); + strbuf_putc(b, ','); + strbuf_json_boolean(b, r->u.msglist.iter.read); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.ack_offset); + strbuf_puts(b, "]"); + r->u.msglist.highest_ack_offset = r->u.msglist.iter.ack_offset; + } + break; + } + if (!strbuf_overrun(b)) { + ++r->u.msglist.rowcount; + enum meshms_status status; + if (meshms_failed(status = meshms_message_iterator_prev(&r->u.msglist.iter))) + return http_request_meshms_response(r, 0, NULL, status); + r->u.msglist.finished = status != MESHMS_STATUS_UPDATED; + } + return 1; + } + r->u.msglist.phase = LIST_END; + } + // fall through... + case LIST_END: + strbuf_puts(b, "\n]\n}\n"); + if (!strbuf_overrun(b)) + r->u.msglist.phase = LIST_DONE; + // fall through... + case LIST_DONE: + return 0; + } + abort(); + return 0; +} + +static HTTP_REQUEST_PARSER restful_meshms_sendmessage_end; +static int send_mime_part_start(struct http_request *); +static int send_mime_part_end(struct http_request *); +static int send_mime_part_header(struct http_request *, const struct mime_part_headers *); +static int send_mime_part_body(struct http_request *, char *, size_t); + +static int restful_meshms_sendmessage(httpd_request *r, const char *remainder) +{ + if (*remainder) + return 404; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_meshms_sendmessage; + // Parse the request body as multipart/form-data. + assert(r->u.sendmsg.current_part == NULL); + assert(!r->u.sendmsg.received_message); + r->http.form_data.handle_mime_part_start = send_mime_part_start; + r->http.form_data.handle_mime_part_end = send_mime_part_end; + r->http.form_data.handle_mime_part_header = send_mime_part_header; + r->http.form_data.handle_mime_body = send_mime_part_body; + // Send the message once the body has arrived. + r->http.handle_content_end = restful_meshms_sendmessage_end; + return 1; +} + +static char PART_MESSAGE[] = "message"; + +static int send_mime_part_start(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + assert(r->u.sendmsg.current_part == NULL); + return 0; +} + +static int send_mime_part_end(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + if (r->u.sendmsg.current_part == PART_MESSAGE) { + if (r->u.sendmsg.message.length == 0) + return http_response_form_part(r, "Invalid (empty)", PART_MESSAGE, NULL, 0); + r->u.sendmsg.received_message = 1; + if (config.debug.httpd) + DEBUGF("received %s = %s", PART_MESSAGE, alloca_toprint(-1, r->u.sendmsg.message.buffer, r->u.sendmsg.message.length)); + } else + FATALF("current_part = %s", alloca_str_toprint(r->u.sendmsg.current_part)); + r->u.sendmsg.current_part = NULL; + return 0; +} + +static int send_mime_part_header(struct http_request *hr, const struct mime_part_headers *h) +{ + httpd_request *r = (httpd_request *) hr; + if (strcmp(h->content_disposition.name, PART_MESSAGE) == 0) { + if (r->u.sendmsg.received_message) + return http_response_form_part(r, "Duplicate", PART_MESSAGE, NULL, 0); + r->u.sendmsg.current_part = PART_MESSAGE; + form_buf_malloc_init(&r->u.sendmsg.message, MESHMS_MESSAGE_MAX_LEN); + } + else + return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0); + return 0; +} + +static int send_mime_part_body(struct http_request *hr, char *buf, size_t len) +{ + httpd_request *r = (httpd_request *) hr; + if (r->u.sendmsg.current_part == PART_MESSAGE) { + form_buf_malloc_accumulate(r, PART_MESSAGE, &r->u.sendmsg.message, buf, len); + } else + FATALF("current_part = %s", alloca_str_toprint(r->u.sendmsg.current_part)); + return 0; +} + +static int restful_meshms_sendmessage_end(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + if (!r->u.sendmsg.received_message) + return http_response_form_part(r, "Missing", PART_MESSAGE, NULL, 0); + assert(r->u.sendmsg.message.length > 0); + assert(r->u.sendmsg.message.length <= MESHMS_MESSAGE_MAX_LEN); + enum meshms_status status; + if (meshms_failed(status = meshms_send_message(&r->sid1, &r->sid2, r->u.sendmsg.message.buffer, r->u.sendmsg.message.length))) + return http_request_meshms_response(r, 0, NULL, status); + return http_request_meshms_response(r, 201, "Message sent", status); +} diff --git a/overlay.c b/overlay.c index 5dbf8533..ad96aa45 100644 --- a/overlay.c +++ b/overlay.c @@ -71,6 +71,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "conf.h" #include "rhizome.h" +#include "httpd.h" #include "strbuf.h" #include "keyring.h" #include "overlay_interface.h" @@ -136,7 +137,7 @@ schedule(&_sched_##X); } } // start the HTTP server if enabled - rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX); + httpd_server_start(HTTPD_PORT, HTTPD_PORT_MAX); // start the dna helper if configured dna_helper_start(); diff --git a/rhizome.c b/rhizome.c index f0264d3f..f40de05d 100644 --- a/rhizome.c +++ b/rhizome.c @@ -53,6 +53,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "conf.h" #include "str.h" #include "rhizome.h" +#include "httpd.h" #include "dataformats.h" int is_rhizome_enabled() @@ -84,7 +85,7 @@ int is_rhizome_advertise_enabled() return config.rhizome.enable && config.rhizome.advertise.enable && rhizome_db - && (is_rhizome_http_server_running() || is_rhizome_mdp_server_running()); + && (is_httpd_server_running() || is_rhizome_mdp_server_running()); } int rhizome_fetch_delay_ms() diff --git a/rhizome.h b/rhizome.h index b67ba248..1899e0bc 100644 --- a/rhizome.h +++ b/rhizome.h @@ -1,5 +1,6 @@ /* -Serval Distributed Numbering Architecture (DNA) +Serval DNA Rhizome file distribution +Copyright (C) 2010-2013 Serval Project Inc. Copyright (C) 2010 Paul Gardner-Stephen This program is free software; you can redistribute it and/or @@ -26,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "uuid.h" #include "str.h" #include "strbuf.h" -#include "http_server.h" #include "nacl.h" #ifndef __RHIZOME_INLINE @@ -55,9 +55,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. // assumed to always be 2^n #define RHIZOME_CRYPT_PAGE_SIZE 4096 -#define RHIZOME_HTTP_PORT 4110 -#define RHIZOME_HTTP_PORT_MAX 4150 - /* Fundamental data type: Rhizome Bundle ID * * @author Andrew Bettison @@ -392,7 +389,6 @@ void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *); #define RHIZOME_SERVICE_MESHMS2 "MeshMS2" extern int64_t rhizome_space; -extern uint16_t rhizome_http_server_port; int log2ll(uint64_t x); int rhizome_configure(); @@ -429,8 +425,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report); int rhizome_manifest_createid(rhizome_manifest *m); int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed); -int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call); - typedef struct sqlite_retry_state { unsigned int limit; // do not retry once elapsed >= limit unsigned int sleep; // number of milliseconds to sleep between retries @@ -689,14 +683,15 @@ struct rhizome_list_cursor { uint64_t rowid_since; // Set by calling the next() function. rhizome_manifest *manifest; - // Private state. + // Private state - implementation that could change. + sqlite_retry_state _retry; sqlite3_stmt *_statement; uint64_t _rowid_current; uint64_t _rowid_last; // for re-opening query }; -int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *); -int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *); +int rhizome_list_open(struct rhizome_list_cursor *); +int rhizome_list_next(struct rhizome_list_cursor *); void rhizome_list_commit(struct rhizome_list_cursor *); void rhizome_list_release(struct rhizome_list_cursor *); @@ -768,118 +763,14 @@ struct rhizome_read uint64_t length; }; -/* Rhizome-specific HTTP request handling. - */ -typedef struct rhizome_http_request -{ - struct http_request http; // MUST BE FIRST ELEMENT - - /* Identify request from others being run. Monotonic counter feeds it. Only - * used for debugging when we write post-.log files for multi-part form - * requests. - */ - unsigned int uuid; - - /* For requests/responses that pertain to a single manifest. - */ - rhizome_manifest *manifest; - - /* Finaliser for union contents (below). - */ - void (*finalise_union)(struct rhizome_http_request *); - - /* Mutually exclusive response arguments. - */ - union { - - /* For receiving Rhizome Direct import request - */ - struct { - // Which part is currently being received - const char *current_part; - // Temporary file currently current part is being written to - int part_fd; - // Which parts have already been received - bool_t received_manifest; - bool_t received_data; - // Name of data file supplied in part's Content-Disposition header, filename - // parameter (if any) - char data_file_name[MIME_FILENAME_MAXLEN + 1]; - } - direct_import; - - /* For receiving RESTful Rhizome insert request - */ - struct { - // Which part is currently being received - const char *current_part; - // Which parts have already been received - bool_t received_author; - bool_t received_secret; - bool_t received_manifest; - bool_t received_payload; - // For storing the "bundle-author" hex SID as we receive it - char author_hex[SID_STRLEN]; - size_t author_hex_len; - sid_t author; - // For storing the "bundle-secret" hex as we receive it - char secret_hex[RHIZOME_BUNDLE_KEY_STRLEN]; - size_t secret_hex_len; - rhizome_bk_t bundle_secret; - // The "force-new" parameter - char force_new_text[5]; // enough for "false" - size_t force_new_text_len; - bool_t force_new; - // For storing the manifest text (malloc/realloc) as we receive it - char *manifest_text; - size_t manifest_text_size; - size_t manifest_len; - // For receiving the payload - enum rhizome_payload_status payload_status; - uint64_t payload_size; - struct rhizome_write write; - } - insert; - - /* For responses that send part or all of a payload. - */ - struct rhizome_read read_state; - - /* For responses that list manifests. - */ - struct { - enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase; - uint64_t rowid_highest; - size_t rowcount; - time_ms_t end_time; - struct rhizome_list_cursor cursor; - } - list; - - } u; - -} rhizome_http_request; - int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, uint64_t offset, size_t count,unsigned char *bytes); -int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid, - int64_t fileLength,int priority); -int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h); -int rhizome_server_free_http_request(rhizome_http_request *r); -int rhizome_server_http_send_bytes(rhizome_http_request *r); -int rhizome_server_parse_http_request(rhizome_http_request *r); -int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response); -int rhizome_server_http_response(rhizome_http_request *r, int result, - const char *mime_type, const char *body, uint64_t bytes); -int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes); -int rhizome_http_server_start(uint16_t port_low, uint16_t port_high); int is_rhizome_enabled(); int is_rhizome_mdp_enabled(); int is_rhizome_http_enabled(); int is_rhizome_advertise_enabled(); int is_rhizome_mdp_server_running(); -int is_rhizome_http_server_running(); typedef struct rhizome_direct_bundle_cursor { /* Where the current fill started */ @@ -918,8 +809,6 @@ int rhizome_direct_get_bars(const rhizome_bid_t *bid_low, const rhizome_bid_t *bid_max, unsigned char *bars_out, int bars_requested); -int rhizome_direct_process_post_multipart_bytes -(rhizome_http_request *r,const char *bytes,int count); typedef struct rhizome_direct_sync_request { struct sched_ent alarm; @@ -1003,18 +892,6 @@ int rhizome_any_fetch_queued(); int rhizome_fetch_status_html(struct strbuf *b); int rhizome_fetch_has_queue_space(unsigned char log2_size); -struct http_response_parts { - uint16_t code; - char *reason; - uint64_t range_start; - uint64_t content_length; - char *content_start; -}; - -#define HTTP_RESPONSE_CONTENT_LENGTH_UNSET UINT64_MAX - -int unpack_http_response(char *response, struct http_response_parts *parts); - /* rhizome storage methods */ int rhizome_exists(const rhizome_filehash_t *hashp); diff --git a/rhizome_database.c b/rhizome_database.c index 9f3da86f..a37ff08f 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1575,7 +1575,7 @@ rollback: * * @author Andrew Bettison */ -int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c) +int rhizome_list_open(struct rhizome_list_cursor *c) { if (config.debug.rhizome) DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64, @@ -1609,18 +1609,19 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c) } if (strbuf_overrun(b)) RETURN(WHYF("SQL command too long: %s", strbuf_str(b))); - c->_statement = sqlite_prepare(retry, strbuf_str(b)); + c->_retry = SQLITE_RETRY_STATE_DEFAULT; + c->_statement = sqlite_prepare(&c->_retry, strbuf_str(b)); if (c->_statement == NULL) RETURN(-1); - if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1) + if (c->service && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1) goto failure; - if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1) + if (c->name && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1) goto failure; - if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1) + if (c->is_sender_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1) goto failure; - if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1) + if (c->is_recipient_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1) goto failure; - if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1) + if (c->_rowid_last && sqlite_bind(&c->_retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1) goto failure; c->manifest = NULL; c->_rowid_current = 0; @@ -1641,7 +1642,7 @@ failure: * * @author Andrew Bettison */ -int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c) +int rhizome_list_next(struct rhizome_list_cursor *c) { if (config.debug.rhizome) DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64, @@ -1654,7 +1655,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c) c->_rowid_last ); IN(); - if (c->_statement == NULL && rhizome_list_open(retry, c) == -1) + if (c->_statement == NULL && rhizome_list_open(c) == -1) RETURN(-1); while (1) { if (c->manifest) { @@ -1662,7 +1663,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c) c->_rowid_current = 0; c->manifest = NULL; } - if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW) + if (sqlite_step_retry(&c->_retry, c->_statement) != SQLITE_ROW) break; assert(sqlite3_column_count(c->_statement) == 6); assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT); diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index 5ec1759b..77cb7d2a 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "rhizome.h" +#include "httpd.h" #include "conf.h" #include "str.h" #include "strbuf.h" @@ -31,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "socket.h" -static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_request *r, char *pathbuf, size_t bufsiz, const char *field) +static int _form_temporary_file_path(struct __sourceloc __whence, httpd_request *r, char *pathbuf, size_t bufsiz, const char *field) { strbuf b = strbuf_local(pathbuf, bufsiz); // TODO: use a temporary directory @@ -45,7 +46,7 @@ static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_r #define form_temporary_file_path(r,buf,field) _form_temporary_file_path(__WHENCE__, (r), (buf), sizeof(buf), (field)) -static void rhizome_direct_clear_temporary_files(rhizome_http_request *r) +static void rhizome_direct_clear_temporary_files(httpd_request *r) { const char *fields[] = { "manifest", "data" }; int i; @@ -59,7 +60,7 @@ static void rhizome_direct_clear_temporary_files(rhizome_http_request *r) static int rhizome_direct_import_end(struct http_request *hr) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; if (!r->u.direct_import.received_manifest) { http_request_simple_response(&r->http, 400, "Missing 'manifest' part"); return 0; @@ -128,7 +129,7 @@ static int rhizome_direct_import_end(struct http_request *hr) int rhizome_direct_enquiry_end(struct http_request *hr) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; if (!r->u.direct_import.received_data) { http_request_simple_response(&r->http, 400, "Missing 'data' part"); return 0; @@ -188,7 +189,7 @@ int rhizome_direct_enquiry_end(struct http_request *hr) static int rhizome_direct_addfile_end(struct http_request *hr) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; // If given a file without a manifest, we should only accept if it we are configured to do so, and // the connection is from localhost. Otherwise people could cause your servald to create // arbitrary bundles, which would be bad. @@ -275,7 +276,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr) if (config.debug.rhizome) DEBUGF("Import sans-manifest appeared to succeed"); /* Respond with the manifest that was added. */ - http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_all_bytes); + http_request_response_static(&r->http, 200, CONTENT_TYPE_TEXT, (const char *)m->manifestdata, m->manifest_all_bytes); /* clean up after ourselves */ if (mout && mout != m) rhizome_manifest_free(mout); @@ -293,7 +294,7 @@ static char PART_DATA[] = "data"; static int rhizome_direct_process_mime_start(struct http_request *hr) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; assert(r->u.direct_import.current_part == NULL); assert(r->u.direct_import.part_fd == -1); return 0; @@ -301,7 +302,7 @@ static int rhizome_direct_process_mime_start(struct http_request *hr) static int rhizome_direct_process_mime_end(struct http_request *hr) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; if (r->u.direct_import.part_fd != -1) { if (close(r->u.direct_import.part_fd) == -1) { WHYF_perror("close(%d)", r->u.direct_import.part_fd); @@ -320,7 +321,7 @@ static int rhizome_direct_process_mime_end(struct http_request *hr) static int rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; if (strcmp(h->content_disposition.name, PART_DATA) == 0) { r->u.direct_import.current_part = PART_DATA; strncpy(r->u.direct_import.data_file_name, @@ -347,7 +348,7 @@ static int rhizome_direct_process_mime_part_header(struct http_request *hr, cons static int rhizome_direct_process_mime_body(struct http_request *hr, char *buf, size_t len) { - rhizome_http_request *r = (rhizome_http_request *) hr; + httpd_request *r = (httpd_request *) hr; if (r->u.direct_import.part_fd != -1) { if (write_all(r->u.direct_import.part_fd, buf, len) == -1) { http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed"); @@ -357,7 +358,7 @@ static int rhizome_direct_process_mime_body(struct http_request *hr, char *buf, return 0; } -int rhizome_direct_import(rhizome_http_request *r, const char *remainder) +int rhizome_direct_import(httpd_request *r, const char *remainder) { if (*remainder) return 404; @@ -374,7 +375,7 @@ int rhizome_direct_import(rhizome_http_request *r, const char *remainder) return 1; } -int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder) +int rhizome_direct_enquiry(httpd_request *r, const char *remainder) { if (*remainder) return 404; @@ -396,7 +397,7 @@ int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder) * loop-holes here, which is part of why we leave it disabled by default, but it will be sufficient * for testing possible uses, including integration with OpenDataKit. */ -int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder) +int rhizome_direct_addfile(httpd_request *r, const char *remainder) { if (*remainder) return 404; @@ -423,7 +424,7 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder) return 1; } -int rhizome_direct_dispatch(rhizome_http_request *r, const char *UNUSED(remainder)) +int rhizome_direct_dispatch(httpd_request *r, const char *UNUSED(remainder)) { if ( config.rhizome.api.addfile.uri_path[0] && strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0 @@ -515,9 +516,9 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) strbuf_sprintf(content_preamble, "--%s\r\n" "Content-Disposition: form-data; name=\"data\"; filename=\"IHAVEs\"\r\n" - "Content-Type: application/octet-stream\r\n" + "Content-Type: %s\r\n" "\r\n", - boundary + boundary, CONTENT_TYPE_BLOB ); strbuf_sprintf(content_postamble, "\r\n--%s--\r\n", boundary); assert(!strbuf_overrun(content_preamble)); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 62ce96f1..79246591 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "conf.h" #include "rhizome.h" +#include "httpd.h" #include "str.h" #include "strbuf_helpers.h" #include "overlay_address.h" diff --git a/rhizome_http.c b/rhizome_http.c index d2480ae4..7c0f8957 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -1,6 +1,6 @@ /* Serval DNA Rhizome HTTP external interface -Copyright (C) 2013 Serval Project Inc. +Copyright (C) 2013,2014 Serval Project Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -17,1197 +17,13 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include -#include -#include -#ifdef HAVE_SYS_FILIO_H -# include -#endif -#include -#include - -#include "serval.h" -#include "overlay_address.h" #include "conf.h" +#include "serval.h" +#include "httpd.h" #include "str.h" #include "strbuf.h" -#include "strbuf_helpers.h" -#include "rhizome.h" -#include "dataformats.h" -#include "http_server.h" -#include "overlay_interface.h" -#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32 - -typedef int HTTP_HANDLER(rhizome_http_request *r, const char *remainder); - -struct http_handler{ - const char *path; - HTTP_HANDLER *parser; -}; - -static HTTP_HANDLER restful_rhizome_bundlelist_json; -static HTTP_HANDLER restful_rhizome_newsince; -static HTTP_HANDLER restful_rhizome_insert; -static HTTP_HANDLER restful_rhizome_; - -static HTTP_HANDLER rhizome_status_page; -static HTTP_HANDLER rhizome_file_page; -static HTTP_HANDLER manifest_by_prefix_page; -static HTTP_HANDLER interface_page; -static HTTP_HANDLER neighbour_page; -static HTTP_HANDLER fav_icon_header; -static HTTP_HANDLER root_page; - -extern HTTP_HANDLER rhizome_direct_import; -extern HTTP_HANDLER rhizome_direct_enquiry; -extern HTTP_HANDLER rhizome_direct_dispatch; - -struct http_handler paths[]={ - {"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json}, - {"/restful/rhizome/newsince/", restful_rhizome_newsince}, - {"/restful/rhizome/insert", restful_rhizome_insert}, - {"/restful/rhizome/", restful_rhizome_}, - {"/rhizome/status", rhizome_status_page}, - {"/rhizome/file/", rhizome_file_page}, - {"/rhizome/import", rhizome_direct_import}, - {"/rhizome/enquiry", rhizome_direct_enquiry}, - {"/rhizome/manifestbyprefix/", manifest_by_prefix_page}, - {"/rhizome/", rhizome_direct_dispatch}, - {"/interface/", interface_page}, - {"/neighbour/", neighbour_page}, - {"/favicon.ico", fav_icon_header}, - {"/", root_page}, -}; - -static int rhizome_dispatch(struct http_request *hr) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - INFOF("RHIZOME HTTP SERVER, %s %s", r->http.verb, r->http.path); - r->http.response.content_generator = NULL; - unsigned i; - for (i = 0; i < NELS(paths); ++i) { - const char *remainder; - if (str_startswith(r->http.path, paths[i].path, &remainder)){ - int result = paths[i].parser(r, remainder); - if (result == -1 || (result >= 200 && result < 600)) - return result; - if (result == 1) - return 0; - if (result) - return WHYF("dispatch function for %s returned invalid result %d", paths[i].path, result); - } - } - return 404; -} - -static HTTP_RENDERER render_manifest_headers; - -struct sched_ent server_alarm; -struct profile_total server_stats = { - .name = "rhizome_server_poll", -}; - -/* - HTTP server and client code for rhizome transfers and rhizome direct. - Selection of either use is made when starting the HTTP server and - specifying the call-back function to use on client connections. - */ - -uint16_t rhizome_http_server_port = 0; -static int rhizome_server_socket = -1; -static int request_count=0; -static time_ms_t rhizome_server_last_start_attempt = -1; - -// Format icon data using: -// od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/ */,/g' -unsigned char favicon_bytes[]={ -0,0,1,0,1,0,16,16,16,0,0,0,0,0,40,1 -,0,0,22,0,0,0,40,0,0,0,16,0,0,0,32,0 -,0,0,1,0,4,0,0,0,0,0,128,0,0,0,0,0 -,0,0,0,0,0,0,16,0,0,0,0,0,0,0,104,158 -,168,0,163,233,247,0,104,161,118,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,17 -,17,17,17,18,34,17,17,18,34,17,17,18,34,17,17,2 -,34,17,17,18,34,17,16,18,34,1,17,17,1,17,1,17 -,1,16,1,16,17,17,17,17,1,17,16,16,17,17,17,17 -,1,17,18,34,17,17,17,16,17,17,2,34,17,17,17,16 -,17,16,18,34,17,17,17,16,17,1,17,1,17,17,17,18 -,34,17,17,16,17,17,17,18,34,17,17,18,34,17,17,18 -,34,17,17,18,34,17,17,16,17,17,17,18,34,17,17,16 -,17,17,17,17,17,0,17,1,17,17,17,17,17,17,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; -int favicon_len=318; - -int is_rhizome_http_server_running() -{ - return rhizome_server_socket != -1; -} - -/* Start the Rhizome HTTP server by creating a socket, binding it to an available port, and - marking it as passive. If called repeatedly and frequently, this function will only try to start - the server after a certain time has elapsed since the last attempt. - Return -1 if an error occurs (message logged). - Return 0 if the server was started. - Return 1 if the server is already started successfully. - Return 2 if the server was not started because it is too soon since last failed attempt. - */ -int rhizome_http_server_start(uint16_t port_low, uint16_t port_high) -{ - if (rhizome_server_socket != -1) - return 1; - - /* Only try to start http server every five seconds. */ - time_ms_t now = gettime_ms(); - if (now < rhizome_server_last_start_attempt + 5000) - return 2; - rhizome_server_last_start_attempt = now; - if (config.debug.rhizome_httpd) - DEBUGF("Starting rhizome HTTP server"); - - uint16_t port; - for (port = port_low; port <= port_high; ++port) { - /* Create a new socket, reusable and non-blocking. */ - if (rhizome_server_socket == -1) { - rhizome_server_socket = socket(AF_INET,SOCK_STREAM,0); - if (rhizome_server_socket == -1) { - WHY_perror("socket"); - goto error; - } - int on=1; - if (setsockopt(rhizome_server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) == -1) { - WHY_perror("setsockopt(REUSEADDR)"); - goto error; - } - if (ioctl(rhizome_server_socket, FIONBIO, (char *)&on) == -1) { - WHY_perror("ioctl(FIONBIO)"); - goto error; - } - } - /* Bind it to the next port we want to try. */ - struct sockaddr_in address; - bzero((char *) &address, sizeof(address)); - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons(port); - if (bind(rhizome_server_socket, (struct sockaddr *) &address, sizeof(address)) == -1) { - if (errno != EADDRINUSE) { - WHY_perror("bind"); - goto error; - } - } else { - /* We bound to a port. The battle is half won. Now we have to successfully listen on that - port, which could also fail with EADDRINUSE, in which case we have to scrap the socket and - create a new one, because once bound, a socket stays bound. - */ - if (listen(rhizome_server_socket, 20) != -1) - goto success; - if (errno != EADDRINUSE) { - WHY_perror("listen"); - goto error; - } - close(rhizome_server_socket); - rhizome_server_socket = -1; - } - } - WHYF("No ports available in range %u to %u", RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX); -error: - if (rhizome_server_socket != -1) { - close(rhizome_server_socket); - rhizome_server_socket = -1; - } - return WHY("Failed to start rhizome HTTP server"); - -success: - if (config.rhizome.http.enable) - INFOF("RHIZOME HTTP SERVER, START port=%"PRIu16" fd=%d", port, rhizome_server_socket); - else - INFOF("HTTP SERVER (LIMITED SERVICE), START port=%"PRIu16" fd=%d", port, rhizome_server_socket); - - rhizome_http_server_port = port; - /* Add Rhizome HTTPd server to list of file descriptors to watch */ - server_alarm.function = rhizome_server_poll; - server_alarm.stats = &server_stats; - server_alarm.poll.fd = rhizome_server_socket; - server_alarm.poll.events = POLLIN; - watch(&server_alarm); - return 0; - -} - -static void finalise_union_read_state(rhizome_http_request *r) -{ - rhizome_read_close(&r->u.read_state); -} - -static void finalise_union_rhizome_insert(rhizome_http_request *r) -{ - if (r->u.insert.manifest_text) { - free(r->u.insert.manifest_text); - r->u.insert.manifest_text = NULL; - } - if (r->u.insert.write.blob_fd != -1) - rhizome_fail_write(&r->u.insert.write); -} - -static void rhizome_server_finalise_http_request(struct http_request *hr) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - if (r->manifest) { - rhizome_manifest_free(r->manifest); - r->manifest = NULL; - } - if (r->finalise_union) { - r->finalise_union(r); - r->finalise_union = NULL; - } - request_count--; -} - -static int rhizome_dispatch(struct http_request *); - -static unsigned int rhizome_http_request_uuid_counter = 0; - -void rhizome_server_poll(struct sched_ent *alarm) -{ - if (alarm->poll.revents & (POLLIN | POLLOUT)) { - struct sockaddr addr; - unsigned int addr_len = sizeof addr; - int sock; - if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) == -1) { - if (errno && errno != EAGAIN) - WARN_perror("accept"); - } else { - struct sockaddr_in *peerip=NULL; - if (addr.sa_family == AF_INET) { - peerip = (struct sockaddr_in *)&addr; // network order - INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u", - addr_len, peerip->sin_family, peerip->sin_port, - ((unsigned char*)&peerip->sin_addr.s_addr)[0], - ((unsigned char*)&peerip->sin_addr.s_addr)[1], - ((unsigned char*)&peerip->sin_addr.s_addr)[2], - ((unsigned char*)&peerip->sin_addr.s_addr)[3] - ); - } else { - INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u data=%s", - addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data) - ); - } - rhizome_http_request *request = emalloc_zero(sizeof(rhizome_http_request)); - if (request == NULL) { - WHY("Cannot respond to HTTP request, out of memory"); - close(sock); - } else { - request_count++; - request->uuid = rhizome_http_request_uuid_counter++; - if (peerip) - request->http.client_sockaddr_in = *peerip; - request->http.handle_headers = rhizome_dispatch; - request->http.debug_flag = &config.debug.rhizome_httpd; - request->http.disable_tx_flag = &config.debug.rhizome_nohttptx; - request->http.finalise = rhizome_server_finalise_http_request; - request->http.free = free; - request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT; - http_request_init(&request->http, sock); - } - } - } - if (alarm->poll.revents & (POLLHUP | POLLERR)) { - INFO("Error on tcp listen socket"); - } -} - -int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call) -{ - IN(); - const char *bufend = buf + len; - const char *p = buf; - size_t tail = read_since_last_call + 4; - if (tail < len) - p = bufend - tail; - int count = 0; - for (; p != bufend; ++p) { - switch (*p) { - case '\n': - if (++count==2) - RETURN(p - buf); - case '\r': // ignore CR - case '\0': // ignore NUL (telnet inserts them) - break; - default: - count = 0; - break; - } - } - RETURN(0); - OUT(); -} - -static int is_from_loopback(const struct http_request *r) -{ - return r->client_sockaddr_in.sin_family == AF_INET - && ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET; -} - -/* Return 1 if the given authorization credentials are acceptable. - * Return 0 if not. - */ -static int is_authorized(const struct http_client_authorization *auth) -{ - if (auth->scheme != BASIC) - return 0; - unsigned i; - for (i = 0; i != config.rhizome.api.restful.users.ac; ++i) { - if ( strcmp(config.rhizome.api.restful.users.av[i].key, auth->credentials.basic.user) == 0 - && strcmp(config.rhizome.api.restful.users.av[i].value.password, auth->credentials.basic.password) == 0 - ) - return 1; - } - return 0; -} - -static int authorize(struct http_request *r) -{ - if (!is_from_loopback(r)) - return 403; - if (!is_authorized(&r->request_header.authorization)) { - r->response.header.www_authenticate.scheme = BASIC; - r->response.header.www_authenticate.realm = "Serval Rhizome"; - return 401; - } - return 0; -} - -#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8)) -#define alloca_list_token(rowid) list_token_to_str(alloca(LIST_TOKEN_STRLEN + 1), (rowid)) - -static char *list_token_to_str(char *buf, uint64_t rowid) -{ - struct iovec iov[2]; - iov[0].iov_base = rhizome_db_uuid.u.binary; - iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; - iov[1].iov_base = &rowid; - iov[1].iov_len = sizeof rowid; - size_t n = base64url_encodev(buf, iov, 2); - assert(n == LIST_TOKEN_STRLEN); - buf[n] = '\0'; - return buf; -} - -static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp) -{ - unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; - if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token) - return 0; - if (cmp_uuid_t(&rhizome_db_uuid, (uuid_t *) &token) != 0) - return 0; - memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); - return 1; -} - -static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content; - -static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder) -{ - if (!is_rhizome_http_enabled()) - return 403; - if (*remainder) - return 404; - if (r->http.verb != HTTP_VERB_GET) - return 405; - int ret = authorize(&r->http); - if (ret) - return ret; - r->u.list.phase = LIST_HEADER; - r->u.list.rowcount = 0; - bzero(&r->u.list.cursor, sizeof r->u.list.cursor); - http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content); - return 1; -} - -static int restful_rhizome_newsince(rhizome_http_request *r, const char *remainder) -{ - if (!is_rhizome_http_enabled()) - return 403; - uint64_t rowid; - const char *end = NULL; - if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0) - return 404; - if (r->http.verb != HTTP_VERB_GET) - return 405; - int ret = authorize(&r->http); - if (ret) - return ret; - r->u.list.phase = LIST_HEADER; - r->u.list.rowcount = 0; - bzero(&r->u.list.cursor, sizeof r->u.list.cursor); - r->u.list.cursor.rowid_since = rowid; - r->u.list.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000; - http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content); - return 1; -} - -static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b) -{ - const char *headers[] = { - ".token", - "_id", - "service", - "id", - "version", - "date", - ".inserttime", - ".author", - ".fromhere", - "filesize", - "filehash", - "sender", - "recipient", - "name" - }; - switch (r->u.list.phase) { - case LIST_HEADER: - strbuf_puts(b, "{\n\"header\":["); - unsigned i; - for (i = 0; i != NELS(headers); ++i) { - if (i) - strbuf_putc(b, ','); - strbuf_json_string(b, headers[i]); - } - strbuf_puts(b, "],\n\"rows\":["); - if (!strbuf_overrun(b)) - r->u.list.phase = LIST_ROWS; - return 1; - case LIST_ROWS: - { - int ret = rhizome_list_next(retry, &r->u.list.cursor); - if (ret == -1) - return -1; - if (ret == 0) { - time_ms_t now; - if (r->u.list.cursor.rowid_since == 0 || (now = gettime_ms()) >= r->u.list.end_time) { - strbuf_puts(b, "\n]\n}\n"); - if (!strbuf_overrun(b)) - r->u.list.phase = LIST_DONE; - return 0; - } - time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms; - if (wake_at > r->u.list.end_time) - wake_at = r->u.list.end_time; - http_request_pause_response(&r->http, wake_at); - return 0; - } - rhizome_manifest *m = r->u.list.cursor.manifest; - assert(m->filesize != RHIZOME_SIZE_UNSET); - rhizome_lookup_author(m); - if (r->u.list.rowcount != 0) - strbuf_putc(b, ','); - strbuf_puts(b, "\n["); - if (m->rowid > r->u.list.rowid_highest) { - strbuf_json_string(b, alloca_list_token(m->rowid)); - r->u.list.rowid_highest = m->rowid; - } else - strbuf_json_null(b); - strbuf_putc(b, ','); - strbuf_sprintf(b, "%"PRIu64, m->rowid); - strbuf_putc(b, ','); - strbuf_json_string(b, m->service); - strbuf_putc(b, ','); - strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); - strbuf_putc(b, ','); - strbuf_sprintf(b, "%"PRIu64, m->version); - strbuf_putc(b, ','); - if (m->has_date) - strbuf_sprintf(b, "%"PRItime_ms_t, m->date); - else - strbuf_json_null(b); - strbuf_putc(b, ','); - strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime); - switch (m->authorship) { - case AUTHOR_LOCAL: - case AUTHOR_AUTHENTIC: - strbuf_json_hex(b, m->author.binary, sizeof m->author.binary); - strbuf_puts(b, ",1,"); - break; - default: - strbuf_json_null(b); - strbuf_puts(b, ",1,"); - break; - } - strbuf_sprintf(b, "%"PRIu64, m->filesize); - strbuf_putc(b, ','); - strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary); - strbuf_putc(b, ','); - strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary); - strbuf_putc(b, ','); - strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary); - strbuf_putc(b, ','); - strbuf_json_string(b, m->name); - strbuf_puts(b, "]"); - if (!strbuf_overrun(b)) { - rhizome_list_commit(&r->u.list.cursor); - ++r->u.list.rowcount; - } - return 1; - } - case LIST_DONE: - return 0; - } - abort(); -} - -static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - assert(bufsz > 0); - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - int ret = rhizome_list_open(&retry, &r->u.list.cursor); - if (ret == -1) - return -1; - strbuf b = strbuf_local((char *)buf, bufsz); - while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) { - if (strbuf_overrun(b)) { - if (config.debug.rhizome) - DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b)); - result->need = strbuf_count(b) + 1 - result->generated; - break; - } - result->generated = strbuf_len(b); - if (ret == 0) - break; - } - rhizome_list_release(&r->u.list.cursor); - return ret; -} - -static HTTP_REQUEST_PARSER restful_rhizome_insert_end; -static int insert_mime_part_start(struct http_request *); -static int insert_mime_part_end(struct http_request *); -static int insert_mime_part_header(struct http_request *, const struct mime_part_headers *); -static int insert_mime_part_body(struct http_request *, char *, size_t); - -static int restful_rhizome_insert(rhizome_http_request *r, const char *remainder) -{ - if (*remainder) - return 404; - if (!is_rhizome_http_enabled()) - return 403; - if (r->http.verb != HTTP_VERB_POST) - return 405; - int ret = authorize(&r->http); - if (ret) - return ret; - // Parse the request body as multipart/form-data. - assert(r->u.insert.current_part == NULL); - assert(!r->u.insert.received_author); - assert(!r->u.insert.received_secret); - assert(!r->u.insert.received_manifest); - assert(!r->u.insert.received_payload); - bzero(&r->u.insert.write, sizeof r->u.insert.write); - r->u.insert.write.blob_fd = -1; - r->finalise_union = finalise_union_rhizome_insert; - r->http.form_data.handle_mime_part_start = insert_mime_part_start; - r->http.form_data.handle_mime_part_end = insert_mime_part_end; - r->http.form_data.handle_mime_part_header = insert_mime_part_header; - r->http.form_data.handle_mime_body = insert_mime_part_body; - // Perform the insert once the body has arrived. - r->http.handle_content_end = restful_rhizome_insert_end; - return 1; -} - -static char PART_MANIFEST[] = "manifest"; -static char PART_PAYLOAD[] = "payload"; -static char PART_AUTHOR[] = "bundle-author"; -static char PART_SECRET[] = "bundle-secret"; - -static int insert_mime_part_start(struct http_request *hr) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - assert(r->u.insert.current_part == NULL); - return 0; -} - -static int http_response_form_part(rhizome_http_request *r, const char *what, const char *partname, const char *text, size_t textlen) -{ - if (config.debug.rhizome) - DEBUGF("%s \"%s\" form part %s", what, partname, text ? alloca_toprint(-1, text, textlen) : ""); - strbuf msg = strbuf_alloca(100); - strbuf_sprintf(msg, "%s \"%s\" form part", what, partname); - http_request_simple_response(&r->http, 403, strbuf_str(msg)); - return 403; -} - -static int insert_make_manifest(rhizome_http_request *r) -{ - if (!r->u.insert.received_manifest) - return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); - if ((r->manifest = rhizome_new_manifest())) { - if (r->u.insert.manifest_len == 0) - return 0; - assert(r->u.insert.manifest_len <= sizeof r->manifest->manifestdata); - memcpy(r->manifest->manifestdata, r->u.insert.manifest_text, r->u.insert.manifest_len); - r->manifest->manifest_all_bytes = r->u.insert.manifest_len; - int n = rhizome_manifest_parse(r->manifest); - switch (n) { - case -1: - break; - case 0: - if (!r->manifest->malformed) - return 0; - // fall through - case 1: - http_request_simple_response(&r->http, 403, "Malformed manifest"); - return 403; - default: - WHYF("rhizome_manifest_parse() returned %d", n); - break; - } - } - return 500; -} - -static int insert_mime_part_header(struct http_request *hr, const struct mime_part_headers *h) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - if (strcmp(h->content_disposition.name, PART_AUTHOR) == 0) { - if (r->u.insert.received_author) - return http_response_form_part(r, "Duplicate", PART_AUTHOR, NULL, 0); - r->u.insert.current_part = PART_AUTHOR; - } - else if (strcmp(h->content_disposition.name, PART_SECRET) == 0) { - if (r->u.insert.received_secret) - return http_response_form_part(r, "Duplicate", PART_SECRET, NULL, 0); - r->u.insert.current_part = PART_SECRET; - } - else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) { - // Reject a request if it has a repeated manifest part. - if (r->u.insert.received_manifest) - return http_response_form_part(r, "Duplicate", PART_MANIFEST, NULL, 0); - assert(r->u.insert.manifest_text == NULL); - assert(r->u.insert.manifest_text_size == 0); - assert(r->u.insert.manifest_len == 0); - if ( strcmp(h->content_type.type, "rhizome-manifest") != 0 - || strcmp(h->content_type.subtype, "text") != 0 - ) - return http_response_form_part(r, "Unsupported Content-Type in", PART_MANIFEST, NULL, 0); - r->u.insert.current_part = PART_MANIFEST; - } - else if (strcmp(h->content_disposition.name, PART_PAYLOAD) == 0) { - // Reject a request if it has a repeated payload part. - if (r->u.insert.received_payload) - return http_response_form_part(r, "Duplicate", PART_PAYLOAD, NULL, 0); - // Reject a request if it has a missing manifest part preceding the payload part. - if (!r->u.insert.received_manifest) - return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); - assert(r->manifest != NULL); - r->u.insert.current_part = PART_PAYLOAD; - // If the manifest does not contain a 'name' field, then assign it from the payload filename. - if ( strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0 - && r->manifest->name == NULL - && *h->content_disposition.filename - ) - rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename); - // Start writing the payload content into the Rhizome store. Note: r->manifest->filesize can be - // RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field. - r->u.insert.payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest); - r->u.insert.payload_size = 0; - switch (r->u.insert.payload_status) { - case RHIZOME_PAYLOAD_STATUS_ERROR: - WHYF("rhizome_write_open_manifest() returned %d", r->u.insert.payload_status); - return 500; - case RHIZOME_PAYLOAD_STATUS_STORED: - // TODO: initialise payload hash so it can be compared with stored payload - break; - default: - break; - } - } - else - return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0); - return 0; -} - -static int accumulate_text(rhizome_http_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len) -{ - if (len) { - size_t newlen = *textlenp + len; - if (newlen > textsiz) { - if (config.debug.rhizome) - DEBUGF("Form part \"%s\" too long, %zu bytes overflows maximum %zu by %zu", - partname, newlen, textsiz, (size_t)(newlen - textsiz) - ); - strbuf msg = strbuf_alloca(100); - strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname); - http_request_simple_response(&r->http, 403, strbuf_str(msg)); - return 0; - } - memcpy(textbuf + *textlenp, buf, len); - *textlenp = newlen; - } - return 1; -} - -static int insert_mime_part_body(struct http_request *hr, char *buf, size_t len) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - if (r->u.insert.current_part == PART_AUTHOR) { - accumulate_text(r, PART_AUTHOR, - r->u.insert.author_hex, - sizeof r->u.insert.author_hex, - &r->u.insert.author_hex_len, - buf, len); - } - else if (r->u.insert.current_part == PART_SECRET) { - accumulate_text(r, PART_SECRET, - r->u.insert.secret_hex, - sizeof r->u.insert.secret_hex, - &r->u.insert.secret_hex_len, - buf, len); - } - else if (r->u.insert.current_part == PART_MANIFEST) { - if (len == 0) - return 0; - size_t newlen = r->u.insert.manifest_len + len; - if (newlen > MAX_MANIFEST_BYTES) { - if (config.debug.rhizome) - DEBUGF("manifest too large, %zu bytes overflows maximum %zu by %zu", - newlen, (size_t)MAX_MANIFEST_BYTES, (size_t)(newlen - MAX_MANIFEST_BYTES) - ); - http_request_simple_response(&r->http, 403, "Manifest size overflow"); - return 403; - } - if (newlen > r->u.insert.manifest_text_size) { - if ((r->u.insert.manifest_text = erealloc(r->u.insert.manifest_text, newlen)) == NULL) - return 500; - r->u.insert.manifest_text_size = newlen; - } - memcpy(r->u.insert.manifest_text + r->u.insert.manifest_len, buf, len); - r->u.insert.manifest_len = newlen; - } - else if (r->u.insert.current_part == PART_PAYLOAD) { - r->u.insert.payload_size += len; - switch (r->u.insert.payload_status) { - case RHIZOME_PAYLOAD_STATUS_NEW: - if (rhizome_write_buffer(&r->u.insert.write, (unsigned char *)buf, len) == -1) - return 500; - break; - case RHIZOME_PAYLOAD_STATUS_STORED: - // TODO: calculate payload hash so it can be compared with stored payload - break; - default: - break; - } - } else - FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part)); - return 0; -} - -static int insert_mime_part_end(struct http_request *hr) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - if (r->u.insert.current_part == PART_AUTHOR) { - if ( r->u.insert.author_hex_len != sizeof r->u.insert.author_hex - || strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, NULL) == -1 - ) - return http_response_form_part(r, "Invalid", PART_AUTHOR, r->u.insert.author_hex, r->u.insert.author_hex_len); - r->u.insert.received_author = 1; - if (config.debug.rhizome) - DEBUGF("received %s = %s", PART_AUTHOR, alloca_tohex_sid_t(r->u.insert.author)); - } - else if (r->u.insert.current_part == PART_SECRET) { - if ( r->u.insert.secret_hex_len != sizeof r->u.insert.secret_hex - || strn_to_rhizome_bk_t(&r->u.insert.bundle_secret, r->u.insert.secret_hex, NULL) == -1 - ) - return http_response_form_part(r, "Invalid", PART_SECRET, r->u.insert.secret_hex, r->u.insert.secret_hex_len); - r->u.insert.received_secret = 1; - if (config.debug.rhizome) - DEBUGF("received %s = %s", PART_SECRET, alloca_tohex_rhizome_bk_t(r->u.insert.bundle_secret)); - } - else if (r->u.insert.current_part == PART_MANIFEST) { - r->u.insert.received_manifest = 1; - int result = insert_make_manifest(r); - if (result) - return result; - if (r->manifest->has_id && r->u.insert.received_secret) - rhizome_apply_bundle_secret(r->manifest, &r->u.insert.bundle_secret); - if (r->manifest->service == NULL) - rhizome_manifest_set_service(r->manifest, RHIZOME_SERVICE_FILE); - if (rhizome_fill_manifest(r->manifest, NULL, r->u.insert.received_author ? &r->u.insert.author: NULL) == -1) { - WHY("rhizome_fill_manifest() failed"); - return 500; - } - if (r->manifest->is_journal) { - http_request_simple_response(&r->http, 403, "Insert not supported for journals"); - return 403; - } - assert(r->manifest != NULL); - } - else if (r->u.insert.current_part == PART_PAYLOAD) { - r->u.insert.received_payload = 1; - if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_NEW) - r->u.insert.payload_status = rhizome_finish_write(&r->u.insert.write); - if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) { - WHYF("rhizome_finish_write() returned status = %d", r->u.insert.payload_status); - return 500; - } - } else - FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part)); - r->u.insert.current_part = NULL; - return 0; -} - -static int restful_rhizome_insert_end(struct http_request *hr) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - if (!r->u.insert.received_manifest) - return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); - if (!r->u.insert.received_payload) - return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0); - // Fill in the missing manifest fields and ensure payload and manifest are consistent. - assert(r->manifest != NULL); - assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET); - switch (r->u.insert.payload_status) { - case RHIZOME_PAYLOAD_STATUS_ERROR: - return 500; - case RHIZOME_PAYLOAD_STATUS_NEW: - if (r->manifest->filesize == RHIZOME_SIZE_UNSET) - rhizome_manifest_set_filesize(r->manifest, r->u.insert.write.file_length); - // fall through - case RHIZOME_PAYLOAD_STATUS_STORED: - // TODO: check that stored hash matches received payload's hash - // fall through - case RHIZOME_PAYLOAD_STATUS_EMPTY: - assert(r->manifest->filesize != RHIZOME_SIZE_UNSET); - if (r->u.insert.payload_size == r->manifest->filesize) - break; - // fall through - case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: - { - strbuf msg = strbuf_alloca(200); - strbuf_sprintf(msg, "Payload size (%"PRIu64") contradicts manifest (filesize=%"PRIu64")", r->u.insert.payload_size, r->manifest->filesize); - http_request_simple_response(&r->http, 403, strbuf_str(msg)); - return 403; - } - case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: - http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest"); - return 403; - case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: - http_request_simple_response(&r->http, 403, "Missing bundle secret"); - return 403; - default: - FATALF("payload_status = %d", r->u.insert.payload_status); - } - // Finalise the manifest and add it to the store. - if (r->manifest->filesize) { - if (!r->manifest->has_filehash) - rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id); - else - assert(cmp_rhizome_filehash_t(&r->u.insert.write.id, &r->manifest->filehash) == 0); - } - if (!rhizome_manifest_validate(r->manifest) || r->manifest->malformed) { - http_request_simple_response(&r->http, 403, "Manifest is malformed"); - return 403; - } - if (!r->manifest->haveSecret) { - http_request_simple_response(&r->http, 403, "Missing bundle secret"); - return 403; - } - rhizome_manifest *mout = NULL; - int result; - switch (rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new)) { - case RHIZOME_BUNDLE_STATUS_NEW: - result = 201; - if (mout && mout != r->manifest) - rhizome_manifest_free(mout); - mout = NULL; - break; - case RHIZOME_BUNDLE_STATUS_SAME: - case RHIZOME_BUNDLE_STATUS_OLD: - case RHIZOME_BUNDLE_STATUS_DUPLICATE: - result = 200; - break; - case RHIZOME_BUNDLE_STATUS_INVALID: - result = 403; - break; - case RHIZOME_BUNDLE_STATUS_ERROR: - default: - result = 500; - break; - } - if (mout && mout != r->manifest) { - rhizome_manifest_free(r->manifest); - r->manifest = mout; - } - if (result >= 400) - return result; - rhizome_authenticate_author(r->manifest); - r->http.render_extra_headers = render_manifest_headers; - http_request_response_static(&r->http, result, "rhizome-manifest/text", - (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes - ); - return 0; -} - -static int rhizome_response_content_init_filehash(rhizome_http_request *r, const rhizome_filehash_t *hash); -static int rhizome_response_content_init_payload(rhizome_http_request *r, rhizome_manifest *); - -static HTTP_CONTENT_GENERATOR rhizome_payload_content; - -static HTTP_HANDLER restful_rhizome_bid_rhm; -static HTTP_HANDLER restful_rhizome_bid_raw_bin; -static HTTP_HANDLER restful_rhizome_bid_decrypted_bin; - -static int restful_rhizome_(rhizome_http_request *r, const char *remainder) -{ - if (!is_rhizome_http_enabled()) - return 403; - HTTP_HANDLER *handler = NULL; - rhizome_bid_t bid; - const char *end; - if (strn_to_rhizome_bid_t(&bid, remainder, &end) != -1) { - if (strcmp(end, ".rhm") == 0) { - handler = restful_rhizome_bid_rhm; - remainder = ""; - } else if (strcmp(end, "/raw.bin") == 0) { - handler = restful_rhizome_bid_raw_bin; - remainder = ""; - } else if (strcmp(end, "/decrypted.bin") == 0) { - handler = restful_rhizome_bid_decrypted_bin; - remainder = ""; - } - } - if (handler == NULL) - return 404; - if (r->http.verb != HTTP_VERB_GET) - return 405; - int ret = authorize(&r->http); - if (ret) - return ret; - if ((r->manifest = rhizome_new_manifest()) == NULL) - return 500; - ret = rhizome_retrieve_manifest(&bid, r->manifest); - if (ret == -1) - return 500; - if (ret == 0) { - rhizome_authenticate_author(r->manifest); - r->http.render_extra_headers = render_manifest_headers; - } else { - assert(r->manifest == NULL); - assert(r->http.render_extra_headers == NULL); - } - ret = handler(r, remainder); - return ret; -} - -static int restful_rhizome_bid_rhm(rhizome_http_request *r, const char *remainder) -{ - if (*remainder || r->manifest == NULL) - return 404; - http_request_response_static(&r->http, 200, "rhizome-manifest/text", - (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes - ); - return 1; -} - -static int restful_rhizome_bid_raw_bin(rhizome_http_request *r, const char *remainder) -{ - if (*remainder || r->manifest == NULL) - return 404; - if (r->manifest->filesize == 0) { - http_request_response_static(&r->http, 200, "application/binary", "", 0); - return 1; - } - int ret = rhizome_response_content_init_filehash(r, &r->manifest->filehash); - if (ret) - return ret; - http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content); - return 1; -} - -static int restful_rhizome_bid_decrypted_bin(rhizome_http_request *r, const char *remainder) -{ - if (*remainder || r->manifest == NULL) - return 404; - if (r->manifest->filesize == 0) { - // TODO use Content Type from manifest (once it is implemented) - http_request_response_static(&r->http, 200, "application/binary", "", 0); - return 1; - } - int ret = rhizome_response_content_init_payload(r, r->manifest); - if (ret) - return ret; - // TODO use Content Type from manifest (once it is implemented) - http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content); - return 1; -} - -static int neighbour_page(rhizome_http_request *r, const char *remainder) -{ - if (r->http.verb != HTTP_VERB_GET) - return 405; - char buf[8*1024]; - strbuf b = strbuf_local(buf, sizeof buf); - sid_t neighbour_sid; - if (str_to_sid_t(&neighbour_sid, remainder) == -1) - return 404; - struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0); - if (!neighbour) - return 404; - strbuf_puts(b, ""); - link_neighbour_status_html(b, neighbour); - strbuf_puts(b, ""); - if (strbuf_overrun(b)) - return -1; - http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); - return 1; -} - -static int interface_page(rhizome_http_request *r, const char *remainder) -{ - if (r->http.verb != HTTP_VERB_GET) - return 405; - char buf[8*1024]; - strbuf b=strbuf_local(buf, sizeof buf); - int index=atoi(remainder); - if (index<0 || index>=OVERLAY_MAX_INTERFACES) - return 404; - strbuf_puts(b, ""); - interface_state_html(b, &overlay_interfaces[index]); - strbuf_puts(b, ""); - if (strbuf_overrun(b)) - return -1; - http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); - return 1; -} - -static int rhizome_status_page(rhizome_http_request *r, const char *remainder) -{ - if (!is_rhizome_http_enabled()) - return 403; - if (*remainder) - return 404; - if (r->http.verb != HTTP_VERB_GET) - return 405; - char buf[32*1024]; - strbuf b = strbuf_local(buf, sizeof buf); - strbuf_puts(b, ""); - strbuf_sprintf(b, "%d HTTP requests
", request_count); - strbuf_sprintf(b, "%d Bundles transferring via MDP
", rhizome_cache_count()); - rhizome_fetch_status_html(b); - strbuf_puts(b, ""); - if (strbuf_overrun(b)) - return -1; - http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b)); - return 1; -} - -static int rhizome_response_content_init_read_state(rhizome_http_request *r) -{ - if (r->u.read_state.length == RHIZOME_SIZE_UNSET && rhizome_read(&r->u.read_state, NULL, 0)) { - rhizome_read_close(&r->u.read_state); - return 404; - } - assert(r->u.read_state.length != RHIZOME_SIZE_UNSET); - r->http.response.header.resource_length = r->u.read_state.length; - if (r->http.request_header.content_range_count > 0) { - assert(r->http.request_header.content_range_count == 1); - struct http_range closed; - unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length); - if (n == 0 || http_range_bytes(&closed, 1) == 0) - return 416; // Request Range Not Satisfiable - r->http.response.header.content_range_start = closed.first; - r->http.response.header.content_length = closed.last - closed.first + 1; - r->u.read_state.offset = closed.first; - } else { - r->http.response.header.content_range_start = 0; - r->http.response.header.content_length = r->http.response.header.resource_length; - r->u.read_state.offset = 0; - } - return 0; -} - -static int rhizome_response_content_init_filehash(rhizome_http_request *r, const rhizome_filehash_t *hash) -{ - bzero(&r->u.read_state, sizeof r->u.read_state); - r->u.read_state.blob_fd = -1; - assert(r->finalise_union == NULL); - r->finalise_union = finalise_union_read_state; - enum rhizome_payload_status status = rhizome_open_read(&r->u.read_state, hash); - switch (status) { - case RHIZOME_PAYLOAD_STATUS_EMPTY: - case RHIZOME_PAYLOAD_STATUS_STORED: - break; - case RHIZOME_PAYLOAD_STATUS_NEW: - return 404; - case RHIZOME_PAYLOAD_STATUS_ERROR: - case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: - case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: - case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: - return -1; - default: - FATALF("status = %d", status); - } - return rhizome_response_content_init_read_state(r); -} - -static int rhizome_response_content_init_payload(rhizome_http_request *r, rhizome_manifest *m) -{ - bzero(&r->u.read_state, sizeof r->u.read_state); - r->u.read_state.blob_fd = -1; - assert(r->finalise_union == NULL); - r->finalise_union = finalise_union_read_state; - enum rhizome_payload_status status = rhizome_open_decrypt_read(m, &r->u.read_state); - switch (status) { - case RHIZOME_PAYLOAD_STATUS_EMPTY: - case RHIZOME_PAYLOAD_STATUS_STORED: - break; - case RHIZOME_PAYLOAD_STATUS_NEW: - return 404; - case RHIZOME_PAYLOAD_STATUS_ERROR: - case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: - case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: - case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: - return -1; - default: - FATALF("status = %d", status); - } - return rhizome_response_content_init_read_state(r); -} - -static int rhizome_payload_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) -{ - // Only read multiples of 4k from disk. - const size_t blocksz = 1 << 12; - // Ask for a large buffer for all future reads. - const size_t preferred_bufsz = 16 * blocksz; - // Reads the next part of the payload into the supplied buffer. - rhizome_http_request *r = (rhizome_http_request *) hr; - assert(r->u.read_state.length != RHIZOME_SIZE_UNSET); - assert(r->u.read_state.offset < r->u.read_state.length); - uint64_t remain = r->u.read_state.length - r->u.read_state.offset; - size_t readlen = bufsz; - if (remain < bufsz) - readlen = remain; - else - readlen &= ~(blocksz - 1); - if (readlen > 0) { - ssize_t n = rhizome_read(&r->u.read_state, buf, readlen); - if (n == -1) - return -1; - result->generated = (size_t) n; - } - assert(r->u.read_state.offset <= r->u.read_state.length); - remain = r->u.read_state.length - r->u.read_state.offset; - result->need = remain < preferred_bufsz ? remain : preferred_bufsz; - return remain ? 1 : 0; -} - -static int rhizome_file_page(rhizome_http_request *r, const char *remainder) +int rhizome_file_page(httpd_request *r, const char *remainder) { /* Stream the specified payload */ if (!is_rhizome_http_enabled()) @@ -1226,11 +42,11 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder) int ret = rhizome_response_content_init_filehash(r, &filehash); if (ret) return ret; - http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content); + http_request_response_generated(&r->http, 200, CONTENT_TYPE_BLOB, rhizome_payload_content); return 1; } -static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder) +int manifest_by_prefix_page(httpd_request *r, const char *remainder) { if (!is_rhizome_http_enabled()) return 403; @@ -1247,81 +63,29 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde if (ret == -1) return 500; if (ret == 0) { - http_request_response_static(&r->http, 200, "application/binary", (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes); + http_request_response_static(&r->http, 200, CONTENT_TYPE_BLOB, (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes); return 1; } return 404; } -static int fav_icon_header(rhizome_http_request *r, const char *remainder) -{ - if (*remainder) - return 404; - http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len); - return 1; -} - -static void render_manifest_headers(struct http_request *hr, strbuf sb) -{ - rhizome_http_request *r = (rhizome_http_request *) hr; - rhizome_manifest *m = r->manifest; - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Id: %s\r\n", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Version: %"PRIu64"\r\n", m->version); - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filesize: %"PRIu64"\r\n", m->filesize); - if (m->filesize != 0) - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filehash: %s\r\n", alloca_tohex_rhizome_filehash_t(m->filehash)); - if (m->has_bundle_key) - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-BK: %s\r\n", alloca_tohex_rhizome_bk_t(m->bundle_key)); - if (m->has_date) - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Date: %"PRIu64"\r\n", m->date); - if (m->name) { - strbuf_puts(sb, "Serval-Rhizome-Bundle-Name: "); - strbuf_append_quoted_string(sb, m->name); - strbuf_puts(sb, "\r\n"); - } - if (m->service) - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Service: %s\r\n", m->service); - assert(m->authorship != AUTHOR_LOCAL); - if (m->authorship == AUTHOR_AUTHENTIC) - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Author: %s\r\n", alloca_tohex_sid_t(m->author)); - assert(m->haveSecret); - { - char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; - rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES); - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Secret: %s\r\n", secret); - } - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Rowid: %"PRIu64"\r\n", m->rowid); - strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Inserttime: %"PRIu64"\r\n", m->inserttime); -} - -static int root_page(rhizome_http_request *r, const char *remainder) +int rhizome_status_page(httpd_request *r, const char *remainder) { + if (!is_rhizome_http_enabled()) + return 403; if (*remainder) return 404; if (r->http.verb != HTTP_VERB_GET) return 405; - char temp[8192]; - strbuf b = strbuf_local(temp, sizeof temp); - strbuf_sprintf(b, "" - "

Hello, I'm %s*


" - "Interfaces;
", - alloca_tohex_sid_t_trunc(my_subscriber->sid, 16)); - int i; - for (i=0;i%d: %s, TX: %d, RX: %d
", - i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count); - } - strbuf_puts(b, "Neighbours;
"); - link_neighbour_short_status_html(b, "/neighbour"); - if (is_rhizome_http_enabled()){ - strbuf_puts(b, "Rhizome Status
"); - } + char buf[32*1024]; + strbuf b = strbuf_local(buf, sizeof buf); + strbuf_puts(b, ""); + strbuf_sprintf(b, "%d HTTP requests
", httpd_request_count); + strbuf_sprintf(b, "%d Bundles transferring via MDP
", rhizome_cache_count()); + rhizome_fetch_status_html(b); strbuf_puts(b, ""); - if (strbuf_overrun(b)) { - WHY("HTTP Root page buffer overrun"); - return 500; - } - http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b)); + if (strbuf_overrun(b)) + return -1; + http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, buf, strbuf_len(b)); return 1; } diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c index 130e562c..285bd391 100644 --- a/rhizome_packetformats.c +++ b/rhizome_packetformats.c @@ -222,7 +222,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm) } ob_limitsize(frame->payload, 800); ob_append_byte(frame->payload, 2); - ob_append_ui16(frame->payload, rhizome_http_server_port); + ob_append_ui16(frame->payload, httpd_server_port); int64_t rowid=0; int count = append_bars(frame->payload, &retry, "SELECT BAR,ROWID FROM MANIFESTS ORDER BY ROWID DESC LIMIT 3", @@ -263,7 +263,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){ goto error; ob_limitsize(frame->payload, 800); ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS); - ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0); + ob_append_ui16(frame->payload, is_rhizome_http_enabled()? httpd_server_port : 0); ob_append_ui16(frame->payload, m->manifest_all_bytes); ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes); ob_append_byte(frame->payload, 0xFF); @@ -292,7 +292,7 @@ int overlay_rhizome_saw_advertisements(struct decode_context *context, struct ov int ad_frame_type=ob_get(f->payload); struct socket_address httpaddr = context->addr; if (httpaddr.addr.sa_family == AF_INET) - httpaddr.inet.sin_port = htons(RHIZOME_HTTP_PORT); + httpaddr.inet.sin_port = htons(HTTPD_PORT); rhizome_manifest *m=NULL; int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads); diff --git a/rhizome_restful.c b/rhizome_restful.c new file mode 100644 index 00000000..1770c76f --- /dev/null +++ b/rhizome_restful.c @@ -0,0 +1,780 @@ +/* +Serval DNA Rhizome HTTP RESTful interface +Copyright (C) 2013,2014 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "conf.h" +#include "serval.h" +#include "httpd.h" +#include "strbuf_helpers.h" + +static HTTP_RENDERER render_manifest_headers; + +static void finalise_union_read_state(httpd_request *r) +{ + rhizome_read_close(&r->u.read_state); +} + +static void finalise_union_rhizome_insert(httpd_request *r) +{ + form_buf_malloc_release(&r->u.insert.manifest); + if (r->u.insert.write.blob_fd != -1) + rhizome_fail_write(&r->u.insert.write); +} + +#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8)) +#define alloca_list_token(rowid) list_token_to_str(alloca(LIST_TOKEN_STRLEN + 1), (rowid)) + +static char *list_token_to_str(char *buf, uint64_t rowid) +{ + struct iovec iov[2]; + iov[0].iov_base = rhizome_db_uuid.u.binary; + iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; + iov[1].iov_base = &rowid; + iov[1].iov_len = sizeof rowid; + size_t n = base64url_encodev(buf, iov, 2); + assert(n == LIST_TOKEN_STRLEN); + buf[n] = '\0'; + return buf; +} + +static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp) +{ + unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; + if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token) + return 0; + if (cmp_uuid_t(&rhizome_db_uuid, (uuid_t *) &token) != 0) + return 0; + memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); + return 1; +} + +static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content; + +int restful_rhizome_bundlelist_json(httpd_request *r, const char *remainder) +{ + r->http.response.header.content_type = CONTENT_TYPE_JSON; + if (!is_rhizome_http_enabled()) + return 403; + if (*remainder) + return 404; + if (r->http.verb != HTTP_VERB_GET) + return 405; + int ret = authorize(&r->http); + if (ret) + return ret; + r->u.rhlist.phase = LIST_HEADER; + r->u.rhlist.rowcount = 0; + bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor); + http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content); + return 1; +} + +static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_rhizome_bundlelist_json_content_chunk; + +static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) +{ + httpd_request *r = (httpd_request *) hr; + int ret = rhizome_list_open(&r->u.rhlist.cursor); + if (ret == -1) + return -1; + ret = generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_rhizome_bundlelist_json_content_chunk); + rhizome_list_release(&r->u.rhlist.cursor); + return ret; +} + +int restful_rhizome_newsince(httpd_request *r, const char *remainder) +{ + r->http.response.header.content_type = CONTENT_TYPE_JSON; + if (!is_rhizome_http_enabled()) + return 403; + uint64_t rowid; + const char *end = NULL; + if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0) + return 404; + if (r->http.verb != HTTP_VERB_GET) + return 405; + int ret = authorize(&r->http); + if (ret) + return ret; + r->u.rhlist.phase = LIST_HEADER; + r->u.rhlist.rowcount = 0; + bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor); + r->u.rhlist.cursor.rowid_since = rowid; + r->u.rhlist.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000; + http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content); + return 1; +} + +static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr, strbuf b) +{ + httpd_request *r = (httpd_request *) hr; + const char *headers[] = { + ".token", + "_id", + "service", + "id", + "version", + "date", + ".inserttime", + ".author", + ".fromhere", + "filesize", + "filehash", + "sender", + "recipient", + "name" + }; + switch (r->u.rhlist.phase) { + case LIST_HEADER: + strbuf_puts(b, "{\n\"header\":["); + unsigned i; + for (i = 0; i != NELS(headers); ++i) { + if (i) + strbuf_putc(b, ','); + strbuf_json_string(b, headers[i]); + } + strbuf_puts(b, "],\n\"rows\":["); + if (!strbuf_overrun(b)) + r->u.rhlist.phase = LIST_ROWS; + return 1; + case LIST_ROWS: + { + int ret = rhizome_list_next(&r->u.rhlist.cursor); + if (ret == -1) + return -1; + if (ret == 0) { + time_ms_t now; + if (r->u.rhlist.cursor.rowid_since == 0 || (now = gettime_ms()) >= r->u.rhlist.end_time) { + r->u.rhlist.phase = LIST_END; + return 1; + } + time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms; + if (wake_at > r->u.rhlist.end_time) + wake_at = r->u.rhlist.end_time; + http_request_pause_response(&r->http, wake_at); + return 0; + } + rhizome_manifest *m = r->u.rhlist.cursor.manifest; + assert(m->filesize != RHIZOME_SIZE_UNSET); + rhizome_lookup_author(m); + if (r->u.rhlist.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + if (m->rowid > r->u.rhlist.rowid_highest) { + strbuf_json_string(b, alloca_list_token(m->rowid)); + r->u.rhlist.rowid_highest = m->rowid; + } else + strbuf_json_null(b); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, m->rowid); + strbuf_putc(b, ','); + strbuf_json_string(b, m->service); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, m->version); + strbuf_putc(b, ','); + if (m->has_date) + strbuf_sprintf(b, "%"PRItime_ms_t, m->date); + else + strbuf_json_null(b); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime); + switch (m->authorship) { + case AUTHOR_LOCAL: + case AUTHOR_AUTHENTIC: + strbuf_json_hex(b, m->author.binary, sizeof m->author.binary); + strbuf_puts(b, ",1,"); + break; + default: + strbuf_json_null(b); + strbuf_puts(b, ",1,"); + break; + } + strbuf_sprintf(b, "%"PRIu64, m->filesize); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary); + strbuf_putc(b, ','); + strbuf_json_string(b, m->name); + strbuf_puts(b, "]"); + if (!strbuf_overrun(b)) { + rhizome_list_commit(&r->u.rhlist.cursor); + ++r->u.rhlist.rowcount; + } + } + return 1; + case LIST_END: + strbuf_puts(b, "\n]\n}\n"); + if (!strbuf_overrun(b)) + r->u.rhlist.phase = LIST_DONE; + // fall through... + case LIST_DONE: + return 0; + } + abort(); +} + +static HTTP_REQUEST_PARSER restful_rhizome_insert_end; +static int insert_mime_part_start(struct http_request *); +static int insert_mime_part_end(struct http_request *); +static int insert_mime_part_header(struct http_request *, const struct mime_part_headers *); +static int insert_mime_part_body(struct http_request *, char *, size_t); + +int restful_rhizome_insert(httpd_request *r, const char *remainder) +{ + r->http.response.header.content_type = CONTENT_TYPE_JSON; + if (*remainder) + return 404; + if (!is_rhizome_http_enabled()) + return 403; + if (r->http.verb != HTTP_VERB_POST) + return 405; + int ret = authorize(&r->http); + if (ret) + return ret; + // Parse the request body as multipart/form-data. + assert(r->u.insert.current_part == NULL); + assert(!r->u.insert.received_author); + assert(!r->u.insert.received_secret); + assert(!r->u.insert.received_manifest); + assert(!r->u.insert.received_payload); + bzero(&r->u.insert.write, sizeof r->u.insert.write); + r->u.insert.write.blob_fd = -1; + r->finalise_union = finalise_union_rhizome_insert; + r->http.form_data.handle_mime_part_start = insert_mime_part_start; + r->http.form_data.handle_mime_part_end = insert_mime_part_end; + r->http.form_data.handle_mime_part_header = insert_mime_part_header; + r->http.form_data.handle_mime_body = insert_mime_part_body; + // Perform the insert once the body has arrived. + r->http.handle_content_end = restful_rhizome_insert_end; + return 1; +} + +static char PART_MANIFEST[] = "manifest"; +static char PART_PAYLOAD[] = "payload"; +static char PART_AUTHOR[] = "bundle-author"; +static char PART_SECRET[] = "bundle-secret"; + +static int insert_mime_part_start(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + assert(r->u.insert.current_part == NULL); + return 0; +} + +static int insert_make_manifest(httpd_request *r) +{ + if (!r->u.insert.received_manifest) + return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); + if ((r->manifest = rhizome_new_manifest())) { + if (r->u.insert.manifest.length == 0) + return 0; + assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata); + memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length); + r->manifest->manifest_all_bytes = r->u.insert.manifest.length; + int n = rhizome_manifest_parse(r->manifest); + switch (n) { + case -1: + break; + case 0: + if (!r->manifest->malformed) + return 0; + // fall through + case 1: + http_request_simple_response(&r->http, 403, "Malformed manifest"); + return 403; + default: + WHYF("rhizome_manifest_parse() returned %d", n); + break; + } + } + return 500; +} + +static int insert_mime_part_header(struct http_request *hr, const struct mime_part_headers *h) +{ + httpd_request *r = (httpd_request *) hr; + if (strcmp(h->content_disposition.name, PART_AUTHOR) == 0) { + if (r->u.insert.received_author) + return http_response_form_part(r, "Duplicate", PART_AUTHOR, NULL, 0); + r->u.insert.current_part = PART_AUTHOR; + assert(r->u.insert.author_hex_len == 0); + } + else if (strcmp(h->content_disposition.name, PART_SECRET) == 0) { + if (r->u.insert.received_secret) + return http_response_form_part(r, "Duplicate", PART_SECRET, NULL, 0); + r->u.insert.current_part = PART_SECRET; + assert(r->u.insert.secret_hex_len == 0); + } + else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) { + // Reject a request if it has a repeated manifest part. + if (r->u.insert.received_manifest) + return http_response_form_part(r, "Duplicate", PART_MANIFEST, NULL, 0); + form_buf_malloc_init(&r->u.insert.manifest, MAX_MANIFEST_BYTES); + if ( strcmp(h->content_type.type, "rhizome-manifest") != 0 + || strcmp(h->content_type.subtype, "text") != 0 + ) + return http_response_form_part(r, "Unsupported Content-Type in", PART_MANIFEST, NULL, 0); + r->u.insert.current_part = PART_MANIFEST; + } + else if (strcmp(h->content_disposition.name, PART_PAYLOAD) == 0) { + // Reject a request if it has a repeated payload part. + if (r->u.insert.received_payload) + return http_response_form_part(r, "Duplicate", PART_PAYLOAD, NULL, 0); + // Reject a request if it has a missing manifest part preceding the payload part. + if (!r->u.insert.received_manifest) + return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); + assert(r->manifest != NULL); + r->u.insert.current_part = PART_PAYLOAD; + // If the manifest does not contain a 'name' field, then assign it from the payload filename. + if ( strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0 + && r->manifest->name == NULL + && *h->content_disposition.filename + ) + rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename); + // Start writing the payload content into the Rhizome store. Note: r->manifest->filesize can be + // RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field. + r->u.insert.payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest); + r->u.insert.payload_size = 0; + switch (r->u.insert.payload_status) { + case RHIZOME_PAYLOAD_STATUS_ERROR: + WHYF("rhizome_write_open_manifest() returned %d", r->u.insert.payload_status); + return 500; + case RHIZOME_PAYLOAD_STATUS_STORED: + // TODO: initialise payload hash so it can be compared with stored payload + break; + default: + break; + } + } + else + return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0); + return 0; +} + +static int insert_mime_part_body(struct http_request *hr, char *buf, size_t len) +{ + httpd_request *r = (httpd_request *) hr; + if (r->u.insert.current_part == PART_AUTHOR) { + accumulate_text(r, PART_AUTHOR, + r->u.insert.author_hex, + sizeof r->u.insert.author_hex, + &r->u.insert.author_hex_len, + buf, len); + } + else if (r->u.insert.current_part == PART_SECRET) { + accumulate_text(r, PART_SECRET, + r->u.insert.secret_hex, + sizeof r->u.insert.secret_hex, + &r->u.insert.secret_hex_len, + buf, len); + } + else if (r->u.insert.current_part == PART_MANIFEST) { + form_buf_malloc_accumulate(r, PART_MANIFEST, &r->u.insert.manifest, buf, len); + } + else if (r->u.insert.current_part == PART_PAYLOAD) { + r->u.insert.payload_size += len; + switch (r->u.insert.payload_status) { + case RHIZOME_PAYLOAD_STATUS_NEW: + if (rhizome_write_buffer(&r->u.insert.write, (unsigned char *)buf, len) == -1) + return 500; + break; + case RHIZOME_PAYLOAD_STATUS_STORED: + // TODO: calculate payload hash so it can be compared with stored payload + break; + default: + break; + } + } else + FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part)); + return 0; +} + +static int insert_mime_part_end(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + if (r->u.insert.current_part == PART_AUTHOR) { + if ( r->u.insert.author_hex_len != sizeof r->u.insert.author_hex + || strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, NULL) == -1 + ) + return http_response_form_part(r, "Invalid", PART_AUTHOR, r->u.insert.author_hex, r->u.insert.author_hex_len); + r->u.insert.received_author = 1; + if (config.debug.rhizome) + DEBUGF("received %s = %s", PART_AUTHOR, alloca_tohex_sid_t(r->u.insert.author)); + } + else if (r->u.insert.current_part == PART_SECRET) { + if ( r->u.insert.secret_hex_len != sizeof r->u.insert.secret_hex + || strn_to_rhizome_bk_t(&r->u.insert.bundle_secret, r->u.insert.secret_hex, NULL) == -1 + ) + return http_response_form_part(r, "Invalid", PART_SECRET, r->u.insert.secret_hex, r->u.insert.secret_hex_len); + r->u.insert.received_secret = 1; + if (config.debug.rhizome) + DEBUGF("received %s = %s", PART_SECRET, alloca_tohex_rhizome_bk_t(r->u.insert.bundle_secret)); + } + else if (r->u.insert.current_part == PART_MANIFEST) { + r->u.insert.received_manifest = 1; + int result = insert_make_manifest(r); + if (result) + return result; + if (r->manifest->has_id && r->u.insert.received_secret) + rhizome_apply_bundle_secret(r->manifest, &r->u.insert.bundle_secret); + if (r->manifest->service == NULL) + rhizome_manifest_set_service(r->manifest, RHIZOME_SERVICE_FILE); + if (rhizome_fill_manifest(r->manifest, NULL, r->u.insert.received_author ? &r->u.insert.author: NULL) == -1) { + WHY("rhizome_fill_manifest() failed"); + return 500; + } + if (r->manifest->is_journal) { + http_request_simple_response(&r->http, 403, "Insert not supported for journals"); + return 403; + } + assert(r->manifest != NULL); + } + else if (r->u.insert.current_part == PART_PAYLOAD) { + r->u.insert.received_payload = 1; + if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_NEW) + r->u.insert.payload_status = rhizome_finish_write(&r->u.insert.write); + if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) { + WHYF("rhizome_finish_write() returned status = %d", r->u.insert.payload_status); + return 500; + } + } else + FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part)); + r->u.insert.current_part = NULL; + return 0; +} + +static int restful_rhizome_insert_end(struct http_request *hr) +{ + httpd_request *r = (httpd_request *) hr; + if (!r->u.insert.received_manifest) + return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0); + if (!r->u.insert.received_payload) + return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0); + // Fill in the missing manifest fields and ensure payload and manifest are consistent. + assert(r->manifest != NULL); + assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET); + switch (r->u.insert.payload_status) { + case RHIZOME_PAYLOAD_STATUS_ERROR: + return 500; + case RHIZOME_PAYLOAD_STATUS_NEW: + if (r->manifest->filesize == RHIZOME_SIZE_UNSET) + rhizome_manifest_set_filesize(r->manifest, r->u.insert.write.file_length); + // fall through + case RHIZOME_PAYLOAD_STATUS_STORED: + // TODO: check that stored hash matches received payload's hash + // fall through + case RHIZOME_PAYLOAD_STATUS_EMPTY: + assert(r->manifest->filesize != RHIZOME_SIZE_UNSET); + if (r->u.insert.payload_size == r->manifest->filesize) + break; + // fall through + case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: + { + strbuf msg = strbuf_alloca(200); + strbuf_sprintf(msg, "Payload size (%"PRIu64") contradicts manifest (filesize=%"PRIu64")", r->u.insert.payload_size, r->manifest->filesize); + http_request_simple_response(&r->http, 403, strbuf_str(msg)); + return 403; + } + case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: + http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest"); + return 403; + case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: + http_request_simple_response(&r->http, 403, "Missing bundle secret"); + return 403; + default: + FATALF("payload_status = %d", r->u.insert.payload_status); + } + // Finalise the manifest and add it to the store. + if (r->manifest->filesize) { + if (!r->manifest->has_filehash) + rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id); + else + assert(cmp_rhizome_filehash_t(&r->u.insert.write.id, &r->manifest->filehash) == 0); + } + if (!rhizome_manifest_validate(r->manifest) || r->manifest->malformed) { + http_request_simple_response(&r->http, 403, "Manifest is malformed"); + return 403; + } + if (!r->manifest->haveSecret) { + http_request_simple_response(&r->http, 403, "Missing bundle secret"); + return 403; + } + rhizome_manifest *mout = NULL; + int result; + switch (rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new)) { + case RHIZOME_BUNDLE_STATUS_NEW: + result = 201; + if (mout && mout != r->manifest) + rhizome_manifest_free(mout); + mout = NULL; + break; + case RHIZOME_BUNDLE_STATUS_SAME: + case RHIZOME_BUNDLE_STATUS_OLD: + case RHIZOME_BUNDLE_STATUS_DUPLICATE: + result = 200; + break; + case RHIZOME_BUNDLE_STATUS_INVALID: + result = 403; + break; + case RHIZOME_BUNDLE_STATUS_ERROR: + default: + result = 500; + break; + } + if (mout && mout != r->manifest) { + rhizome_manifest_free(r->manifest); + r->manifest = mout; + } + if (result >= 400) + return result; + rhizome_authenticate_author(r->manifest); + r->http.render_extra_headers = render_manifest_headers; + http_request_response_static(&r->http, result, "rhizome-manifest/text", + (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes + ); + return 0; +} + +static HTTP_HANDLER restful_rhizome_bid_rhm; +static HTTP_HANDLER restful_rhizome_bid_raw_bin; +static HTTP_HANDLER restful_rhizome_bid_decrypted_bin; + +int restful_rhizome_(httpd_request *r, const char *remainder) +{ + r->http.response.header.content_type = CONTENT_TYPE_JSON; + if (!is_rhizome_http_enabled()) + return 403; + HTTP_HANDLER *handler = NULL; + rhizome_bid_t bid; + const char *end; + if (strn_to_rhizome_bid_t(&bid, remainder, &end) != -1) { + if (strcmp(end, ".rhm") == 0) { + handler = restful_rhizome_bid_rhm; + remainder = ""; + } else if (strcmp(end, "/raw.bin") == 0) { + handler = restful_rhizome_bid_raw_bin; + remainder = ""; + } else if (strcmp(end, "/decrypted.bin") == 0) { + handler = restful_rhizome_bid_decrypted_bin; + remainder = ""; + } + } + if (handler == NULL) + return 404; + if (r->http.verb != HTTP_VERB_GET) + return 405; + int ret = authorize(&r->http); + if (ret) + return ret; + if ((r->manifest = rhizome_new_manifest()) == NULL) + return 500; + ret = rhizome_retrieve_manifest(&bid, r->manifest); + if (ret == -1) + return 500; + if (ret == 0) { + rhizome_authenticate_author(r->manifest); + r->http.render_extra_headers = render_manifest_headers; + } else { + assert(r->manifest == NULL); + assert(r->http.render_extra_headers == NULL); + } + ret = handler(r, remainder); + return ret; +} + +static int restful_rhizome_bid_rhm(httpd_request *r, const char *remainder) +{ + if (*remainder || r->manifest == NULL) + return 404; + http_request_response_static(&r->http, 200, "rhizome-manifest/text", + (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes + ); + return 1; +} + +static int restful_rhizome_bid_raw_bin(httpd_request *r, const char *remainder) +{ + if (*remainder || r->manifest == NULL) + return 404; + if (r->manifest->filesize == 0) { + http_request_response_static(&r->http, 200, CONTENT_TYPE_BLOB, "", 0); + return 1; + } + int ret = rhizome_response_content_init_filehash(r, &r->manifest->filehash); + if (ret) + return ret; + http_request_response_generated(&r->http, 200, CONTENT_TYPE_BLOB, rhizome_payload_content); + return 1; +} + +static int restful_rhizome_bid_decrypted_bin(httpd_request *r, const char *remainder) +{ + if (*remainder || r->manifest == NULL) + return 404; + if (r->manifest->filesize == 0) { + // TODO use Content Type from manifest (once it is implemented) + http_request_response_static(&r->http, 200, CONTENT_TYPE_BLOB, "", 0); + return 1; + } + int ret = rhizome_response_content_init_payload(r, r->manifest); + if (ret) + return ret; + // TODO use Content Type from manifest (once it is implemented) + http_request_response_generated(&r->http, 200, CONTENT_TYPE_BLOB, rhizome_payload_content); + return 1; +} + +static int rhizome_response_content_init_read_state(httpd_request *r) +{ + if (r->u.read_state.length == RHIZOME_SIZE_UNSET && rhizome_read(&r->u.read_state, NULL, 0)) { + rhizome_read_close(&r->u.read_state); + return 404; + } + assert(r->u.read_state.length != RHIZOME_SIZE_UNSET); + r->http.response.header.resource_length = r->u.read_state.length; + if (r->http.request_header.content_range_count > 0) { + assert(r->http.request_header.content_range_count == 1); + struct http_range closed; + unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length); + if (n == 0 || http_range_bytes(&closed, 1) == 0) + return 416; // Request Range Not Satisfiable + r->http.response.header.content_range_start = closed.first; + r->http.response.header.content_length = closed.last - closed.first + 1; + r->u.read_state.offset = closed.first; + } else { + r->http.response.header.content_range_start = 0; + r->http.response.header.content_length = r->http.response.header.resource_length; + r->u.read_state.offset = 0; + } + return 0; +} + +int rhizome_response_content_init_filehash(httpd_request *r, const rhizome_filehash_t *hash) +{ + bzero(&r->u.read_state, sizeof r->u.read_state); + r->u.read_state.blob_fd = -1; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_read_state; + enum rhizome_payload_status status = rhizome_open_read(&r->u.read_state, hash); + switch (status) { + case RHIZOME_PAYLOAD_STATUS_EMPTY: + case RHIZOME_PAYLOAD_STATUS_STORED: + break; + case RHIZOME_PAYLOAD_STATUS_NEW: + return 404; + case RHIZOME_PAYLOAD_STATUS_ERROR: + case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: + case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: + case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: + return -1; + default: + FATALF("status = %d", status); + } + return rhizome_response_content_init_read_state(r); +} + +int rhizome_response_content_init_payload(httpd_request *r, rhizome_manifest *m) +{ + bzero(&r->u.read_state, sizeof r->u.read_state); + r->u.read_state.blob_fd = -1; + assert(r->finalise_union == NULL); + r->finalise_union = finalise_union_read_state; + enum rhizome_payload_status status = rhizome_open_decrypt_read(m, &r->u.read_state); + switch (status) { + case RHIZOME_PAYLOAD_STATUS_EMPTY: + case RHIZOME_PAYLOAD_STATUS_STORED: + break; + case RHIZOME_PAYLOAD_STATUS_NEW: + return 404; + case RHIZOME_PAYLOAD_STATUS_ERROR: + case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE: + case RHIZOME_PAYLOAD_STATUS_WRONG_HASH: + case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL: + return -1; + default: + FATALF("status = %d", status); + } + return rhizome_response_content_init_read_state(r); +} + +int rhizome_payload_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) +{ + // Only read multiples of 4k from disk. + const size_t blocksz = 1 << 12; + // Ask for a large buffer for all future reads. + const size_t preferred_bufsz = 16 * blocksz; + // Reads the next part of the payload into the supplied buffer. + httpd_request *r = (httpd_request *) hr; + assert(r->u.read_state.length != RHIZOME_SIZE_UNSET); + assert(r->u.read_state.offset < r->u.read_state.length); + uint64_t remain = r->u.read_state.length - r->u.read_state.offset; + size_t readlen = bufsz; + if (remain < bufsz) + readlen = remain; + else + readlen &= ~(blocksz - 1); + if (readlen > 0) { + ssize_t n = rhizome_read(&r->u.read_state, buf, readlen); + if (n == -1) + return -1; + result->generated = (size_t) n; + } + assert(r->u.read_state.offset <= r->u.read_state.length); + remain = r->u.read_state.length - r->u.read_state.offset; + result->need = remain < preferred_bufsz ? remain : preferred_bufsz; + return remain ? 1 : 0; +} + +static void render_manifest_headers(struct http_request *hr, strbuf sb) +{ + httpd_request *r = (httpd_request *) hr; + rhizome_manifest *m = r->manifest; + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Id: %s\r\n", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Version: %"PRIu64"\r\n", m->version); + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filesize: %"PRIu64"\r\n", m->filesize); + if (m->filesize != 0) + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filehash: %s\r\n", alloca_tohex_rhizome_filehash_t(m->filehash)); + if (m->has_bundle_key) + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-BK: %s\r\n", alloca_tohex_rhizome_bk_t(m->bundle_key)); + if (m->has_date) + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Date: %"PRIu64"\r\n", m->date); + if (m->name) { + strbuf_puts(sb, "Serval-Rhizome-Bundle-Name: "); + strbuf_append_quoted_string(sb, m->name); + strbuf_puts(sb, "\r\n"); + } + if (m->service) + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Service: %s\r\n", m->service); + assert(m->authorship != AUTHOR_LOCAL); + if (m->authorship == AUTHOR_AUTHENTIC) + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Author: %s\r\n", alloca_tohex_sid_t(m->author)); + assert(m->haveSecret); + { + char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; + rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES); + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Secret: %s\r\n", secret); + } + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Rowid: %"PRIu64"\r\n", m->rowid); + strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Inserttime: %"PRIu64"\r\n", m->inserttime); +} + diff --git a/sourcefiles.mk b/sourcefiles.mk index 976771d1..668e4121 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -14,6 +14,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)fdqueue.c \ $(SERVAL_BASE)fifo.c \ $(SERVAL_BASE)golay.c \ + $(SERVAL_BASE)httpd.c \ $(SERVAL_BASE)http_server.c \ $(SERVAL_BASE)keyring.c \ $(SERVAL_BASE)log.c \ @@ -22,6 +23,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)main.c \ $(SERVAL_BASE)radio_link.c \ $(SERVAL_BASE)meshms.c \ + $(SERVAL_BASE)meshms_restful.c \ $(SERVAL_BASE)mdp_client.c \ $(SERVAL_BASE)mdp_net.c \ $(SERVAL_BASE)msp_client.c \ @@ -58,6 +60,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)rhizome_direct_http.c \ $(SERVAL_BASE)rhizome_fetch.c \ $(SERVAL_BASE)rhizome_http.c \ + $(SERVAL_BASE)rhizome_restful.c \ $(SERVAL_BASE)rhizome_packetformats.c \ $(SERVAL_BASE)rhizome_store.c \ $(SERVAL_BASE)rhizome_sync.c \ diff --git a/str.c b/str.c index c673e721..12fe5830 100644 --- a/str.c +++ b/str.c @@ -128,10 +128,10 @@ static size_t _base64_encodev(const char symbols[], char *dstBase64, const struc if (place) *dst++ = symbols[buf]; switch (place) { - case 2: - *dst++ = symbols[64]; case 1: *dst++ = symbols[64]; + case 2: + *dst++ = symbols[64]; } return dst - dstBase64; } diff --git a/strbuf.h b/strbuf.h index b83c60b2..ff8f27ec 100644 --- a/strbuf.h +++ b/strbuf.h @@ -82,6 +82,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include +#include #ifndef __STRBUF_INLINE # if __GNUC__ && !__GNUC_STDC_INLINE__ @@ -132,6 +133,29 @@ typedef const struct strbuf *const_strbuf; */ #define strbuf_alloca(size) strbuf_make(alloca(SIZEOF_STRBUF + (size)), SIZEOF_STRBUF + (size)) +/** Convenience macro that calls strbuf_alloca() to allocate a large enough + * buffer to hold the entire content produced by a given expression that + * appends to the strbuf. The first strbuf_alloca() will use the supplied + * initial length, and if that overruns, then a second strbuf_alloca() will use + * the strbuf_count() from the first pass, so as long as the expression is + * stable (ie, always produces the same output), the final assert() will not + * be triggered. + * + * strbuf b; + * STRBUF_ALLOCA_FIT(b, 20, (strbuf_append_variable_content(b, ...))); + * + * @author Andrew Bettison + */ +#define STRBUF_ALLOCA_FIT(__SB, __INITIAL_LEN, __EXPR) \ + do { \ + __SB = strbuf_alloca((__INITIAL_LEN) + 1); \ + __EXPR; \ + if (strbuf_overrun(__SB)) { \ + __SB = strbuf_alloca(strbuf_count(__SB) + 1); \ + __EXPR; \ + } \ + assert(!strbuf_overrun(__SB)); \ + } while (0) /** Convenience macro for filling a strbuf from the calling function's * printf(3)-like variadic arguments. diff --git a/strbuf_helpers.c b/strbuf_helpers.c index e2ed5ec3..341aba87 100644 --- a/strbuf_helpers.c +++ b/strbuf_helpers.c @@ -459,42 +459,86 @@ strbuf strbuf_append_quoted_string(strbuf sb, const char *str) return sb; } +static void _html_char(strbuf sb, char c) +{ + if (c == '&') + strbuf_puts(sb, "&"); + else if (c == '<') + strbuf_puts(sb, "<"); + else if (c == '>') + strbuf_puts(sb, ">"); + else if (c == '"') + strbuf_puts(sb, """); + else if (c == '\'') + strbuf_puts(sb, "'"); + else if (iscntrl(c)) + strbuf_sprintf(sb, "&#%u;", (unsigned char) c); + else + strbuf_putc(sb, c); +} + +strbuf strbuf_html_escape(strbuf sb, const char *str, size_t strlen) +{ + for (; strlen; --strlen, ++str) + _html_char(sb, *str); + return sb; +} + strbuf strbuf_json_null(strbuf sb) { strbuf_puts(sb, "null"); return sb; } +strbuf strbuf_json_boolean(strbuf sb, int boolean) +{ + strbuf_puts(sb, boolean ? "true" : "false"); + return sb; +} + +static void _json_char(strbuf sb, char c) +{ + if (c == '"' || c == '\\') { + strbuf_putc(sb, '\\'); + strbuf_putc(sb, c); + } + else if (c == '\b') + strbuf_puts(sb, "\\b"); + else if (c == '\f') + strbuf_puts(sb, "\\f"); + else if (c == '\n') + strbuf_puts(sb, "\\n"); + else if (c == '\r') + strbuf_puts(sb, "\\r"); + else if (c == '\t') + strbuf_puts(sb, "\\t"); + else if (iscntrl(c)) + strbuf_sprintf(sb, "\\u%04X", (unsigned char) c); + else + strbuf_putc(sb, c); +} + strbuf strbuf_json_string(strbuf sb, const char *str) { if (str) { strbuf_putc(sb, '"'); - for (; *str; ++str) { - if (*str == '"' || *str == '\\') { - strbuf_putc(sb, '\\'); - strbuf_putc(sb, *str); - } - else if (*str == '\b') - strbuf_puts(sb, "\\b"); - else if (*str == '\f') - strbuf_puts(sb, "\\f"); - else if (*str == '\n') - strbuf_puts(sb, "\\n"); - else if (*str == '\r') - strbuf_puts(sb, "\\r"); - else if (*str == '\t') - strbuf_puts(sb, "\\t"); - else if (iscntrl(*str)) - strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str); - else - strbuf_putc(sb, *str); - } + for (; *str; ++str) + _json_char(sb, *str); strbuf_putc(sb, '"'); } else strbuf_json_null(sb); return sb; } +strbuf strbuf_json_string_len(strbuf sb, const char *str, size_t strlen) +{ + strbuf_putc(sb, '"'); + for (; strlen; --strlen, ++str) + _json_char(sb, *str); + strbuf_putc(sb, '"'); + return sb; +} + strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len) { if (buf) { @@ -510,6 +554,60 @@ strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len) return sb; } +strbuf strbuf_json_atom(strbuf sb, const struct json_atom *atom) +{ + switch (atom->type) { + case JSON_NULL: + return strbuf_json_null(sb); + case JSON_BOOLEAN: + return strbuf_json_boolean(sb, atom->u.boolean); + case JSON_INTEGER: + strbuf_sprintf(sb, "%"PRId64, atom->u.integer); + return sb; + case JSON_STRING_NULTERM: + return strbuf_json_string(sb, atom->u.string.content); + case JSON_STRING_LENGTH: + return strbuf_json_string_len(sb, atom->u.string.content, atom->u.string.length); + } + abort(); +} + +strbuf strbuf_json_atom_as_text(strbuf sb, const struct json_atom *atom) +{ + switch (atom->type) { + case JSON_NULL: + return strbuf_json_null(sb); + case JSON_BOOLEAN: + return strbuf_puts(sb, atom->u.boolean ? "True" : "False"); + case JSON_INTEGER: + strbuf_sprintf(sb, "%"PRId64, atom->u.integer); + return sb; + case JSON_STRING_NULTERM: + return strbuf_puts(sb, atom->u.string.content); + case JSON_STRING_LENGTH: + return strbuf_ncat(sb, atom->u.string.content, atom->u.string.length); + } + abort(); +} + +strbuf strbuf_json_atom_as_html(strbuf sb, const struct json_atom *atom) +{ + switch (atom->type) { + case JSON_NULL: + return strbuf_json_null(sb); + case JSON_BOOLEAN: + return strbuf_json_boolean(sb, atom->u.boolean); + case JSON_INTEGER: + strbuf_sprintf(sb, "%"PRId64, atom->u.integer); + return sb; + case JSON_STRING_NULTERM: + return strbuf_html_escape(sb, atom->u.string.content, strlen(atom->u.string.content)); + case JSON_STRING_LENGTH: + return strbuf_html_escape(sb, atom->u.string.content, atom->u.string.length); + } + abort(); +} + strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels) { unsigned i; diff --git a/strbuf_helpers.h b/strbuf_helpers.h index 4fee86cb..09ef461b 100644 --- a/strbuf_helpers.h +++ b/strbuf_helpers.h @@ -162,12 +162,33 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt); */ strbuf strbuf_append_quoted_string(strbuf sb, const char *str); +/* Escape HTML entities. + * @author Andrew Bettison + */ +strbuf strbuf_html_escape(strbuf sb, const char *, size_t); + /* Append various JSON elements. * @author Andrew Bettison */ strbuf strbuf_json_null(strbuf sb); -strbuf strbuf_json_string(strbuf sb, const char *str); +strbuf strbuf_json_boolean(strbuf sb, int boolean); +strbuf strbuf_json_string(strbuf sb, const char *str); // str can be NULL +strbuf strbuf_json_string_len(strbuf sb, const char *str, size_t strlen); // str cannot be NULL strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len); +struct json_atom { + enum json_atomic_type { JSON_NULL, JSON_BOOLEAN, JSON_INTEGER, JSON_STRING_NULTERM, JSON_STRING_LENGTH } type; + union { + int boolean; + int64_t integer; + struct { + const char *content; + size_t length; + } string; + } u; +}; +strbuf strbuf_json_atom(strbuf sb, const struct json_atom *); +strbuf strbuf_json_atom_as_html(strbuf sb, const struct json_atom *); +strbuf strbuf_json_atom_as_text(strbuf sb, const struct json_atom *); /* Append a representation of a struct http_range[] array. * @author Andrew Bettison diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh index b0c4db15..a2b8d78b 100644 --- a/testdefs_rhizome.sh +++ b/testdefs_rhizome.sh @@ -400,13 +400,13 @@ compute_filehash() { rhizome_http_server_started() { local logvar=LOG${1#+} - $GREP 'RHIZOME HTTP SERVER,.*START.*port=[0-9]' "${!logvar}" + $GREP 'HTTP SERVER START.*port=[0-9].*services=[^ ]*\' "${!logvar}" } get_rhizome_server_port() { local _var="$1" local _logvar=LOG${2#+} - local _port=$($SED -n -e '/RHIZOME HTTP SERVER.*START/s/.*port=\([0-9]\{1,\}\).*/\1/p' "${!_logvar}" | $SED -n '$p') + local _port=$($SED -n -e '/HTTP SERVER START/s/.*port=\([0-9]\{1,\}\).*services=[^ ]*\.*/\1/p' "${!_logvar}" | $SED -n '$p') assert --message="instance $2 Rhizome HTTP server port number is known" [ -n "$_port" ] if [ -n "$_var" ]; then eval "$_var=\$_port" diff --git a/tests/meshms b/tests/meshms index 0c7002c2..6b98df8e 100755 --- a/tests/meshms +++ b/tests/meshms @@ -201,4 +201,21 @@ test_listConversations() { assertStdoutLineCount '==' 5 } +doc_sendNoIdentity="Send message from unknown identity" +setup_sendNoIdentity() { + setup_servald + set_instance +A + create_identities 2 + setup_logging + SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF +} +test_sendNoIdentity() { + executeOk_servald meshms send message $SIDA1 $SIDX "First message" + execute $servald meshms send message $SIDX $SIDA1 "First reply" + assertExitStatus --stderr == 2 + executeOk_servald meshms list messages $SIDA1 $SIDX + execute $servald meshms list messages $SIDX $SIDA1 + assertExitStatus --stderr == 2 +} + runTests "$@" diff --git a/tests/rhizomechannels b/tests/rhizomechannels index 2024e1e3..673203f7 100755 --- a/tests/rhizomechannels +++ b/tests/rhizomechannels @@ -41,7 +41,7 @@ configure_servald_server() { set log.show_pid on \ set log.show_time on \ set debug.rhizome on \ - set debug.rhizome_httpd on \ + set debug.httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on \ set server.respawn_on_crash off \ diff --git a/tests/rhizomehttp b/tests/rhizomehttp index 0714ffe2..cabdd4b5 100755 --- a/tests/rhizomehttp +++ b/tests/rhizomehttp @@ -24,11 +24,51 @@ source "${0%/*}/../testdefs_rhizome.sh" shopt -s extglob +assertJq() { + local json="$1" + local jqscript="$2" + assert --message="$jqscript" --dump-on-fail="$json" [ "$(jq "$jqscript" "$json")" = true ] +} + +assertJqCmp() { + local opts=() + while [ $# -gt 0 ]; do + case "$1" in + --) shift; break;; + --*) opts+=("$1"); shift;; + *) break;; + esac + done + [ $# -eq 3 ] || error "invalid arguments" + local json="$1" + local jqscript="$2" + local file="$3" + jq --raw-output "$jqscript" "$json" >"$TFWTMP/jqcmp.tmp" + assert --dump-on-fail="$TFWTMP/jqcmp.tmp" --dump-on-fail="$file" "${opts[@]}" cmp "$TFWTMP/jqcmp.tmp" "$file" +} + +assertJqGrep() { + local opts=() + while [ $# -gt 0 ]; do + case "$1" in + --) shift; break;; + --*) opts+=("$1"); shift;; + *) break;; + esac + done + [ $# -eq 3 ] || error "invalid arguments" + local json="$1" + local jqscript="$2" + local pattern="$3" + jq "$jqscript" "$json" >"$TFWTMP/jqgrep.tmp" + assertGrep "${opts[@]}" "$TFWTMP/jqgrep.tmp" "$pattern" +} + setup() { CR=' ' VT=' ' setup_curl 7 - setup_jq 1.2 + setup_jq 1.3 setup_servald set_instance +A set_rhizome_config @@ -36,6 +76,7 @@ setup() { set rhizome.api.restful.users.harry.password potter \ set rhizome.api.restful.users.ron.password weasley \ set rhizome.api.restful.users.hermione.password grainger + set_extra_config if [ -z "$IDENTITY_COUNT" ]; then create_single_identity else @@ -56,10 +97,14 @@ teardown() { report_all_servald_servers } +set_extra_config() { + : +} + set_rhizome_config() { executeOk_servald config \ + set debug.http_server on \ set debug.httpd on \ - set debug.rhizome_httpd on \ set debug.rhizome_manifest on \ set debug.externalblobs on \ set debug.rhizome on \ @@ -76,6 +121,8 @@ test_AuthBasicMissing() { "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" assertStdoutIs '401' assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$" + assertJq http.output 'contains({"http_status_code": 401})' + assertJq http.output 'contains({"http_status_message": ""})' } teardown_AuthBasicMissing() { tfw_cat http.headers http.output @@ -92,6 +139,8 @@ test_AuthBasicWrong() { "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" assertStdoutIs '401' assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$" + assertJq http.output 'contains({"http_status_code": 401})' + assertJq http.output 'contains({"http_status_message": ""})' executeOk curl \ --silent --fail --show-error --write-out '%{http_code}' \ --output http.output \ @@ -144,52 +193,38 @@ add_bundles() { done } -transform_bundlelist_json() { - # The following jq(1) incantation transforms the JSON array in - # bundlelist.json from the following form (which is optimised for - # transmission size): +transform_list_json() { + # The following jq(1) incantation transforms a JSON array in from the + # following form (which is optimised for transmission size): # { - # "header":[ - # ".token", "_id", "service", "id", "version","date",".inserttime", - # ".author",".fromhere","filesize","filehash","sender","recipient", - # "name" - # ], + # "header":[ "label1", "label2", "label3", ... ], # "rows":[ - # [ "xx", rowid1, "service1", bundleid1, version1, .... ], - # [ null, rowid2, "service2", bundleid2, version2, .... ], + # [ row1value1, row1value2, row1value3, ... ], + # [ row2value1, row2value2, row2value3, ... ], # ... - # [ null, rowidN, "serviceN", bundleidN, versionN, .... ] + # [ rowNvalue1, rowNvalue2, rowNvalue3, ... ] # ] # } # # into an array of JSON objects: # [ # { - # "__index": 0, - # ".token": "xx", - # "_id": rowid1, - # "service": service1, - # "id": bundleid1, - # "version": version1, + # "label1": row1value1, + # "label2": row1value2, + # "label3": row1value3, # ... # }, # { - # "__index": 1, - # ".token": null, - # "_id": rowid2, - # "service": service2, - # "id": bundleid2, - # "version": version2, + # "label1": row2value1, + # "label2": row2value2, + # "label3": row2value3, # ... # }, # ... # { - # "__index": 2, - # ".token": null, - # "_id": rowidN, - # "service": serviceN, - # "id": bundleidN, - # "version": versionN, + # "label1": rowNvalue1, + # "label2": rowNvalue2, + # "label3": rowNvalue3, # ... # } # ] @@ -206,12 +241,6 @@ transform_bundlelist_json() { ' "$1" >"$2" } -assertJq() { - local json="$1" - local jqscript="$2" - assert --message="$jqscript" [ "$(jq "$jqscript" "$json")" = true ] -} - doc_RhizomeList="HTTP RESTful list Rhizome bundles as JSON" setup_RhizomeList() { setup @@ -229,7 +258,7 @@ test_RhizomeList() { tfw_cat http.headers bundlelist.json tfw_preserve bundlelist.json assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ] - transform_bundlelist_json bundlelist.json array_of_objects.json + transform_list_json bundlelist.json array_of_objects.json tfw_preserve array_of_objects.json for ((n = 0; n != NBUNDLES; ++n)); do if [ "${ROWID[$n]}" -eq "$ROWID_MAX" ]; then @@ -256,20 +285,13 @@ test_RhizomeList() { done } -curl_newsince() { - curl \ - --silent --fail --show-error \ - --no-buffer \ - --output "$1" \ - --basic --user harry:potter \ - "http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json" -} - doc_RhizomeNewSince="HTTP RESTful list Rhizome bundles since token as JSON" setup_RhizomeNewSince() { + set_extra_config() { + executeOk_servald config set rhizome.api.restful.newsince_timeout 60s \ + set rhizome.api.restful.newsince_poll_ms 500 + } setup - executeOk_servald config set rhizome.api.restful.newsince_timeout 60s - executeOk_servald config set rhizome.api.restful.newsince_poll_ms 500 add_bundles 0 5 executeOk curl \ --silent --fail --show-error \ @@ -278,19 +300,24 @@ setup_RhizomeNewSince() { --basic --user harry:potter \ "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json" assert [ "$(jq '.rows | length' bundlelist.json)" = 6 ] - transform_bundlelist_json bundlelist.json array_of_objects.json + transform_list_json bundlelist.json array_of_objects.json token=$(jq --raw-output '.[0][".token"]' array_of_objects.json) assert [ -n "$token" ] } test_RhizomeNewSince() { - fork %curl1 curl_newsince newsince1.json - fork %curl2 curl_newsince newsince2.json - fork %curl3 curl_newsince newsince3.json + for i in 1 2 3; do + fork %curl$i curl \ + --silent --fail --show-error \ + --no-buffer \ + --output newsince$i.json \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json" + done wait_until [ -e newsince1.json -a -e newsince2.json -a -e newsince3.json ] add_bundles 6 10 - wait_until --timeout=10 grep "${BID[10]}" newsince1.json - wait_until --timeout=10 grep "${BID[10]}" newsince2.json - wait_until --timeout=10 grep "${BID[10]}" newsince3.json + for i in 1 2 3; do + wait_until --timeout=10 grep "${BID[10]}" newsince$i.json + done fork_terminate_all fork_wait_all for i in 1 2 3; do @@ -298,7 +325,7 @@ test_RhizomeNewSince() { echo ']}' >>newsince$i.json assert [ $(jq . newsince$i.json | wc -c) -ne 0 ] fi - transform_bundlelist_json newsince$i.json objects$i.json + transform_list_json newsince$i.json objects$i.json tfw_preserve newsince$i.json objects$i.json for ((n = 0; n <= 5; ++n)); do assertJq objects$i.json "contains([{id:\"${BID[$n]}\"}]) | not" @@ -529,7 +556,8 @@ test_RhizomeInsert() { assertStdoutIs 201 else assertStdoutIs 403 - assertGrep --ignore-case nfile$n.manifest "missing bundle secret" + assertJq nfile$n.manifest 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case nfile$n.manifest '.http_status_message' "missing bundle secret" fi done } @@ -593,7 +621,7 @@ setup_RhizomeInsertLarge() { create_file file1 50m } test_RhizomeInsertLarge() { - execute curl \ + execute --timeout=120 curl \ --silent --show-error --write-out '%{http_code}' \ --output file1.manifest \ --dump-header http.header \ @@ -628,7 +656,8 @@ test_RhizomeInsertMissingManifest() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'missing.*manifest.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*manifest.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -650,7 +679,8 @@ test_RhizomeInsertIncorrectManifestType() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'unsupported content-type.*manifest.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'unsupported content-type.*manifest.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -675,7 +705,8 @@ test_RhizomeInsertDuplicateManifest() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'duplicate.*manifest.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*manifest.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -698,7 +729,8 @@ test_RhizomeInsertJournal() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'not supported.*journal' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'not supported.*journal' executeOk_servald rhizome list assert_rhizome_list } @@ -720,7 +752,8 @@ test_RhizomeInsertMissingPayload() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'missing.*payload.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*payload.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -745,7 +778,8 @@ test_RhizomeInsertDuplicatePayload() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'duplicate.*payload.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*payload.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -768,7 +802,8 @@ test_RhizomeInsertPartOrder() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'missing.*manifest.*form.*part' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*manifest.*form.*part' executeOk_servald rhizome list assert_rhizome_list } @@ -792,8 +827,9 @@ test_RhizomeInsertPartUnsupported() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'unsupported.*form.*part' - assertGrep http.body 'happyhappy' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'unsupported.*form.*part' + assertJqGrep http.body '.http_status_message' 'happyhappy' executeOk_servald rhizome list assert_rhizome_list } @@ -818,7 +854,8 @@ test_RhizomeInsertIncorrectFilesize() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'payload size.*contradicts manifest' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'payload size.*contradicts manifest' execute curl \ --silent --show-error --write-out '%{http_code}' \ --output http.body \ @@ -853,29 +890,420 @@ test_RhizomeInsertIncorrectFilehash() { tfw_cat http.header http.body assertExitStatus == 0 assertStdoutIs 403 - assertGrep --ignore-case http.body 'payload hash.*contradicts manifest' + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'payload hash.*contradicts manifest' executeOk_servald rhizome list assert_rhizome_list } doc_MeshmsListConversations="HTTP RESTful list MeshMS conversations as JSON" -Xtest_MeshmsListConversations() { - : +setup_MeshmsListConversations() { + IDENTITY_COUNT=5 + setup + # create 3 threads, with all permutations of incoming and outgoing messages + executeOk_servald meshms send message $SIDA1 $SIDA2 "Message1" + executeOk_servald meshms send message $SIDA3 $SIDA1 "Message2" + executeOk_servald meshms send message $SIDA1 $SIDA4 "Message3" + executeOk_servald meshms send message $SIDA4 $SIDA1 "Message4" +} +test_MeshmsListConversations() { + executeOk curl \ + --silent --fail --show-error \ + --output conversationlist1.json \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/conversationlist.json" + tfw_cat http.headers conversationlist1.json + tfw_preserve conversationlist1.json + assert [ "$(jq '.rows | length' conversationlist1.json)" = 3 ] + transform_list_json conversationlist1.json conversations1.json + tfw_preserve conversations1.json + assertJq conversations1.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA2\", + read: true, + last_message: 0, + read_offset: 0 + } + ])" + assertJq conversations1.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA3\", + read: false, + last_message: 11, + read_offset: 0 + } + ])" + assertJq conversations1.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA4\", + read: false, + last_message: 14, + read_offset: 0 + } + ])" + # mark all incoming messages as read + executeOk_servald meshms read messages $SIDA1 + tfw_cat --stderr + executeOk curl \ + --silent --fail --show-error \ + --output conversationlist2.json \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/conversationlist.json" + tfw_cat http.headers conversationlist2.json + tfw_preserve conversationlist2.json + assert [ "$(jq '.rows | length' conversationlist2.json)" = 3 ] + transform_list_json conversationlist2.json conversations2.json + tfw_preserve conversations2.json + assertJq conversations2.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA2\", + read: true, + last_message: 0, + read_offset: 0 + } + ])" + assertJq conversations2.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA3\", + read: true, + last_message: 11, + read_offset: 11 + } + ])" + assertJq conversations2.json \ + "contains([ + { my_sid: \"$SIDA1\", + their_sid: \"$SIDA4\", + read: true, + last_message: 14, + read_offset: 14 + } + ])" +} + +# Create a file that contains no blank lines. +create_message_file() { + create_file "$1" $2 + sed -i -e '/^$/d' "$1" +} + +# Add a sequence of messages of varying sizes up to 1 KiB. +add_messages() { + local symbols="$1" + shift + local texts=("$@") + local sent_since_ack=0 + local i n size msize + local size=0 + for ((i = 0; i < ${#symbols}; ++i)); do + local sym="${symbols:$i:1}" + let size+=379 + let msize=size%1021 + let n=NMESSAGE++ + local text="${texts[$i]}" + case $sym in + '>'|'<') + if [ -n "$text" ]; then + echo "$text" >text$n + else + create_message_file text$n $msize + text="$(') + MESSAGE[$n]=">" + executeOk_servald meshms send message $SIDA1 $SIDA2 "$text" + let ++sent_since_ack + let ++NSENT + ;; + '<') + MESSAGE[$n]="<" + executeOk_servald meshms send message $SIDA2 $SIDA1 "$text" + let ++NRECV + ;; + 'A') + MESSAGE[$n]=ACK + [ $i -ne 0 -a $sent_since_ack -eq 0 ] && error "two ACKs in a row (at position $i)" + executeOk_servald meshms list messages $SIDA2 $SIDA1 + let ++NACK + ;; + *) + error "invalid message symbol '$sym' (at position $i)" + ;; + esac + done } doc_MeshmsListMessages="HTTP RESTful list MeshMS messages in one conversation as JSON" -Xtest_MeshmsListMessages() { - : +setup_MeshmsListMessages() { + IDENTITY_COUNT=2 + setup + add_messages '><>>A>A<>><><><>>>A>A><<<><>>A<<>' + let NROWS=NSENT+NRECV+(NACK?1:0) + executeOk_servald meshms list messages $SIDA1 $SIDA2 + delivered_offset=$(sed -n -e '/^[0-9]\+:[0-9]\+:ACK:delivered$/{n;s/^[0-9]\+:\([0-9]\+\):>:.*/\1/p;q}' "$TFWSTDOUT") + [ -z "$delivered_offset" ] && delivered_offset=0 + read_offset=$(sed -n -e 's/^[0-9]\+:\([0-9]\+\):MARK:read$/\1/p' "$TFWSTDOUT") + [ -z "$read_offset" ] && read_offset=0 +} +test_MeshmsListMessages() { + executeOk curl \ + --silent --fail --show-error \ + --output messagelist.json \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json" + tfw_cat http.headers messagelist.json + tfw_preserve messagelist.json + assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ] + transform_list_json messagelist.json messages.json + tfw_preserve messages.json + seen_ack=false + let i=0 + for ((j = NMESSAGE-1; j >= 0; --j)); do + case ${MESSAGE[$j]} in + 'ACK') $seen_ack && continue + esac + assertJq messages.json '(.['$i'].token | length) > 0' + assertJq messages.json '.['$i'].my_sid == "'$SIDA1'"' + assertJq messages.json '.['$i'].their_sid == "'$SIDA2'"' + case ${MESSAGE[$j]} in + '>') + assertJq messages.json '.['$i'].type == ">"' + assertJqCmp messages.json '.['$i'].text' text$j + assertJq messages.json '.['$i'].delivered == (.['$i'].offset <= '$delivered_offset')' + let ++i + ;; + '<') + assertJq messages.json '.['$i'].type == "<"' + assertJqCmp messages.json '.['$i'].text' text$j + assertJq messages.json '.['$i'].read == (.['$i'].offset <= '$read_offset')' + let ++i + ;; + 'ACK') + assertJq messages.json '.['$i'].type == "ACK"' + assertJq messages.json '.['$i'].text == null' + assertJq messages.json '.['$i'].ack_offset == '$delivered_offset + let ++i + seen_ack=true + ;; + esac + done } -doc_MeshmsListMessagesSince="HTTP RESTful list MeshMS messages in one conversation since token as JSON" -Xtest_MeshmsListMessagesSince() { - : +doc_MeshmsListMessagesNoIdentity="HTTP RESTful list MeshMS messages from unknown identity" +setup_MeshmsListMessagesNoIdentity() { + setup + SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF +} +test_MeshmsListMessagesNoIdentity() { + execute curl \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDX/$SIDA/messagelist.json" + tfw_cat http.header http.body + assertExitStatus == 0 + assertStdoutIs 403 + assertJq http.body 'contains({"http_status_code": 403})' + assertJq http.body 'contains({"meshms_status_code": 2})' + assertJqGrep --ignore-case http.body '.http_status_message' 'identity.*unknown' +} + +doc_MeshmsListMessagesNewSince="HTTP RESTful list MeshMS messages in one conversation since token as JSON" +setup_MeshmsListMessagesNewSince() { + IDENTITY_COUNT=2 + set_extra_config() { + executeOk_servald config set rhizome.api.restful.newsince_timeout 1s \ + set rhizome.api.restful.newsince_poll_ms 500 + } + setup + add_messages '><>>A>A<>><><><>>>A>A><<<><>>A<<>' + let NROWS=NSENT+NRECV+(NACK?1:0) + executeOk curl \ + --silent --fail --show-error \ + --output messagelist.json \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json" + assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ] + transform_list_json messagelist.json messages.json + tfw_preserve messages.json + for ((i = 0; i < NROWS; i += 3)); do + token[$i]=$(jq --raw-output '.['$i'].token' messages.json) + done +} +test_MeshmsListMessagesNewSince() { + for ((i = 0; i < NROWS; i += 3)); do + # At most five requests going at once + [ $i -ge 15 ] && fork_wait %curl$((i-15)) + fork %curl$i executeOk curl \ + --silent --fail --show-error \ + --output messagelist$i.json \ + --dump-header http.headers$i \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/newsince/${token[$i]}/messagelist.json" + done + fork_wait_all + for ((i = 0; i < NROWS; i += 3)); do + transform_list_json messagelist$i.json messages$i.json + tfw_preserve messages$i.json + { echo '{"a":'; cat messages.json; echo ',"b":'; cat messages$i.json; echo '}'; } >tmp.json + assertJq tmp.json '.a[:'$i'] == .b' + done +} + +grepall() { + local pattern="$1" + shift + for file; do + grep "$pattern" "$file" || return $? + done + return 0 +} + +doc_MeshmsListMessagesNewSinceArrival="HTTP RESTful list newly arriving MeshMS messages in one conversation as JSON" +setup_MeshmsListMessagesNewSinceArrival() { + IDENTITY_COUNT=2 + set_extra_config() { + executeOk_servald config set rhizome.api.restful.newsince_timeout 60s \ + set rhizome.api.restful.newsince_poll_ms 500 + } + setup + add_messages '><>A>' + let NROWS=NSENT+NRECV+(NACK?1:0) + executeOk curl \ + --silent --fail --show-error \ + --output messagelist.json \ + --dump-header http.headers \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json" + assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ] + transform_list_json messagelist.json messages.json + tfw_preserve messages.json + token=$(jq --raw-output '.[0].token' messages.json) + assert [ -n "$token" ] +} +test_MeshmsListMessagesNewSinceArrival() { + for i in 1 2 3; do + fork %curl$i executeOk curl \ + --silent --fail --show-error \ + --no-buffer \ + --output newsince$i.json \ + --basic --user harry:potter \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/newsince/$token/messagelist.json" + done + wait_until [ -e newsince1.json -a -e newsince2.json -a -e newsince3.json ] + for message in '>Rumplestiltskin' 'A' 'Eulenspiegel'; do + add_messages "${message:0:1}" "${message:1}" + wait_until --timeout=60 grepall "${message:1}" newsince{1,2,3}.json + done + fork_terminate_all + fork_wait_all +} +teardown_MeshmsListMessagesNewSinceArrival() { + tfw_preserve newsince{1,2,3}.json } doc_MeshmsSend="HTTP RESTful send MeshMS message" -Xtest_MeshmsSend() { - : +setup_MeshmsSend() { + IDENTITY_COUNT=2 + setup +} +test_MeshmsSend() { + executeOk curl \ + --silent --fail --show-error \ + --output sendmessage.json \ + --basic --user harry:potter \ + --form "message=Hello World" \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/sendmessage" + executeOk_servald meshms list messages $SIDA1 $SIDA2 + assertStdoutGrep --matches=1 ':>:Hello World' + executeOk curl \ + --silent --fail --show-error \ + --output sendmessage.json \ + --basic --user ron:weasley \ + --form "message=Hello back!" \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage" + executeOk_servald meshms list messages $SIDA1 $SIDA2 + assertStdoutGrep --matches=1 ':>:Hello World$' + assertStdoutGrep --matches=1 ':<:Hello back!$' +} + +doc_MeshmsSendMissingMessage="HTTP RESTful MeshMS send missing 'message' form part " +setup_MeshmsSendMissingMessage() { + IDENTITY_COUNT=2 + setup +} +test_MeshmsSendMissingMessage() { + execute curl \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --data '' \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage" + tfw_cat http.header http.body + assertExitStatus == 0 + assertStdoutIs 403 + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*message.*form.*part' + executeOk_servald meshms list messages $SIDA1 $SIDA2 + assertStdoutLineCount '==' 2 +} + +doc_MeshmsSendDuplicateMessage="HTTP RESTful MeshMS send missing 'message' form part " +setup_MeshmsSendDuplicateMessage() { + IDENTITY_COUNT=2 + setup +} +test_MeshmsSendDuplicateMessage() { + execute curl \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --form "message=Hello one" \ + --form "message=Hello two" \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage" + tfw_cat http.header http.body + assertExitStatus == 0 + assertStdoutIs 403 + assertJq http.body 'contains({"http_status_code": 403})' + assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*message.*form.*part' + executeOk_servald meshms list messages $SIDA1 $SIDA2 + assertStdoutLineCount '==' 2 +} + +doc_MeshmsSendNoIdentity="HTTP RESTful MeshMS send from unknown identity" +setup_MeshmsSendNoIdentity() { + setup + SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF +} +test_MeshmsSendNoIdentity() { + execute curl \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --form "message=Hello" \ + "http://$addr_localhost:$PORTA/restful/meshms/$SIDX/$SIDA/sendmessage" + tfw_cat http.header http.body + assertExitStatus == 0 + assertStdoutIs 403 + assertJq http.body 'contains({"http_status_code": 403})' + assertJq http.body 'contains({"meshms_status_code": 2})' + assertJqGrep --ignore-case http.body '.http_status_message' 'identity.*unknown' } runTests "$@" diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol index f88a9f0e..6afec2bd 100755 --- a/tests/rhizomeprotocol +++ b/tests/rhizomeprotocol @@ -42,8 +42,8 @@ configure_servald_server() { set log.console.show_pid on \ set log.console.show_time on \ set debug.rhizome on \ + set debug.http_server on \ set debug.httpd on \ - set debug.rhizome_httpd on \ set debug.rhizome_manifest on \ set debug.rhizome_ads on \ set debug.rhizome_tx on \ @@ -180,7 +180,7 @@ setup_MDPTransportFailOver() { setup_common foreach_instance +A +B \ executeOk_servald config \ - set debug.rhizome_nohttptx 1 \ + set debug.nohttptx 1 \ set rhizome.mdp.enable 1 set_instance +A rhizome_add_file file1 2048 @@ -583,7 +583,7 @@ setup_direct() { set log.console.level debug \ set log.console.show_time on \ set debug.rhizome on \ - set debug.rhizome_httpd on \ + set debug.httpd on \ set debug.rhizome_tx on \ set debug.rhizome_rx on rhizome_add_file fileB1 2000 diff --git a/tests/rhizomestress b/tests/rhizomestress index 6fb5d53f..e974a5bb 100755 --- a/tests/rhizomestress +++ b/tests/rhizomestress @@ -41,7 +41,7 @@ configure_servald_server() { set log.file.show_pid on \ set log.file.show_time on \ set debug.rhizome off \ - set debug.rhizome_httpd off \ + set debug.httpd off \ set debug.rhizome_tx off \ set debug.rhizome_rx off \ set server.respawn_on_crash off \ @@ -149,7 +149,7 @@ setup_StressRhizomeDirect() { executeOk_servald config \ set log.file.show_time on \ set debug.rhizome off \ - set debug.rhizome_httpd off \ + set debug.httpd off \ set debug.rhizome_tx off \ set debug.rhizome_rx off \ set server.respawn_on_crash off \