Merge branch 'naf4' into 'development'

New RESTful HTTP API for MeshMS

commandline.c

@@ -1952,8 +1952,7 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
       return WHYF("Invalid <recipient: %s", recipient_hex);
     cursor.is_recipient_set = 1;
   }
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  if (rhizome_list_open(&cursor) == -1) {
-  if (rhizome_list_open(&retry, &cursor) == -1) {
     keyring_free(keyring);
     return -1;
   }
@@ -1975,7 +1974,7 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
   cli_columns(context, NELS(headers), headers);
   size_t rowcount = 0;
   int n;
-  while ((n = rhizome_list_next(&retry, &cursor)) == 1) {
+  while ((n = rhizome_list_next(&cursor)) == 1) {
     ++rowcount;
     if (rowcount <= rowoffset)
       continue;

conf.h

@@ -232,6 +232,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf.h"
 #include "serval.h"
 #include "rhizome.h"
+#include "httpd.h"
 
 #define CONFIG_FILE_MAX_SIZE        (32 * 1024)
 #define INTERFACE_NAME_STRLEN       40

conf_schema.c

@@ -153,7 +153,7 @@ int cf_opt_rhizome_peer_from_uri(struct config_rhizome_peer *rpeer, const char *
   }
   const char *host;
   size_t hostlen;
-  uint16_t port = RHIZOME_HTTP_PORT;
+  uint16_t port = HTTPD_PORT;
   if (!str_uri_authority_hostname(auth, &host, &hostlen))
     return CFINVALID;
   str_uri_authority_port(auth, &port);

conf_schema.h

@@ -233,7 +233,9 @@ ATOM(bool_t, dnaresponses,              0, boolean,, "")
 ATOM(bool_t, dnahelper,                 0, boolean,, "")
 ATOM(bool_t, queues,                    0, boolean,, "")
 ATOM(bool_t, timing,                    0, boolean,, "")
+ATOM(bool_t, http_server,               0, boolean,, "")
 ATOM(bool_t, httpd,                     0, boolean,, "")
+ATOM(bool_t, nohttptx,                  0, boolean,, "")
 ATOM(bool_t, io,                        0, boolean,, "")
 ATOM(bool_t, verbose_io,                0, boolean,, "")
 ATOM(bool_t, interactive_io,            0, boolean,, "")
@@ -263,11 +265,9 @@ ATOM(bool_t, packetconstruction,        0, boolean,, "")
 ATOM(bool_t, rhizome,                   0, boolean,, "")
 ATOM(bool_t, rhizome_manifest,          0, boolean,, "")
 ATOM(bool_t, rhizome_sql_bind,          0, boolean,, "")
-ATOM(bool_t, rhizome_httpd,             0, boolean,, "")
 ATOM(bool_t, rhizome_tx,                0, boolean,, "")
 ATOM(bool_t, rhizome_rx,                0, boolean,, "")
 ATOM(bool_t, rhizome_ads,               0, boolean,, "")
-ATOM(bool_t, rhizome_nohttptx,          0, boolean,, "")
 ATOM(bool_t, rhizome_mdp_rx,            0, boolean,, "")
 ATOM(bool_t, subscriber,                0, boolean,, "")
 ATOM(bool_t, throttling,                0, boolean,, "")
@@ -367,7 +367,7 @@ END_STRUCT
 STRUCT(rhizome_peer)
 STRING(25,                  protocol,   "http", protocol,, "Protocol name")
 STRING(256,                 host,       "", str_nonempty, MANDATORY, "Host name or IP address")
-ATOM(uint16_t,              port,       RHIZOME_HTTP_PORT, uint16_nonzero,, "Port number")
+ATOM(uint16_t,              port,       HTTPD_PORT, uint16_nonzero,, "Port number")
 END_STRUCT
 
 ARRAY(peerlist,)

headerfiles.mk

@@ -5,6 +5,8 @@ HDRS=	fifo.h \
 	overlay_packet.h \
 	overlay_interface.h \
 	rhizome.h \
+	httpd.h \
+	meshms.h \
 	serval.h \
 	keyring.h \
 	socket.h \

http_server.c

@@ -64,17 +64,22 @@ static struct {
 #undef VERB_ENTRY
 };
 
+const char CONTENT_TYPE_TEXT[] = "text/plain";
+const char CONTENT_TYPE_HTML[] = "text/html";
+const char CONTENT_TYPE_JSON[] = "application/json";
+const char CONTENT_TYPE_BLOB[] = "application/octet-stream";
+
 static struct profile_total http_server_stats = {
   .name = "http_server_poll",
 };
 
 #define DEBUG_DUMP_PARSED(r) do { \
-      if (config.debug.httpd) \
+      if (config.debug.http_server) \
 	DEBUGF("%s %s HTTP/%u.%u", r->verb ? r->verb : "NULL", alloca_str_toprint(r->path), r->version_major, r->version_minor);\
     } while (0)
 
 #define DEBUG_DUMP_PARSER(r) do { \
-      if (config.debug.httpd) \
+      if (config.debug.http_server) \
 	DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \
 	    (int)(r->parsed - r->received), alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \
 	    (int)(r->cursor - r->received), alloca_toprint(50, r->cursor, r->end - r->cursor), \
@@ -1208,14 +1213,14 @@ static int http_request_parse_body_form_data(struct http_request *r)
   int at_start = 0;
   switch (r->form_data_state) {
     case START:
-      if (config.debug.httpd)
+      if (config.debug.http_server)
 	DEBUGF("START");
       // The logic here allows for a missing initial CRLF before the first boundary line.
       at_start = 1;
       r->form_data_state = PREAMBLE;
       // fall through
     case PREAMBLE: {
-	if (config.debug.httpd)
+	if (config.debug.http_server)
 	  DEBUGF("PREAMBLE");
 	char *start = r->parsed;
 	for (; at_start || _skip_to_crlf(r); at_start = 0) {
@@ -1241,7 +1246,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
       }
       return 100; // need more data
     case HEADER: {
-      if (config.debug.httpd)
+      if (config.debug.http_server)
 	DEBUGF("HEADER");
 	// If not at a CRLF, then we are skipping through an over-long header that didn't
 	// fit into the buffer.  Just discard bytes up to the next CRLF.
@@ -1365,7 +1370,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
       }
       return 400;
     case BODY:
-      if (config.debug.httpd)
+      if (config.debug.http_server)
 	DEBUGF("BODY");
       char *start = r->parsed;
       while (_skip_to_crlf(r)) {
@@ -1396,7 +1401,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
       _INVOKE_HANDLER_BUF_LEN(handle_mime_body, start, r->parsed);
       return 100; // need more data
   case EPILOGUE:
-      if (config.debug.httpd)
+      if (config.debug.http_server)
 	DEBUGF("EPILOGUE");
     r->cursor = r->end;
     assert(r->cursor >= r->parsed);
@@ -1785,6 +1790,50 @@ static const char *httpResultString(int response_code)
   }
 }
 
+static strbuf strbuf_status_body(strbuf sb, struct http_response *hr, const char *message)
+{
+  if (   hr->header.content_type == CONTENT_TYPE_TEXT
+      || (hr->header.content_type && strcmp(hr->header.content_type, CONTENT_TYPE_TEXT) == 0)
+  ) {
+    hr->header.content_type = CONTENT_TYPE_TEXT;
+    strbuf_sprintf(sb, "%03u %s", hr->result_code, message);
+    if (hr->result_extra_label) {
+      strbuf_puts(sb, "\r\n");
+      strbuf_puts(sb, hr->result_extra_label);
+      strbuf_puts(sb, "=");
+      strbuf_json_atom_as_text(sb, &hr->result_extra_value);
+    }
+    strbuf_puts(sb, "\r\n");
+  }
+  else if (    hr->header.content_type == CONTENT_TYPE_JSON
+           || (hr->header.content_type && strcmp(hr->header.content_type, CONTENT_TYPE_JSON) == 0)
+  ) {
+    hr->header.content_type = CONTENT_TYPE_JSON;
+    strbuf_sprintf(sb, "{\n \"http_status_code\": %u,\n \"http_status_message\": ", hr->result_code);
+    strbuf_json_string(sb, message);
+    if (hr->result_extra_label) {
+      strbuf_puts(sb, ",\n ");
+      strbuf_json_string(sb, hr->result_extra_label);
+      strbuf_puts(sb, ": ");
+      strbuf_json_atom_as_html(sb, &hr->result_extra_value);
+    }
+    strbuf_puts(sb, "\n}");
+  }
+  else {
+    hr->header.content_type = CONTENT_TYPE_HTML;
+    strbuf_sprintf(sb, "<html>\n<h1>%03u %s</h1>", hr->result_code, message);
+    if (hr->result_extra_label) {
+      strbuf_puts(sb, "\n<dl><dt>");
+      strbuf_html_escape(sb, hr->result_extra_label, strlen(hr->result_extra_label));
+      strbuf_puts(sb, "</dt><dd>");
+      strbuf_json_atom_as_html(sb, &hr->result_extra_value);
+      strbuf_puts(sb, "</dd></dl>");
+    }
+    strbuf_puts(sb, "\n</html>");
+  }
+  return sb;
+}
+
 /* Render the HTTP response into the current response buffer.  Return 1 if it fits, 0 if it does
  * not.  The buffer response_pointer may be NULL, in which case no response is rendered, but the
  * content_length is still computed
@@ -1833,16 +1882,16 @@ static int _render_response(struct http_request *r)
     }
   } else {
     // If no content is supplied at all, then render a standard, short body based solely on result
-    // code.
+    // code, consistent with the response Content-Type if already set (HTML if not set).
     assert(hr.header.content_length == CONTENT_LENGTH_UNKNOWN);
     assert(hr.header.resource_length == CONTENT_LENGTH_UNKNOWN);
     assert(hr.header.content_range_start == 0);
     assert(hr.result_code != 206);
-    strbuf cb = strbuf_alloca(100 + strlen(result_string));
+    strbuf cb;
-    strbuf_sprintf(cb, "<html><h1>%03u %s</h1></html>", hr.result_code, result_string);
+    STRBUF_ALLOCA_FIT(cb, 40 + strlen(result_string), (strbuf_status_body(cb, &hr, result_string)));
     hr.content = strbuf_str(cb);
-    hr.header.content_type = "text/html";
+    hr.header.content_length = strbuf_len(cb);
-    hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
+    hr.header.resource_length = hr.header.content_length;
     hr.header.content_range_start = 0;
   }
   assert(hr.header.content_type != NULL);
@@ -2045,25 +2094,21 @@ void http_request_response_generated(struct http_request *r, int result, const c
 }
 
 /* Start sending a short response back to the client.  The result code must be either a success
- * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'body' argument
+ * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'message'
- * may be a bare message which is enclosed in an HTML envelope to form the response content, so it
+ * argument may be a bare message which is enclosed in an HTML envelope to form the response
- * may contain HTML markup.  If the 'body' argument is NULL, then the response content is generated
+ * content, so it may contain HTML markup.  If the 'message' argument is NULL, then the response
- * automatically from the result code.
+ * content is generated automatically from the result code.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-void http_request_simple_response(struct http_request *r, uint16_t result, const char *body)
+void http_request_simple_response(struct http_request *r, uint16_t result, const char *message)
 {
   assert(r->phase == RECEIVE);
-  strbuf h = NULL;
-  if (body) {
-    size_t html_len = strlen(body) + 40;
-    h = strbuf_alloca(html_len);
-    strbuf_sprintf(h, "<html><h1>%03u %s</h1></html>", result, body);
-  }
   r->response.result_code = result;
-  r->response.header.content_type = "text/html";
   r->response.header.content_range_start = 0;
+  strbuf h = NULL;
+  if (message)
+    STRBUF_ALLOCA_FIT(h, 40 + strlen(message), (strbuf_status_body(h, &r->response, message)));
   if (h) {
     r->response.header.resource_length = r->response.header.content_length = strbuf_len(h);
     r->response.content = strbuf_str(h);
@@ -2071,3 +2116,28 @@ void http_request_simple_response(struct http_request *r, uint16_t result, const
   r->response.content_generator = NULL;
   http_request_start_response(r);
 }
+
+int generate_http_content_from_strbuf_chunks(
+  struct http_request *r,
+  char *buf,
+  size_t bufsz,
+  struct http_content_generator_result *result,
+  HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER *chunker
+)
+{
+  assert(bufsz > 0);
+  strbuf b = strbuf_local((char *)buf, bufsz);
+  int ret;
+  while ((ret = chunker(r, b)) != -1) {
+    if (strbuf_overrun(b)) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b));
+      result->need = strbuf_count(b) + 1 - result->generated;
+      break;
+    }
+    result->generated = strbuf_len(b);
+    if (ret == 0)
+      break;
+  }
+  return ret;
+}

http_server.h

@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <limits.h>
 #include "constants.h"
 #include "strbuf.h"
+#include "strbuf_helpers.h"
 #include "fdqueue.h"
 
 /* Generic HTTP request handling.
@@ -56,6 +57,11 @@ http_size_t http_range_bytes(const struct http_range *range, unsigned nranges);
 
 #define CONTENT_LENGTH_UNKNOWN   UINT64_MAX
 
+extern const char CONTENT_TYPE_TEXT[];
+extern const char CONTENT_TYPE_HTML[];
+extern const char CONTENT_TYPE_JSON[];
+extern const char CONTENT_TYPE_BLOB[];
+
 struct mime_content_type {
   char type[64];
   char subtype[64];
@@ -63,7 +69,6 @@ struct mime_content_type {
   char charset[31];
 };
 
-
 struct http_client_authorization {
   enum http_authorization_scheme { NOAUTH = 0, BASIC } scheme;
   union {
@@ -105,6 +110,8 @@ typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, siz
 
 struct http_response {
   uint16_t result_code;
+  const char *result_extra_label;
+  struct json_atom result_extra_value;
   struct http_response_headers header;
   const char *content;
   HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content
@@ -152,6 +159,9 @@ void http_request_response_static(struct http_request *r, int result, const char
 void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
 
+typedef int (HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER)(struct http_request *, strbuf);
+int generate_http_content_from_strbuf_chunks(struct http_request *, char *, size_t, struct http_content_generator_result *, HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER *);
+
 typedef int HTTP_REQUEST_PARSER(struct http_request *);
 typedef void HTTP_RENDERER(struct http_request *, strbuf);
 

httpd.c

@@ -0,0 +1,507 @@
+/*
+Serval DNA HTTP external interface
+Copyright (C) 2014 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "serval.h"
+#include "conf.h"
+#include "httpd.h"
+#include "overlay_address.h"
+#include "overlay_interface.h"
+
+#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
+
+static HTTP_HANDLER root_page;
+static HTTP_HANDLER fav_icon_header;
+static HTTP_HANDLER interface_page;
+static HTTP_HANDLER neighbour_page;
+
+HTTP_HANDLER restful_rhizome_bundlelist_json;
+HTTP_HANDLER restful_rhizome_newsince;
+HTTP_HANDLER restful_rhizome_insert;
+HTTP_HANDLER restful_rhizome_;
+HTTP_HANDLER restful_meshms_;
+
+HTTP_HANDLER rhizome_status_page;
+HTTP_HANDLER rhizome_file_page;
+HTTP_HANDLER manifest_by_prefix_page;
+
+HTTP_HANDLER rhizome_direct_import;
+HTTP_HANDLER rhizome_direct_enquiry;
+HTTP_HANDLER rhizome_direct_dispatch;
+
+struct http_handler {
+  const char *path;
+  HTTP_HANDLER *parser;
+};
+
+struct http_handler paths[]={
+  {"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json},
+  {"/restful/rhizome/newsince/", restful_rhizome_newsince},
+  {"/restful/rhizome/insert", restful_rhizome_insert},
+  {"/restful/rhizome/", restful_rhizome_},
+  {"/restful/meshms/", restful_meshms_},
+  {"/rhizome/status", rhizome_status_page},
+  {"/rhizome/file/", rhizome_file_page},
+  {"/rhizome/import", rhizome_direct_import},
+  {"/rhizome/enquiry", rhizome_direct_enquiry},
+  {"/rhizome/manifestbyprefix/", manifest_by_prefix_page},
+  {"/rhizome/", rhizome_direct_dispatch},
+  {"/interface/", interface_page},
+  {"/neighbour/", neighbour_page},
+  {"/favicon.ico", fav_icon_header},
+  {"/", root_page},
+};
+
+static int httpd_dispatch(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  INFOF("HTTP SERVER, %s %s", r->http.verb, r->http.path);
+  r->http.response.content_generator = NULL;
+  unsigned i;
+  for (i = 0; i < NELS(paths); ++i) {
+    const char *remainder;
+    if (str_startswith(r->http.path, paths[i].path, &remainder)){
+      int result = paths[i].parser(r, remainder);
+      if (result == -1 || (result >= 200 && result < 600))
+	return result;
+      if (result == 1)
+	return 0;
+      if (result)
+	return WHYF("dispatch function for %s returned invalid result %d", paths[i].path, result);
+    }
+  }
+  return 404;
+}
+
+void httpd_server_poll(struct sched_ent *);
+struct sched_ent server_alarm;
+struct profile_total server_stats = {
+  .name = "httpd_server_poll",
+};
+
+uint16_t httpd_server_port = 0;
+unsigned int httpd_request_count = 0;
+
+static int httpd_server_socket = -1;
+static time_ms_t httpd_server_last_start_attempt = -1;
+
+// Format icon data using:
+//   od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/  */,/g'
+unsigned char favicon_bytes[]={
+0,0,1,0,1,0,16,16,16,0,0,0,0,0,40,1
+,0,0,22,0,0,0,40,0,0,0,16,0,0,0,32,0
+,0,0,1,0,4,0,0,0,0,0,128,0,0,0,0,0
+,0,0,0,0,0,0,16,0,0,0,0,0,0,0,104,158
+,168,0,163,233,247,0,104,161,118,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,17
+,17,17,17,18,34,17,17,18,34,17,17,18,34,17,17,2
+,34,17,17,18,34,17,16,18,34,1,17,17,1,17,1,17
+,1,16,1,16,17,17,17,17,1,17,16,16,17,17,17,17
+,1,17,18,34,17,17,17,16,17,17,2,34,17,17,17,16
+,17,16,18,34,17,17,17,16,17,1,17,1,17,17,17,18
+,34,17,17,16,17,17,17,18,34,17,17,18,34,17,17,18
+,34,17,17,18,34,17,17,16,17,17,17,18,34,17,17,16
+,17,17,17,17,17,0,17,1,17,17,17,17,17,17,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+int favicon_len=318;
+
+int is_httpd_server_running()
+{
+  return httpd_server_socket != -1;
+}
+
+/* Start the Rhizome HTTP server by creating a socket, binding it to an available port, and
+   marking it as passive.  If called repeatedly and frequently, this function will only try to start
+   the server after a certain time has elapsed since the last attempt.
+   Return -1 if an error occurs (message logged).
+   Return 0 if the server was started.
+   Return 1 if the server is already started successfully.
+   Return 2 if the server was not started because it is too soon since last failed attempt.
+ */
+int httpd_server_start(uint16_t port_low, uint16_t port_high)
+{
+  if (httpd_server_socket != -1)
+    return 1;
+
+  /* Only try to start http server every five seconds. */
+  time_ms_t now = gettime_ms();
+  if (now < httpd_server_last_start_attempt + 5000)
+    return 2;
+  httpd_server_last_start_attempt  = now;
+  if (config.debug.httpd)
+    DEBUGF("Starting HTTP server");
+
+  uint16_t port;
+  for (port = port_low; port <= port_high; ++port) {
+    /* Create a new socket, reusable and non-blocking. */
+    if (httpd_server_socket == -1) {
+      httpd_server_socket = socket(AF_INET,SOCK_STREAM,0);
+      if (httpd_server_socket == -1) {
+	WHY_perror("socket");
+	goto error;
+      }
+      int on=1;
+      if (setsockopt(httpd_server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) == -1) {
+	WHY_perror("setsockopt(REUSEADDR)");
+	goto error;
+      }
+      if (ioctl(httpd_server_socket, FIONBIO, (char *)&on) == -1) {
+	WHY_perror("ioctl(FIONBIO)");
+	goto error;
+      }
+    }
+    /* Bind it to the next port we want to try. */
+    struct sockaddr_in address;
+    bzero((char *) &address, sizeof(address));
+    address.sin_family = AF_INET;
+    address.sin_addr.s_addr = INADDR_ANY;
+    address.sin_port = htons(port);
+    if (bind(httpd_server_socket, (struct sockaddr *) &address, sizeof(address)) == -1) {
+      if (errno != EADDRINUSE) {
+	WHY_perror("bind");
+	goto error;
+      }
+    } else {
+      /* We bound to a port.  The battle is half won.  Now we have to successfully listen on that
+	port, which could also fail with EADDRINUSE, in which case we have to scrap the socket and
+	create a new one, because once bound, a socket stays bound.
+      */
+      if (listen(httpd_server_socket, 20) != -1)
+	goto success;
+      if (errno != EADDRINUSE) {
+	WHY_perror("listen");
+	goto error;
+      }
+      close(httpd_server_socket);
+      httpd_server_socket = -1;
+    }
+  }
+  WHYF("No ports available in range %u to %u", HTTPD_PORT, HTTPD_PORT_MAX);
+error:
+  if (httpd_server_socket != -1) {
+    close(httpd_server_socket);
+    httpd_server_socket = -1;
+  }
+  return WHY("Failed to start HTTP server");
+
+success:
+  INFOF("HTTP SERVER START port=%"PRIu16" fd=%d services=RESTful%s%s",
+      port,
+      httpd_server_socket,
+      config.rhizome.http.enable ? ",Rhizome" : "",
+      config.rhizome.api.addfile.uri_path[0] ? ",RhizomeDirect" : ""
+    );
+  httpd_server_port = port;
+  /* Add Rhizome HTTPd server to list of file descriptors to watch */
+  server_alarm.function = httpd_server_poll;
+  server_alarm.stats = &server_stats;
+  server_alarm.poll.fd = httpd_server_socket;
+  server_alarm.poll.events = POLLIN;
+  watch(&server_alarm);
+  return 0;
+}
+
+static void httpd_server_finalise_http_request(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (r->manifest) {
+    rhizome_manifest_free(r->manifest);
+    r->manifest = NULL;
+  }
+  if (r->finalise_union) {
+    r->finalise_union(r);
+    r->finalise_union = NULL;
+  }
+  if (httpd_request_count)
+    --httpd_request_count;
+}
+
+static int httpd_dispatch(struct http_request *);
+
+static unsigned int http_request_uuid_counter = 0;
+
+void httpd_server_poll(struct sched_ent *alarm)
+{
+  if (alarm->poll.revents & (POLLIN | POLLOUT)) {
+    struct sockaddr addr;
+    unsigned int addr_len = sizeof addr;
+    int sock;
+    if ((sock = accept(httpd_server_socket, &addr, &addr_len)) == -1) {
+      if (errno && errno != EAGAIN)
+	WARN_perror("accept");
+    } else {
+      struct sockaddr_in *peerip=NULL;
+      if (addr.sa_family == AF_INET) {
+	peerip = (struct sockaddr_in *)&addr; // network order
+	INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
+	    addr_len, peerip->sin_family, peerip->sin_port,
+	    ((unsigned char*)&peerip->sin_addr.s_addr)[0],
+	    ((unsigned char*)&peerip->sin_addr.s_addr)[1],
+	    ((unsigned char*)&peerip->sin_addr.s_addr)[2],
+	    ((unsigned char*)&peerip->sin_addr.s_addr)[3]
+	  );
+      } else {
+	INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u data=%s",
+	    addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data)
+	  );
+      }
+      httpd_request *request = emalloc_zero(sizeof(httpd_request));
+      if (request == NULL) {
+	WHY("Cannot respond to HTTP request, out of memory");
+	close(sock);
+      } else {
+	++httpd_request_count;
+	request->uuid = http_request_uuid_counter++;
+	if (peerip)
+	  request->http.client_sockaddr_in = *peerip;
+	request->http.handle_headers = httpd_dispatch;
+	request->http.debug_flag = &config.debug.httpd;
+	request->http.disable_tx_flag = &config.debug.nohttptx;
+	request->http.finalise = httpd_server_finalise_http_request;
+	request->http.free = free;
+	request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT;
+	http_request_init(&request->http, sock);
+      }
+    }
+  }
+  if (alarm->poll.revents & (POLLHUP | POLLERR)) {
+    INFO("Error on tcp listen socket");
+  }
+}
+
+int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
+{
+  IN();
+  const char *bufend = buf + len;
+  const char *p = buf;
+  size_t tail = read_since_last_call + 4;
+  if (tail < len)
+    p = bufend - tail;
+  int count = 0;
+  for (; p != bufend; ++p) {
+    switch (*p) {
+      case '\n':
+	if (++count==2)
+	  RETURN(p - buf);
+      case '\r': // ignore CR
+      case '\0': // ignore NUL (telnet inserts them)
+	break;
+      default:
+	count = 0;
+	break;
+    }
+  }
+  RETURN(0);
+  OUT();
+}
+
+static int is_from_loopback(const struct http_request *r)
+{
+  return   r->client_sockaddr_in.sin_family == AF_INET
+	&& ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET;
+}
+
+/* Return 1 if the given authorization credentials are acceptable.
+ * Return 0 if not.
+ */
+static int is_authorized(const struct http_client_authorization *auth)
+{
+  if (auth->scheme != BASIC)
+    return 0;
+  unsigned i;
+  for (i = 0; i != config.rhizome.api.restful.users.ac; ++i) {
+    if (   strcmp(config.rhizome.api.restful.users.av[i].key, auth->credentials.basic.user) == 0
+	&& strcmp(config.rhizome.api.restful.users.av[i].value.password, auth->credentials.basic.password) == 0
+    )
+      return 1;
+  }
+  return 0;
+}
+
+int authorize(struct http_request *r)
+{
+  if (!is_from_loopback(r))
+    return 403;
+  if (!is_authorized(&r->request_header.authorization)) {
+    r->response.header.www_authenticate.scheme = BASIC;
+    r->response.header.www_authenticate.realm = "Serval Rhizome";
+    return 401;
+  }
+  return 0;
+}
+
+int accumulate_text(httpd_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len)
+{
+  if (len) {
+    size_t newlen = *textlenp + len;
+    if (newlen > textsiz) {
+      if (config.debug.httpd)
+	DEBUGF("Form part \"%s\" too long, %zu bytes overflows maximum %zu by %zu",
+	    partname, newlen, textsiz, (size_t)(newlen - textsiz)
+	  );
+      strbuf msg = strbuf_alloca(100);
+      strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname);
+      http_request_simple_response(&r->http, 403, strbuf_str(msg));
+      return 0;
+    }
+    memcpy(textbuf + *textlenp, buf, len);
+    *textlenp = newlen;
+  }
+  return 1;
+}
+
+int form_buf_malloc_init(struct form_buf_malloc *f, size_t size_limit)
+{
+  assert(f->buffer == NULL);
+  assert(f->buffer_alloc_size == 0);
+  assert(f->length == 0);
+  f->size_limit = size_limit;
+  return 0;
+}
+
+int form_buf_malloc_accumulate(httpd_request *r, const char *partname, struct form_buf_malloc *f, const char *buf, size_t len)
+{
+  if (len == 0)
+    return 0;
+  size_t newlen = f->length + len;
+  if (newlen > f->size_limit) {
+    if (config.debug.httpd)
+      DEBUGF("form part \"%s\" overflow, %zu bytes exceeds limit %zu by %zu",
+	  partname, newlen, f->size_limit, (size_t)(newlen - f->size_limit)
+	);
+    strbuf msg = strbuf_alloca(100);
+    strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname);
+    http_request_simple_response(&r->http, 403, strbuf_str(msg));
+    return 403;
+  }
+  if (newlen > f->buffer_alloc_size) {
+    if ((f->buffer = erealloc(f->buffer, newlen)) == NULL) {
+      http_request_simple_response(&r->http, 500, NULL);
+      return 500;
+    }
+    f->buffer_alloc_size = newlen;
+  }
+  memcpy(f->buffer + f->length, buf, len);
+  f->length = newlen;
+  return 0;
+}
+
+void form_buf_malloc_release(struct form_buf_malloc *f)
+{
+  if (f->buffer) {
+    free(f->buffer);
+    f->buffer = NULL;
+  }
+  f->buffer_alloc_size = 0;
+  f->length = 0;
+  f->size_limit = 0;
+}
+
+int http_response_form_part(httpd_request *r, const char *what, const char *partname, const char *text, size_t textlen)
+{
+  if (config.debug.httpd)
+    DEBUGF("%s \"%s\" form part %s", what, partname, text ? alloca_toprint(-1, text, textlen) : "");
+  strbuf msg = strbuf_alloca(100);
+  strbuf_sprintf(msg, "%s \"%s\" form part", what, partname);
+  http_request_simple_response(&r->http, 403, strbuf_str(msg));
+  return 403;
+}
+
+static int root_page(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  char temp[8192];
+  strbuf b = strbuf_local(temp, sizeof temp);
+  strbuf_sprintf(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>"
+	   "<h1>Hello, I'm %s*</h1><br>"
+	   "Interfaces;<br>",
+	   alloca_tohex_sid_t_trunc(my_subscriber->sid, 16));
+  int i;
+  for (i=0;i<OVERLAY_MAX_INTERFACES;i++){
+    if (overlay_interfaces[i].state==INTERFACE_STATE_UP)
+      strbuf_sprintf(b, "<a href=\"/interface/%d\">%d: %s, TX: %d, RX: %d</a><br>",
+	i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count);
+  }
+  strbuf_puts(b, "Neighbours;<br>");
+  link_neighbour_short_status_html(b, "/neighbour");
+  if (is_rhizome_http_enabled()){
+    strbuf_puts(b, "<a href=\"/rhizome/status\">Rhizome Status</a><br>");
+  }
+  strbuf_puts(b, "</body></html>");
+  if (strbuf_overrun(b)) {
+    WHY("HTTP Root page buffer overrun");
+    return 500;
+  }
+  http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, temp, strbuf_len(b));
+  return 1;
+}
+
+static int fav_icon_header(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
+  return 1;
+}
+
+static int neighbour_page(httpd_request *r, const char *remainder)
+{
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  char buf[8*1024];
+  strbuf b = strbuf_local(buf, sizeof buf);
+  sid_t neighbour_sid;
+  if (str_to_sid_t(&neighbour_sid, remainder) == -1)
+    return 404;
+  struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0);
+  if (!neighbour)
+    return 404;
+  strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
+  link_neighbour_status_html(b, neighbour);
+  strbuf_puts(b, "</body></html>");
+  if (strbuf_overrun(b))
+    return -1;
+  http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, buf, strbuf_len(b));
+  return 1;
+}
+
+static int interface_page(httpd_request *r, const char *remainder)
+{
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  char buf[8*1024];
+  strbuf b=strbuf_local(buf, sizeof buf);
+  int index=atoi(remainder);
+  if (index<0 || index>=OVERLAY_MAX_INTERFACES)
+    return 404;
+  strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
+  interface_state_html(b, &overlay_interfaces[index]);
+  strbuf_puts(b, "</body></html>");
+  if (strbuf_overrun(b))
+    return -1;
+  http_request_response_static(&r->http, 200, CONTENT_TYPE_HTML, buf, strbuf_len(b));
+  return 1;
+}

httpd.h

@@ -0,0 +1,214 @@
+/*
+Serval DNA Rhizome HTTP interface
+Copyright (C) 2013-2014 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#ifndef __SERVAL_DNA__HTTPD_H
+#define __SERVAL_DNA__HTTPD_H
+
+#include "rhizome.h"
+#include "meshms.h"
+#include "http_server.h"
+
+int is_httpd_server_running();
+
+#define HTTPD_PORT 4110
+#define HTTPD_PORT_MAX 4150
+
+extern uint16_t httpd_server_port;
+extern unsigned int httpd_request_count;
+
+enum list_phase { LIST_HEADER = 0, LIST_ROWS, LIST_END, LIST_DONE };
+
+struct form_buf_malloc {
+  char *buffer;
+  size_t size_limit; // == 0 means no limit
+  size_t buffer_alloc_size;
+  size_t length;
+};
+
+struct httpd_request;
+
+int form_buf_malloc_init(struct form_buf_malloc *, size_t size_limit);
+int form_buf_malloc_accumulate(struct httpd_request *, const char *partname, struct form_buf_malloc *, const char *, size_t);
+void form_buf_malloc_release(struct form_buf_malloc *);
+
+typedef struct httpd_request
+{
+  struct http_request http; // MUST BE FIRST ELEMENT
+
+  /* Identify request from others being run.  Monotonic counter feeds it.  Only
+   * used for debugging when we write post-<uuid>.log files for multi-part form
+   * requests.
+   */
+  unsigned int uuid;
+
+  /* For requests/responses that pertain to a single manifest.
+   */
+  rhizome_manifest *manifest;
+
+  /* For requests/responses that contain one or two SIDs.
+   */
+  sid_t sid1;
+  sid_t sid2;
+
+  /* For requests/responses that contain a Rhizome Bundle ID.
+   */
+  rhizome_bid_t bid;
+
+  /* For requests/responses that contain a 64-bit unsigned integer (eg, SQLite ROWID, byte offset).
+   */
+  uint64_t ui64;
+
+  /* Finaliser for union contents (below).
+   */
+  void (*finalise_union)(struct httpd_request *);
+
+  /* Mutually exclusive response arguments.
+   */
+  union {
+
+    /* For receiving Rhizome Direct import request
+     */
+    struct {
+      // Which part is currently being received
+      const char *current_part;
+      // Temporary file currently current part is being written to
+      int part_fd;
+      // Which parts have already been received
+      bool_t received_manifest;
+      bool_t received_data;
+      // Name of data file supplied in part's Content-Disposition header, filename
+      // parameter (if any)
+      char data_file_name[MIME_FILENAME_MAXLEN + 1];
+    }
+      direct_import;
+
+    /* For receiving RESTful Rhizome insert request
+     */
+    struct {
+      // Which part is currently being received
+      const char *current_part;
+      // Which parts have already been received
+      bool_t received_author;
+      bool_t received_secret;
+      bool_t received_manifest;
+      bool_t received_payload;
+      // For storing the "bundle-author" hex SID as we receive it
+      char author_hex[SID_STRLEN];
+      size_t author_hex_len;
+      sid_t author;
+      // For storing the "bundle-secret" hex as we receive it
+      char secret_hex[RHIZOME_BUNDLE_KEY_STRLEN];
+      size_t secret_hex_len;
+      rhizome_bk_t bundle_secret;
+      // The "force-new" parameter
+      char force_new_text[5]; // enough for "false"
+      size_t force_new_text_len;
+      bool_t force_new;
+      // For storing the manifest text (malloc/realloc) as we receive it
+      struct form_buf_malloc manifest;
+      // For receiving the payload
+      enum rhizome_payload_status payload_status;
+      uint64_t payload_size;
+      struct rhizome_write write;
+    }
+      insert;
+
+    /* For responses that send part or all of a payload.
+    */
+    struct rhizome_read read_state;
+
+    /* For responses that list manifests.
+    */
+    struct {
+      enum list_phase phase;
+      uint64_t rowid_highest;
+      size_t rowcount;
+      time_ms_t end_time;
+      struct rhizome_list_cursor cursor;
+    }
+      rhlist;
+
+    /* For responses that list MeshMS conversations.
+    */
+    struct {
+      enum list_phase phase;
+      size_t rowcount;
+      struct meshms_conversations *conv;
+      struct meshms_conversation_iterator iter;
+    }
+      mclist;
+
+    /* For responses that list MeshMS messages in a single conversation.
+    */
+    struct {
+      enum meshms_which_ply token_which_ply;
+      uint64_t token_offset;
+      enum meshms_which_ply latest_which_ply;
+      uint64_t latest_offset;
+      time_ms_t end_time;
+      uint64_t highest_ack_offset;
+      enum list_phase phase;
+      size_t rowcount;
+      struct meshms_message_iterator iter;
+      int finished;
+    }
+      msglist;
+
+    /* For responses that send a MeshMS message.
+    */
+    struct {
+      // Which part is currently being received
+      const char *current_part;
+      // Which parts have already been received
+      bool_t received_message;
+      // The text of the message to send
+      struct form_buf_malloc message;
+    }
+      sendmsg;
+
+  } u;
+
+} httpd_request;
+
+int httpd_server_start(uint16_t port_low, uint16_t port_high);
+
+typedef int HTTP_HANDLER(httpd_request *r, const char *remainder);
+
+int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
+int authorize(struct http_request *r);
+int http_response_form_part(httpd_request *r, const char *what, const char *partname, const char *text, size_t textlen);
+int accumulate_text(httpd_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len);
+
+int rhizome_response_content_init_filehash(httpd_request *r, const rhizome_filehash_t *hash);
+int rhizome_response_content_init_payload(httpd_request *r, rhizome_manifest *);
+HTTP_CONTENT_GENERATOR rhizome_payload_content;
+
+struct http_response_parts {
+  uint16_t code;
+  char *reason;
+  uint64_t range_start;
+  uint64_t content_length;
+  char *content_start;
+};
+
+#define HTTP_RESPONSE_CONTENT_LENGTH_UNSET UINT64_MAX
+
+int unpack_http_response(char *response, struct http_response_parts *parts);
+
+#endif // __SERVAL_DNA__HTTPD_H

meshms.h

@@ -0,0 +1,184 @@
+/*
+Serval DNA MeshMS
+Copyright (C) 2014 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#ifndef __SERVAL_DNA__MESHMS_H
+#define __SERVAL_DNA__MESHMS_H
+
+#ifndef __MESHMS_INLINE
+# if __GNUC__ && !__GNUC_STDC_INLINE__
+#  define __MESHMS_INLINE extern inline
+# else
+#  define __MESHMS_INLINE inline
+# endif
+#endif
+
+#include "serval.h"
+#include "rhizome.h"
+
+#define MESHMS_MESSAGE_MAX_LEN  4095
+
+/* The result of a MeshMS operation.  Negative indicates failure, zero or
+ * positive success.
+ */
+enum meshms_status {
+    MESHMS_STATUS_ERROR = -1, // unexpected error (underlying failure)
+    MESHMS_STATUS_OK = 0, // operation succeeded, no bundle changed
+    MESHMS_STATUS_UPDATED = 1, // operation succeeded, bundle updated
+    MESHMS_STATUS_SID_LOCKED = 2, // cannot decode or send messages for that SID
+    MESHMS_STATUS_PROTOCOL_FAULT = 3, // missing or faulty ply bundle
+};
+
+__MESHMS_INLINE int meshms_failed(enum meshms_status status) {
+    return status != MESHMS_STATUS_OK && status != MESHMS_STATUS_UPDATED;
+}
+
+// the manifest details for one half of a conversation
+struct meshms_ply {
+  rhizome_bid_t bundle_id;
+  uint64_t version;
+  uint64_t tail;
+  uint64_t size;
+};
+
+struct meshms_conversations {
+  // binary tree
+  struct meshms_conversations *_left;
+  struct meshms_conversations *_right;
+  // keeping a pointer to parent node here means the traversal iterator does not need a stack, so
+  // there is no fixed limit on the tree depth
+  struct meshms_conversations *_parent;
+  
+  // who are we talking to?
+  sid_t them;
+  
+  char found_my_ply;
+  struct meshms_ply my_ply;
+  
+  char found_their_ply;
+  struct meshms_ply their_ply;
+  
+  // what is the offset of their last message
+  uint64_t their_last_message;
+  // what is the last message we marked as read
+  uint64_t read_offset;
+  // our cached value for the last known size of their ply
+  uint64_t their_size;
+};
+
+// cursor state for reading one half of a conversation
+struct meshms_ply_read {
+  // rhizome payload
+  struct rhizome_read read;
+  // block buffer
+  struct rhizome_read_buffer buff;
+  // details of the current record
+  uint64_t record_end_offset;
+  uint16_t record_length;
+  size_t record_size;
+  char type;
+  // raw record data
+  unsigned char *record;
+};
+
+/* Fetch the list of all MeshMS conversations into a binary tree whose nodes
+ * are all allocated by malloc(3).
+ */
+enum meshms_status meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv);
+void meshms_free_conversations(struct meshms_conversations *conv);
+
+/* For iterating over a binary tree of all MeshMS conversations, as created by
+ * meshms_conversations_list().
+ *
+ *      struct meshms_conversation_iterator it;
+ *      meshms_conversation_iterator_start(&it, conv);
+ *      while (it.current) {
+ *          ...
+ *          meshms_conversation_iterator_advance(&it);
+ *      }
+ */
+struct meshms_conversation_iterator {
+  struct meshms_conversations *current;
+};
+void meshms_conversation_iterator_start(struct meshms_conversation_iterator *, struct meshms_conversations *);
+void meshms_conversation_iterator_advance(struct meshms_conversation_iterator *);
+
+/* For iterating through the messages in a single MeshMS conversation; both
+ * plys threaded (interleaved) in the order as seen by the sender.  The
+ * meshms_message_iterator_prev() function returns MESHMS_STATUS_UPDATED if it
+ * advances the iterator to a message, or MESHMS_STATUS_OK if there are no more
+ * messages.  Any other return value indicates failure.
+ *
+ *      struct meshms_message_iterator it;
+ *      enum meshms_status status;
+ *      if (meshms_failed(status = meshms_message_iterator_open(&it, &sender_sid, &recip_sid)))
+ *          return -1;
+ *      while ((status = meshms_message_iterator_prev(&it)) == MESHMS_STATUS_UPDATED) {
+ *          ...
+ *      }
+ *      meshms_message_iterator_close(&it);
+ *      if (meshms_failed(status))
+ *          return -1;
+ *      ...
+ */
+struct meshms_message_iterator {
+  // Public fields that remain fixed for the life of the iterator:
+  const sid_t *my_sid;
+  const sid_t *their_sid;
+  const rhizome_bid_t *my_ply_bid;
+  const rhizome_bid_t *their_ply_bid;
+  uint64_t latest_ack_offset; // offset in remote (their) ply of most recent ACK
+  uint64_t latest_ack_my_offset; // offset in my ply of most recent message ACKed by them
+  uint64_t read_offset; // offset in remote (their) ply of most recent message read by me
+  // The following public fields change per message:
+  enum meshms_which_ply { MY_PLY, THEIR_PLY } which_ply;
+  enum { MESSAGE_SENT, MESSAGE_RECEIVED, ACK_RECEIVED } type;
+  // For MESSAGE_SENT 'offset' is the byte position within the local ply
+  // (mine).  For MESSAGE_RECEIVED and ACK_RECEIVED, it is the byte position
+  // within the remote ply (theirs).
+  uint64_t offset;
+  const char *text; // text of UTF8 message (NUL terminated)
+  size_t text_length; // excluding terminating NUL
+  union {
+    bool_t delivered; // for MESSAGE_SENT
+    bool_t read; // for MESSAGE_RECEIVED
+    uint64_t ack_offset; // for ACK_RECEIVED
+  };
+  // Private implementation -- could change, so don't use them.
+  sid_t _my_sid;
+  struct meshms_conversations *_conv;
+  rhizome_manifest *_my_manifest;
+  rhizome_manifest *_their_manifest;
+  struct meshms_ply_read _my_reader;
+  struct meshms_ply_read _their_reader;
+  uint64_t _end_range;
+  bool_t _in_ack;
+};
+enum meshms_status meshms_message_iterator_open(struct meshms_message_iterator *, const sid_t *me, const sid_t *them);
+int meshms_message_iterator_is_open(const struct meshms_message_iterator *);
+void meshms_message_iterator_close(struct meshms_message_iterator *);
+enum meshms_status meshms_message_iterator_prev(struct meshms_message_iterator *);
+
+/* Append a message ('message_len' bytes of UTF8 at 'message') to the sender's
+ * ply in the conversation between 'sender' and 'recipient'.  If no
+ * conversation (ply bundle) exists, then create it.  Returns 0 on success, -1
+ * on error (already logged).
+ */
+enum meshms_status meshms_send_message(const sid_t *sender, const sid_t *recipient, const char *message, size_t message_len);
+
+#endif // __SERVAL_DNA__MESHMS_H

meshms_restful.c

@@ -0,0 +1,558 @@
+/*
+Serval DNA MeshMS HTTP RESTful interface
+Copyright (C) 2013,2014 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "conf.h"
+#include "serval.h"
+#include "httpd.h"
+#include "strbuf_helpers.h"
+
+static void finalise_union_meshms_conversationlist(httpd_request *r)
+{
+  meshms_free_conversations(r->u.mclist.conv);
+  r->u.mclist.conv = NULL;
+}
+
+static void finalise_union_meshms_messagelist(httpd_request *r)
+{
+  meshms_message_iterator_close(&r->u.msglist.iter);
+}
+
+static void finalise_union_meshms_sendmessage(httpd_request *r)
+{
+  form_buf_malloc_release(&r->u.sendmsg.message);
+}
+
+#define MESHMS_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(rhizome_bid_t) + sizeof(uint64_t)))
+#define alloca_meshms_token(bid, offset) meshms_token_to_str(alloca(MESHMS_TOKEN_STRLEN + 1), (bid), (offset))
+
+static char *meshms_token_to_str(char *buf, const rhizome_bid_t *bid, uint64_t offset)
+{
+  struct iovec iov[2];
+  iov[0].iov_base = (void *) bid->binary;
+  iov[0].iov_len = sizeof bid->binary;
+  iov[1].iov_base = &offset;
+  iov[1].iov_len = sizeof offset;
+  size_t n = base64url_encodev(buf, iov, 2);
+  assert(n == MESHMS_TOKEN_STRLEN);
+  buf[n] = '\0';
+  return buf;
+}
+
+static int strn_to_meshms_token(const char *str, rhizome_bid_t *bidp, uint64_t *offsetp, const char **afterp)
+{
+  unsigned char token[sizeof bidp->binary + sizeof *offsetp];
+  if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token)
+    return 0;
+  memcpy(bidp->binary, token, sizeof bidp->binary);
+  memcpy(offsetp, token + sizeof bidp->binary, sizeof *offsetp);
+  return 1;
+}
+
+static int http_request_meshms_response(struct httpd_request *r, uint16_t result, const char *message, enum meshms_status status)
+{
+  r->http.response.result_extra_label = "meshms_status_code";
+  r->http.response.result_extra_value.type = JSON_INTEGER;
+  r->http.response.result_extra_value.u.integer = status;
+  switch (status) {
+    case MESHMS_STATUS_OK:
+      if (!result)
+	result = 200;
+      if (!message)
+	message = "OK";
+      break;
+    case MESHMS_STATUS_UPDATED:
+      if (!result)
+	result = 201;
+      if (!message)
+	message = "Updated";
+      break;
+    case MESHMS_STATUS_SID_LOCKED:
+      if (!result)
+	result = 403;
+      if (!message)
+	message = "Identity unknown";
+      break;
+    case MESHMS_STATUS_PROTOCOL_FAULT:
+      if (!result)
+	result = 403;
+      if (!message)
+	message = "MeshMS protocol fault";
+      break;
+    case MESHMS_STATUS_ERROR:
+      if (!result)
+	result = 500;
+      break;
+    default:
+      WHYF("Invalid MeshMS status code %d", status);
+      result = 500;
+      break;
+  }
+  assert(result != 0);
+  http_request_simple_response(&r->http, result, message);
+  return result;
+}
+
+static HTTP_HANDLER restful_meshms_conversationlist_json;
+static HTTP_HANDLER restful_meshms_messagelist_json;
+static HTTP_HANDLER restful_meshms_newsince_messagelist_json;
+static HTTP_HANDLER restful_meshms_sendmessage;
+
+int restful_meshms_(httpd_request *r, const char *remainder)
+{
+  r->http.response.header.content_type = CONTENT_TYPE_JSON;
+  if (!is_rhizome_http_enabled())
+    return 403;
+  const char *verb = HTTP_VERB_GET;
+  HTTP_HANDLER *handler = NULL;
+  const char *end;
+  if (strn_to_sid_t(&r->sid1, remainder, &end) != -1) {
+    remainder = end;
+    if (strcmp(remainder, "/conversationlist.json") == 0) {
+      handler = restful_meshms_conversationlist_json;
+      remainder = "";
+    } else if (*remainder == '/' && strn_to_sid_t(&r->sid2, remainder + 1, &end) != -1) {
+      remainder = end;
+      if (strcmp(remainder, "/messagelist.json") == 0) {
+	handler = restful_meshms_messagelist_json;
+	remainder = "";
+      }
+      else if (   str_startswith(remainder, "/newsince/", &end)
+	       && strn_to_meshms_token(end, &r->bid, &r->ui64, &end)
+	       && strcmp(end, "/messagelist.json") == 0
+      ) {
+	handler = restful_meshms_newsince_messagelist_json;
+	remainder = "";
+      }
+      else if (strcmp(remainder, "/sendmessage") == 0) {
+	handler = restful_meshms_sendmessage;
+	verb = HTTP_VERB_POST;
+	remainder = "";
+      }
+    }
+  }
+  if (handler == NULL)
+    return 404;
+  if (r->http.verb != verb)
+    return 405;
+  int ret = authorize(&r->http);
+  if (ret)
+    return ret;
+  ret = handler(r, remainder);
+  return ret;
+}
+
+static HTTP_CONTENT_GENERATOR restful_meshms_conversationlist_json_content;
+
+static int restful_meshms_conversationlist_json(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_meshms_conversationlist;
+  r->u.mclist.phase = LIST_HEADER;
+  r->u.mclist.rowcount = 0;
+  r->u.mclist.conv = NULL;
+  enum meshms_status status;
+  if (meshms_failed(status = meshms_conversations_list(&r->sid1, NULL, &r->u.mclist.conv)))
+    return http_request_meshms_response(r, 0, NULL, status);
+  meshms_conversation_iterator_start(&r->u.mclist.iter, r->u.mclist.conv);
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_conversationlist_json_content);
+  return 1;
+}
+
+static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_conversationlist_json_content_chunk;
+
+static int restful_meshms_conversationlist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
+{
+  return generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_meshms_conversationlist_json_content_chunk);
+}
+
+static int restful_meshms_conversationlist_json_content_chunk(struct http_request *hr, strbuf b)
+{
+  httpd_request *r = (httpd_request *) hr;
+  // The "my_sid" and "their_sid" per-conversation fields allow the same JSON structure to be used
+  // in a future, non-SID-specific request, eg, to list all conversations for all currently open
+  // identities.
+  const char *headers[] = {
+    "_id",
+    "my_sid",
+    "their_sid",
+    "read",
+    "last_message",
+    "read_offset"
+  };
+  switch (r->u.mclist.phase) {
+    case LIST_HEADER:
+      strbuf_puts(b, "{\n\"header\":[");
+      unsigned i;
+      for (i = 0; i != NELS(headers); ++i) {
+	if (i)
+	  strbuf_putc(b, ',');
+	strbuf_json_string(b, headers[i]);
+      }
+      strbuf_puts(b, "],\n\"rows\":[");
+      if (!strbuf_overrun(b))
+	r->u.mclist.phase = LIST_ROWS;
+      return 1;
+    case LIST_ROWS:
+      if (r->u.mclist.iter.current == NULL) {
+	r->u.mclist.phase = LIST_END;
+	// fall through...
+      } else {
+	if (r->u.mclist.rowcount != 0)
+	  strbuf_putc(b, ',');
+	strbuf_puts(b, "\n[");
+	strbuf_sprintf(b, "%u", r->u.mclist.rowcount);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, r->sid1.binary, sizeof r->sid1.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, r->u.mclist.iter.current->them.binary, sizeof r->u.mclist.iter.current->them.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_boolean(b, r->u.mclist.iter.current->read_offset >= r->u.mclist.iter.current->their_last_message);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRIu64, r->u.mclist.iter.current->their_last_message);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRIu64, r->u.mclist.iter.current->read_offset);
+	strbuf_puts(b, "]");
+	if (!strbuf_overrun(b)) {
+	  meshms_conversation_iterator_advance(&r->u.mclist.iter);
+	  ++r->u.mclist.rowcount;
+	}
+	return 1;
+      }
+      // fall through...
+    case LIST_END:
+      strbuf_puts(b, "\n]\n}\n");
+      if (!strbuf_overrun(b))
+	r->u.mclist.phase = LIST_DONE;
+      // fall through...
+    case LIST_DONE:
+      return 0;
+  }
+  abort();
+  return 0;
+}
+
+static HTTP_CONTENT_GENERATOR restful_meshms_messagelist_json_content;
+
+static enum meshms_status reopen_meshms_message_iterator(httpd_request *r)
+{
+  if (!meshms_message_iterator_is_open(&r->u.msglist.iter)) {
+    enum meshms_status status;
+    if (   meshms_failed(status = meshms_message_iterator_open(&r->u.msglist.iter, &r->sid1, &r->sid2))
+	|| meshms_failed(status = meshms_message_iterator_prev(&r->u.msglist.iter))
+    )
+      return status;
+    r->u.msglist.finished = status != MESHMS_STATUS_UPDATED;
+    if (!r->u.msglist.finished) {
+      r->u.msglist.latest_which_ply = r->u.msglist.iter.which_ply;
+      r->u.msglist.latest_offset = r->u.msglist.iter.offset;
+    }
+  }
+  return MESHMS_STATUS_OK;
+}
+
+static int restful_meshms_messagelist_json(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_meshms_messagelist;
+  r->u.msglist.rowcount = 0;
+  r->u.msglist.phase = LIST_HEADER;
+  r->u.msglist.token_offset = 0;
+  r->u.msglist.end_time = 0;
+  enum meshms_status status;
+  if (meshms_failed(status = reopen_meshms_message_iterator(r)))
+    return http_request_meshms_response(r, 0, NULL, status);
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_messagelist_json_content);
+  return 1;
+}
+
+static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_meshms_messagelist;
+  r->u.msglist.rowcount = 0;
+  r->u.msglist.phase = LIST_HEADER;
+  enum meshms_status status;
+  if (meshms_failed(status = reopen_meshms_message_iterator(r)))
+    return http_request_meshms_response(r, 0, NULL, status);
+  if (cmp_rhizome_bid_t(&r->bid, r->u.msglist.iter.my_ply_bid) == 0)
+    r->u.msglist.token_which_ply = MY_PLY;
+  else if (cmp_rhizome_bid_t(&r->bid, r->u.msglist.iter.their_ply_bid) == 0)
+    r->u.msglist.token_which_ply = THEIR_PLY;
+  else {
+    http_request_simple_response(&r->http, 404, "Unmatched token");
+    return 404;
+  }
+  r->u.msglist.token_offset = r->ui64;
+  r->u.msglist.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000;
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_meshms_messagelist_json_content);
+  return 1;
+}
+
+static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_messagelist_json_content_chunk;
+
+static int restful_meshms_messagelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (meshms_failed(reopen_meshms_message_iterator(r)))
+    return -1;
+  return generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_meshms_messagelist_json_content_chunk);
+}
+
+static int restful_meshms_messagelist_json_content_chunk(struct http_request *hr, strbuf b)
+{
+  httpd_request *r = (httpd_request *) hr;
+  // Include "my_sid" and "their_sid" per-message, so that the same JSON structure can be used by a
+  // future, non-SID-specific request (eg, to get all messages for all currently open identities).
+  const char *headers[] = {
+    "type",
+    "my_sid",
+    "their_sid",
+    "offset",
+    "token",
+    "text",
+    "delivered",
+    "read",
+    "ack_offset"
+  };
+  switch (r->u.msglist.phase) {
+    case LIST_HEADER:
+      strbuf_puts(b, "{\n");
+      if (!r->u.msglist.end_time) {
+	strbuf_sprintf(b, "\"read_offset\":%"PRIu64",\n\"latest_ack_offset\":%"PRIu64",\n",
+	    r->u.msglist.iter.read_offset,
+	    r->u.msglist.iter.latest_ack_my_offset
+	  );
+      }
+      strbuf_puts(b, "\"header\":[");
+      unsigned i;
+      for (i = 0; i != NELS(headers); ++i) {
+	if (i)
+	  strbuf_putc(b, ',');
+	strbuf_json_string(b, headers[i]);
+      }
+      strbuf_puts(b, "],\n\"rows\":[");
+      if (!strbuf_overrun(b))
+	r->u.msglist.phase = r->u.msglist.finished ? LIST_END : LIST_ROWS;
+      return 1;
+    case LIST_ROWS:
+      {
+	if (   r->u.msglist.finished
+	    || (r->u.msglist.token_which_ply == r->u.msglist.iter.which_ply && r->u.msglist.iter.offset <= r->u.msglist.token_offset)
+	) {
+	    time_ms_t now;
+	    if (r->u.msglist.end_time && (now = gettime_ms()) < r->u.msglist.end_time) {
+	      r->u.msglist.token_which_ply = r->u.msglist.latest_which_ply;
+	      r->u.msglist.token_offset = r->u.msglist.latest_offset;
+	      meshms_message_iterator_close(&r->u.msglist.iter);
+	      time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms;
+	      if (wake_at > r->u.msglist.end_time)
+		wake_at = r->u.msglist.end_time;
+	      http_request_pause_response(&r->http, wake_at);
+	      return 0;
+	    }
+	} else {
+	  switch (r->u.msglist.iter.type) {
+	    case MESSAGE_SENT:
+	      if (r->u.msglist.rowcount != 0)
+		strbuf_putc(b, ',');
+	      strbuf_puts(b, "\n[");
+	      strbuf_json_string(b, ">");
+	      strbuf_putc(b, ',');
+	      strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary);
+	      strbuf_putc(b, ',');
+	      strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary);
+	      strbuf_putc(b, ',');
+	      strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset);
+	      strbuf_putc(b, ',');
+	      strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->my_ply.bundle_id, r->u.msglist.iter.offset));
+	      strbuf_putc(b, ',');
+	      strbuf_json_string(b, r->u.msglist.iter.text);
+	      strbuf_putc(b, ',');
+	      strbuf_json_boolean(b, r->u.msglist.iter.delivered);
+	      strbuf_putc(b, ',');
+	      strbuf_json_boolean(b, 0);
+	      strbuf_putc(b, ',');
+	      strbuf_json_null(b);
+	      strbuf_puts(b, "]");
+	      break;
+	    case MESSAGE_RECEIVED:
+	      if (r->u.msglist.rowcount != 0)
+		strbuf_putc(b, ',');
+	      strbuf_puts(b, "\n[");
+	      strbuf_json_string(b, "<");
+	      strbuf_putc(b, ',');
+	      strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary);
+	      strbuf_putc(b, ',');
+	      strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary);
+	      strbuf_putc(b, ',');
+	      strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset);
+	      strbuf_putc(b, ',');
+	      strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->their_ply.bundle_id, r->u.msglist.iter.offset));
+	      strbuf_putc(b, ',');
+	      strbuf_json_string(b, r->u.msglist.iter.text);
+	      strbuf_putc(b, ',');
+	      strbuf_json_boolean(b, 1);
+	      strbuf_putc(b, ',');
+	      strbuf_json_boolean(b, r->u.msglist.iter.read);
+	      strbuf_putc(b, ',');
+	      strbuf_json_null(b);
+	      strbuf_puts(b, "]");
+	      break;
+	    case ACK_RECEIVED:
+	      // Don't send old (irrelevant) ACKs.
+	      if (r->u.msglist.iter.ack_offset > r->u.msglist.highest_ack_offset) {
+		if (r->u.msglist.rowcount != 0)
+		  strbuf_putc(b, ',');
+		strbuf_puts(b, "\n[");
+		strbuf_json_string(b, "ACK");
+		strbuf_putc(b, ',');
+		strbuf_json_hex(b, r->u.msglist.iter.my_sid->binary, sizeof r->u.msglist.iter.my_sid->binary);
+		strbuf_putc(b, ',');
+		strbuf_json_hex(b, r->u.msglist.iter.their_sid->binary, sizeof r->u.msglist.iter.their_sid->binary);
+		strbuf_putc(b, ',');
+		strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.offset);
+		strbuf_putc(b, ',');
+		strbuf_json_string(b, alloca_meshms_token(&r->u.msglist.iter._conv->their_ply.bundle_id, r->u.msglist.iter.offset));
+		strbuf_putc(b, ',');
+		strbuf_json_string(b, r->u.msglist.iter.text);
+		strbuf_putc(b, ',');
+		strbuf_json_boolean(b, 1);
+		strbuf_putc(b, ',');
+		strbuf_json_boolean(b, r->u.msglist.iter.read);
+		strbuf_putc(b, ',');
+		strbuf_sprintf(b, "%"PRIu64, r->u.msglist.iter.ack_offset);
+		strbuf_puts(b, "]");
+		r->u.msglist.highest_ack_offset = r->u.msglist.iter.ack_offset;
+	      }
+	      break;
+	  }
+	  if (!strbuf_overrun(b)) {
+	    ++r->u.msglist.rowcount;
+	    enum meshms_status status;
+	    if (meshms_failed(status = meshms_message_iterator_prev(&r->u.msglist.iter)))
+	      return http_request_meshms_response(r, 0, NULL, status);
+	    r->u.msglist.finished = status != MESHMS_STATUS_UPDATED;
+	  }
+	  return 1;
+	}
+	r->u.msglist.phase = LIST_END;
+      }
+      // fall through...
+    case LIST_END:
+      strbuf_puts(b, "\n]\n}\n");
+      if (!strbuf_overrun(b))
+	r->u.msglist.phase = LIST_DONE;
+      // fall through...
+    case LIST_DONE:
+      return 0;
+  }
+  abort();
+  return 0;
+}
+
+static HTTP_REQUEST_PARSER restful_meshms_sendmessage_end;
+static int send_mime_part_start(struct http_request *);
+static int send_mime_part_end(struct http_request *);
+static int send_mime_part_header(struct http_request *, const struct mime_part_headers *);
+static int send_mime_part_body(struct http_request *, char *, size_t);
+
+static int restful_meshms_sendmessage(httpd_request *r, const char *remainder)
+{
+  if (*remainder)
+    return 404;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_meshms_sendmessage;
+  // Parse the request body as multipart/form-data.
+  assert(r->u.sendmsg.current_part == NULL);
+  assert(!r->u.sendmsg.received_message);
+  r->http.form_data.handle_mime_part_start = send_mime_part_start;
+  r->http.form_data.handle_mime_part_end = send_mime_part_end;
+  r->http.form_data.handle_mime_part_header = send_mime_part_header;
+  r->http.form_data.handle_mime_body = send_mime_part_body;
+  // Send the message once the body has arrived.
+  r->http.handle_content_end = restful_meshms_sendmessage_end;
+  return 1;
+}
+
+static char PART_MESSAGE[] = "message";
+
+static int send_mime_part_start(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  assert(r->u.sendmsg.current_part == NULL);
+  return 0;
+}
+
+static int send_mime_part_end(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (r->u.sendmsg.current_part == PART_MESSAGE) {
+    if (r->u.sendmsg.message.length == 0)
+      return http_response_form_part(r, "Invalid (empty)", PART_MESSAGE, NULL, 0);
+    r->u.sendmsg.received_message = 1;
+    if (config.debug.httpd)
+      DEBUGF("received %s = %s", PART_MESSAGE, alloca_toprint(-1, r->u.sendmsg.message.buffer, r->u.sendmsg.message.length));
+  } else
+    FATALF("current_part = %s", alloca_str_toprint(r->u.sendmsg.current_part));
+  r->u.sendmsg.current_part = NULL;
+  return 0;
+}
+
+static int send_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (strcmp(h->content_disposition.name, PART_MESSAGE) == 0) {
+    if (r->u.sendmsg.received_message)
+      return http_response_form_part(r, "Duplicate", PART_MESSAGE, NULL, 0);
+    r->u.sendmsg.current_part = PART_MESSAGE;
+    form_buf_malloc_init(&r->u.sendmsg.message, MESHMS_MESSAGE_MAX_LEN);
+  }
+  else
+    return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0);
+  return 0;
+}
+
+static int send_mime_part_body(struct http_request *hr, char *buf, size_t len)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (r->u.sendmsg.current_part == PART_MESSAGE) {
+    form_buf_malloc_accumulate(r, PART_MESSAGE, &r->u.sendmsg.message, buf, len);
+  } else
+    FATALF("current_part = %s", alloca_str_toprint(r->u.sendmsg.current_part));
+  return 0;
+}
+
+static int restful_meshms_sendmessage_end(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (!r->u.sendmsg.received_message)
+    return http_response_form_part(r, "Missing", PART_MESSAGE, NULL, 0);
+  assert(r->u.sendmsg.message.length > 0);
+  assert(r->u.sendmsg.message.length <= MESHMS_MESSAGE_MAX_LEN);
+  enum meshms_status status;
+  if (meshms_failed(status = meshms_send_message(&r->sid1, &r->sid2, r->u.sendmsg.message.buffer, r->u.sendmsg.message.length)))
+    return http_request_meshms_response(r, 0, NULL, status);
+  return http_request_meshms_response(r, 201, "Message sent", status);
+}

overlay.c

@@ -71,6 +71,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "conf.h"
 #include "rhizome.h"
+#include "httpd.h"
 #include "strbuf.h"
 #include "keyring.h"
 #include "overlay_interface.h"
@@ -136,7 +137,7 @@ schedule(&_sched_##X); }
   }
 
   // start the HTTP server if enabled
-  rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX);    
+  httpd_server_start(HTTPD_PORT, HTTPD_PORT_MAX);    
 
   // start the dna helper if configured
   dna_helper_start();

rhizome.c

@@ -53,6 +53,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "conf.h"
 #include "str.h"
 #include "rhizome.h"
+#include "httpd.h"
 #include "dataformats.h"
 
 int is_rhizome_enabled()
@@ -84,7 +85,7 @@ int is_rhizome_advertise_enabled()
   return config.rhizome.enable
     &&   config.rhizome.advertise.enable
     &&   rhizome_db
-    &&   (is_rhizome_http_server_running() || is_rhizome_mdp_server_running());
+    &&   (is_httpd_server_running() || is_rhizome_mdp_server_running());
 }
 
 int rhizome_fetch_delay_ms()

rhizome.h

@@ -1,5 +1,6 @@
 /*
-Serval Distributed Numbering Architecture (DNA)
+Serval DNA Rhizome file distribution
+Copyright (C) 2010-2013 Serval Project Inc.
 Copyright (C) 2010 Paul Gardner-Stephen
  
 This program is free software; you can redistribute it and/or
@@ -26,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "uuid.h"
 #include "str.h"
 #include "strbuf.h"
-#include "http_server.h"
 #include "nacl.h"
 
 #ifndef __RHIZOME_INLINE
@@ -55,9 +55,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 // assumed to always be 2^n
 #define RHIZOME_CRYPT_PAGE_SIZE         4096
 
-#define RHIZOME_HTTP_PORT 4110
-#define RHIZOME_HTTP_PORT_MAX 4150
-
 /* Fundamental data type: Rhizome Bundle ID
  *
  * @author Andrew Bettison <andrew@servalproject.com>
@@ -392,7 +389,6 @@ void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
 #define     RHIZOME_SERVICE_MESHMS2  "MeshMS2"
 
 extern int64_t rhizome_space;
-extern uint16_t rhizome_http_server_port;
 
 int log2ll(uint64_t x);
 int rhizome_configure();
@@ -429,8 +425,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report);
 int rhizome_manifest_createid(rhizome_manifest *m);
 int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
 
-int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
-
 typedef struct sqlite_retry_state {
   unsigned int limit; // do not retry once elapsed >= limit
   unsigned int sleep; // number of milliseconds to sleep between retries
@@ -689,14 +683,15 @@ struct rhizome_list_cursor {
   uint64_t rowid_since;
   // Set by calling the next() function.
   rhizome_manifest *manifest;
-  // Private state.
+  // Private state - implementation that could change.
+  sqlite_retry_state _retry;
   sqlite3_stmt *_statement;
   uint64_t _rowid_current;
   uint64_t _rowid_last; // for re-opening query
 };
 
-int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
+int rhizome_list_open(struct rhizome_list_cursor *);
-int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *);
+int rhizome_list_next(struct rhizome_list_cursor *);
 void rhizome_list_commit(struct rhizome_list_cursor *);
 void rhizome_list_release(struct rhizome_list_cursor *);
 
@@ -768,118 +763,14 @@ struct rhizome_read
   uint64_t length;
 };
 
-/* Rhizome-specific HTTP request handling.
- */
-typedef struct rhizome_http_request
-{
-  struct http_request http; // MUST BE FIRST ELEMENT
-
-  /* Identify request from others being run.  Monotonic counter feeds it.  Only
-   * used for debugging when we write post-<uuid>.log files for multi-part form
-   * requests.
-   */
-  unsigned int uuid;
-
-  /* For requests/responses that pertain to a single manifest.
-   */
-  rhizome_manifest *manifest;
-
-  /* Finaliser for union contents (below).
-   */
-  void (*finalise_union)(struct rhizome_http_request *);
-
-  /* Mutually exclusive response arguments.
-   */
-  union {
-
-    /* For receiving Rhizome Direct import request
-     */
-    struct {
-      // Which part is currently being received
-      const char *current_part;
-      // Temporary file currently current part is being written to
-      int part_fd;
-      // Which parts have already been received
-      bool_t received_manifest;
-      bool_t received_data;
-      // Name of data file supplied in part's Content-Disposition header, filename
-      // parameter (if any)
-      char data_file_name[MIME_FILENAME_MAXLEN + 1];
-    }
-      direct_import;
-
-    /* For receiving RESTful Rhizome insert request
-     */
-    struct {
-      // Which part is currently being received
-      const char *current_part;
-      // Which parts have already been received
-      bool_t received_author;
-      bool_t received_secret;
-      bool_t received_manifest;
-      bool_t received_payload;
-      // For storing the "bundle-author" hex SID as we receive it
-      char author_hex[SID_STRLEN];
-      size_t author_hex_len;
-      sid_t author;
-      // For storing the "bundle-secret" hex as we receive it
-      char secret_hex[RHIZOME_BUNDLE_KEY_STRLEN];
-      size_t secret_hex_len;
-      rhizome_bk_t bundle_secret;
-      // The "force-new" parameter
-      char force_new_text[5]; // enough for "false"
-      size_t force_new_text_len;
-      bool_t force_new;
-      // For storing the manifest text (malloc/realloc) as we receive it
-      char *manifest_text;
-      size_t manifest_text_size;
-      size_t manifest_len;
-      // For receiving the payload
-      enum rhizome_payload_status payload_status;
-      uint64_t payload_size;
-      struct rhizome_write write;
-    }
-      insert;
-
-    /* For responses that send part or all of a payload.
-    */
-    struct rhizome_read read_state;
-
-    /* For responses that list manifests.
-    */
-    struct {
-      enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase;
-      uint64_t rowid_highest;
-      size_t rowcount;
-      time_ms_t end_time;
-      struct rhizome_list_cursor cursor;
-    }
-      list;
-
-  } u;
-
-} rhizome_http_request;
-
 int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, 
 			     uint64_t offset, size_t count,unsigned char *bytes);
-int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
-					 int64_t fileLength,int priority);
-int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h);
-int rhizome_server_free_http_request(rhizome_http_request *r);
-int rhizome_server_http_send_bytes(rhizome_http_request *r);
-int rhizome_server_parse_http_request(rhizome_http_request *r);
-int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response);
-int rhizome_server_http_response(rhizome_http_request *r, int result, 
-    const char *mime_type, const char *body, uint64_t bytes);
-int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes);
-int rhizome_http_server_start(uint16_t port_low, uint16_t port_high);
 
 int is_rhizome_enabled();
 int is_rhizome_mdp_enabled();
 int is_rhizome_http_enabled();
 int is_rhizome_advertise_enabled();
 int is_rhizome_mdp_server_running();
-int is_rhizome_http_server_running();
 
 typedef struct rhizome_direct_bundle_cursor {
   /* Where the current fill started */
@@ -918,8 +809,6 @@ int rhizome_direct_get_bars(const rhizome_bid_t *bid_low,
 			    const rhizome_bid_t *bid_max,
 			    unsigned char *bars_out,
 			    int bars_requested);
-int rhizome_direct_process_post_multipart_bytes
-(rhizome_http_request *r,const char *bytes,int count);
 
 typedef struct rhizome_direct_sync_request {
   struct sched_ent alarm;
@@ -1003,18 +892,6 @@ int rhizome_any_fetch_queued();
 int rhizome_fetch_status_html(struct strbuf *b);
 int rhizome_fetch_has_queue_space(unsigned char log2_size);
 
-struct http_response_parts {
-  uint16_t code;
-  char *reason;
-  uint64_t range_start;
-  uint64_t content_length;
-  char *content_start;
-};
-
-#define HTTP_RESPONSE_CONTENT_LENGTH_UNSET UINT64_MAX
-
-int unpack_http_response(char *response, struct http_response_parts *parts);
-
 /* rhizome storage methods */
 
 int rhizome_exists(const rhizome_filehash_t *hashp);

rhizome_database.c

@@ -1575,7 +1575,7 @@ rollback:
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
+int rhizome_list_open(struct rhizome_list_cursor *c)
 {
   if (config.debug.rhizome)
     DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64,
@@ -1609,18 +1609,19 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
   }
   if (strbuf_overrun(b))
     RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
-  c->_statement = sqlite_prepare(retry, strbuf_str(b));
+  c->_retry = SQLITE_RETRY_STATE_DEFAULT;
+  c->_statement = sqlite_prepare(&c->_retry, strbuf_str(b));
   if (c->_statement == NULL)
     RETURN(-1);
-  if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
+  if (c->service && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
     goto failure;
-  if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
+  if (c->name && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
     goto failure;
-  if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
+  if (c->is_sender_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
     goto failure;
-  if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
+  if (c->is_recipient_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
     goto failure;
-  if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
+  if (c->_rowid_last && sqlite_bind(&c->_retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
     goto failure;
   c->manifest = NULL;
   c->_rowid_current = 0;
@@ -1641,7 +1642,7 @@ failure:
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
+int rhizome_list_next(struct rhizome_list_cursor *c)
 {
   if (config.debug.rhizome)
     DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64,
@@ -1654,7 +1655,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
 	c->_rowid_last
       );
   IN();
-  if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
+  if (c->_statement == NULL && rhizome_list_open(c) == -1)
     RETURN(-1);
   while (1) {
     if (c->manifest) {
@@ -1662,7 +1663,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
       c->_rowid_current = 0;
       c->manifest = NULL;
     }
-    if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW)
+    if (sqlite_step_retry(&c->_retry, c->_statement) != SQLITE_ROW)
       break;
     assert(sqlite3_column_count(c->_statement) == 6);
     assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);

rhizome_direct_http.c

@@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include "serval.h"
 #include "rhizome.h"
+#include "httpd.h"
 #include "conf.h"
 #include "str.h"
 #include "strbuf.h"
@@ -31,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "socket.h"
 
 
-static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_request *r, char *pathbuf, size_t bufsiz, const char *field)
+static int _form_temporary_file_path(struct __sourceloc __whence, httpd_request *r, char *pathbuf, size_t bufsiz, const char *field)
 {
   strbuf b = strbuf_local(pathbuf, bufsiz);
   // TODO: use a temporary directory
@@ -45,7 +46,7 @@ static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_r
 
 #define form_temporary_file_path(r,buf,field) _form_temporary_file_path(__WHENCE__, (r), (buf), sizeof(buf), (field))
 
-static void rhizome_direct_clear_temporary_files(rhizome_http_request *r)
+static void rhizome_direct_clear_temporary_files(httpd_request *r)
 {
   const char *fields[] = { "manifest", "data" };
   int i;
@@ -59,7 +60,7 @@ static void rhizome_direct_clear_temporary_files(rhizome_http_request *r)
 
 static int rhizome_direct_import_end(struct http_request *hr)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   if (!r->u.direct_import.received_manifest) {
     http_request_simple_response(&r->http, 400, "Missing 'manifest' part");
     return 0;
@@ -128,7 +129,7 @@ static int rhizome_direct_import_end(struct http_request *hr)
 
 int rhizome_direct_enquiry_end(struct http_request *hr)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   if (!r->u.direct_import.received_data) {
     http_request_simple_response(&r->http, 400, "Missing 'data' part");
     return 0;
@@ -188,7 +189,7 @@ int rhizome_direct_enquiry_end(struct http_request *hr)
 
 static int rhizome_direct_addfile_end(struct http_request *hr)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   // If given a file without a manifest, we should only accept if it we are configured to do so, and
   // the connection is from localhost.  Otherwise people could cause your servald to create
   // arbitrary bundles, which would be bad.
@@ -275,7 +276,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
     if (config.debug.rhizome)
       DEBUGF("Import sans-manifest appeared to succeed");
     /* Respond with the manifest that was added. */
-    http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_all_bytes);
+    http_request_response_static(&r->http, 200, CONTENT_TYPE_TEXT, (const char *)m->manifestdata, m->manifest_all_bytes);
     /* clean up after ourselves */
     if (mout && mout != m)
       rhizome_manifest_free(mout);
@@ -293,7 +294,7 @@ static char PART_DATA[] = "data";
 
 static int rhizome_direct_process_mime_start(struct http_request *hr)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   assert(r->u.direct_import.current_part == NULL);
   assert(r->u.direct_import.part_fd == -1);
   return 0;
@@ -301,7 +302,7 @@ static int rhizome_direct_process_mime_start(struct http_request *hr)
 
 static int rhizome_direct_process_mime_end(struct http_request *hr)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   if (r->u.direct_import.part_fd != -1) {
     if (close(r->u.direct_import.part_fd) == -1) {
       WHYF_perror("close(%d)", r->u.direct_import.part_fd);
@@ -320,7 +321,7 @@ static int rhizome_direct_process_mime_end(struct http_request *hr)
 
 static int rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   if (strcmp(h->content_disposition.name, PART_DATA) == 0) {
     r->u.direct_import.current_part = PART_DATA;
     strncpy(r->u.direct_import.data_file_name,
@@ -347,7 +348,7 @@ static int rhizome_direct_process_mime_part_header(struct http_request *hr, cons
 
 static int rhizome_direct_process_mime_body(struct http_request *hr, char *buf, size_t len)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
+  httpd_request *r = (httpd_request *) hr;
   if (r->u.direct_import.part_fd != -1) {
     if (write_all(r->u.direct_import.part_fd, buf, len) == -1) {
       http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed");
@@ -357,7 +358,7 @@ static int rhizome_direct_process_mime_body(struct http_request *hr, char *buf,
   return 0;
 }
 
-int rhizome_direct_import(rhizome_http_request *r, const char *remainder)
+int rhizome_direct_import(httpd_request *r, const char *remainder)
 {
   if (*remainder)
     return 404;
@@ -374,7 +375,7 @@ int rhizome_direct_import(rhizome_http_request *r, const char *remainder)
   return 1;
 }
 
-int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
+int rhizome_direct_enquiry(httpd_request *r, const char *remainder)
 {
   if (*remainder)
     return 404;
@@ -396,7 +397,7 @@ int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
  * loop-holes here, which is part of why we leave it disabled by default, but it will be sufficient
  * for testing possible uses, including integration with OpenDataKit.
  */
-int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
+int rhizome_direct_addfile(httpd_request *r, const char *remainder)
 {
   if (*remainder)
     return 404;
@@ -423,7 +424,7 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
   return 1;
 }
 
-int rhizome_direct_dispatch(rhizome_http_request *r, const char *UNUSED(remainder))
+int rhizome_direct_dispatch(httpd_request *r, const char *UNUSED(remainder))
 {
   if (   config.rhizome.api.addfile.uri_path[0]
       && strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0
@@ -515,9 +516,9 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
   strbuf_sprintf(content_preamble,
       "--%s\r\n"
       "Content-Disposition: form-data; name=\"data\"; filename=\"IHAVEs\"\r\n"
-      "Content-Type: application/octet-stream\r\n"
+      "Content-Type: %s\r\n"
       "\r\n",
-      boundary
+      boundary, CONTENT_TYPE_BLOB
     );
   strbuf_sprintf(content_postamble, "\r\n--%s--\r\n", boundary);
   assert(!strbuf_overrun(content_preamble));

rhizome_fetch.c

@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "conf.h"
 #include "rhizome.h"
+#include "httpd.h"
 #include "str.h"
 #include "strbuf_helpers.h"
 #include "overlay_address.h"

rhizome_packetformats.c

@@ -222,7 +222,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm)
   }
   ob_limitsize(frame->payload, 800);
   ob_append_byte(frame->payload, 2);
-  ob_append_ui16(frame->payload, rhizome_http_server_port);
+  ob_append_ui16(frame->payload, httpd_server_port);
   int64_t rowid=0;
   int count = append_bars(frame->payload, &retry, 
 			  "SELECT BAR,ROWID FROM MANIFESTS ORDER BY ROWID DESC LIMIT 3", 
@@ -263,7 +263,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){
     goto error;
   ob_limitsize(frame->payload, 800);
   ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS);
-  ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0);
+  ob_append_ui16(frame->payload, is_rhizome_http_enabled()? httpd_server_port : 0);
   ob_append_ui16(frame->payload, m->manifest_all_bytes);
   ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes);
   ob_append_byte(frame->payload, 0xFF);
@@ -292,7 +292,7 @@ int overlay_rhizome_saw_advertisements(struct decode_context *context, struct ov
   int ad_frame_type=ob_get(f->payload);
   struct socket_address httpaddr = context->addr;
   if (httpaddr.addr.sa_family == AF_INET)
-    httpaddr.inet.sin_port = htons(RHIZOME_HTTP_PORT);
+    httpaddr.inet.sin_port = htons(HTTPD_PORT);
   rhizome_manifest *m=NULL;
 
   int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);

rhizome_restful.c

@@ -0,0 +1,780 @@
+/*
+Serval DNA Rhizome HTTP RESTful interface
+Copyright (C) 2013,2014 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "conf.h"
+#include "serval.h"
+#include "httpd.h"
+#include "strbuf_helpers.h"
+
+static HTTP_RENDERER render_manifest_headers;
+
+static void finalise_union_read_state(httpd_request *r)
+{
+  rhizome_read_close(&r->u.read_state);
+}
+
+static void finalise_union_rhizome_insert(httpd_request *r)
+{
+  form_buf_malloc_release(&r->u.insert.manifest);
+  if (r->u.insert.write.blob_fd != -1)
+    rhizome_fail_write(&r->u.insert.write);
+}
+
+#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8))
+#define alloca_list_token(rowid) list_token_to_str(alloca(LIST_TOKEN_STRLEN + 1), (rowid))
+
+static char *list_token_to_str(char *buf, uint64_t rowid)
+{
+  struct iovec iov[2];
+  iov[0].iov_base = rhizome_db_uuid.u.binary;
+  iov[0].iov_len = sizeof rhizome_db_uuid.u.binary;
+  iov[1].iov_base = &rowid;
+  iov[1].iov_len = sizeof rowid;
+  size_t n = base64url_encodev(buf, iov, 2);
+  assert(n == LIST_TOKEN_STRLEN);
+  buf[n] = '\0';
+  return buf;
+}
+
+static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp)
+{
+  unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp];
+  if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token)
+    return 0;
+  if (cmp_uuid_t(&rhizome_db_uuid, (uuid_t *) &token) != 0)
+    return 0;
+  memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp);
+  return 1;
+}
+
+static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
+
+int restful_rhizome_bundlelist_json(httpd_request *r, const char *remainder)
+{
+  r->http.response.header.content_type = CONTENT_TYPE_JSON;
+  if (!is_rhizome_http_enabled())
+    return 403;
+  if (*remainder)
+    return 404;
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  int ret = authorize(&r->http);
+  if (ret)
+    return ret;
+  r->u.rhlist.phase = LIST_HEADER;
+  r->u.rhlist.rowcount = 0;
+  bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor);
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content);
+  return 1;
+}
+
+static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_rhizome_bundlelist_json_content_chunk;
+
+static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
+{
+  httpd_request *r = (httpd_request *) hr;
+  int ret = rhizome_list_open(&r->u.rhlist.cursor);
+  if (ret == -1)
+    return -1;
+  ret = generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, restful_rhizome_bundlelist_json_content_chunk);
+  rhizome_list_release(&r->u.rhlist.cursor);
+  return ret;
+}
+
+int restful_rhizome_newsince(httpd_request *r, const char *remainder)
+{
+  r->http.response.header.content_type = CONTENT_TYPE_JSON;
+  if (!is_rhizome_http_enabled())
+    return 403;
+  uint64_t rowid;
+  const char *end = NULL;
+  if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0)
+    return 404;
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  int ret = authorize(&r->http);
+  if (ret)
+    return ret;
+  r->u.rhlist.phase = LIST_HEADER;
+  r->u.rhlist.rowcount = 0;
+  bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor);
+  r->u.rhlist.cursor.rowid_since = rowid;
+  r->u.rhlist.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000;
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content);
+  return 1;
+}
+
+static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr, strbuf b)
+{
+  httpd_request *r = (httpd_request *) hr;
+  const char *headers[] = {
+    ".token",
+    "_id",
+    "service",
+    "id",
+    "version",
+    "date",
+    ".inserttime",
+    ".author",
+    ".fromhere",
+    "filesize",
+    "filehash",
+    "sender",
+    "recipient",
+    "name"
+  };
+  switch (r->u.rhlist.phase) {
+    case LIST_HEADER:
+      strbuf_puts(b, "{\n\"header\":[");
+      unsigned i;
+      for (i = 0; i != NELS(headers); ++i) {
+	if (i)
+	  strbuf_putc(b, ',');
+	strbuf_json_string(b, headers[i]);
+      }
+      strbuf_puts(b, "],\n\"rows\":[");
+      if (!strbuf_overrun(b))
+	r->u.rhlist.phase = LIST_ROWS;
+      return 1;
+    case LIST_ROWS:
+      {
+	int ret = rhizome_list_next(&r->u.rhlist.cursor);
+	if (ret == -1)
+	  return -1;
+	if (ret == 0) {
+	  time_ms_t now;
+	  if (r->u.rhlist.cursor.rowid_since == 0 || (now = gettime_ms()) >= r->u.rhlist.end_time) {
+	    r->u.rhlist.phase = LIST_END;
+	    return 1;
+	  }
+	  time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms;
+	  if (wake_at > r->u.rhlist.end_time)
+	    wake_at = r->u.rhlist.end_time;
+	  http_request_pause_response(&r->http, wake_at);
+	  return 0;
+	}
+	rhizome_manifest *m = r->u.rhlist.cursor.manifest;
+	assert(m->filesize != RHIZOME_SIZE_UNSET);
+	rhizome_lookup_author(m);
+	if (r->u.rhlist.rowcount != 0)
+	  strbuf_putc(b, ',');
+	strbuf_puts(b, "\n[");
+	if (m->rowid > r->u.rhlist.rowid_highest) {
+	  strbuf_json_string(b, alloca_list_token(m->rowid));
+	  r->u.rhlist.rowid_highest = m->rowid;
+	} else
+	  strbuf_json_null(b);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRIu64, m->rowid);
+	strbuf_putc(b, ',');
+	strbuf_json_string(b, m->service);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRIu64, m->version);
+	strbuf_putc(b, ',');
+	if (m->has_date)
+	  strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
+	else
+	  strbuf_json_null(b);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
+	switch (m->authorship) {
+	  case AUTHOR_LOCAL:
+	  case AUTHOR_AUTHENTIC:
+	    strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
+	    strbuf_puts(b, ",1,");
+	    break;
+	  default:
+	    strbuf_json_null(b);
+	    strbuf_puts(b, ",1,");
+	    break;
+	}
+	strbuf_sprintf(b, "%"PRIu64, m->filesize);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_string(b, m->name);
+	strbuf_puts(b, "]");
+	if (!strbuf_overrun(b)) {
+	  rhizome_list_commit(&r->u.rhlist.cursor);
+	  ++r->u.rhlist.rowcount;
+	}
+      }
+      return 1;
+    case LIST_END:
+      strbuf_puts(b, "\n]\n}\n");
+      if (!strbuf_overrun(b))
+	r->u.rhlist.phase = LIST_DONE;
+      // fall through...
+    case LIST_DONE:
+      return 0;
+  }
+  abort();
+}
+
+static HTTP_REQUEST_PARSER restful_rhizome_insert_end;
+static int insert_mime_part_start(struct http_request *);
+static int insert_mime_part_end(struct http_request *);
+static int insert_mime_part_header(struct http_request *, const struct mime_part_headers *);
+static int insert_mime_part_body(struct http_request *, char *, size_t);
+
+int restful_rhizome_insert(httpd_request *r, const char *remainder)
+{
+  r->http.response.header.content_type = CONTENT_TYPE_JSON;
+  if (*remainder)
+    return 404;
+  if (!is_rhizome_http_enabled())
+    return 403;
+  if (r->http.verb != HTTP_VERB_POST)
+    return 405;
+  int ret = authorize(&r->http);
+  if (ret)
+    return ret;
+  // Parse the request body as multipart/form-data.
+  assert(r->u.insert.current_part == NULL);
+  assert(!r->u.insert.received_author);
+  assert(!r->u.insert.received_secret);
+  assert(!r->u.insert.received_manifest);
+  assert(!r->u.insert.received_payload);
+  bzero(&r->u.insert.write, sizeof r->u.insert.write);
+  r->u.insert.write.blob_fd = -1;
+  r->finalise_union = finalise_union_rhizome_insert;
+  r->http.form_data.handle_mime_part_start = insert_mime_part_start;
+  r->http.form_data.handle_mime_part_end = insert_mime_part_end;
+  r->http.form_data.handle_mime_part_header = insert_mime_part_header;
+  r->http.form_data.handle_mime_body = insert_mime_part_body;
+  // Perform the insert once the body has arrived.
+  r->http.handle_content_end = restful_rhizome_insert_end;
+  return 1;
+}
+
+static char PART_MANIFEST[] = "manifest";
+static char PART_PAYLOAD[] = "payload";
+static char PART_AUTHOR[] = "bundle-author";
+static char PART_SECRET[] = "bundle-secret";
+
+static int insert_mime_part_start(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  assert(r->u.insert.current_part == NULL);
+  return 0;
+}
+
+static int insert_make_manifest(httpd_request *r)
+{
+  if (!r->u.insert.received_manifest)
+    return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
+  if ((r->manifest = rhizome_new_manifest())) {
+    if (r->u.insert.manifest.length == 0)
+      return 0;
+    assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata);
+    memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length);
+    r->manifest->manifest_all_bytes = r->u.insert.manifest.length;
+    int n = rhizome_manifest_parse(r->manifest);
+    switch (n) {
+      case -1:
+	break;
+      case 0:
+	if (!r->manifest->malformed)
+	  return 0;
+	// fall through
+      case 1:
+	http_request_simple_response(&r->http, 403, "Malformed manifest");
+	return 403;
+      default:
+	WHYF("rhizome_manifest_parse() returned %d", n);
+	break;
+    }
+  }
+  return 500;
+}
+
+static int insert_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (strcmp(h->content_disposition.name, PART_AUTHOR) == 0) {
+    if (r->u.insert.received_author)
+      return http_response_form_part(r, "Duplicate", PART_AUTHOR, NULL, 0);
+    r->u.insert.current_part = PART_AUTHOR;
+    assert(r->u.insert.author_hex_len == 0);
+  }
+  else if (strcmp(h->content_disposition.name, PART_SECRET) == 0) {
+    if (r->u.insert.received_secret)
+      return http_response_form_part(r, "Duplicate", PART_SECRET, NULL, 0);
+    r->u.insert.current_part = PART_SECRET;
+    assert(r->u.insert.secret_hex_len == 0);
+  }
+  else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) {
+    // Reject a request if it has a repeated manifest part.
+    if (r->u.insert.received_manifest)
+      return http_response_form_part(r, "Duplicate", PART_MANIFEST, NULL, 0);
+    form_buf_malloc_init(&r->u.insert.manifest, MAX_MANIFEST_BYTES);
+    if (   strcmp(h->content_type.type, "rhizome-manifest") != 0
+	|| strcmp(h->content_type.subtype, "text") != 0
+    )
+      return http_response_form_part(r, "Unsupported Content-Type in", PART_MANIFEST, NULL, 0);
+    r->u.insert.current_part = PART_MANIFEST;
+  }
+  else if (strcmp(h->content_disposition.name, PART_PAYLOAD) == 0) {
+    // Reject a request if it has a repeated payload part.
+    if (r->u.insert.received_payload)
+      return http_response_form_part(r, "Duplicate", PART_PAYLOAD, NULL, 0);
+    // Reject a request if it has a missing manifest part preceding the payload part.
+    if (!r->u.insert.received_manifest)
+      return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
+    assert(r->manifest != NULL);
+    r->u.insert.current_part = PART_PAYLOAD;
+    // If the manifest does not contain a 'name' field, then assign it from the payload filename.
+    if (   strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0
+	&& r->manifest->name == NULL
+	&& *h->content_disposition.filename
+    )
+      rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
+    // Start writing the payload content into the Rhizome store.  Note: r->manifest->filesize can be
+    // RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field.
+    r->u.insert.payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
+    r->u.insert.payload_size = 0;
+    switch (r->u.insert.payload_status) {
+      case RHIZOME_PAYLOAD_STATUS_ERROR:
+	WHYF("rhizome_write_open_manifest() returned %d", r->u.insert.payload_status);
+	return 500;
+      case RHIZOME_PAYLOAD_STATUS_STORED:
+	// TODO: initialise payload hash so it can be compared with stored payload
+	break;
+      default:
+	break;
+    }
+  }
+  else
+    return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0);
+  return 0;
+}
+
+static int insert_mime_part_body(struct http_request *hr, char *buf, size_t len)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (r->u.insert.current_part == PART_AUTHOR) {
+    accumulate_text(r, PART_AUTHOR,
+		    r->u.insert.author_hex,
+		    sizeof r->u.insert.author_hex,
+		    &r->u.insert.author_hex_len,
+		    buf, len);
+  }
+  else if (r->u.insert.current_part == PART_SECRET) {
+    accumulate_text(r, PART_SECRET,
+		    r->u.insert.secret_hex,
+		    sizeof r->u.insert.secret_hex,
+		    &r->u.insert.secret_hex_len,
+		    buf, len);
+  }
+  else if (r->u.insert.current_part == PART_MANIFEST) {
+    form_buf_malloc_accumulate(r, PART_MANIFEST, &r->u.insert.manifest, buf, len);
+  }
+  else if (r->u.insert.current_part == PART_PAYLOAD) {
+    r->u.insert.payload_size += len;
+    switch (r->u.insert.payload_status) {
+      case RHIZOME_PAYLOAD_STATUS_NEW:
+	if (rhizome_write_buffer(&r->u.insert.write, (unsigned char *)buf, len) == -1)
+	  return 500;
+	break;
+      case RHIZOME_PAYLOAD_STATUS_STORED:
+	// TODO: calculate payload hash so it can be compared with stored payload
+	break;
+      default:
+	break;
+    }
+  } else
+    FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part));
+  return 0;
+}
+
+static int insert_mime_part_end(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (r->u.insert.current_part == PART_AUTHOR) {
+    if (   r->u.insert.author_hex_len != sizeof r->u.insert.author_hex
+	|| strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, NULL) == -1
+    )
+      return http_response_form_part(r, "Invalid", PART_AUTHOR, r->u.insert.author_hex, r->u.insert.author_hex_len);
+    r->u.insert.received_author = 1;
+    if (config.debug.rhizome)
+      DEBUGF("received %s = %s", PART_AUTHOR, alloca_tohex_sid_t(r->u.insert.author));
+  }
+  else if (r->u.insert.current_part == PART_SECRET) {
+    if (   r->u.insert.secret_hex_len != sizeof r->u.insert.secret_hex
+	|| strn_to_rhizome_bk_t(&r->u.insert.bundle_secret, r->u.insert.secret_hex, NULL) == -1
+    )
+      return http_response_form_part(r, "Invalid", PART_SECRET, r->u.insert.secret_hex, r->u.insert.secret_hex_len);
+    r->u.insert.received_secret = 1;
+    if (config.debug.rhizome)
+      DEBUGF("received %s = %s", PART_SECRET, alloca_tohex_rhizome_bk_t(r->u.insert.bundle_secret));
+  }
+  else if (r->u.insert.current_part == PART_MANIFEST) {
+    r->u.insert.received_manifest = 1;
+    int result = insert_make_manifest(r);
+    if (result)
+      return result;
+    if (r->manifest->has_id && r->u.insert.received_secret)
+      rhizome_apply_bundle_secret(r->manifest, &r->u.insert.bundle_secret);
+    if (r->manifest->service == NULL)
+      rhizome_manifest_set_service(r->manifest, RHIZOME_SERVICE_FILE);
+    if (rhizome_fill_manifest(r->manifest, NULL, r->u.insert.received_author ? &r->u.insert.author: NULL) == -1) {
+      WHY("rhizome_fill_manifest() failed");
+      return 500;
+    }
+    if (r->manifest->is_journal) {
+      http_request_simple_response(&r->http, 403, "Insert not supported for journals");
+      return 403;
+    }
+    assert(r->manifest != NULL);
+  }
+  else if (r->u.insert.current_part == PART_PAYLOAD) {
+    r->u.insert.received_payload = 1;
+    if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_NEW)
+      r->u.insert.payload_status = rhizome_finish_write(&r->u.insert.write);
+    if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
+      WHYF("rhizome_finish_write() returned status = %d", r->u.insert.payload_status);
+      return 500;
+    }
+  } else
+    FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part));
+  r->u.insert.current_part = NULL;
+  return 0;
+}
+
+static int restful_rhizome_insert_end(struct http_request *hr)
+{
+  httpd_request *r = (httpd_request *) hr;
+  if (!r->u.insert.received_manifest)
+    return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
+  if (!r->u.insert.received_payload)
+    return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0);
+  // Fill in the missing manifest fields and ensure payload and manifest are consistent.
+  assert(r->manifest != NULL);
+  assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET);
+  switch (r->u.insert.payload_status) {
+    case RHIZOME_PAYLOAD_STATUS_ERROR:
+      return 500;
+    case RHIZOME_PAYLOAD_STATUS_NEW:
+      if (r->manifest->filesize == RHIZOME_SIZE_UNSET)
+	rhizome_manifest_set_filesize(r->manifest, r->u.insert.write.file_length);
+      // fall through
+    case RHIZOME_PAYLOAD_STATUS_STORED:
+      // TODO: check that stored hash matches received payload's hash
+      // fall through
+    case RHIZOME_PAYLOAD_STATUS_EMPTY:
+      assert(r->manifest->filesize != RHIZOME_SIZE_UNSET);
+      if (r->u.insert.payload_size == r->manifest->filesize)
+	break;
+      // fall through
+    case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
+      {
+	strbuf msg = strbuf_alloca(200);
+	strbuf_sprintf(msg, "Payload size (%"PRIu64") contradicts manifest (filesize=%"PRIu64")", r->u.insert.payload_size, r->manifest->filesize);
+	http_request_simple_response(&r->http, 403, strbuf_str(msg));
+	return 403;
+      }
+    case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
+      http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest");
+      return 403;
+    case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
+      http_request_simple_response(&r->http, 403, "Missing bundle secret");
+      return 403;
+    default:
+      FATALF("payload_status = %d", r->u.insert.payload_status);
+  }
+  // Finalise the manifest and add it to the store.
+  if (r->manifest->filesize) {
+    if (!r->manifest->has_filehash)
+      rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id);
+    else
+      assert(cmp_rhizome_filehash_t(&r->u.insert.write.id, &r->manifest->filehash) == 0);
+  }
+  if (!rhizome_manifest_validate(r->manifest) || r->manifest->malformed) {
+    http_request_simple_response(&r->http, 403, "Manifest is malformed");
+    return 403;
+  }
+  if (!r->manifest->haveSecret) {
+    http_request_simple_response(&r->http, 403, "Missing bundle secret");
+    return 403;
+  }
+  rhizome_manifest *mout = NULL;
+  int result;
+  switch (rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new)) {
+    case RHIZOME_BUNDLE_STATUS_NEW:
+      result = 201;
+      if (mout && mout != r->manifest)
+	rhizome_manifest_free(mout);
+      mout = NULL;
+      break;
+    case RHIZOME_BUNDLE_STATUS_SAME:
+    case RHIZOME_BUNDLE_STATUS_OLD:
+    case RHIZOME_BUNDLE_STATUS_DUPLICATE:
+      result = 200;
+      break;
+    case RHIZOME_BUNDLE_STATUS_INVALID:
+      result = 403;
+      break;
+    case RHIZOME_BUNDLE_STATUS_ERROR:
+    default:
+      result = 500;
+      break;
+  }
+  if (mout && mout != r->manifest) {
+    rhizome_manifest_free(r->manifest);
+    r->manifest = mout;
+  }
+  if (result >= 400)
+    return result;
+  rhizome_authenticate_author(r->manifest);
+  r->http.render_extra_headers = render_manifest_headers;
+  http_request_response_static(&r->http, result, "rhizome-manifest/text",
+      (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
+    );
+  return 0;
+}
+
+static HTTP_HANDLER restful_rhizome_bid_rhm;
+static HTTP_HANDLER restful_rhizome_bid_raw_bin;
+static HTTP_HANDLER restful_rhizome_bid_decrypted_bin;
+
+int restful_rhizome_(httpd_request *r, const char *remainder)
+{
+  r->http.response.header.content_type = CONTENT_TYPE_JSON;
+  if (!is_rhizome_http_enabled())
+    return 403;
+  HTTP_HANDLER *handler = NULL;
+  rhizome_bid_t bid;
+  const char *end;
+  if (strn_to_rhizome_bid_t(&bid, remainder, &end) != -1) {
+    if (strcmp(end, ".rhm") == 0) {
+      handler = restful_rhizome_bid_rhm;
+      remainder = "";
+    } else if (strcmp(end, "/raw.bin") == 0) {
+      handler = restful_rhizome_bid_raw_bin;
+      remainder = "";
+    } else if (strcmp(end, "/decrypted.bin") == 0) {
+      handler = restful_rhizome_bid_decrypted_bin;
+      remainder = "";
+    }
+  }
+  if (handler == NULL)
+    return 404;
+  if (r->http.verb != HTTP_VERB_GET)
+    return 405;
+  int ret = authorize(&r->http);
+  if (ret)
+    return ret;
+  if ((r->manifest = rhizome_new_manifest()) == NULL)
+    return 500;
+  ret = rhizome_retrieve_manifest(&bid, r->manifest);
+  if (ret == -1)
+    return 500;
+  if (ret == 0) {
+    rhizome_authenticate_author(r->manifest);
+    r->http.render_extra_headers = render_manifest_headers;
+  } else {
+    assert(r->manifest == NULL);
+    assert(r->http.render_extra_headers == NULL);
+  }
+  ret = handler(r, remainder);
+  return ret;
+}
+
+static int restful_rhizome_bid_rhm(httpd_request *r, const char *remainder)
+{
+  if (*remainder || r->manifest == NULL)
+    return 404;
+  http_request_response_static(&r->http, 200, "rhizome-manifest/text",
+      (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
+    );
+  return 1;
+}
+
+static int restful_rhizome_bid_raw_bin(httpd_request *r, const char *remainder)
+{
+  if (*remainder || r->manifest == NULL)
+    return 404;
+  if (r->manifest->filesize == 0) {
+    http_request_response_static(&r->http, 200, CONTENT_TYPE_BLOB, "", 0);
+    return 1;
+  }
+  int ret = rhizome_response_content_init_filehash(r, &r->manifest->filehash);
+  if (ret)
+    return ret;
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_BLOB, rhizome_payload_content);
+  return 1;
+}
+
+static int restful_rhizome_bid_decrypted_bin(httpd_request *r, const char *remainder)
+{
+  if (*remainder || r->manifest == NULL)
+    return 404;
+  if (r->manifest->filesize == 0) {
+    // TODO use Content Type from manifest (once it is implemented)
+    http_request_response_static(&r->http, 200, CONTENT_TYPE_BLOB, "", 0);
+    return 1;
+  }
+  int ret = rhizome_response_content_init_payload(r, r->manifest);
+  if (ret)
+    return ret;
+  // TODO use Content Type from manifest (once it is implemented)
+  http_request_response_generated(&r->http, 200, CONTENT_TYPE_BLOB, rhizome_payload_content);
+  return 1;
+}
+
+static int rhizome_response_content_init_read_state(httpd_request *r)
+{
+  if (r->u.read_state.length == RHIZOME_SIZE_UNSET && rhizome_read(&r->u.read_state, NULL, 0)) {
+    rhizome_read_close(&r->u.read_state);
+    return 404;
+  }
+  assert(r->u.read_state.length != RHIZOME_SIZE_UNSET);
+  r->http.response.header.resource_length = r->u.read_state.length;
+  if (r->http.request_header.content_range_count > 0) {
+    assert(r->http.request_header.content_range_count == 1);
+    struct http_range closed;
+    unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
+    if (n == 0 || http_range_bytes(&closed, 1) == 0)
+      return 416; // Request Range Not Satisfiable
+    r->http.response.header.content_range_start = closed.first;
+    r->http.response.header.content_length = closed.last - closed.first + 1;
+    r->u.read_state.offset = closed.first;
+  } else {
+    r->http.response.header.content_range_start = 0;
+    r->http.response.header.content_length = r->http.response.header.resource_length;
+    r->u.read_state.offset = 0;
+  }
+  return 0;
+}
+
+int rhizome_response_content_init_filehash(httpd_request *r, const rhizome_filehash_t *hash)
+{
+  bzero(&r->u.read_state, sizeof r->u.read_state);
+  r->u.read_state.blob_fd = -1;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_read_state;
+  enum rhizome_payload_status status = rhizome_open_read(&r->u.read_state, hash);
+  switch (status) {
+    case RHIZOME_PAYLOAD_STATUS_EMPTY:
+    case RHIZOME_PAYLOAD_STATUS_STORED:
+      break;
+    case RHIZOME_PAYLOAD_STATUS_NEW:
+      return 404;
+    case RHIZOME_PAYLOAD_STATUS_ERROR:
+    case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
+    case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
+    case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
+      return -1;
+    default:
+      FATALF("status = %d", status);
+  }
+  return rhizome_response_content_init_read_state(r);
+}
+
+int rhizome_response_content_init_payload(httpd_request *r, rhizome_manifest *m)
+{
+  bzero(&r->u.read_state, sizeof r->u.read_state);
+  r->u.read_state.blob_fd = -1;
+  assert(r->finalise_union == NULL);
+  r->finalise_union = finalise_union_read_state;
+  enum rhizome_payload_status status = rhizome_open_decrypt_read(m, &r->u.read_state);
+  switch (status) {
+    case RHIZOME_PAYLOAD_STATUS_EMPTY:
+    case RHIZOME_PAYLOAD_STATUS_STORED:
+      break;
+    case RHIZOME_PAYLOAD_STATUS_NEW:
+      return 404;
+    case RHIZOME_PAYLOAD_STATUS_ERROR:
+    case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
+    case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
+    case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
+      return -1;
+    default:
+      FATALF("status = %d", status);
+  }
+  return rhizome_response_content_init_read_state(r);
+}
+
+int rhizome_payload_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
+{
+  // Only read multiples of 4k from disk.
+  const size_t blocksz = 1 << 12;
+  // Ask for a large buffer for all future reads.
+  const size_t preferred_bufsz = 16 * blocksz;
+  // Reads the next part of the payload into the supplied buffer.
+  httpd_request *r = (httpd_request *) hr;
+  assert(r->u.read_state.length != RHIZOME_SIZE_UNSET);
+  assert(r->u.read_state.offset < r->u.read_state.length);
+  uint64_t remain = r->u.read_state.length - r->u.read_state.offset;
+  size_t readlen = bufsz;
+  if (remain < bufsz)
+    readlen = remain;
+  else
+    readlen &= ~(blocksz - 1);
+  if (readlen > 0) {
+    ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
+    if (n == -1)
+      return -1;
+    result->generated = (size_t) n;
+  }
+  assert(r->u.read_state.offset <= r->u.read_state.length);
+  remain = r->u.read_state.length - r->u.read_state.offset;
+  result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
+  return remain ? 1 : 0;
+}
+
+static void render_manifest_headers(struct http_request *hr, strbuf sb)
+{
+  httpd_request *r = (httpd_request *) hr;
+  rhizome_manifest *m = r->manifest;
+  strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Id: %s\r\n", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
+  strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Version: %"PRIu64"\r\n", m->version);
+  strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filesize: %"PRIu64"\r\n", m->filesize);
+  if (m->filesize != 0)
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filehash: %s\r\n", alloca_tohex_rhizome_filehash_t(m->filehash));
+  if (m->has_bundle_key)
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-BK: %s\r\n", alloca_tohex_rhizome_bk_t(m->bundle_key));
+  if (m->has_date)
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Date: %"PRIu64"\r\n", m->date);
+  if (m->name) {
+    strbuf_puts(sb, "Serval-Rhizome-Bundle-Name: ");
+    strbuf_append_quoted_string(sb, m->name);
+    strbuf_puts(sb, "\r\n");
+  }
+  if (m->service)
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Service: %s\r\n", m->service);
+  assert(m->authorship != AUTHOR_LOCAL);
+  if (m->authorship == AUTHOR_AUTHENTIC)
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Author: %s\r\n", alloca_tohex_sid_t(m->author));
+  assert(m->haveSecret);
+  {
+    char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
+    rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
+    strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Secret: %s\r\n", secret);
+  }
+  strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Rowid: %"PRIu64"\r\n", m->rowid);
+  strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Inserttime: %"PRIu64"\r\n", m->inserttime);
+}
+

sourcefiles.mk

@@ -14,6 +14,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)fdqueue.c \
 	$(SERVAL_BASE)fifo.c \
 	$(SERVAL_BASE)golay.c \
+	$(SERVAL_BASE)httpd.c \
 	$(SERVAL_BASE)http_server.c \
 	$(SERVAL_BASE)keyring.c \
 	$(SERVAL_BASE)log.c \
@@ -22,6 +23,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)main.c \
 	$(SERVAL_BASE)radio_link.c \
 	$(SERVAL_BASE)meshms.c \
+	$(SERVAL_BASE)meshms_restful.c \
 	$(SERVAL_BASE)mdp_client.c \
 	$(SERVAL_BASE)mdp_net.c \
 	$(SERVAL_BASE)msp_client.c \
@@ -58,6 +60,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)rhizome_direct_http.c \
 	$(SERVAL_BASE)rhizome_fetch.c \
 	$(SERVAL_BASE)rhizome_http.c \
+	$(SERVAL_BASE)rhizome_restful.c \
 	$(SERVAL_BASE)rhizome_packetformats.c \
 	$(SERVAL_BASE)rhizome_store.c \
 	$(SERVAL_BASE)rhizome_sync.c \

str.c

@@ -128,10 +128,10 @@ static size_t _base64_encodev(const char symbols[], char *dstBase64, const struc
   if (place)
     *dst++ = symbols[buf];
   switch (place) {
-    case 2:
-      *dst++ = symbols[64];
     case 1:
       *dst++ = symbols[64];
+    case 2:
+      *dst++ = symbols[64];
   }
   return dst - dstBase64;
 }

strbuf.h

@@ -82,6 +82,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <string.h>
 #include <stdarg.h>
 #include <alloca.h>
+#include <assert.h>
 
 #ifndef __STRBUF_INLINE
 # if __GNUC__ && !__GNUC_STDC_INLINE__
@@ -132,6 +133,29 @@ typedef const struct strbuf *const_strbuf;
  */
 #define strbuf_alloca(size) strbuf_make(alloca(SIZEOF_STRBUF + (size)), SIZEOF_STRBUF + (size))
 
+/** Convenience macro that calls strbuf_alloca() to allocate a large enough
+ * buffer to hold the entire content produced by a given expression that
+ * appends to the strbuf.  The first strbuf_alloca() will use the supplied
+ * initial length, and if that overruns, then a second strbuf_alloca() will use
+ * the strbuf_count() from the first pass, so as long as the expression is
+ * stable (ie, always produces the same output), the final assert() will not
+ * be triggered.
+ *
+ *      strbuf b;
+ *      STRBUF_ALLOCA_FIT(b, 20, (strbuf_append_variable_content(b, ...)));
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+#define STRBUF_ALLOCA_FIT(__SB, __INITIAL_LEN, __EXPR) \
+    do { \
+        __SB = strbuf_alloca((__INITIAL_LEN) + 1); \
+        __EXPR; \
+        if (strbuf_overrun(__SB)) { \
+            __SB = strbuf_alloca(strbuf_count(__SB) + 1); \
+            __EXPR; \
+        } \
+        assert(!strbuf_overrun(__SB)); \
+    } while (0)
 
 /** Convenience macro for filling a strbuf from the calling function's
  * printf(3)-like variadic arguments.

strbuf_helpers.c

@@ -459,42 +459,86 @@ strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
   return sb;
 }
 
+static void _html_char(strbuf sb, char c)
+{
+  if (c == '&')
+    strbuf_puts(sb, "&");
+  else if (c == '<')
+    strbuf_puts(sb, "&lt;");
+  else if (c == '>')
+    strbuf_puts(sb, "&gt;");
+  else if (c == '"')
+    strbuf_puts(sb, "&quot;");
+  else if (c == '\'')
+    strbuf_puts(sb, "&apos;");
+  else if (iscntrl(c))
+    strbuf_sprintf(sb, "&#%u;", (unsigned char) c);
+  else
+    strbuf_putc(sb, c);
+}
+
+strbuf strbuf_html_escape(strbuf sb, const char *str, size_t strlen)
+{
+  for (; strlen; --strlen, ++str)
+    _html_char(sb, *str);
+  return sb;
+}
+
 strbuf strbuf_json_null(strbuf sb)
 {
   strbuf_puts(sb, "null");
   return sb;
 }
 
+strbuf strbuf_json_boolean(strbuf sb, int boolean)
+{
+  strbuf_puts(sb, boolean ? "true" : "false");
+  return sb;
+}
+
+static void _json_char(strbuf sb, char c)
+{
+  if (c == '"' || c == '\\') {
+    strbuf_putc(sb, '\\');
+    strbuf_putc(sb, c);
+  }
+  else if (c == '\b')
+    strbuf_puts(sb, "\\b");
+  else if (c == '\f')
+    strbuf_puts(sb, "\\f");
+  else if (c == '\n')
+    strbuf_puts(sb, "\\n");
+  else if (c == '\r')
+    strbuf_puts(sb, "\\r");
+  else if (c == '\t')
+    strbuf_puts(sb, "\\t");
+  else if (iscntrl(c))
+    strbuf_sprintf(sb, "\\u%04X", (unsigned char) c);
+  else
+    strbuf_putc(sb, c);
+}
+
 strbuf strbuf_json_string(strbuf sb, const char *str)
 {
   if (str) {
     strbuf_putc(sb, '"');
-    for (; *str; ++str) {
+    for (; *str; ++str)
-      if (*str == '"' || *str == '\\') {
+      _json_char(sb, *str);
-	strbuf_putc(sb, '\\');
-	strbuf_putc(sb, *str);
-      }
-      else if (*str == '\b')
-	strbuf_puts(sb, "\\b");
-      else if (*str == '\f')
-	strbuf_puts(sb, "\\f");
-      else if (*str == '\n')
-	strbuf_puts(sb, "\\n");
-      else if (*str == '\r')
-	strbuf_puts(sb, "\\r");
-      else if (*str == '\t')
-	strbuf_puts(sb, "\\t");
-      else if (iscntrl(*str))
-	strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str);
-      else
-	strbuf_putc(sb, *str);
-    }
     strbuf_putc(sb, '"');
   } else
     strbuf_json_null(sb);
   return sb;
 }
 
+strbuf strbuf_json_string_len(strbuf sb, const char *str, size_t strlen)
+{
+  strbuf_putc(sb, '"');
+  for (; strlen; --strlen, ++str)
+    _json_char(sb, *str);
+  strbuf_putc(sb, '"');
+  return sb;
+}
+
 strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
 {
   if (buf) {
@@ -510,6 +554,60 @@ strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
   return sb;
 }
 
+strbuf strbuf_json_atom(strbuf sb, const struct json_atom *atom)
+{
+  switch (atom->type) {
+    case JSON_NULL:
+      return strbuf_json_null(sb);
+    case JSON_BOOLEAN:
+      return strbuf_json_boolean(sb, atom->u.boolean);
+    case JSON_INTEGER:
+      strbuf_sprintf(sb, "%"PRId64, atom->u.integer);
+      return sb;
+    case JSON_STRING_NULTERM:
+      return strbuf_json_string(sb, atom->u.string.content);
+    case JSON_STRING_LENGTH:
+      return strbuf_json_string_len(sb, atom->u.string.content, atom->u.string.length);
+  }
+  abort();
+}
+
+strbuf strbuf_json_atom_as_text(strbuf sb, const struct json_atom *atom)
+{
+  switch (atom->type) {
+    case JSON_NULL:
+      return strbuf_json_null(sb);
+    case JSON_BOOLEAN:
+      return strbuf_puts(sb, atom->u.boolean ? "True" : "False");
+    case JSON_INTEGER:
+      strbuf_sprintf(sb, "%"PRId64, atom->u.integer);
+      return sb;
+    case JSON_STRING_NULTERM:
+      return strbuf_puts(sb, atom->u.string.content);
+    case JSON_STRING_LENGTH:
+      return strbuf_ncat(sb, atom->u.string.content, atom->u.string.length);
+  }
+  abort();
+}
+
+strbuf strbuf_json_atom_as_html(strbuf sb, const struct json_atom *atom)
+{
+  switch (atom->type) {
+    case JSON_NULL:
+      return strbuf_json_null(sb);
+    case JSON_BOOLEAN:
+      return strbuf_json_boolean(sb, atom->u.boolean);
+    case JSON_INTEGER:
+      strbuf_sprintf(sb, "%"PRId64, atom->u.integer);
+      return sb;
+    case JSON_STRING_NULTERM:
+      return strbuf_html_escape(sb, atom->u.string.content, strlen(atom->u.string.content));
+    case JSON_STRING_LENGTH:
+      return strbuf_html_escape(sb, atom->u.string.content, atom->u.string.length);
+  }
+  abort();
+}
+
 strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
 {
   unsigned i;

strbuf_helpers.h

@@ -162,12 +162,33 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
  */
 strbuf strbuf_append_quoted_string(strbuf sb, const char *str);
 
+/* Escape HTML entities.
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+strbuf strbuf_html_escape(strbuf sb, const char *, size_t);
+
 /* Append various JSON elements.
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 strbuf strbuf_json_null(strbuf sb);
-strbuf strbuf_json_string(strbuf sb, const char *str);
+strbuf strbuf_json_boolean(strbuf sb, int boolean);
+strbuf strbuf_json_string(strbuf sb, const char *str); // str can be NULL
+strbuf strbuf_json_string_len(strbuf sb, const char *str, size_t strlen); // str cannot be NULL
 strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len);
+struct json_atom {
+    enum json_atomic_type { JSON_NULL, JSON_BOOLEAN, JSON_INTEGER, JSON_STRING_NULTERM, JSON_STRING_LENGTH } type;
+    union {
+        int boolean;
+        int64_t integer;
+        struct {
+            const char *content;
+            size_t length;
+        } string;
+    } u;
+};
+strbuf strbuf_json_atom(strbuf sb, const struct json_atom *);
+strbuf strbuf_json_atom_as_html(strbuf sb, const struct json_atom *);
+strbuf strbuf_json_atom_as_text(strbuf sb, const struct json_atom *);
 
 /* Append a representation of a struct http_range[] array.
  * @author Andrew Bettison <andrew@servalproject.com>

testdefs_rhizome.sh

@@ -400,13 +400,13 @@ compute_filehash() {
 
 rhizome_http_server_started() {
    local logvar=LOG${1#+}
-   $GREP 'RHIZOME HTTP SERVER,.*START.*port=[0-9]' "${!logvar}"
+   $GREP 'HTTP SERVER START.*port=[0-9].*services=[^ ]*\<Rhizome\>' "${!logvar}"
 }
 
 get_rhizome_server_port() {
    local _var="$1"
    local _logvar=LOG${2#+}
-   local _port=$($SED -n -e '/RHIZOME HTTP SERVER.*START/s/.*port=\([0-9]\{1,\}\).*/\1/p' "${!_logvar}" | $SED -n '$p')
+   local _port=$($SED -n -e '/HTTP SERVER START/s/.*port=\([0-9]\{1,\}\).*services=[^ ]*\<Rhizome\>.*/\1/p' "${!_logvar}" | $SED -n '$p')
    assert --message="instance $2 Rhizome HTTP server port number is known" [ -n "$_port" ]
    if [ -n "$_var" ]; then
       eval "$_var=\$_port"

tests/meshms

@@ -201,4 +201,21 @@ test_listConversations() {
    assertStdoutLineCount '==' 5
 }
 
+doc_sendNoIdentity="Send message from unknown identity"
+setup_sendNoIdentity() {
+   setup_servald
+   set_instance +A
+   create_identities 2
+   setup_logging
+   SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
+}
+test_sendNoIdentity() {
+   executeOk_servald meshms send message $SIDA1 $SIDX "First message"
+   execute $servald meshms send message $SIDX $SIDA1 "First reply"
+   assertExitStatus --stderr == 2
+   executeOk_servald meshms list messages $SIDA1 $SIDX
+   execute $servald meshms list messages $SIDX $SIDA1
+   assertExitStatus --stderr == 2
+}
+
 runTests "$@"

tests/rhizomechannels

@@ -41,7 +41,7 @@ configure_servald_server() {
       set log.show_pid on \
       set log.show_time on \
       set debug.rhizome on \
-      set debug.rhizome_httpd on \
+      set debug.httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set server.respawn_on_crash off \

tests/rhizomehttp

@@ -24,11 +24,51 @@ source "${0%/*}/../testdefs_rhizome.sh"
 
 shopt -s extglob
 
+assertJq() {
+   local json="$1"
+   local jqscript="$2"
+   assert --message="$jqscript" --dump-on-fail="$json" [ "$(jq "$jqscript" "$json")" = true ]
+}
+
+assertJqCmp() {
+   local opts=()
+   while [ $# -gt 0 ]; do
+      case "$1" in
+      --) shift; break;;
+      --*) opts+=("$1"); shift;;
+      *) break;;
+      esac
+   done
+   [ $# -eq 3 ] || error "invalid arguments"
+   local json="$1"
+   local jqscript="$2"
+   local file="$3"
+   jq --raw-output "$jqscript" "$json" >"$TFWTMP/jqcmp.tmp"
+   assert --dump-on-fail="$TFWTMP/jqcmp.tmp" --dump-on-fail="$file" "${opts[@]}" cmp "$TFWTMP/jqcmp.tmp" "$file"
+}
+
+assertJqGrep() {
+   local opts=()
+   while [ $# -gt 0 ]; do
+      case "$1" in
+      --) shift; break;;
+      --*) opts+=("$1"); shift;;
+      *) break;;
+      esac
+   done
+   [ $# -eq 3 ] || error "invalid arguments"
+   local json="$1"
+   local jqscript="$2"
+   local pattern="$3"
+   jq "$jqscript" "$json" >"$TFWTMP/jqgrep.tmp"
+   assertGrep "${opts[@]}" "$TFWTMP/jqgrep.tmp" "$pattern"
+}
+
 setup() {
    CR='
'
    VT='	'
    setup_curl 7
-   setup_jq 1.2
+   setup_jq 1.3
    setup_servald
    set_instance +A
    set_rhizome_config
@@ -36,6 +76,7 @@ setup() {
       set rhizome.api.restful.users.harry.password potter \
       set rhizome.api.restful.users.ron.password weasley \
       set rhizome.api.restful.users.hermione.password grainger
+   set_extra_config
    if [ -z "$IDENTITY_COUNT" ]; then
       create_single_identity
    else
@@ -56,10 +97,14 @@ teardown() {
    report_all_servald_servers
 }
 
+set_extra_config() {
+   :
+}
+
 set_rhizome_config() {
    executeOk_servald config \
+      set debug.http_server on \
       set debug.httpd on \
-      set debug.rhizome_httpd on \
       set debug.rhizome_manifest on \
       set debug.externalblobs on \
       set debug.rhizome on \
@@ -76,6 +121,8 @@ test_AuthBasicMissing() {
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
    assertStdoutIs '401'
    assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$"
+   assertJq http.output 'contains({"http_status_code": 401})'
+   assertJq http.output 'contains({"http_status_message": ""})'
 }
 teardown_AuthBasicMissing() {
    tfw_cat http.headers http.output
@@ -92,6 +139,8 @@ test_AuthBasicWrong() {
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
    assertStdoutIs '401'
    assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$"
+   assertJq http.output 'contains({"http_status_code": 401})'
+   assertJq http.output 'contains({"http_status_message": ""})'
    executeOk curl \
          --silent --fail --show-error --write-out '%{http_code}' \
          --output http.output \
@@ -144,52 +193,38 @@ add_bundles() {
    done
 }
 
-transform_bundlelist_json() {
+transform_list_json() {
-   # The following jq(1) incantation transforms the JSON array in
+   # The following jq(1) incantation transforms a JSON array in from the
-   # bundlelist.json from the following form (which is optimised for
+   # following form (which is optimised for transmission size):
-   # transmission size):
    #     {
-   #        "header":[
+   #        "header":[ "label1", "label2", "label3", ... ],
-   #              ".token", "_id", "service", "id", "version","date",".inserttime",
-   #              ".author",".fromhere","filesize","filehash","sender","recipient",
-   #              "name"
-   #           ],
    #        "rows":[
-   #              [  "xx", rowid1, "service1", bundleid1, version1, .... ],
+   #              [  row1value1, row1value2, row1value3, ... ],
-   #              [  null, rowid2, "service2", bundleid2, version2, .... ],
+   #              [  row2value1, row2value2, row2value3, ... ],
    #                 ...
-   #              [  null, rowidN, "serviceN", bundleidN, versionN, .... ]
+   #              [  rowNvalue1, rowNvalue2, rowNvalue3, ... ]
    #           ]
    #     }
    #
    # into an array of JSON objects:
    #     [
    #        {
-   #           "__index": 0,
+   #           "label1": row1value1,
-   #           ".token": "xx",
+   #           "label2": row1value2,
-   #           "_id": rowid1,
+   #           "label3": row1value3,
-   #           "service": service1,
-   #           "id": bundleid1,
-   #           "version": version1,
    #           ...
    #        },
    #        {
-   #           "__index": 1,
+   #           "label1": row2value1,
-   #           ".token": null,
+   #           "label2": row2value2,
-   #           "_id": rowid2,
+   #           "label3": row2value3,
-   #           "service": service2,
-   #           "id": bundleid2,
-   #           "version": version2,
    #           ...
    #        },
    #        ...
    #        {
-   #           "__index": 2,
+   #           "label1": rowNvalue1,
-   #           ".token": null,
+   #           "label2": rowNvalue2,
-   #           "_id": rowidN,
+   #           "label3": rowNvalue3,
-   #           "service": serviceN,
-   #           "id": bundleidN,
-   #           "version": versionN,
    #           ...
    #        }
    #     ]
@@ -206,12 +241,6 @@ transform_bundlelist_json() {
       ' "$1" >"$2"
 }
 
-assertJq() {
-   local json="$1"
-   local jqscript="$2"
-   assert --message="$jqscript" [ "$(jq "$jqscript" "$json")" = true ]
-}
-
 doc_RhizomeList="HTTP RESTful list Rhizome bundles as JSON"
 setup_RhizomeList() {
    setup
@@ -229,7 +258,7 @@ test_RhizomeList() {
    tfw_cat http.headers bundlelist.json
    tfw_preserve bundlelist.json
    assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ]
-   transform_bundlelist_json bundlelist.json array_of_objects.json
+   transform_list_json bundlelist.json array_of_objects.json
    tfw_preserve array_of_objects.json
    for ((n = 0; n != NBUNDLES; ++n)); do
       if [ "${ROWID[$n]}" -eq "$ROWID_MAX" ]; then
@@ -256,20 +285,13 @@ test_RhizomeList() {
    done
 }
 
-curl_newsince() {
-   curl \
-         --silent --fail --show-error \
-         --no-buffer \
-         --output "$1" \
-         --basic --user harry:potter \
-         "http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json"
-}
-
 doc_RhizomeNewSince="HTTP RESTful list Rhizome bundles since token as JSON"
 setup_RhizomeNewSince() {
+   set_extra_config() {
+      executeOk_servald config set rhizome.api.restful.newsince_timeout 60s \
+                               set rhizome.api.restful.newsince_poll_ms 500
+   }
    setup
-   executeOk_servald config set rhizome.api.restful.newsince_timeout 60s
-   executeOk_servald config set rhizome.api.restful.newsince_poll_ms 500
    add_bundles 0 5
    executeOk curl \
          --silent --fail --show-error \
@@ -278,19 +300,24 @@ setup_RhizomeNewSince() {
          --basic --user harry:potter \
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
    assert [ "$(jq '.rows | length' bundlelist.json)" = 6 ]
-   transform_bundlelist_json bundlelist.json array_of_objects.json
+   transform_list_json bundlelist.json array_of_objects.json
    token=$(jq --raw-output '.[0][".token"]' array_of_objects.json)
    assert [ -n "$token" ]
 }
 test_RhizomeNewSince() {
-   fork %curl1 curl_newsince newsince1.json
+   for i in 1 2 3; do
-   fork %curl2 curl_newsince newsince2.json
+      fork %curl$i curl \
-   fork %curl3 curl_newsince newsince3.json
+            --silent --fail --show-error \
+            --no-buffer \
+            --output newsince$i.json \
+            --basic --user harry:potter \
+            "http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json"
+   done
    wait_until [ -e newsince1.json -a -e newsince2.json -a -e newsince3.json ]
    add_bundles 6 10
-   wait_until --timeout=10 grep "${BID[10]}" newsince1.json
+   for i in 1 2 3; do
-   wait_until --timeout=10 grep "${BID[10]}" newsince2.json
+      wait_until --timeout=10 grep "${BID[10]}" newsince$i.json
-   wait_until --timeout=10 grep "${BID[10]}" newsince3.json
+   done
    fork_terminate_all
    fork_wait_all
    for i in 1 2 3; do
@@ -298,7 +325,7 @@ test_RhizomeNewSince() {
          echo ']}' >>newsince$i.json
          assert [ $(jq . newsince$i.json | wc -c) -ne 0 ]
       fi
-      transform_bundlelist_json newsince$i.json objects$i.json
+      transform_list_json newsince$i.json objects$i.json
       tfw_preserve newsince$i.json objects$i.json
       for ((n = 0; n <= 5; ++n)); do
          assertJq objects$i.json "contains([{id:\"${BID[$n]}\"}]) | not"
@@ -529,7 +556,8 @@ test_RhizomeInsert() {
          assertStdoutIs 201
       else
          assertStdoutIs 403
-         assertGrep --ignore-case nfile$n.manifest "missing bundle secret"
+         assertJq nfile$n.manifest 'contains({"http_status_code": 403})'
+         assertJqGrep --ignore-case nfile$n.manifest '.http_status_message' "missing bundle secret"
       fi
    done
 }
@@ -593,7 +621,7 @@ setup_RhizomeInsertLarge() {
    create_file file1 50m
 }
 test_RhizomeInsertLarge() {
-   execute curl \
+   execute --timeout=120 curl \
          --silent --show-error --write-out '%{http_code}' \
          --output file1.manifest \
          --dump-header http.header \
@@ -628,7 +656,8 @@ test_RhizomeInsertMissingManifest() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'missing.*manifest.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*manifest.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -650,7 +679,8 @@ test_RhizomeInsertIncorrectManifestType() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'unsupported content-type.*manifest.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'unsupported content-type.*manifest.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -675,7 +705,8 @@ test_RhizomeInsertDuplicateManifest() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'duplicate.*manifest.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*manifest.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -698,7 +729,8 @@ test_RhizomeInsertJournal() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'not supported.*journal'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'not supported.*journal'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -720,7 +752,8 @@ test_RhizomeInsertMissingPayload() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'missing.*payload.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*payload.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -745,7 +778,8 @@ test_RhizomeInsertDuplicatePayload() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'duplicate.*payload.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*payload.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -768,7 +802,8 @@ test_RhizomeInsertPartOrder() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'missing.*manifest.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*manifest.*form.*part'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -792,8 +827,9 @@ test_RhizomeInsertPartUnsupported() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'unsupported.*form.*part'
+   assertJq http.body 'contains({"http_status_code": 403})'
-   assertGrep http.body 'happyhappy'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'unsupported.*form.*part'
+   assertJqGrep http.body '.http_status_message' 'happyhappy'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
@@ -818,7 +854,8 @@ test_RhizomeInsertIncorrectFilesize() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'payload size.*contradicts manifest'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'payload size.*contradicts manifest'
    execute curl \
          --silent --show-error --write-out '%{http_code}' \
          --output http.body \
@@ -853,29 +890,420 @@ test_RhizomeInsertIncorrectFilehash() {
    tfw_cat http.header http.body
    assertExitStatus == 0
    assertStdoutIs 403
-   assertGrep --ignore-case http.body 'payload hash.*contradicts manifest'
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'payload hash.*contradicts manifest'
    executeOk_servald rhizome list
    assert_rhizome_list
 }
 
 doc_MeshmsListConversations="HTTP RESTful list MeshMS conversations as JSON"
-Xtest_MeshmsListConversations() {
+setup_MeshmsListConversations() {
-   :
+   IDENTITY_COUNT=5
+   setup
+   # create 3 threads, with all permutations of incoming and outgoing messages
+   executeOk_servald meshms send message $SIDA1 $SIDA2 "Message1"
+   executeOk_servald meshms send message $SIDA3 $SIDA1 "Message2"
+   executeOk_servald meshms send message $SIDA1 $SIDA4 "Message3"
+   executeOk_servald meshms send message $SIDA4 $SIDA1 "Message4"
+}
+test_MeshmsListConversations() {
+   executeOk curl \
+         --silent --fail --show-error \
+         --output conversationlist1.json \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/conversationlist.json"
+   tfw_cat http.headers conversationlist1.json
+   tfw_preserve conversationlist1.json
+   assert [ "$(jq '.rows | length' conversationlist1.json)" = 3 ]
+   transform_list_json conversationlist1.json conversations1.json
+   tfw_preserve conversations1.json
+   assertJq conversations1.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA2\",
+                  read: true,
+                  last_message: 0,
+                  read_offset: 0
+               }
+            ])"
+   assertJq conversations1.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA3\",
+                  read: false,
+                  last_message: 11,
+                  read_offset: 0
+               }
+            ])"
+   assertJq conversations1.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA4\",
+                  read: false,
+                  last_message: 14,
+                  read_offset: 0
+               }
+            ])"
+   # mark all incoming messages as read
+   executeOk_servald meshms read messages $SIDA1
+   tfw_cat --stderr
+   executeOk curl \
+         --silent --fail --show-error \
+         --output conversationlist2.json \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/conversationlist.json"
+   tfw_cat http.headers conversationlist2.json
+   tfw_preserve conversationlist2.json
+   assert [ "$(jq '.rows | length' conversationlist2.json)" = 3 ]
+   transform_list_json conversationlist2.json conversations2.json
+   tfw_preserve conversations2.json
+   assertJq conversations2.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA2\",
+                  read: true,
+                  last_message: 0,
+                  read_offset: 0
+               }
+            ])"
+   assertJq conversations2.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA3\",
+                  read: true,
+                  last_message: 11,
+                  read_offset: 11
+               }
+            ])"
+   assertJq conversations2.json \
+            "contains([
+               {  my_sid: \"$SIDA1\",
+                  their_sid: \"$SIDA4\",
+                  read: true,
+                  last_message: 14,
+                  read_offset: 14
+               }
+            ])"
+}
+
+# Create a file that contains no blank lines.
+create_message_file() {
+   create_file "$1" $2
+   sed -i -e '/^$/d' "$1"
+}
+
+# Add a sequence of messages of varying sizes up to 1 KiB.
+add_messages() {
+   local symbols="$1"
+   shift
+   local texts=("$@")
+   local sent_since_ack=0
+   local i n size msize
+   local size=0
+   for ((i = 0; i < ${#symbols}; ++i)); do
+      local sym="${symbols:$i:1}"
+      let size+=379
+      let msize=size%1021
+      let n=NMESSAGE++
+      local text="${texts[$i]}"
+      case $sym in
+      '>'|'<')
+         if [ -n "$text" ]; then
+            echo "$text" >text$n
+         else
+            create_message_file text$n $msize
+            text="$(<text$n)"
+         fi
+         ;;
+      esac
+      case $sym in
+      '>')
+         MESSAGE[$n]=">"
+         executeOk_servald meshms send message $SIDA1 $SIDA2 "$text"
+         let ++sent_since_ack
+         let ++NSENT
+         ;;
+      '<')
+         MESSAGE[$n]="<"
+         executeOk_servald meshms send message $SIDA2 $SIDA1 "$text"
+         let ++NRECV
+         ;;
+      'A')
+         MESSAGE[$n]=ACK
+         [ $i -ne 0 -a $sent_since_ack -eq 0 ] && error "two ACKs in a row (at position $i)"
+         executeOk_servald meshms list messages $SIDA2 $SIDA1
+         let ++NACK
+         ;;
+      *)
+         error "invalid message symbol '$sym' (at position $i)"
+         ;;
+      esac
+   done
 }
 
 doc_MeshmsListMessages="HTTP RESTful list MeshMS messages in one conversation as JSON"
-Xtest_MeshmsListMessages() {
+setup_MeshmsListMessages() {
-   :
+   IDENTITY_COUNT=2
+   setup
+   add_messages '><>>A>A<>><><><>>>A>A><<<<A<>><>>A<<>'
+   let NROWS=NSENT+NRECV+(NACK?1:0)
+   executeOk_servald meshms list messages $SIDA1 $SIDA2
+   delivered_offset=$(sed -n -e '/^[0-9]\+:[0-9]\+:ACK:delivered$/{n;s/^[0-9]\+:\([0-9]\+\):>:.*/\1/p;q}' "$TFWSTDOUT")
+   [ -z "$delivered_offset" ] && delivered_offset=0
+   read_offset=$(sed -n -e 's/^[0-9]\+:\([0-9]\+\):MARK:read$/\1/p' "$TFWSTDOUT")
+   [ -z "$read_offset" ] && read_offset=0
+}
+test_MeshmsListMessages() {
+   executeOk curl \
+         --silent --fail --show-error \
+         --output messagelist.json \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json"
+   tfw_cat http.headers messagelist.json
+   tfw_preserve messagelist.json
+   assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ]
+   transform_list_json messagelist.json messages.json
+   tfw_preserve messages.json
+   seen_ack=false
+   let i=0
+   for ((j = NMESSAGE-1; j >= 0; --j)); do
+      case ${MESSAGE[$j]} in
+      'ACK') $seen_ack && continue
+      esac
+      assertJq messages.json '(.['$i'].token | length) > 0'
+      assertJq messages.json '.['$i'].my_sid == "'$SIDA1'"'
+      assertJq messages.json '.['$i'].their_sid == "'$SIDA2'"'
+      case ${MESSAGE[$j]} in
+      '>')
+         assertJq messages.json '.['$i'].type == ">"'
+         assertJqCmp messages.json '.['$i'].text' text$j
+         assertJq messages.json '.['$i'].delivered == (.['$i'].offset <= '$delivered_offset')'
+         let ++i
+         ;;
+      '<')
+         assertJq messages.json '.['$i'].type == "<"'
+         assertJqCmp messages.json '.['$i'].text' text$j
+         assertJq messages.json '.['$i'].read == (.['$i'].offset <= '$read_offset')'
+         let ++i
+         ;;
+      'ACK')
+         assertJq messages.json '.['$i'].type == "ACK"'
+         assertJq messages.json '.['$i'].text == null'
+         assertJq messages.json '.['$i'].ack_offset == '$delivered_offset
+         let ++i
+         seen_ack=true
+         ;;
+      esac
+   done
 }
 
-doc_MeshmsListMessagesSince="HTTP RESTful list MeshMS messages in one conversation since token as JSON"
+doc_MeshmsListMessagesNoIdentity="HTTP RESTful list MeshMS messages from unknown identity"
-Xtest_MeshmsListMessagesSince() {
+setup_MeshmsListMessagesNoIdentity() {
-   :
+   setup
+   SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
+}
+test_MeshmsListMessagesNoIdentity() {
+   execute curl \
+         --silent --show-error --write-out '%{http_code}' \
+         --output http.body \
+         --dump-header http.header \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDX/$SIDA/messagelist.json"
+   tfw_cat http.header http.body
+   assertExitStatus == 0
+   assertStdoutIs 403
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJq http.body 'contains({"meshms_status_code": 2})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'identity.*unknown'
+}
+
+doc_MeshmsListMessagesNewSince="HTTP RESTful list MeshMS messages in one conversation since token as JSON"
+setup_MeshmsListMessagesNewSince() {
+   IDENTITY_COUNT=2
+   set_extra_config() {
+      executeOk_servald config set rhizome.api.restful.newsince_timeout 1s \
+                               set rhizome.api.restful.newsince_poll_ms 500
+   }
+   setup
+   add_messages '><>>A>A<>><><><>>>A>A><<<<A<>><>>A<<>'
+   let NROWS=NSENT+NRECV+(NACK?1:0)
+   executeOk curl \
+         --silent --fail --show-error \
+         --output messagelist.json \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json"
+   assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ]
+   transform_list_json messagelist.json messages.json
+   tfw_preserve messages.json
+   for ((i = 0; i < NROWS; i += 3)); do
+      token[$i]=$(jq --raw-output '.['$i'].token' messages.json)
+   done
+}
+test_MeshmsListMessagesNewSince() {
+   for ((i = 0; i < NROWS; i += 3)); do
+      # At most five requests going at once
+      [ $i -ge 15 ] && fork_wait %curl$((i-15))
+      fork %curl$i executeOk curl \
+            --silent --fail --show-error \
+            --output messagelist$i.json \
+            --dump-header http.headers$i \
+            --basic --user harry:potter \
+            "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/newsince/${token[$i]}/messagelist.json"
+   done
+   fork_wait_all
+   for ((i = 0; i < NROWS; i += 3)); do
+      transform_list_json messagelist$i.json messages$i.json
+      tfw_preserve messages$i.json
+      { echo '{"a":'; cat messages.json; echo ',"b":'; cat messages$i.json; echo '}'; } >tmp.json
+      assertJq tmp.json '.a[:'$i'] == .b'
+   done
+}
+
+grepall() {
+   local pattern="$1"
+   shift
+   for file; do
+      grep "$pattern" "$file" || return $?
+   done
+   return 0
+}
+
+doc_MeshmsListMessagesNewSinceArrival="HTTP RESTful list newly arriving MeshMS messages in one conversation as JSON"
+setup_MeshmsListMessagesNewSinceArrival() {
+   IDENTITY_COUNT=2
+   set_extra_config() {
+      executeOk_servald config set rhizome.api.restful.newsince_timeout 60s \
+                               set rhizome.api.restful.newsince_poll_ms 500
+   }
+   setup
+   add_messages '><>A>'
+   let NROWS=NSENT+NRECV+(NACK?1:0)
+   executeOk curl \
+         --silent --fail --show-error \
+         --output messagelist.json \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/messagelist.json"
+   assert [ "$(jq '.rows | length' messagelist.json)" = $NROWS ]
+   transform_list_json messagelist.json messages.json
+   tfw_preserve messages.json
+   token=$(jq --raw-output '.[0].token' messages.json)
+   assert [ -n "$token" ]
+}
+test_MeshmsListMessagesNewSinceArrival() {
+   for i in 1 2 3; do
+      fork %curl$i executeOk curl \
+            --silent --fail --show-error \
+            --no-buffer \
+            --output newsince$i.json \
+            --basic --user harry:potter \
+            "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/newsince/$token/messagelist.json"
+   done
+   wait_until [ -e newsince1.json -a -e newsince2.json -a -e newsince3.json ]
+   for message in '>Rumplestiltskin' 'A' '<Howdydoody' '>Eulenspiegel'; do
+      add_messages "${message:0:1}" "${message:1}"
+      wait_until --timeout=60 grepall "${message:1}" newsince{1,2,3}.json
+   done
+   fork_terminate_all
+   fork_wait_all
+}
+teardown_MeshmsListMessagesNewSinceArrival() {
+   tfw_preserve newsince{1,2,3}.json
 }
 
 doc_MeshmsSend="HTTP RESTful send MeshMS message"
-Xtest_MeshmsSend() {
+setup_MeshmsSend() {
-   :
+   IDENTITY_COUNT=2
+   setup
+}
+test_MeshmsSend() {
+   executeOk curl \
+         --silent --fail --show-error \
+         --output sendmessage.json \
+         --basic --user harry:potter \
+         --form "message=Hello World" \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA1/$SIDA2/sendmessage"
+   executeOk_servald meshms list messages $SIDA1 $SIDA2
+   assertStdoutGrep --matches=1 ':>:Hello World'
+   executeOk curl \
+         --silent --fail --show-error \
+         --output sendmessage.json \
+         --basic --user ron:weasley \
+         --form "message=Hello back!" \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage"
+   executeOk_servald meshms list messages $SIDA1 $SIDA2
+   assertStdoutGrep --matches=1 ':>:Hello World$'
+   assertStdoutGrep --matches=1 ':<:Hello back!$'
+}
+
+doc_MeshmsSendMissingMessage="HTTP RESTful MeshMS send missing 'message' form part "
+setup_MeshmsSendMissingMessage() {
+   IDENTITY_COUNT=2
+   setup
+}
+test_MeshmsSendMissingMessage() {
+   execute curl \
+         --silent --show-error --write-out '%{http_code}' \
+         --output http.body \
+         --dump-header http.header \
+         --basic --user harry:potter \
+         --data '' \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage"
+   tfw_cat http.header http.body
+   assertExitStatus == 0
+   assertStdoutIs 403
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'missing.*message.*form.*part'
+   executeOk_servald meshms list messages $SIDA1 $SIDA2
+   assertStdoutLineCount '==' 2
+}
+
+doc_MeshmsSendDuplicateMessage="HTTP RESTful MeshMS send missing 'message' form part "
+setup_MeshmsSendDuplicateMessage() {
+   IDENTITY_COUNT=2
+   setup
+}
+test_MeshmsSendDuplicateMessage() {
+   execute curl \
+         --silent --show-error --write-out '%{http_code}' \
+         --output http.body \
+         --dump-header http.header \
+         --basic --user harry:potter \
+         --form "message=Hello one" \
+         --form "message=Hello two" \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDA2/$SIDA1/sendmessage"
+   tfw_cat http.header http.body
+   assertExitStatus == 0
+   assertStdoutIs 403
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'duplicate.*message.*form.*part'
+   executeOk_servald meshms list messages $SIDA1 $SIDA2
+   assertStdoutLineCount '==' 2
+}
+
+doc_MeshmsSendNoIdentity="HTTP RESTful MeshMS send from unknown identity"
+setup_MeshmsSendNoIdentity() {
+   setup
+   SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
+}
+test_MeshmsSendNoIdentity() {
+   execute curl \
+         --silent --show-error --write-out '%{http_code}' \
+         --output http.body \
+         --dump-header http.header \
+         --basic --user harry:potter \
+         --form "message=Hello" \
+         "http://$addr_localhost:$PORTA/restful/meshms/$SIDX/$SIDA/sendmessage"
+   tfw_cat http.header http.body
+   assertExitStatus == 0
+   assertStdoutIs 403
+   assertJq http.body 'contains({"http_status_code": 403})'
+   assertJq http.body 'contains({"meshms_status_code": 2})'
+   assertJqGrep --ignore-case http.body '.http_status_message' 'identity.*unknown'
 }
 
 runTests "$@"

tests/rhizomeprotocol

@@ -42,8 +42,8 @@ configure_servald_server() {
       set log.console.show_pid on \
       set log.console.show_time on \
       set debug.rhizome on \
+      set debug.http_server on \
       set debug.httpd on \
-      set debug.rhizome_httpd on \
       set debug.rhizome_manifest on \
       set debug.rhizome_ads on \
       set debug.rhizome_tx on \
@@ -180,7 +180,7 @@ setup_MDPTransportFailOver() {
    setup_common
    foreach_instance +A +B \
      executeOk_servald config \
-       set debug.rhizome_nohttptx 1 \
+       set debug.nohttptx 1 \
        set rhizome.mdp.enable 1
    set_instance +A
    rhizome_add_file file1 2048
@@ -583,7 +583,7 @@ setup_direct() {
       set log.console.level debug \
       set log.console.show_time on \
       set debug.rhizome on \
-      set debug.rhizome_httpd on \
+      set debug.httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on
    rhizome_add_file fileB1 2000

tests/rhizomestress

@@ -41,7 +41,7 @@ configure_servald_server() {
       set log.file.show_pid on \
       set log.file.show_time on \
       set debug.rhizome off \
-      set debug.rhizome_httpd off \
+      set debug.httpd off \
       set debug.rhizome_tx off \
       set debug.rhizome_rx off \
       set server.respawn_on_crash off \
@@ -149,7 +149,7 @@ setup_StressRhizomeDirect() {
       executeOk_servald config \
          set log.file.show_time on \
          set debug.rhizome off \
-         set debug.rhizome_httpd off \
+         set debug.httpd off \
          set debug.rhizome_tx off \
          set debug.rhizome_rx off \
          set server.respawn_on_crash off \