From 9606b4b1b8f6eb045f3cad7356723ad263c0e88b Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 10:21:11 +1030
Subject: [PATCH 01/21] Refactor "rhizome list" command

Move output formatting into commandline.c, leave database query in
rhizome_database.c
---
 commandline.c      |  89 ++++++++++++++++++++++++++++++++++-----
 rhizome.h          |  26 ++++++++++--
 rhizome_database.c | 103 ++++-----------------------------------------
 3 files changed, 111 insertions(+), 107 deletions(-)

diff --git a/commandline.c b/commandline.c
index 21e37b7f..a98ef988 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1818,25 +1818,94 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
 {
   if (config.debug.verbose)
     DEBUG_cli_parsed(parsed);
-  const char *service, *name, *sender_sid, *recipient_sid, *offset, *limit;
+  const char *service = NULL, *name = NULL, *sender_hex = NULL, *recipient_hex = NULL, *offset_ascii = NULL, *limit_ascii = NULL;
   cli_arg(parsed, "service", &service, NULL, "");
   cli_arg(parsed, "name", &name, NULL, "");
-  cli_arg(parsed, "sender_sid", &sender_sid, cli_optional_sid, "");
-  cli_arg(parsed, "recipient_sid", &recipient_sid, cli_optional_sid, "");
-  cli_arg(parsed, "offset", &offset, cli_uint, "0");
-  cli_arg(parsed, "limit", &limit, cli_uint, "0");
+  cli_arg(parsed, "sender_sid", &sender_hex, cli_optional_sid, "");
+  cli_arg(parsed, "recipient_sid", &recipient_hex, cli_optional_sid, "");
+  cli_arg(parsed, "offset", &offset_ascii, cli_uint, "0");
+  cli_arg(parsed, "limit", &limit_ascii, cli_uint, "0");
   /* Create the instance directory if it does not yet exist */
   if (create_serval_instance_dir() == -1)
     return -1;
   if (!(keyring = keyring_open_instance_cli(parsed)))
     return -1;
-    
-  int r=-1;
-  if (rhizome_opendb() != -1){
-    r=rhizome_list_manifests(context, service, name, sender_sid, recipient_sid, atoi(offset), atoi(limit), 0);
+  if (rhizome_opendb() == -1) {
+    keyring_free(keyring);
+    return -1;
   }
+  size_t rowlimit = atoi(limit_ascii);
+  size_t rowoffset = atoi(offset_ascii);
+  struct rhizome_list_cursor cursor;
+  bzero(&cursor, sizeof cursor);
+  cursor.service = service && service[0] ? service : NULL;
+  cursor.name = name && name[0] ? name : NULL;
+  if (sender_hex && sender_hex[0]) {
+    if (str_to_sid_t(&cursor.sender, sender_hex) == -1)
+      return WHYF("Invalid <sender>: %s", sender_hex);
+    cursor.is_sender_set = 1;
+  }
+  if (recipient_hex && recipient_hex[0]) {
+    if (str_to_sid_t(&cursor.recipient, recipient_hex) == -1)
+      return WHYF("Invalid <recipient: %s", recipient_hex);
+    cursor.is_recipient_set = 1;
+  }
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  if (rhizome_list_open(&retry, &cursor) == -1) {
+    keyring_free(keyring);
+    return -1;
+  }
+  const char *headers[]={
+    "_id",
+    "service",
+    "id",
+    "version",
+    "date",
+    ".inserttime",
+    ".author",
+    ".fromhere",
+    "filesize",
+    "filehash",
+    "sender",
+    "recipient",
+    "name"
+  };
+  cli_columns(context, NELS(headers), headers);
+  while (rhizome_list_next(&retry, &cursor) == 1) {
+    rhizome_manifest *m = cursor.manifest;
+    assert(m->filesize != RHIZOME_SIZE_UNSET);
+    if (cursor.rowcount < rowoffset)
+      continue;
+    if (rowlimit != 0 && cursor.rowcount > rowlimit)
+      break;
+    rhizome_lookup_author(m);
+    cli_put_long(context, cursor.rowid, ":");
+    cli_put_string(context, m->service, ":");
+    cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
+    cli_put_long(context, m->version, ":");
+    cli_put_long(context, m->has_date ? m->date : 0, ":");
+    cli_put_long(context, m->inserttime, ":");
+    switch (m->authorship) {
+      case AUTHOR_LOCAL:
+      case AUTHOR_AUTHENTIC:
+	cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
+	cli_put_long(context, 1, ":");
+	break;
+      default:
+	cli_put_string(context, NULL, ":");
+	cli_put_long(context, 0, ":");
+	break;
+    }
+    cli_put_long(context, m->filesize, ":");
+    cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
+    cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
+    cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
+    cli_put_string(context, m->name, "\n");
+  }
+  rhizome_list_release(&cursor);
+  cli_row_count(context, cursor.rowcount);
   keyring_free(keyring);
-  return r;
+  return 0;
 }
 
 int app_keyring_create(const struct cli_parsed *parsed, struct cli_context *context)
diff --git a/rhizome.h b/rhizome.h
index 93981aab..ea712679 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -564,9 +564,6 @@ int64_t rhizome_bar_version(const unsigned char *bar);
 uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar);
 int rhizome_is_bar_interesting(unsigned char *bar);
 int rhizome_is_manifest_interesting(rhizome_manifest *m);
-int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
-			   const char *sender_sid, const char *recipient_sid,
-			   size_t rowlimit, size_t rowoffset, char count_rows);
 int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m);
 int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m);
 int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
@@ -613,6 +610,29 @@ int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned cha
 int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int timeout);
 int rhizome_ignore_manifest_check(unsigned char *bid_prefix, int prefix_len);
 
+/* Rhizome list cursor for iterating over all or a subset of manifests in the store.
+ */
+struct rhizome_list_cursor {
+  // Query parameters that narrow the set of listed bundles.
+  const char *service;
+  const char *name;
+  bool_t is_sender_set;
+  bool_t is_recipient_set;
+  sid_t sender;
+  sid_t recipient;
+  // Set by calling the next() function.
+  int64_t rowid;
+  rhizome_manifest *manifest;
+  size_t rowcount;
+  // Private state.
+  sqlite3_stmt *_statement;
+  unsigned _offset;
+};
+
+int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
+int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *);
+void rhizome_list_release(struct rhizome_list_cursor *);
+
 /* one manifest is required per candidate, plus a few spare.
    so MAX_RHIZOME_MANIFESTS must be > MAX_CANDIDATES. 
 */
diff --git a/rhizome_database.c b/rhizome_database.c
index f6791e30..4005b312 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1407,27 +1407,12 @@ rollback:
   return -1;
 }
 
-struct rhizome_list_cursor {
-  // Query parameters that narrow the set of listed bundles.
-  const char *service;
-  const char *name;
-  sid_t sender;
-  sid_t recipient;
-  // Set by calling the next() function.
-  int64_t rowid;
-  rhizome_manifest *manifest;
-  size_t rowcount;
-  // Private state.
-  sqlite3_stmt *_statement;
-  unsigned _offset;
-};
-
 /* The cursor struct must be zerofilled and the query parameters optionally filled in prior to
  * calling this function.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-static int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
+int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
 {
   IN();
   strbuf b = strbuf_alloca(1024);
@@ -1436,9 +1421,9 @@ static int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_curs
     strbuf_puts(b, " AND service = @service");
   if (cursor->name)
     strbuf_puts(b, " AND name like @name");
-  if (!is_sid_t_any(cursor->sender))
+  if (cursor->is_sender_set)
     strbuf_puts(b, " AND sender = @sender");
-  if (!is_sid_t_any(cursor->recipient))
+  if (cursor->is_recipient_set)
     strbuf_puts(b, " AND recipient = @recipient");
   strbuf_puts(b, " ORDER BY inserttime DESC LIMIT -1 OFFSET @offset");
   if (strbuf_overrun(b))
@@ -1452,9 +1437,9 @@ static int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_curs
     goto failure;
   if (cursor->name && sqlite_bind(retry, cursor->_statement, NAMED|STATIC_TEXT, "@name", cursor->name, END) == -1)
     goto failure;
-  if (!is_sid_t_any(cursor->sender) && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@sender", &cursor->sender, END) == -1)
+  if (cursor->is_sender_set && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@sender", &cursor->sender, END) == -1)
     goto failure;
-  if (!is_sid_t_any(cursor->recipient) && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@recipient", &cursor->recipient, END) == -1)
+  if (cursor->is_recipient_set && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@recipient", &cursor->recipient, END) == -1)
     goto failure;
   cursor->manifest = NULL;
   RETURN(0);
@@ -1466,7 +1451,7 @@ failure:
   OUT();
 }
 
-static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
+int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
 {
   IN();
   if (cursor->_statement == NULL && rhizome_list_open(retry, cursor) == -1)
@@ -1516,9 +1501,9 @@ static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_curs
     rhizome_manifest_set_inserttime(m, q_inserttime);
     if (cursor->service && !(m->service && strcasecmp(cursor->service, m->service) == 0))
       continue;
-    if (!is_sid_t_any(cursor->sender) && !(m->has_sender && cmp_sid_t(&cursor->sender, &m->sender) == 0))
+    if (cursor->is_sender_set && !(m->has_sender && cmp_sid_t(&cursor->sender, &m->sender) == 0))
       continue;
-    if (!is_sid_t_any(cursor->recipient) && !(m->has_recipient && cmp_sid_t(&cursor->recipient, &m->recipient) == 0))
+    if (cursor->is_recipient_set && !(m->has_recipient && cmp_sid_t(&cursor->recipient, &m->recipient) == 0))
       continue;
     // Don't do rhizome_verify_author(m); too CPU expensive for a listing.  Save that for when
     // the bundle is extracted or exported.
@@ -1529,7 +1514,7 @@ static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_curs
   OUT();
 }
 
-static void rhizome_list_release(struct rhizome_list_cursor *cursor)
+void rhizome_list_release(struct rhizome_list_cursor *cursor)
 {
   if (cursor->manifest) {
     rhizome_manifest_free(cursor->manifest);
@@ -1541,76 +1526,6 @@ static void rhizome_list_release(struct rhizome_list_cursor *cursor)
   }
 }
 
-int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
-			   const char *sender_hex, const char *recipient_hex,
-			   size_t rowlimit, size_t rowoffset, char count_rows)
-{
-  IN();
-  struct rhizome_list_cursor cursor;
-  bzero(&cursor, sizeof cursor);
-  cursor.service = service && service[0] ? service : NULL;
-  cursor.name = name && name[0] ? name : NULL;
-  if (sender_hex && *sender_hex && str_to_sid_t(&cursor.sender, sender_hex) == -1)
-    RETURN(WHYF("Invalid sender SID: %s", sender_hex));
-  if (recipient_hex && *recipient_hex && str_to_sid_t(&cursor.recipient, recipient_hex) == -1)
-    RETURN(WHYF("Invalid recipient SID: %s", recipient_hex));
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  if (rhizome_list_open(&retry, &cursor) == -1)
-    RETURN(-1);
-  const char *names[]={
-    "_id",
-    "service",
-    "id",
-    "version",
-    "date",
-    ".inserttime",
-    ".author",
-    ".fromhere",
-    "filesize",
-    "filehash",
-    "sender",
-    "recipient",
-    "name"
-  };
-  cli_columns(context, NELS(names), names);
-  while (rhizome_list_next(&retry, &cursor) == 1) {
-    rhizome_manifest *m = cursor.manifest;
-    assert(m->filesize != RHIZOME_SIZE_UNSET);
-    if (cursor.rowcount < rowoffset)
-      continue;
-    if (rowlimit == 0 || cursor.rowcount <= rowlimit) {
-      rhizome_lookup_author(m);
-      cli_put_long(context, cursor.rowid, ":");
-      cli_put_string(context, m->service, ":");
-      cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
-      cli_put_long(context, m->version, ":");
-      cli_put_long(context, m->has_date ? m->date : 0, ":");
-      cli_put_long(context, m->inserttime, ":");
-      switch (m->authorship) {
-	case AUTHOR_LOCAL:
-	case AUTHOR_AUTHENTIC:
-	  cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
-	  cli_put_long(context, 1, ":");
-	  break;
-	default:
-	  cli_put_string(context, NULL, ":");
-	  cli_put_long(context, 0, ":");
-	  break;
-      }
-      cli_put_long(context, m->filesize, ":");
-      cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
-      cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
-      cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
-      cli_put_string(context, m->name, "\n");
-    } else if (!count_rows)
-      break;
-  }
-  rhizome_list_release(&cursor);
-  cli_row_count(context, cursor.rowcount);
-  RETURN(0);
-  OUT();
-}
-
 void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
 {
   (void) tohex(out, byteCount * 2, in);

From 4f9cbeab204d7b1bc41c4d8d5d5df6e9f838aaaa Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 17:14:09 +1030
Subject: [PATCH 02/21] Remove unused elements from struct rhizome_http_request

Also improve comments
---
 rhizome.h | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index ea712679..f50c4273 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -707,35 +707,27 @@ typedef struct rhizome_http_request
 {
   struct http_request http; // MUST BE FIRST ELEMENT
 
-  /* Identify request from others being run.
-     Monotonic counter feeds it.  Only used for debugging when we write
-     post-<uuid>.log files for multi-part form requests. */
+  /* Identify request from others being run.  Monotonic counter feeds it.  Only
+   * used for debugging when we write post-<uuid>.log files for multi-part form
+   * requests.
+   */
   unsigned int uuid;
 
-  struct rhizome_read read_state;
-  
-  /* File currently being written to while decoding POST multipart form */
+  /* For receiving a POST multipart form:
+   */
+  // Which part is currently being received
   enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
+  // Temporary file currently current part is being written to
   int part_fd;
-  /* Which parts have been received in POST multipart form */
+  // Which parts have already been received
   bool_t received_manifest;
   bool_t received_data;
-  /* Name of data file supplied */
+  // Name of data file supplied in part's Content-Disposition header, filename
+  // parameter (if any)
   char data_file_name[MIME_FILENAME_MAXLEN + 1];
 
-  /* The source specification data which are used in different ways by different 
-   request types */
-  char source[1024];
-  int64_t source_index;
-  int64_t source_count;
-  int source_record_size;
-  unsigned int source_flags;
-  
-  const char *sql_table;
-  const char *sql_row;
-  int64_t rowid;
-  /* source_index used for offset in blob */
-  int64_t blob_end; 
+  // For responses that serve a payload.
+  struct rhizome_read read_state;
   
 } rhizome_http_request;
 

From 47988bfe0ffd22897e21246f6720a736527a0bd8 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 17:18:38 +1030
Subject: [PATCH 03/21] Make Content-Length optional in HTTP responses

To eventually support the /restful/rhizome/bundlelist.json query which
will not compute in advance the length of its response.
---
 http_server.c | 50 ++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/http_server.c b/http_server.c
index 272c7418..393d1763 100644
--- a/http_server.c
+++ b/http_server.c
@@ -1825,30 +1825,54 @@ static int _render_response(struct http_request *r)
   struct http_response hr = r->response;
   assert(hr.result_code >= 100);
   assert(hr.result_code < 600);
+  // Status code 401 must be accompanied by a WWW-Authenticate header.
   if (hr.result_code == 401)
     assert(hr.header.www_authenticate.scheme != NOAUTH);
   const char *result_string = httpResultString(hr.result_code);
   strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size);
-  if (hr.content == NULL && hr.content_generator == NULL) {
+  // Cannot specify both static (pre-rendered) content AND generated content.
+  assert(!(hr.content && hr.content_generator));
+  if (hr.content || hr.content_generator) {
+    // With static (pre-rendered) content, the content length is mandatory (so we know how much data
+    // follows the 'hr.content' pointer.  Generated content will generally not send a Content-Length
+    // header, nor send partial content, but they might.
+    if (hr.content)
+      assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
+    // Ensure that all partial content fields are consistent.  If content length or resource length
+    // are unknown, there can be no range field.
+    if (   hr.header.content_length != CONTENT_LENGTH_UNKNOWN
+	&& hr.header.resource_length != CONTENT_LENGTH_UNKNOWN
+    ) {
+      assert(hr.header.content_length <= hr.header.resource_length);
+      assert(hr.header.content_range_start + hr.header.content_length <= hr.header.resource_length);
+    } else {
+      assert(hr.header.content_range_start == 0);
+    }
+    // Convert a 200 status code into 206 if only partial content is being sent.  This saves page
+    // handlers having to decide between 200 (OK) and 206 (Partial Content), they can just set the
+    // content and resource length fields and pass 200 to http_request_response_static(), and this
+    // logic will change it to 206 if appropriate.
+    if (   hr.header.content_length != CONTENT_LENGTH_UNKNOWN
+	&& hr.header.resource_length != CONTENT_LENGTH_UNKNOWN
+	&& hr.header.content_length > 0
+	&& hr.header.content_length < hr.header.resource_length
+    ) {
+      if (hr.result_code == 200)
+	hr.result_code = 206; // Partial Content
+    }
+  } else {
+    // If no content is supplied at all, then render a standard, short body based solely on result
+    // code.
     assert(hr.header.content_length == CONTENT_LENGTH_UNKNOWN);
     assert(hr.header.resource_length == CONTENT_LENGTH_UNKNOWN);
     assert(hr.header.content_range_start == 0);
+    assert(hr.result_code != 206);
     strbuf cb = strbuf_alloca(100 + strlen(result_string));
     strbuf_sprintf(cb, "<html><h1>%03u %s</h1></html>", hr.result_code, result_string);
     hr.content = strbuf_str(cb);
-    hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
     hr.header.content_type = "text/html";
+    hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
     hr.header.content_range_start = 0;
-  } else {
-    assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
-    assert(hr.header.resource_length != CONTENT_LENGTH_UNKNOWN);
-    assert(hr.header.content_length <= hr.header.resource_length);
-    assert(hr.header.content_range_start + hr.header.content_length <= hr.header.resource_length);
-    // To save page handlers having to decide between 200 (OK) and 206 (Partial Content), they can
-    // just set the content range fields and pass 200 to http_request_response_static(), and this
-    // logic will change it to 206 if appropriate.
-    if (hr.header.content_length > 0 && hr.header.content_length < hr.header.resource_length && hr.result_code == 200)
-      hr.result_code = 206; // Partial Content
   }
   assert(hr.header.content_type != NULL);
   assert(hr.header.content_type[0]);
@@ -1865,6 +1889,8 @@ static int _render_response(struct http_request *r)
   if (hr.result_code == 206) {
     // Must only use result code 206 (Partial Content) if the content is in fact less than the whole
     // resource length.
+    assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
+    assert(hr.header.resource_length != CONTENT_LENGTH_UNKNOWN);
     assert(hr.header.content_length > 0);
     assert(hr.header.content_length < hr.header.resource_length);
     strbuf_sprintf(sb,

From 051eca477501d0f90aef41014c71d6732efe4f24 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 17:22:06 +1030
Subject: [PATCH 04/21] Improve Rhizome list cursor

Remove 'rowcount' element

Order by descending ROWID, which is functionally the same as descending
inserttime but more reliable

Replace '_offset' cursor element with rowid first-last range to record
the expanding window of rows already returned, which allows release and
re-open of cursor mid-listing without missing rows or producing duplicates
---
 commandline.c      |  63 +++++++++++++------------
 rhizome.h          |   4 +-
 rhizome_database.c | 112 +++++++++++++++++++++++++--------------------
 3 files changed, 99 insertions(+), 80 deletions(-)

diff --git a/commandline.c b/commandline.c
index a98ef988..7f1b8204 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1871,40 +1871,45 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
     "name"
   };
   cli_columns(context, NELS(headers), headers);
-  while (rhizome_list_next(&retry, &cursor) == 1) {
-    rhizome_manifest *m = cursor.manifest;
-    assert(m->filesize != RHIZOME_SIZE_UNSET);
-    if (cursor.rowcount < rowoffset)
+  size_t rowcount = 0;
+  int n;
+  while ((n = rhizome_list_next(&retry, &cursor)) == 1) {
+    ++rowcount;
+    if (rowcount <= rowoffset)
       continue;
-    if (rowlimit != 0 && cursor.rowcount > rowlimit)
-      break;
-    rhizome_lookup_author(m);
-    cli_put_long(context, cursor.rowid, ":");
-    cli_put_string(context, m->service, ":");
-    cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
-    cli_put_long(context, m->version, ":");
-    cli_put_long(context, m->has_date ? m->date : 0, ":");
-    cli_put_long(context, m->inserttime, ":");
-    switch (m->authorship) {
-      case AUTHOR_LOCAL:
-      case AUTHOR_AUTHENTIC:
-	cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
-	cli_put_long(context, 1, ":");
-	break;
-      default:
-	cli_put_string(context, NULL, ":");
-	cli_put_long(context, 0, ":");
-	break;
+    if (rowlimit == 0 || rowcount <= rowoffset + rowlimit) {
+      rhizome_manifest *m = cursor.manifest;
+      assert(m->filesize != RHIZOME_SIZE_UNSET);
+      rhizome_lookup_author(m);
+      cli_put_long(context, cursor.rowid, ":");
+      cli_put_string(context, m->service, ":");
+      cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
+      cli_put_long(context, m->version, ":");
+      cli_put_long(context, m->has_date ? m->date : 0, ":");
+      cli_put_long(context, m->inserttime, ":");
+      switch (m->authorship) {
+	case AUTHOR_LOCAL:
+	case AUTHOR_AUTHENTIC:
+	  cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
+	  cli_put_long(context, 1, ":");
+	  break;
+	default:
+	  cli_put_string(context, NULL, ":");
+	  cli_put_long(context, 0, ":");
+	  break;
+      }
+      cli_put_long(context, m->filesize, ":");
+      cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
+      cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
+      cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
+      cli_put_string(context, m->name, "\n");
     }
-    cli_put_long(context, m->filesize, ":");
-    cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
-    cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
-    cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
-    cli_put_string(context, m->name, "\n");
   }
   rhizome_list_release(&cursor);
-  cli_row_count(context, cursor.rowcount);
   keyring_free(keyring);
+  if (n == -1)
+    return -1;
+  cli_row_count(context, rowcount);
   return 0;
 }
 
diff --git a/rhizome.h b/rhizome.h
index f50c4273..194a8f66 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -623,10 +623,10 @@ struct rhizome_list_cursor {
   // Set by calling the next() function.
   int64_t rowid;
   rhizome_manifest *manifest;
-  size_t rowcount;
   // Private state.
   sqlite3_stmt *_statement;
-  unsigned _offset;
+  int64_t _rowid_first;
+  int64_t _rowid_last;
 };
 
 int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
diff --git a/rhizome_database.c b/rhizome_database.c
index 4005b312..bb13897f 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1298,6 +1298,10 @@ int rhizome_store_bundle(rhizome_manifest *m)
 
   time_ms_t now = gettime_ms();
 
+  // The INSERT OR REPLACE statement will delete a row with the same ID (primary key) if it exists,
+  // so a new autoincremented ROWID will be allocated whether or not the manifest with this ID is
+  // already in the table.  Other code depends on this property: that ROWID is monotonically
+  // increasing with time and unique.
   sqlite3_stmt *stmt;
   if ((stmt = sqlite_prepare_bind(&retry,
 	"INSERT OR REPLACE INTO MANIFESTS("
@@ -1412,70 +1416,81 @@ rollback:
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
+int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
 {
   IN();
   strbuf b = strbuf_alloca(1024);
   strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
-  if (cursor->service)
+  if (c->service)
     strbuf_puts(b, " AND service = @service");
-  if (cursor->name)
+  if (c->name)
     strbuf_puts(b, " AND name like @name");
-  if (cursor->is_sender_set)
+  if (c->is_sender_set)
     strbuf_puts(b, " AND sender = @sender");
-  if (cursor->is_recipient_set)
+  if (c->is_recipient_set)
     strbuf_puts(b, " AND recipient = @recipient");
-  strbuf_puts(b, " ORDER BY inserttime DESC LIMIT -1 OFFSET @offset");
+  if (c->_rowid_first) {
+    assert(c->_rowid_last);
+    assert(c->_rowid_last <= c->_rowid_first);
+    strbuf_puts(b, " AND (rowid > @first OR rowid < @last)");
+  }
+  strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first
   if (strbuf_overrun(b))
     RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
-  cursor->_statement = sqlite_prepare(retry, strbuf_str(b));
-  if (cursor->_statement == NULL)
+  c->_statement = sqlite_prepare(retry, strbuf_str(b));
+  if (c->_statement == NULL)
     RETURN(-1);
-  if (sqlite_bind(retry, cursor->_statement, NAMED|INT, "@offset", cursor->_offset, END) == -1)
+  if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
     goto failure;
-  if (cursor->service && sqlite_bind(retry, cursor->_statement, NAMED|STATIC_TEXT, "@service", cursor->service, END) == -1)
+  if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
     goto failure;
-  if (cursor->name && sqlite_bind(retry, cursor->_statement, NAMED|STATIC_TEXT, "@name", cursor->name, END) == -1)
+  if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
     goto failure;
-  if (cursor->is_sender_set && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@sender", &cursor->sender, END) == -1)
+  if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
     goto failure;
-  if (cursor->is_recipient_set && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@recipient", &cursor->recipient, END) == -1)
+  if (   c->_rowid_first
+      && sqlite_bind(retry, c->_statement, NAMED|INT64, "@first", c->_rowid_first,
+					   NAMED|INT64, "@last", c->_rowid_last, END) == -1
+  )
     goto failure;
-  cursor->manifest = NULL;
+  c->manifest = NULL;
   RETURN(0);
   OUT();
 failure:
-  sqlite3_finalize(cursor->_statement);
-  cursor->_statement = NULL;
+  sqlite3_finalize(c->_statement);
+  c->_statement = NULL;
   RETURN(-1);
   OUT();
 }
 
-int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
+int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
 {
   IN();
-  if (cursor->_statement == NULL && rhizome_list_open(retry, cursor) == -1)
+  if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
     RETURN(-1);
-  while (sqlite_step_retry(retry, cursor->_statement) == SQLITE_ROW) {
-    ++cursor->_offset;
-    if (cursor->manifest) {
-      rhizome_manifest_free(cursor->manifest);
-      cursor->manifest = NULL;
+  while (sqlite_step_retry(retry, c->_statement) == SQLITE_ROW) {
+    if (c->manifest) {
+      rhizome_manifest_free(c->manifest);
+      c->manifest = NULL;
     }
-    assert(sqlite3_column_count(cursor->_statement) == 6);
-    assert(sqlite3_column_type(cursor->_statement, 0) == SQLITE_TEXT);
-    assert(sqlite3_column_type(cursor->_statement, 1) == SQLITE_BLOB);
-    assert(sqlite3_column_type(cursor->_statement, 2) == SQLITE_INTEGER);
-    assert(sqlite3_column_type(cursor->_statement, 3) == SQLITE_INTEGER);
-    assert(sqlite3_column_type(cursor->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(cursor->_statement, 4) == SQLITE_NULL);
-    assert(sqlite3_column_type(cursor->_statement, 5) == SQLITE_INTEGER);
-    const char *q_manifestid = (const char *) sqlite3_column_text(cursor->_statement, 0);
-    const char *manifestblob = (char *) sqlite3_column_blob(cursor->_statement, 1);
-    size_t manifestblobsize = sqlite3_column_bytes(cursor->_statement, 1); // must call after sqlite3_column_blob()
-    int64_t q_version = sqlite3_column_int64(cursor->_statement, 2);
-    int64_t q_inserttime = sqlite3_column_int64(cursor->_statement, 3);
-    const char *q_author = (const char *) sqlite3_column_text(cursor->_statement, 4);
-    cursor->rowid = sqlite3_column_int64(cursor->_statement, 5);
+    assert(sqlite3_column_count(c->_statement) == 6);
+    assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
+    assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
+    assert(sqlite3_column_type(c->_statement, 2) == SQLITE_INTEGER);
+    assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
+    assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
+    assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
+    const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
+    const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
+    size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
+    int64_t q_version = sqlite3_column_int64(c->_statement, 2);
+    int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
+    const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
+    c->rowid = sqlite3_column_int64(c->_statement, 5);
+    if (c->rowid > c->_rowid_first)
+      c->_rowid_first = c->rowid;
+    if (c->_rowid_last == 0 || c->rowid < c->_rowid_last)
+      c->_rowid_last = c->rowid;
     sid_t *author = NULL;
     if (q_author) {
       author = alloca(sizeof *author);
@@ -1484,7 +1499,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cur
 	continue;
       }
     }
-    rhizome_manifest *m = cursor->manifest = rhizome_new_manifest();
+    rhizome_manifest *m = c->manifest = rhizome_new_manifest();
     if (m == NULL)
       RETURN(-1);
     if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
@@ -1499,30 +1514,29 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cur
     if (author)
       rhizome_manifest_set_author(m, author);
     rhizome_manifest_set_inserttime(m, q_inserttime);
-    if (cursor->service && !(m->service && strcasecmp(cursor->service, m->service) == 0))
+    if (c->service && !(m->service && strcasecmp(c->service, m->service) == 0))
       continue;
-    if (cursor->is_sender_set && !(m->has_sender && cmp_sid_t(&cursor->sender, &m->sender) == 0))
+    if (c->is_sender_set && !(m->has_sender && cmp_sid_t(&c->sender, &m->sender) == 0))
       continue;
-    if (cursor->is_recipient_set && !(m->has_recipient && cmp_sid_t(&cursor->recipient, &m->recipient) == 0))
+    if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0))
       continue;
     // Don't do rhizome_verify_author(m); too CPU expensive for a listing.  Save that for when
     // the bundle is extracted or exported.
-    ++cursor->rowcount;
     RETURN(1);
   }
   RETURN(0);
   OUT();
 }
 
-void rhizome_list_release(struct rhizome_list_cursor *cursor)
+void rhizome_list_release(struct rhizome_list_cursor *c)
 {
-  if (cursor->manifest) {
-    rhizome_manifest_free(cursor->manifest);
-    cursor->manifest = NULL;
+  if (c->manifest) {
+    rhizome_manifest_free(c->manifest);
+    c->manifest = NULL;
   }
-  if (cursor->_statement) {
-    sqlite3_finalize(cursor->_statement);
-    cursor->_statement = NULL;
+  if (c->_statement) {
+    sqlite3_finalize(c->_statement);
+    c->_statement = NULL;
   }
 }
 

From d337542067a633daf2852a078fe0f2629e88a5f3 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 23:39:24 +1030
Subject: [PATCH 05/21] Improve HTTP server generated content logic

Support generated content with an unspecified Content-Length.  Generator
functions return 1 if there is more content to come, and 0 if they have
just produced the last piece of content.
---
 http_server.c  | 54 +++++++++++++++++++++++++++++++++-----------------
 http_server.h  |  1 +
 rhizome_http.c |  2 +-
 3 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/http_server.c b/http_server.c
index 393d1763..60e3de7c 100644
--- a/http_server.c
+++ b/http_server.c
@@ -1642,16 +1642,22 @@ static void http_request_receive(struct http_request *r)
  */
 static void http_request_send_response(struct http_request *r)
 {
-  assert(r->response_sent <= r->response_length);
-  while (r->response_sent < r->response_length) {
+  if (r->response_length != CONTENT_LENGTH_UNKNOWN)
+    assert(r->response_sent <= r->response_length);
+  int g = 1;
+  while (g) {
+    if (r->response_length != CONTENT_LENGTH_UNKNOWN && r->response_sent == r->response_length)
+      break;
     assert(r->response_buffer_sent <= r->response_buffer_length);
     if (r->response_buffer_sent == r->response_buffer_length) {
+      g = 0;
       if (r->response.content_generator) {
 	// Content generator must fill or partly fill response_buffer and set response_buffer_sent
 	// and response_buffer_length.  May also malloc() a bigger buffer and set response_buffer to
 	// point to it.
 	r->response_buffer_sent = r->response_buffer_length = 0;
-	if (r->response.content_generator(r) == -1) {
+	g = r->response.content_generator(r);
+	if (g == -1) {
 	  if (r->debug_flag && *r->debug_flag)
 	    DEBUG("Content generation error, closing connection");
 	  http_request_finalise(r);
@@ -1659,12 +1665,17 @@ static void http_request_send_response(struct http_request *r)
 	}
 	assert(r->response_buffer_sent <= r->response_buffer_length);
 	if (r->response_buffer_sent == r->response_buffer_length) {
-	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
-	      r->response_sent, r->response_length, r->response_length - r->response_sent);
+	  if (g == 0)
+	    break;
+	  if (r->response_length != CONTENT_LENGTH_UNKNOWN)
+	    WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
+		r->response_sent, r->response_length, r->response_length - r->response_sent);
+	  else
+	    WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"", r->response_sent);
 	  http_request_finalise(r);
 	  return;
 	}
-      } else {
+      } else if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
 	WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes",
 	    r->response_length, r->response_length - r->response_sent);
 	http_request_finalise(r);
@@ -1900,7 +1911,8 @@ static int _render_response(struct http_request *r)
 	  hr.header.resource_length
 	);
   }
-  strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
+  if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN)
+    strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
   const char *scheme = NULL;
   switch (hr.header.www_authenticate.scheme) {
     case NOAUTH: break;
@@ -1913,17 +1925,23 @@ static int _render_response(struct http_request *r)
     strbuf_puts(sb, "\r\n");
   }
   strbuf_puts(sb, "\r\n");
-  if (strbuf_overrun(sb))
-    return 0;
-  r->response_length = strbuf_len(sb) + hr.header.content_length;
+  if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN)
+    r->response_length = strbuf_count(sb) + hr.header.content_length;
+  else
+    r->response_length = CONTENT_LENGTH_UNKNOWN;
   if (hr.content) {
-    if (r->response_buffer_size < r->response_length)
-      return 0;
-    bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
-    r->response_buffer_length = r->response_length;
+    assert(r->response_length != CONTENT_LENGTH_UNKNOWN);
+    r->response_buffer_need = r->response_length + 1;
   } else {
-    r->response_buffer_length = strbuf_len(sb);
+    assert(hr.content_generator);
+    r->response_buffer_need = strbuf_count(sb) + 1;
   }
+  if (r->response_buffer_size < r->response_buffer_need)
+    return 0;
+  assert(!strbuf_overrun(sb));
+  if (hr.content)
+    bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
+  r->response_buffer_length = r->response_buffer_need;
   r->response_buffer_sent = 0;
   return 1;
 }
@@ -1942,9 +1960,9 @@ static void http_request_render_response(struct http_request *r)
   // rendered headers, so after this step, whether or not the buffer was overrun, we know the total
   // length of the response.
   if (!_render_response(r)) {
-    // If the response did not fit into the existing buffer, then allocate a large buffer from the
-    // heap and try rendering again.
-    if (http_request_set_response_bufsize(r, r->response_length + 1) == -1)
+    // If the static response did not fit into the existing buffer, then allocate a large buffer
+    // from the heap and try rendering again.
+    if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1)
       WHY("Cannot render HTTP response, out of memory");
     else if (!_render_response(r))
       FATAL("Re-render of HTTP response overflowed buffer");
diff --git a/http_server.h b/http_server.h
index 20e8d439..50b671c8 100644
--- a/http_server.h
+++ b/http_server.h
@@ -192,6 +192,7 @@ struct http_request {
   http_size_t response_length; // total response bytes (header + content)
   http_size_t response_sent; // for counting up to response_length
   char *response_buffer;
+  size_t response_buffer_need;
   size_t response_buffer_size;
   size_t response_buffer_length;
   size_t response_buffer_sent;
diff --git a/rhizome_http.c b/rhizome_http.c
index 391cc2b7..73c73937 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -443,7 +443,7 @@ static int rhizome_file_content(struct http_request *hr)
     return -1;
   assert((size_t) len <= r->http.response_buffer_size);
   r->http.response_buffer_length += (size_t) len;
-  return 0;
+  return 1;
 }
 
 static int rhizome_file_page(rhizome_http_request *r, const char *remainder)

From 9ecf9891fb14334fc468639af013d6555acf654f Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 23:39:53 +1030
Subject: [PATCH 06/21] Add strbuf JSON helper functions

---
 strbuf_helpers.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++
 strbuf_helpers.h |  7 +++++++
 2 files changed, 59 insertions(+)

diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 7d04446b..530be30d 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -36,6 +36,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <sys/uio.h>
 #include "http_server.h"
 #include "strbuf_helpers.h"
+#include "str.h"
 
 static inline strbuf _toprint(strbuf sb, char c)
 {
@@ -414,6 +415,57 @@ strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
   return sb;
 }
 
+strbuf strbuf_json_null(strbuf sb)
+{
+  strbuf_puts(sb, "null");
+  return sb;
+}
+
+strbuf strbuf_json_string(strbuf sb, const char *str)
+{
+  if (str) {
+    strbuf_putc(sb, '"');
+    for (; *str; ++str) {
+      if (*str == '"' || *str == '\\') {
+	strbuf_putc(sb, '\\');
+	strbuf_putc(sb, *str);
+      }
+      else if (*str == '\b')
+	strbuf_puts(sb, "\\b");
+      else if (*str == '\f')
+	strbuf_puts(sb, "\\f");
+      else if (*str == '\n')
+	strbuf_puts(sb, "\\n");
+      else if (*str == '\r')
+	strbuf_puts(sb, "\\r");
+      else if (*str == '\t')
+	strbuf_puts(sb, "\\t");
+      else if (iscntrl(*str))
+	strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str);
+      else
+	strbuf_putc(sb, *str);
+    }
+    strbuf_putc(sb, '"');
+  } else
+    strbuf_json_null(sb);
+  return sb;
+}
+
+strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
+{
+  if (buf) {
+    strbuf_putc(sb, '"');
+    size_t i;
+    for (i = 0; i != len; ++i) {
+      strbuf_putc(sb, hexdigit[*buf >> 4]);
+      strbuf_putc(sb, hexdigit[*buf++ & 0xf]);
+    }
+    strbuf_putc(sb, '"');
+  } else
+    strbuf_json_null(sb);
+  return sb;
+}
+
 strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
 {
   unsigned i;
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index fb003dd0..5dca8a90 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -151,6 +151,13 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
  */
 strbuf strbuf_append_quoted_string(strbuf sb, const char *str);
 
+/* Append various JSON elements.
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+strbuf strbuf_json_null(strbuf sb);
+strbuf strbuf_json_string(strbuf sb, const char *str);
+strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len);
+
 /* Append a representation of a struct http_range[] array.
  * @author Andrew Bettison <andrew@servalproject.com>
  */

From 1b906f3f118cde97598a36be2ae9c210c7320ac8 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 7 Nov 2013 23:40:56 +1030
Subject: [PATCH 07/21] Implement HTTP /restful/rhizome/bundlelist.json

Only tested for one bundle.
---
 rhizome.h      |  14 +++++-
 rhizome_http.c | 121 ++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index 194a8f66..93ac6746 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -726,8 +726,18 @@ typedef struct rhizome_http_request
   // parameter (if any)
   char data_file_name[MIME_FILENAME_MAXLEN + 1];
 
-  // For responses that serve a payload.
-  struct rhizome_read read_state;
+  union {
+    /* For responses that send part or all of a payload.
+    */
+    struct rhizome_read read_state;
+
+    /* For responses that list manifests.
+    */
+    struct {
+        size_t rowcount;
+        struct rhizome_list_cursor cursor;
+    } list;
+  } u;
   
 } rhizome_http_request;
 
diff --git a/rhizome_http.c b/rhizome_http.c
index 73c73937..33f4efe8 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "overlay_address.h"
 #include "conf.h"
 #include "str.h"
+#include "strbuf_helpers.h"
 #include "rhizome.h"
 #include "http_server.h"
 
@@ -231,7 +232,7 @@ success:
 static void rhizome_server_finalise_http_request(struct http_request *_r)
 {
   rhizome_http_request *r = (rhizome_http_request *) _r;
-  rhizome_read_close(&r->read_state);
+  rhizome_read_close(&r->u.read_state);
   request_count--;
 }
 
@@ -272,8 +273,8 @@ void rhizome_server_poll(struct sched_ent *alarm)
 	request_count++;
 	request->uuid = rhizome_http_request_uuid_counter++;
 	request->data_file_name[0] = '\0';
-	request->read_state.blob_fd = -1;
-	request->read_state.blob_rowid = -1;
+	request->u.read_state.blob_fd = -1;
+	request->u.read_state.blob_rowid = -1;
 	if (peerip)
 	  request->http.client_sockaddr_in = *peerip;
 	request->http.handle_headers = rhizome_dispatch;
@@ -334,6 +335,8 @@ static int is_authorized(struct http_client_authorization *auth)
   return 0;
 }
 
+static int restful_rhizome_bundlelist_json_content(struct http_request *);
+
 static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
 {
   if (!is_rhizome_http_enabled())
@@ -350,10 +353,96 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
     http_request_simple_response(&r->http, 401, NULL);
     return 0;
   }
-  http_request_simple_response(&r->http, 200, NULL);
+  r->u.list.rowcount = 0;
+  bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
+  http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
   return 0;
 }
 
+static int restful_rhizome_bundlelist_json_content(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  assert(r->http.response_buffer_sent == 0);
+  assert(r->http.response_buffer_length == 0);
+  strbuf b = strbuf_local(r->http.response_buffer, r->http.response_buffer_size);
+  const char *headers[]={
+    "_id",
+    "service",
+    "id",
+    "version",
+    "date",
+    ".inserttime",
+    ".author",
+    ".fromhere",
+    "filesize",
+    "filehash",
+    "sender",
+    "recipient",
+    "name"
+  };
+  if (r->u.list.rowcount == 0) {
+    strbuf_puts(b, "[[");
+    unsigned i;
+    for (i = 0; i != NELS(headers); ++i) {
+      if (i)
+	strbuf_putc(b, ',');
+      strbuf_json_string(b, headers[i]);
+    }
+    strbuf_puts(b, "]");
+  }
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  int ret = rhizome_list_next(&retry, &r->u.list.cursor);
+  if (ret == 0) {
+    strbuf_puts(b, "\n]\n");
+  } else if (ret == 1) {
+    ++r->u.list.rowcount;
+    rhizome_manifest *m = r->u.list.cursor.manifest;
+    assert(m->filesize != RHIZOME_SIZE_UNSET);
+    rhizome_lookup_author(m);
+    strbuf_puts(b, ",\n [");
+    strbuf_sprintf(b, "%"PRIu64, r->u.list.cursor.rowid);
+    strbuf_putc(b, ',');
+    strbuf_json_string(b, m->service);
+    strbuf_putc(b, ',');
+    strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
+    strbuf_putc(b, ',');
+    strbuf_sprintf(b, "%"PRId64, m->version);
+    strbuf_putc(b, ',');
+    if (m->has_date)
+      strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
+    else
+      strbuf_json_null(b);
+    strbuf_putc(b, ',');
+    strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
+    switch (m->authorship) {
+      case AUTHOR_LOCAL:
+      case AUTHOR_AUTHENTIC:
+	strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
+	strbuf_puts(b, ",1,");
+	break;
+      default:
+	strbuf_json_null(b);
+	strbuf_puts(b, ",1,");
+	break;
+    }
+    strbuf_sprintf(b, "%"PRIu64, m->filesize);
+    strbuf_putc(b, ',');
+    strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
+    strbuf_putc(b, ',');
+    strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
+    strbuf_putc(b, ',');
+    strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
+    strbuf_putc(b, ',');
+    strbuf_json_string(b, m->name);
+    strbuf_puts(b, "]");
+  }
+  if (strbuf_overrun(b))
+    ret = WHY("HTTP response buffer overrun");
+  rhizome_list_release(&r->u.list.cursor);
+  r->http.response_buffer_length = strbuf_len(b);
+  return ret;
+}
+
 static int neighbour_page(rhizome_http_request *r, const char *remainder)
 {
   if (r->http.verb != HTTP_VERB_GET) {
@@ -425,8 +514,8 @@ static int rhizome_file_content(struct http_request *hr)
   rhizome_http_request *r = (rhizome_http_request *) hr;
   assert(r->http.response_buffer_sent == 0);
   assert(r->http.response_buffer_length == 0);
-  assert(r->read_state.offset < r->read_state.length);
-  uint64_t readlen = r->read_state.length - r->read_state.offset;
+  assert(r->u.read_state.offset < r->u.read_state.length);
+  uint64_t readlen = r->u.read_state.length - r->u.read_state.offset;
   size_t suggested_size = 64 * 1024;
   if (suggested_size > readlen)
     suggested_size = readlen;
@@ -436,7 +525,7 @@ static int rhizome_file_content(struct http_request *hr)
     http_request_set_response_bufsize(&r->http, 1);
   if (r->http.response_buffer == NULL)
     return -1;
-  ssize_t len = rhizome_read(&r->read_state,
+  ssize_t len = rhizome_read(&r->u.read_state,
 		         (unsigned char *)r->http.response_buffer,
 			 r->http.response_buffer_size);
   if (len == -1)
@@ -464,35 +553,35 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
   rhizome_filehash_t filehash;
   if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
     return 1;
-  bzero(&r->read_state, sizeof r->read_state);
-  int n = rhizome_open_read(&r->read_state, &filehash);
+  bzero(&r->u.read_state, sizeof r->u.read_state);
+  int n = rhizome_open_read(&r->u.read_state, &filehash);
   if (n == -1) {
     http_request_simple_response(&r->http, 500, NULL);
     return 0;
   }
   if (n != 0)
     return 1;
-  if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) {
-    rhizome_read_close(&r->read_state);
+  if (r->u.read_state.length == -1 && rhizome_read(&r->u.read_state, NULL, 0)) {
+    rhizome_read_close(&r->u.read_state);
     return 1;
   }
-  assert(r->read_state.length != -1);
-  r->http.response.header.resource_length = r->read_state.length;
+  assert(r->u.read_state.length != -1);
+  r->http.response.header.resource_length = r->u.read_state.length;
   if (r->http.request_header.content_range_count > 0) {
     assert(r->http.request_header.content_range_count == 1);
     struct http_range closed;
-    unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length);
+    unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
     if (n == 0 || http_range_bytes(&closed, 1) == 0) {
       http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
       return 0;
     }
     r->http.response.header.content_range_start = closed.first;
     r->http.response.header.content_length = closed.last - closed.first + 1;
-    r->read_state.offset = closed.first;
+    r->u.read_state.offset = closed.first;
   } else {
     r->http.response.header.content_range_start = 0;
     r->http.response.header.content_length = r->http.response.header.resource_length;
-    r->read_state.offset = 0;
+    r->u.read_state.offset = 0;
   }
   http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
   return 0;

From 32ce7f5ed98ae3f2582fcbc79244d749fd861698 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 8 Nov 2013 18:25:33 +1030
Subject: [PATCH 08/21] Do not catch SIGQUIT

So that it can be used to make a core dump during manual debugging
---
 server.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server.c b/server.c
index 56323b24..402263ab 100644
--- a/server.c
+++ b/server.c
@@ -108,11 +108,9 @@ int server(const struct cli_parsed *parsed)
   sigemptyset(&sig.sa_mask); // Block the same signals during handler
   sigaddset(&sig.sa_mask, SIGHUP);
   sigaddset(&sig.sa_mask, SIGINT);
-  sigaddset(&sig.sa_mask, SIGQUIT);
   sig.sa_flags = 0;
   sigaction(SIGHUP, &sig, NULL);
   sigaction(SIGINT, &sig, NULL);
-  sigaction(SIGQUIT, &sig, NULL);
 
   /* Record PID to advertise that the server is now running */
   char filename[1024];

From a62b6f9250f44c37394c56f20eeaa8b5c5d260f9 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 8 Nov 2013 18:29:49 +1030
Subject: [PATCH 09/21] Improve Rhizome list cursor

Separate function rhizome_list_commit() which, if not called, causes the
next list re-query to include the last row that was fetched with
rhizome_list_next(), rather than excluding it.
---
 rhizome.h          |  1 +
 rhizome_database.c | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index 93ac6746..ba205c10 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -631,6 +631,7 @@ struct rhizome_list_cursor {
 
 int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
 int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *);
+void rhizome_list_commit(struct rhizome_list_cursor *);
 void rhizome_list_release(struct rhizome_list_cursor *);
 
 /* one manifest is required per candidate, plus a few spare.
diff --git a/rhizome_database.c b/rhizome_database.c
index bb13897f..f4d7941c 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1487,10 +1487,6 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
     const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
     c->rowid = sqlite3_column_int64(c->_statement, 5);
-    if (c->rowid > c->_rowid_first)
-      c->_rowid_first = c->rowid;
-    if (c->_rowid_last == 0 || c->rowid < c->_rowid_last)
-      c->_rowid_last = c->rowid;
     sid_t *author = NULL;
     if (q_author) {
       author = alloca(sizeof *author);
@@ -1528,6 +1524,15 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
   OUT();
 }
 
+void rhizome_list_commit(struct rhizome_list_cursor *c)
+{
+  assert(c->rowid != 0);
+  if (c->rowid > c->_rowid_first)
+    c->_rowid_first = c->rowid;
+  if (c->_rowid_last == 0 || c->rowid < c->_rowid_last)
+    c->_rowid_last = c->rowid;
+}
+
 void rhizome_list_release(struct rhizome_list_cursor *c)
 {
   if (c->manifest) {

From d5b48f5a9ed87135bec3d95f2d474e95742008c4 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 8 Nov 2013 18:30:11 +1030
Subject: [PATCH 10/21] Improve HTTP server generated content logic

Content generator functions now take arguments describing the buffer
they are to fill, and respond with a struct containing the number of
bytes filled, and the number of free bytes needed before being called
again.

The HTTP response logic now fills the buffer as much as possible before
calling write(2) by topping it up instead of waiting for it to be
completely emptied before generating more content.
---
 http_server.c  | 153 +++++++++++++++++++++++++--------------
 http_server.h  |  11 ++-
 rhizome.h      |   1 +
 rhizome_http.c | 189 ++++++++++++++++++++++++++++---------------------
 4 files changed, 215 insertions(+), 139 deletions(-)

diff --git a/http_server.c b/http_server.c
index 60e3de7c..1bb1cc04 100644
--- a/http_server.c
+++ b/http_server.c
@@ -118,6 +118,8 @@ void http_request_init(struct http_request *r, int sockfd)
 void http_request_free_response_buffer(struct http_request *r)
 {
   if (r->response_free_buffer) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Free response buffer of %zu bytes", r->response_buffer_size);
     r->response_free_buffer(r->response_buffer);
     r->response_free_buffer = NULL;
   }
@@ -127,6 +129,8 @@ void http_request_free_response_buffer(struct http_request *r)
 
 int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
 {
+  // Don't allocate a new buffer if the existing one contains content.
+  assert(r->response_buffer_sent == r->response_buffer_length);
   const char *const bufe = r->buffer + sizeof r->buffer;
   assert(r->received < bufe);
   size_t rbufsiz = bufe - r->received;
@@ -134,6 +138,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
     http_request_free_response_buffer(r);
     r->response_buffer = (char *) r->received;
     r->response_buffer_size = rbufsiz;
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Static response buffer %zu bytes", r->response_buffer_size);
     return 0;
   }
   if (bufsiz != r->response_buffer_size) {
@@ -142,6 +148,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
       return -1;
     r->response_free_buffer = free;
     r->response_buffer_size = bufsiz;
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Allocated response buffer %zu bytes", r->response_buffer_size);
   }
   assert(r->response_buffer_size >= bufsiz);
   assert(r->response_buffer != NULL);
@@ -1642,56 +1650,87 @@ static void http_request_receive(struct http_request *r)
  */
 static void http_request_send_response(struct http_request *r)
 {
-  if (r->response_length != CONTENT_LENGTH_UNKNOWN)
-    assert(r->response_sent <= r->response_length);
-  int g = 1;
-  while (g) {
-    if (r->response_length != CONTENT_LENGTH_UNKNOWN && r->response_sent == r->response_length)
-      break;
+  while (1) {
+    if (r->response_length != CONTENT_LENGTH_UNKNOWN)
+      assert(r->response_sent <= r->response_length);
     assert(r->response_buffer_sent <= r->response_buffer_length);
-    if (r->response_buffer_sent == r->response_buffer_length) {
-      g = 0;
-      if (r->response.content_generator) {
-	// Content generator must fill or partly fill response_buffer and set response_buffer_sent
-	// and response_buffer_length.  May also malloc() a bigger buffer and set response_buffer to
-	// point to it.
-	r->response_buffer_sent = r->response_buffer_length = 0;
-	g = r->response.content_generator(r);
-	if (g == -1) {
-	  if (r->debug_flag && *r->debug_flag)
-	    DEBUG("Content generation error, closing connection");
-	  http_request_finalise(r);
-	  return;
-	}
-	assert(r->response_buffer_sent <= r->response_buffer_length);
-	if (r->response_buffer_sent == r->response_buffer_length) {
-	  if (g == 0)
-	    break;
-	  if (r->response_length != CONTENT_LENGTH_UNKNOWN)
-	    WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
-		r->response_sent, r->response_length, r->response_length - r->response_sent);
-	  else
-	    WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"", r->response_sent);
-	  http_request_finalise(r);
-	  return;
-	}
-      } else if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
-	WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes",
-	    r->response_length, r->response_length - r->response_sent);
-	http_request_finalise(r);
-	return;
-      }
+    uint64_t remaining = CONTENT_LENGTH_UNKNOWN;
+    size_t unsent = r->response_buffer_length - r->response_buffer_sent;
+    if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
+      remaining = r->response_length - r->response_sent;
+      assert(unsent <= remaining);
+      assert(r->response_buffer_need <= remaining);
+      if (remaining == 0)
+	break; // no more to generate
     }
-    assert(r->response_buffer_sent < r->response_buffer_length);
-    size_t bytes = r->response_buffer_length - r->response_buffer_sent;
-    if (r->response_sent + bytes > r->response_length) {
-      WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t"  bytes -- truncating",
-	  r->response_length,
-	  r->response_sent + bytes - r->response_length);
-      bytes = r->response_length - r->response_sent;
+    if (unsent == 0)
+      r->response_buffer_sent = r->response_buffer_length = 0;
+    if (r->response.content_generator) {
+      // If the buffer is smaller than the content generator needs, and it contains no unsent
+      // content, then allocate a larger buffer.
+      if (r->response_buffer_need > r->response_buffer_size && unsent == 0) {
+	if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1) {
+	  WHYF("HTTP response truncated at offset=%"PRIhttp_size_t" due to insufficient buffer space",
+	      r->response_sent);
+	  http_request_finalise(r);
+	  return;
+	}
+      }
+      // If there are some sent bytes at the start of the buffer and only a few unsent bytes, then
+      // move the unsent content to the start of the buffer to make more room.
+      if (r->response_buffer_sent > 0 && unsent < 128) {
+	memmove(r->response_buffer, r->response_buffer + r->response_buffer_sent, unsent);
+	r->response_buffer_length -= r->response_buffer_sent;
+	r->response_buffer_sent = 0;
+      }
+      // If there is enough unfilled room at the end of the buffer, then fill the buffer with some
+      // more content.
+      assert(r->response_buffer_length <= r->response_buffer_size);
+      size_t unfilled = r->response_buffer_size - r->response_buffer_length;
+      if (unfilled > 0 && unfilled >= r->response_buffer_need) {
+	// The content generator must fill or partly fill the part of the buffer we indicate and
+	// return the number of bytes appended.  If it returns zero, it means it has no more
+	// content (EOF), and must not be called again.  If the return value exceeds the buffer size
+	// we supply, it gives the amount of free space the generator needs in order to append; the
+	// generator will not append any bytes until that much free space is available.  If returns
+	// -1, it means an unrecoverable error occurred, and the generator must not be called again.
+	struct http_content_generator_result result;
+	bzero(&result, sizeof result);
+	int ret = r->response.content_generator(r, (unsigned char *) r->response_buffer + r->response_buffer_length, unfilled, &result);
+	if (ret == -1) {
+	  WHY("Content generation error, closing connection");
+	  http_request_finalise(r);
+	  return;
+	}
+	if (result.generated == 0 && result.need <= unfilled) {
+	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t, r->response_sent);
+	  http_request_finalise(r);
+	  return;
+	}
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer", result.generated, result.need);
+	assert(result.generated <= unfilled);
+	r->response_buffer_length += result.generated;
+	r->response_buffer_need = result.need;
+	if (ret == 0)
+	  r->response.content_generator = NULL;
+	continue;
+      }
+    } else if (remaining != CONTENT_LENGTH_UNKNOWN && unsent < remaining) {
+      WHYF("HTTP response generator finished prematurely at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
+	  r->response_sent, r->response_length, remaining);
+      http_request_finalise(r);
+      return;
+    } else if (unsent == 0)
+      break;
+    assert(unsent > 0);
+    if (remaining != CONTENT_LENGTH_UNKNOWN && unsent > remaining) {
+      WHYF("HTTP response overruns Content-Length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating",
+	  r->response_length, unsent - remaining);
+      unsent = remaining;
     }
     sigPipeFlag = 0;
-    ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes);
+    ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, unsent);
     if (written == -1) {
       if (r->debug_flag && *r->debug_flag)
 	DEBUG("HTTP socket write error, closing connection");
@@ -1719,8 +1758,8 @@ static void http_request_send_response(struct http_request *r)
     r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
     unschedule(&r->alarm);
     schedule(&r->alarm);
-    // If we wrote less than we tried, then go back to polling.
-    if (written < (size_t) bytes)
+    // If we wrote less than we tried, then go back to polling, otherwise keep generating content.
+    if (written < (size_t) unsent)
       return;
   }
   if (r->debug_flag && *r->debug_flag)
@@ -1929,19 +1968,22 @@ static int _render_response(struct http_request *r)
     r->response_length = strbuf_count(sb) + hr.header.content_length;
   else
     r->response_length = CONTENT_LENGTH_UNKNOWN;
+  r->response_buffer_need = strbuf_count(sb) + 1; // the header and the strbuf terminating NUL
   if (hr.content) {
     assert(r->response_length != CONTENT_LENGTH_UNKNOWN);
-    r->response_buffer_need = r->response_length + 1;
-  } else {
+    if (r->response_buffer_need < r->response_length)
+      r->response_buffer_need = r->response_length;
+  } else
     assert(hr.content_generator);
-    r->response_buffer_need = strbuf_count(sb) + 1;
-  }
   if (r->response_buffer_size < r->response_buffer_need)
-    return 0;
+    return 0; // doesn't fit
   assert(!strbuf_overrun(sb));
-  if (hr.content)
+  if (hr.content) {
     bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
-  r->response_buffer_length = r->response_buffer_need;
+    r->response_buffer_length = r->response_length;
+  } else {
+    r->response_buffer_length = strbuf_count(sb);
+  }
   r->response_buffer_sent = 0;
   return 1;
 }
@@ -2022,6 +2064,7 @@ static void http_request_start_response(struct http_request *r)
       return;
     }
   }
+  r->response_buffer_need = 0;
   r->response_sent = 0;
   if (r->debug_flag && *r->debug_flag)
     DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_buffer_length));
diff --git a/http_server.h b/http_server.h
index 50b671c8..475a363e 100644
--- a/http_server.h
+++ b/http_server.h
@@ -96,13 +96,18 @@ struct http_response_headers {
   struct http_www_authenticate www_authenticate;
 };
 
-typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *);
+struct http_content_generator_result {
+  size_t generated;
+  size_t need;
+};
+
+typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, size_t, struct http_content_generator_result *);
 
 struct http_response {
   uint16_t result_code;
   struct http_response_headers header;
   const char *content;
-  HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content
+  HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content
 };
 
 #define MIME_FILENAME_MAXLEN 127
@@ -139,7 +144,7 @@ void http_request_free_response_buffer(struct http_request *r);
 int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
 void http_request_finalise(struct http_request *r);
 void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
-void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR);
+void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
 
 typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
diff --git a/rhizome.h b/rhizome.h
index ba205c10..6ef569f8 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -735,6 +735,7 @@ typedef struct rhizome_http_request
     /* For responses that list manifests.
     */
     struct {
+        enum { LIST_HEADER = 0, LIST_BODY, LIST_DONE } phase;
         size_t rowcount;
         struct rhizome_list_cursor cursor;
     } list;
diff --git a/rhizome_http.c b/rhizome_http.c
index 33f4efe8..39104a2a 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -335,7 +335,7 @@ static int is_authorized(struct http_client_authorization *auth)
   return 0;
 }
 
-static int restful_rhizome_bundlelist_json_content(struct http_request *);
+static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
 
 static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
 {
@@ -353,19 +353,16 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
     http_request_simple_response(&r->http, 401, NULL);
     return 0;
   }
+  r->u.list.phase = LIST_HEADER;
   r->u.list.rowcount = 0;
   bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
   http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
   return 0;
 }
 
-static int restful_rhizome_bundlelist_json_content(struct http_request *hr)
+static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
 {
-  rhizome_http_request *r = (rhizome_http_request *) hr;
-  assert(r->http.response_buffer_sent == 0);
-  assert(r->http.response_buffer_length == 0);
-  strbuf b = strbuf_local(r->http.response_buffer, r->http.response_buffer_size);
-  const char *headers[]={
+  const char *headers[] = {
     "_id",
     "service",
     "id",
@@ -380,66 +377,102 @@ static int restful_rhizome_bundlelist_json_content(struct http_request *hr)
     "recipient",
     "name"
   };
-  if (r->u.list.rowcount == 0) {
-    strbuf_puts(b, "[[");
-    unsigned i;
-    for (i = 0; i != NELS(headers); ++i) {
-      if (i)
+  switch (r->u.list.phase) {
+    case LIST_HEADER:
+      strbuf_puts(b, "[[");
+      unsigned i;
+      for (i = 0; i != NELS(headers); ++i) {
+	if (i)
+	  strbuf_putc(b, ',');
+	strbuf_json_string(b, headers[i]);
+      }
+      strbuf_puts(b, "]");
+      if (strbuf_overrun(b))
+	return 0;
+      r->u.list.phase = LIST_BODY;
+      return 1;
+    case LIST_BODY:
+      {
+	int ret = rhizome_list_next(retry, &r->u.list.cursor);
+	if (ret == -1)
+	  return -1;
+	if (ret == 0) {
+	  strbuf_puts(b, "\n]\n");
+	  if (strbuf_overrun(b))
+	    return 0;
+	  r->u.list.phase = LIST_DONE;
+	  return 0;
+	}
+	rhizome_manifest *m = r->u.list.cursor.manifest;
+	assert(m->filesize != RHIZOME_SIZE_UNSET);
+	rhizome_lookup_author(m);
+	strbuf_puts(b, ",\n [");
+	strbuf_sprintf(b, "%"PRIu64, r->u.list.cursor.rowid);
 	strbuf_putc(b, ',');
-      strbuf_json_string(b, headers[i]);
-    }
-    strbuf_puts(b, "]");
+	strbuf_json_string(b, m->service);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRId64, m->version);
+	strbuf_putc(b, ',');
+	if (m->has_date)
+	  strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
+	else
+	  strbuf_json_null(b);
+	strbuf_putc(b, ',');
+	strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
+	switch (m->authorship) {
+	  case AUTHOR_LOCAL:
+	  case AUTHOR_AUTHENTIC:
+	    strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
+	    strbuf_puts(b, ",1,");
+	    break;
+	  default:
+	    strbuf_json_null(b);
+	    strbuf_puts(b, ",1,");
+	    break;
+	}
+	strbuf_sprintf(b, "%"PRIu64, m->filesize);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
+	strbuf_putc(b, ',');
+	strbuf_json_string(b, m->name);
+	strbuf_puts(b, "]");
+	if (strbuf_overrun(b))
+	  return 0;
+	rhizome_list_commit(&r->u.list.cursor);
+	++r->u.list.rowcount;
+	return 1;
+      }
+    case LIST_DONE:
+      break;
   }
+  return 0;
+}
+
+static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  assert(bufsz > 0);
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  int ret = rhizome_list_next(&retry, &r->u.list.cursor);
-  if (ret == 0) {
-    strbuf_puts(b, "\n]\n");
-  } else if (ret == 1) {
-    ++r->u.list.rowcount;
-    rhizome_manifest *m = r->u.list.cursor.manifest;
-    assert(m->filesize != RHIZOME_SIZE_UNSET);
-    rhizome_lookup_author(m);
-    strbuf_puts(b, ",\n [");
-    strbuf_sprintf(b, "%"PRIu64, r->u.list.cursor.rowid);
-    strbuf_putc(b, ',');
-    strbuf_json_string(b, m->service);
-    strbuf_putc(b, ',');
-    strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
-    strbuf_putc(b, ',');
-    strbuf_sprintf(b, "%"PRId64, m->version);
-    strbuf_putc(b, ',');
-    if (m->has_date)
-      strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
-    else
-      strbuf_json_null(b);
-    strbuf_putc(b, ',');
-    strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
-    switch (m->authorship) {
-      case AUTHOR_LOCAL:
-      case AUTHOR_AUTHENTIC:
-	strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
-	strbuf_puts(b, ",1,");
-	break;
-      default:
-	strbuf_json_null(b);
-	strbuf_puts(b, ",1,");
-	break;
+  int ret = rhizome_list_open(&retry, &r->u.list.cursor);
+  if (ret == -1)
+    return -1;
+  strbuf b = strbuf_local((char *)buf, bufsz);
+  while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) {
+    if (strbuf_overrun(b)) {
+      result->need = strbuf_count(b) + 1 - result->generated;
+      break;
     }
-    strbuf_sprintf(b, "%"PRIu64, m->filesize);
-    strbuf_putc(b, ',');
-    strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
-    strbuf_putc(b, ',');
-    strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
-    strbuf_putc(b, ',');
-    strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
-    strbuf_putc(b, ',');
-    strbuf_json_string(b, m->name);
-    strbuf_puts(b, "]");
+    result->generated = strbuf_len(b);
+    if (ret == 0)
+      break;
   }
-  if (strbuf_overrun(b))
-    ret = WHY("HTTP response buffer overrun");
   rhizome_list_release(&r->u.list.cursor);
-  r->http.response_buffer_length = strbuf_len(b);
   return ret;
 }
 
@@ -509,30 +542,24 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
   return 0;
 }
 
-static int rhizome_file_content(struct http_request *hr)
+static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
 {
+  // Reads the next part of the payload into the supplied buffer.
   rhizome_http_request *r = (rhizome_http_request *) hr;
-  assert(r->http.response_buffer_sent == 0);
-  assert(r->http.response_buffer_length == 0);
   assert(r->u.read_state.offset < r->u.read_state.length);
-  uint64_t readlen = r->u.read_state.length - r->u.read_state.offset;
-  size_t suggested_size = 64 * 1024;
-  if (suggested_size > readlen)
-    suggested_size = readlen;
-  if (r->http.response_buffer_size < suggested_size)
-    http_request_set_response_bufsize(&r->http, suggested_size);
-  if (r->http.response_buffer == NULL)
-    http_request_set_response_bufsize(&r->http, 1);
-  if (r->http.response_buffer == NULL)
+  size_t readlen = r->u.read_state.length - r->u.read_state.offset;
+  if (readlen > bufsz)
+    readlen = bufsz;
+  ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
+  if (n == -1)
     return -1;
-  ssize_t len = rhizome_read(&r->u.read_state,
-		         (unsigned char *)r->http.response_buffer,
-			 r->http.response_buffer_size);
-  if (len == -1)
-    return -1;
-  assert((size_t) len <= r->http.response_buffer_size);
-  r->http.response_buffer_length += (size_t) len;
-  return 1;
+  result->generated = (size_t) n;
+  // Ask for a large buffer for all future reads.
+  const size_t preferred_bufsz = 64 * 1024;
+  assert(r->u.read_state.offset < r->u.read_state.length);
+  size_t remain = r->u.read_state.length - r->u.read_state.offset;
+  result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
+  return remain ? 1 : 0;
 }
 
 static int rhizome_file_page(rhizome_http_request *r, const char *remainder)

From ca66a387ddf7c8749a5cd26c66bf9a2b82cbe1a7 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 11 Nov 2013 16:16:04 +1030
Subject: [PATCH 11/21] Add "date" output field to Rhizome commands

To wit: "add file" and "import bundle"
---
 commandline.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/commandline.c b/commandline.c
index 7f1b8204..e6e669de 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1442,6 +1442,10 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     cli_field_name(context, "BK", ":");
     cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n");
   }
+  if (mout->has_date) {
+    cli_field_name(context, "date", ":");
+    cli_put_long(context, mout->date, "\n");
+  }
   cli_field_name(context, "version", ":");
   cli_put_long(context, mout->version, "\n");
   cli_field_name(context, "filesize", ":");
@@ -1545,6 +1549,10 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
   }
   cli_field_name(context, "version", ":");
   cli_put_long(context, m->version, "\n");
+  if (m->has_date) {
+    cli_field_name(context, "date", ":");
+    cli_put_long(context, m->date, "\n");
+  }
   cli_field_name(context, "filesize", ":");
   cli_put_long(context, m->filesize, "\n");
   assert(m->filesize != RHIZOME_SIZE_UNSET);

From 701f14fdf94b1057dccd9796d79016319c2fb664 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 11 Nov 2013 16:17:16 +1030
Subject: [PATCH 12/21] Improve test defs: extract_stdout_keyvalue_optional()

Logs its variable assignment
---
 testdefs.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/testdefs.sh b/testdefs.sh
index 2e083640..def06464 100644
--- a/testdefs.sh
+++ b/testdefs.sh
@@ -57,14 +57,17 @@ extract_stdout_keyvalue_optional() {
    esac
    local _label_re=$(escape_grep_basic "$_label")
    local _delim_re=$(escape_grep_basic "$_delim")
-   local _line=$(replayStdout | $GREP "^$_label_re$_delim_re")
+   local _line=$($GREP "^$_label_re$_delim_re" "$TFWSTDOUT")
    local _value=
    local _return=1
    if [ -n "$_line" ]; then
       _value="${_line#*$_delim}"
       _return=0
    fi
-   [ -n "$_var" ] && eval $_var="\$_value"
+   if [ -n "$_var" ]; then
+      eval $_var="\$_value"
+      eval tfw_log "$_var=\$(shellarg "\${$_var}")"
+   fi
    return $_return
 }
 

From 6b961c56ce24a46ad58ebe744ca87e47c49adadd Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 11 Nov 2013 16:21:26 +1030
Subject: [PATCH 13/21] Fix HTTP /restful/rhizome/bundlelist.json

Write test case assertions using jq(1) utility, increase from four
bundles to 100.

Fix bugs in HTTP server content generation logic.

Make payload content generator read payload 4KiB at a time, to
always read on filesystem block boundaries for performance.  Increase
size of payload in relevant test case.
---
 http_server.c         |  8 +++--
 rhizome_http.c        | 44 +++++++++++++++----------
 testdefs_rhizome.sh   | 16 +++++++++
 tests/rhizomehttp     | 77 ++++++++++++++++++++++++++++++++++++++++---
 tests/rhizomeprotocol |  2 +-
 5 files changed, 121 insertions(+), 26 deletions(-)

diff --git a/http_server.c b/http_server.c
index 1bb1cc04..277acdf0 100644
--- a/http_server.c
+++ b/http_server.c
@@ -1656,6 +1656,8 @@ static void http_request_send_response(struct http_request *r)
     assert(r->response_buffer_sent <= r->response_buffer_length);
     uint64_t remaining = CONTENT_LENGTH_UNKNOWN;
     size_t unsent = r->response_buffer_length - r->response_buffer_sent;
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("HTTP response buffer contains %zu bytes unsent", unsent);
     if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
       remaining = r->response_length - r->response_sent;
       assert(unsent <= remaining);
@@ -1703,17 +1705,17 @@ static void http_request_send_response(struct http_request *r)
 	  return;
 	}
 	if (result.generated == 0 && result.need <= unfilled) {
-	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t, r->response_sent);
+	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t" (ret=%d)", r->response_sent, ret);
 	  http_request_finalise(r);
 	  return;
 	}
 	if (r->debug_flag && *r->debug_flag)
-	  DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer", result.generated, result.need);
+	  DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer (ret=%d)", result.generated, result.need, ret);
 	assert(result.generated <= unfilled);
 	r->response_buffer_length += result.generated;
 	r->response_buffer_need = result.need;
 	if (ret == 0)
-	  r->response.content_generator = NULL;
+	  r->response.content_generator = NULL; // ensure we never invoke again
 	continue;
       }
     } else if (remaining != CONTENT_LENGTH_UNKNOWN && unsent < remaining) {
diff --git a/rhizome_http.c b/rhizome_http.c
index 39104a2a..98d3c629 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -387,9 +387,8 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	strbuf_json_string(b, headers[i]);
       }
       strbuf_puts(b, "]");
-      if (strbuf_overrun(b))
-	return 0;
-      r->u.list.phase = LIST_BODY;
+      if (!strbuf_overrun(b))
+	r->u.list.phase = LIST_BODY;
       return 1;
     case LIST_BODY:
       {
@@ -442,10 +441,10 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	strbuf_putc(b, ',');
 	strbuf_json_string(b, m->name);
 	strbuf_puts(b, "]");
-	if (strbuf_overrun(b))
-	  return 0;
-	rhizome_list_commit(&r->u.list.cursor);
-	++r->u.list.rowcount;
+	if (!strbuf_overrun(b)) {
+	  rhizome_list_commit(&r->u.list.cursor);
+	  ++r->u.list.rowcount;
+	}
 	return 1;
       }
     case LIST_DONE:
@@ -465,6 +464,8 @@ static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsi
   strbuf b = strbuf_local((char *)buf, bufsz);
   while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) {
     if (strbuf_overrun(b)) {
+      if (config.debug.rhizome)
+	DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b));
       result->need = strbuf_count(b) + 1 - result->generated;
       break;
     }
@@ -544,20 +545,27 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
 
 static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
 {
+  // Only read multiples of 4k from disk.
+  const size_t blocksz = 1 << 12;
+  // Ask for a large buffer for all future reads.
+  const size_t preferred_bufsz = 16 * blocksz;
   // Reads the next part of the payload into the supplied buffer.
   rhizome_http_request *r = (rhizome_http_request *) hr;
   assert(r->u.read_state.offset < r->u.read_state.length);
-  size_t readlen = r->u.read_state.length - r->u.read_state.offset;
-  if (readlen > bufsz)
-    readlen = bufsz;
-  ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
-  if (n == -1)
-    return -1;
-  result->generated = (size_t) n;
-  // Ask for a large buffer for all future reads.
-  const size_t preferred_bufsz = 64 * 1024;
-  assert(r->u.read_state.offset < r->u.read_state.length);
-  size_t remain = r->u.read_state.length - r->u.read_state.offset;
+  uint64_t remain = r->u.read_state.length - r->u.read_state.offset;
+  size_t readlen = bufsz;
+  if (remain < bufsz)
+    readlen = remain;
+  else
+    readlen &= ~(blocksz - 1);
+  if (readlen > 0) {
+    ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
+    if (n == -1)
+      return -1;
+    result->generated = (size_t) n;
+  }
+  assert(r->u.read_state.offset <= r->u.read_state.length);
+  remain = r->u.read_state.length - r->u.read_state.offset;
   result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
   return remain ? 1 : 0;
 }
diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh
index 959d2a49..db06a360 100644
--- a/testdefs_rhizome.sh
+++ b/testdefs_rhizome.sh
@@ -306,6 +306,18 @@ extract_stdout_BK() {
    extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
 }
 
+extract_stdout_date() {
+   extract_stdout_keyvalue "$1" date "$rexp_date"
+}
+
+extract_stdout_filesize() {
+   extract_stdout_keyvalue "$1" filesize "$rexp_filesize"
+}
+
+extract_stdout_filehash() {
+   extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
+}
+
 extract_manifest() {
    local _var="$1"
    local _manifestfile="$2"
@@ -346,6 +358,10 @@ extract_manifest_version() {
    extract_manifest "$1" "$2" version "$rexp_version"
 }
 
+extract_manifest_date() {
+   extract_manifest "$1" "$2" date "$rexp_date"
+}
+
 extract_manifest_crypt() {
    extract_manifest "$1" "$2" crypt "$rexp_crypt"
 }
diff --git a/tests/rhizomehttp b/tests/rhizomehttp
index e436e5cc..d698da9f 100755
--- a/tests/rhizomehttp
+++ b/tests/rhizomehttp
@@ -99,20 +99,89 @@ teardown_AuthBasicWrong() {
    teardown
 }
 
-doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
+doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
 setup_RhizomeList() {
-   for n in 1 2 3 4; do
-      create_file file$n ${n}k
+   setup
+   NBUNDLES=100
+   for ((n = 0; n != NBUNDLES; ++n)); do
+      create_file file$n $((1000 + $n))
       executeOk_servald rhizome add file $SIDA file$n file$n.manifest
+      extract_stdout_manifestid BID[$n]
+      extract_stdout_version VERSION[$n]
+      extract_stdout_filesize SIZE[$n]
+      extract_stdout_filehash HASH[$n]
+      extract_stdout_date DATE[$n]
    done
 }
 test_RhizomeList() {
    executeOk curl \
          --silent --fail --show-error \
-         --output http.output \
+         --output bundlelist.json \
          --dump-header http.headers \
          --basic --user harry:potter \
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+   tfw_cat http.headers bundlelist.json
+   tfw_preserve bundlelist.json
+   assert [ "$(jq 'length' bundlelist.json)" = $((NBUNDLES+1)) ]
+   # The following jq(1) incantation transforms the JSON array in
+   # bundlelist.json from the following form (which is optimised for
+   # transmission size):
+   #     [
+   #        [  "_id", "service", "id", "version","date",".inserttime",
+   #           ".author",".fromhere","filesize","filehash","sender","recipient",
+   #           "name"
+   #        ],
+   #        [  rowid1, "service1", bundleid1, version1, .... ],
+   #        [  rowid2, "service2", bundleid2, version2, .... ],
+   #           ...
+   #        [  rowidN, "serviceN", bundleidN, versionN, .... ]
+   #     ]
+   #
+   # into an array of JSON objects:
+   #     [
+   #        {
+   #           "_id": rowid1,
+   #           "service": service1,
+   #           "id": bundleid1,
+   #           "version": version1,
+   #           ...
+   #        },
+   #        {
+   #           "_id": rowid2,
+   #           "service": service2,
+   #           "id": bundleid2,
+   #           "version": version2,
+   #           ...
+   #        },
+   #        ...
+   #
+   #        {
+   #           "_id": rowidN,
+   #           "service": serviceN,
+   #           "id": bundleidN,
+   #           "version": versionN,
+   #           ...
+   #        }
+   #     ]
+   # which is much easier to test with jq(1) expressions.
+   jq '[ .[0] as $h | .[1:][] as $d | [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] | from_entries ]' \
+      bundlelist.json >array_of_objects.json
+   #tfw_cat array_of_objects.json
+   for ((n = 0; n != NBUNDLES; ++n)); do
+      jqscript="contains([
+                  {  name:\"file$n\",
+                     service:\"file\",
+                     id:\"${BID[$n]}\",
+                     version:${VERSION[$n]},
+                     filesize:${SIZE[$n]},
+                     filehash:\"${HASH[$n]}\",
+                     date:${DATE[$n]},
+                     \".fromhere\":1,
+                     \".author\":\"$SIDA\",
+                  }
+               ])"
+      assert [ "$(jq "$jqscript" array_of_objects.json)" = true ]
+   done
 }
 
 doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 2b81620e..15e8665f 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -75,7 +75,7 @@ doc_FileTransfer="New bundle and update transfer to one node"
 setup_FileTransfer() {
    setup_common
    set_instance +A
-   rhizome_add_file file1 2048
+   rhizome_add_file file1 250000
    start_servald_instances +A +B
    foreach_instance +A assert_peers_are_instances +B
    foreach_instance +B assert_peers_are_instances +A

From 13634f8748997b05f1002bd026bcbd398d194fe2 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 11 Nov 2013 18:13:38 +1030
Subject: [PATCH 14/21] Add ROWID field to struct rhizome_manifest

New ".rowid" output field from rhizome add, import, extract, export
operations.  (Also added missing ".inserttime" and "date" fields to
some operations.)

Use new "rhizome add file" .rowid output field to check output of of
/restful/rhizome/bundlelist.json
---
 commandline.c       | 32 +++++++++++------
 rhizome.h           |  9 ++++-
 rhizome_bundle.c    |  5 +++
 rhizome_crypto.c    |  6 ++--
 rhizome_database.c  | 20 ++++++-----
 rhizome_http.c      |  2 +-
 testdefs_rhizome.sh |  8 +++++
 tests/rhizomehttp   |  2 ++
 tests/rhizomeops    | 86 +++++++++++++++++++++++++++++++--------------
 9 files changed, 119 insertions(+), 51 deletions(-)

diff --git a/commandline.c b/commandline.c
index e6e669de..701bb694 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1438,6 +1438,10 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     cli_field_name(context, ".author", ":");
     cli_put_string(context, alloca_tohex_sid_t(mout->author), "\n");
   }
+  cli_field_name(context, ".rowid", ":");
+  cli_put_long(context, m->rowid, "\n");
+  cli_field_name(context, ".inserttime", ":");
+  cli_put_long(context, m->inserttime, "\n");
   if (mout->has_bundle_key) {
     cli_field_name(context, "BK", ":");
     cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n");
@@ -1727,6 +1731,19 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
     }
     cli_field_name(context, "manifestid", ":");
     cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
+    cli_field_name(context, "version", ":");
+    cli_put_long(context, m->version, "\n");
+    if (m->has_date) {
+      cli_field_name(context, "date", ":");
+      cli_put_long(context, m->date, "\n");
+    }
+    cli_field_name(context, "filesize", ":");
+    cli_put_long(context, m->filesize, "\n");
+    assert(m->filesize != RHIZOME_SIZE_UNSET);
+    if (m->filesize != 0) {
+      cli_field_name(context, "filehash", ":");
+      cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
+    }
     if (m->haveSecret) {
       char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
       rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
@@ -1738,19 +1755,12 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
       cli_field_name(context, ".author", ":");
       cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
     }
-    cli_field_name(context, "version", ":");
-    cli_put_long(context, m->version, "\n");
-    cli_field_name(context, "inserttime", ":");
+    cli_field_name(context, ".rowid", ":");
+    cli_put_long(context, m->rowid, "\n");
+    cli_field_name(context, ".inserttime", ":");
     cli_put_long(context, m->inserttime, "\n");
     cli_field_name(context, ".readonly", ":");
     cli_put_long(context, m->haveSecret?0:1, "\n");
-    cli_field_name(context, "filesize", ":");
-    cli_put_long(context, m->filesize, "\n");
-    assert(m->filesize != RHIZOME_SIZE_UNSET);
-    if (m->filesize != 0) {
-      cli_field_name(context, "filehash", ":");
-      cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
-    }
   }
   
   int retfile=0;
@@ -1889,7 +1899,7 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
       rhizome_manifest *m = cursor.manifest;
       assert(m->filesize != RHIZOME_SIZE_UNSET);
       rhizome_lookup_author(m);
-      cli_put_long(context, cursor.rowid, ":");
+      cli_put_long(context, m->rowid, ":");
       cli_put_string(context, m->service, ":");
       cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
       cli_put_long(context, m->version, ":");
diff --git a/rhizome.h b/rhizome.h
index 6ef569f8..8f5c2055 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -291,6 +291,12 @@ typedef struct rhizome_manifest
   sid_t sender;
   sid_t recipient;
 
+  /* Local data, not encapsulated in the bundle.  The ROWID of the SQLite
+   * MANIFESTS table row in which this manifest is stored.  Zero if the
+   * manifest has not been stored yet.
+   */
+  uint64_t rowid;
+
   /* Local data, not encapsulated in the bundle.  The system time of the most
    * recent INSERT or UPDATE of the manifest into the store.  Zero if the manifest
    * has not been stored yet.
@@ -342,6 +348,7 @@ typedef struct rhizome_manifest
 #define rhizome_manifest_set_recipient(m,v)     _rhizome_manifest_set_recipient(__WHENCE__,(m),(v))
 #define rhizome_manifest_del_recipient(m)       _rhizome_manifest_del_recipient(__WHENCE__,(m))
 #define rhizome_manifest_set_crypt(m,v)         _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_rowid(m,v)         _rhizome_manifest_set_rowid(__WHENCE__,(m),(v))
 #define rhizome_manifest_set_inserttime(m,v)    _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v))
 #define rhizome_manifest_set_author(m,v)        _rhizome_manifest_set_author(__WHENCE__,(m),(v))
 #define rhizome_manifest_del_author(m)          _rhizome_manifest_del_author(__WHENCE__,(m))
@@ -364,6 +371,7 @@ void _rhizome_manifest_del_sender(struct __sourceloc, rhizome_manifest *);
 void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *);
 void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *);
 void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
+void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t);
 void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t);
 void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
 void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
@@ -621,7 +629,6 @@ struct rhizome_list_cursor {
   sid_t sender;
   sid_t recipient;
   // Set by calling the next() function.
-  int64_t rowid;
   rhizome_manifest *manifest;
   // Private state.
   sqlite3_stmt *_statement;
diff --git a/rhizome_bundle.c b/rhizome_bundle.c
index 59aaebae..3d4b3db2 100644
--- a/rhizome_bundle.c
+++ b/rhizome_bundle.c
@@ -310,6 +310,11 @@ void _rhizome_manifest_set_crypt(struct __sourceloc __whence, rhizome_manifest *
   m->payloadEncryption = flag;
 }
 
+void _rhizome_manifest_set_rowid(struct __sourceloc __whence, rhizome_manifest *m, uint64_t rowid)
+{
+  m->rowid = rowid;
+}
+
 void _rhizome_manifest_set_inserttime(struct __sourceloc __whence, rhizome_manifest *m, time_ms_t time)
 {
   m->inserttime = time;
diff --git a/rhizome_crypto.c b/rhizome_crypto.c
index 1fb9c28b..774d23be 100644
--- a/rhizome_crypto.c
+++ b/rhizome_crypto.c
@@ -383,11 +383,11 @@ void rhizome_find_bundle_author_and_secret(rhizome_manifest *m)
 	rhizome_manifest_set_author(m, authorSidp);
 	m->authorship = AUTHOR_AUTHENTIC;
 	// if this bundle is already in the database, update the author.
-	if (m->inserttime)
+	if (m->rowid)
 	  sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
-	      "UPDATE MANIFESTS SET author = ? WHERE id = ?;",
+	      "UPDATE MANIFESTS SET author = ? WHERE rowid = ?;",
 	      SID_T, &m->author,
-	      RHIZOME_BID_T, &m->cryptoSignPublic,
+	      INT64, m->rowid,
 	      END);
 	RETURNVOID; // bingo
       }
diff --git a/rhizome_database.c b/rhizome_database.c
index f4d7941c..bb2b6a0a 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1343,6 +1343,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
     goto rollback;
   sqlite3_finalize(stmt);
   stmt = NULL;
+  rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db));
   rhizome_manifest_set_inserttime(m, now);
 
 //  if (serverMode)
@@ -1486,7 +1487,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     int64_t q_version = sqlite3_column_int64(c->_statement, 2);
     int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
     const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
-    c->rowid = sqlite3_column_int64(c->_statement, 5);
+    uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
     sid_t *author = NULL;
     if (q_author) {
       author = alloca(sizeof *author);
@@ -1509,6 +1510,7 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     }
     if (author)
       rhizome_manifest_set_author(m, author);
+    rhizome_manifest_set_rowid(m, q_rowid);
     rhizome_manifest_set_inserttime(m, q_inserttime);
     if (c->service && !(m->service && strcasecmp(c->service, m->service) == 0))
       continue;
@@ -1526,11 +1528,11 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
 
 void rhizome_list_commit(struct rhizome_list_cursor *c)
 {
-  assert(c->rowid != 0);
-  if (c->rowid > c->_rowid_first)
-    c->_rowid_first = c->rowid;
-  if (c->_rowid_last == 0 || c->rowid < c->_rowid_last)
-    c->_rowid_last = c->rowid;
+  assert(c->manifest->rowid != 0);
+  if (c->manifest->rowid > c->_rowid_first)
+    c->_rowid_first = c->manifest->rowid;
+  if (c->_rowid_last == 0 || c->manifest->rowid < c->_rowid_last)
+    c->_rowid_last = c->manifest->rowid;
 }
 
 void rhizome_list_release(struct rhizome_list_cursor *c)
@@ -1667,6 +1669,7 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
   int64_t q_inserttime = sqlite3_column_int64(statement, 3);
   const char *q_author = (const char *) sqlite3_column_text(statement, 4);
   size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
+  uint64_t q_rowid = sqlite3_column_int64(statement, 5);
   if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1)
     return WHYF("Manifest %s exists but is invalid", q_id);
   if (q_author) {
@@ -1678,6 +1681,7 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
   }
   if (m->version != q_version)
     WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version);
+  rhizome_manifest_set_rowid(m, q_rowid);
   rhizome_manifest_set_inserttime(m, q_inserttime);
   return 0;
 }
@@ -1693,7 +1697,7 @@ int rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m)
 {
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
-      "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id = ?",
+      "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id = ?",
       RHIZOME_BID_T, bidp,
       END);
   if (!statement)
@@ -1723,7 +1727,7 @@ int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned pr
   like[prefix_strlen] = '%';
   like[prefix_strlen + 1] = '\0';
   sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
-      "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id like ?",
+      "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id like ?",
       TEXT, like,
       END);
   if (!statement)
diff --git a/rhizome_http.c b/rhizome_http.c
index 98d3c629..8f26957c 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -406,7 +406,7 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	assert(m->filesize != RHIZOME_SIZE_UNSET);
 	rhizome_lookup_author(m);
 	strbuf_puts(b, ",\n [");
-	strbuf_sprintf(b, "%"PRIu64, r->u.list.cursor.rowid);
+	strbuf_sprintf(b, "%"PRIu64, m->rowid);
 	strbuf_putc(b, ',');
 	strbuf_json_string(b, m->service);
 	strbuf_putc(b, ',');
diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh
index db06a360..631371cd 100644
--- a/testdefs_rhizome.sh
+++ b/testdefs_rhizome.sh
@@ -302,6 +302,14 @@ extract_stdout_secret() {
    extract_stdout_keyvalue "$1" .secret "$rexp_bundlesecret"
 }
 
+extract_stdout_rowid() {
+   extract_stdout_keyvalue "$1" .rowid "$rexp_rowid"
+}
+
+extract_stdout_inserttime() {
+   extract_stdout_keyvalue "$1" .inserttime "$rexp_date"
+}
+
 extract_stdout_BK() {
    extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
 }
diff --git a/tests/rhizomehttp b/tests/rhizomehttp
index d698da9f..a9df4d62 100755
--- a/tests/rhizomehttp
+++ b/tests/rhizomehttp
@@ -111,6 +111,7 @@ setup_RhizomeList() {
       extract_stdout_filesize SIZE[$n]
       extract_stdout_filehash HASH[$n]
       extract_stdout_date DATE[$n]
+      extract_stdout_rowid ROWID[$n]
    done
 }
 test_RhizomeList() {
@@ -176,6 +177,7 @@ test_RhizomeList() {
                      filesize:${SIZE[$n]},
                      filehash:\"${HASH[$n]}\",
                      date:${DATE[$n]},
+                     _id:${ROWID[$n]},
                      \".fromhere\":1,
                      \".author\":\"$SIDA\",
                   }
diff --git a/tests/rhizomeops b/tests/rhizomeops
index 62892c2c..0668db07 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -208,23 +208,27 @@ setup_ExtractManifestAfterAdd() {
    setup_rhizome
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
+   extract_stdout_rowid rowid
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
+   extract_manifest_date date file1.manifest
 }
 test_ExtractManifestAfterAdd() {
    executeOk_servald rhizome export manifest $manifestid file1x.manifest
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
@@ -238,23 +242,27 @@ setup_ExtractManifestFileAfterAdd() {
    setup_rhizome
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
+   extract_stdout_rowid rowid
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
+   extract_manifest_date date file1.manifest
 }
 test_ExtractManifestFileAfterAdd() {
    executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
@@ -271,29 +279,35 @@ setup_ExtractManifestFileFromExtBlob() {
    executeOk_servald config set rhizome.external_blobs 1
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
+   extract_stdout_rowid rowid1
    executeOk_servald config set rhizome.external_blobs 0
    echo "Another test file" >file2
    executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
+   extract_stdout_rowid rowid2
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
    extract_manifest_id manifestid1 file1.manifest
    extract_manifest_version version1 file1.manifest
    extract_manifest_filehash filehash1 file1.manifest
+   extract_manifest_date date1 file1.manifest
    extract_manifest_id manifestid2 file2.manifest
    extract_manifest_version version2 file2.manifest
    extract_manifest_filehash filehash2 file2.manifest
+   extract_manifest_date date2 file2.manifest
 }
 test_ExtractManifestFileFromExtBlob() {
    executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
    assertStdoutGrep --matches=1 "^version:$version1\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date1\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
@@ -303,14 +317,16 @@ test_ExtractManifestFileFromExtBlob() {
    assert diff file1 file1x
    executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    local size=$(( $(cat file2 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
    assertStdoutGrep --matches=1 "^version:$version2\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date2\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
@@ -342,26 +358,30 @@ setup_ExtractManifestToStdout() {
    setup_rhizome
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
+   extract_stdout_rowid rowid
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
+   extract_manifest_date date file1.manifest
 }
 test_ExtractManifestToStdout() {
    executeOk_servald rhizome export manifest $manifestid -
-   assertStdoutLineCount '>=' 9
+   assertStdoutLineCount '>=' 11
    local size=$(( $(cat file1 | wc -c) + 0 ))
-   assertStdoutGrep --line=..9 --matches=1 "^service:file$"
-   assertStdoutGrep --line=..9 --matches=1 "^manifestid:$manifestid\$"
-   assertStdoutGrep --line=..9 --matches=1 "^version:$version\$"
-   assertStdoutGrep --line=..9 --matches=1 "^inserttime:$rexp_date\$"
-   assertStdoutGrep --line=..9 --matches=1 "^filehash:$filehash\$"
-   assertStdoutGrep --line=..9 --matches=1 "^filesize:$size\$"
-   assertStdoutGrep --line=..9 --matches=1 "^\.author:$SIDB1\$"
-   assertStdoutGrep --line=..9 --matches=1 "^\.secret:$rexp_bundlesecret\$"
-   assertStdoutGrep --line=..9 --matches=1 "^\.readonly:0\$"
-   assertStdoutGrep --line=10 --matches=1 "^manifest:"
-   replayStdout | $SED -n '10s/^manifest://p' >file1x.manifest
-   replayStdout | $SED -n '11,$p' >>file1x.manifest
+   assertStdoutGrep --line=..11 --matches=1 "^service:file$"
+   assertStdoutGrep --line=..11 --matches=1 "^manifestid:$manifestid\$"
+   assertStdoutGrep --line=..11 --matches=1 "^version:$version\$"
+   assertStdoutGrep --line=..11 --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --line=..11 --matches=1 "^\.inserttime:$rexp_date\$"
+   assertStdoutGrep --line=..11 --matches=1 "^filehash:$filehash\$"
+   assertStdoutGrep --line=..11 --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --line=..11 --matches=1 "^date:$date\$"
+   assertStdoutGrep --line=..11 --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --line=..11 --matches=1 "^\.secret:$rexp_bundlesecret\$"
+   assertStdoutGrep --line=..11 --matches=1 "^\.readonly:0\$"
+   assertStdoutGrep --line=12 --matches=1 "^manifest:"
+   replayStdout | $SED -n '12s/^manifest://p' >file1x.manifest
+   replayStdout | $SED -n '13,$p' >>file1x.manifest
    cat file1.manifest >file1n.manifest
    echo >>file1n.manifest
    tfw_cat file1n.manifest file1x.manifest
@@ -374,23 +394,27 @@ setup_ExtractManifestAfterAddNoAuthor() {
    setup_rhizome
    echo "A test file" >file1
    executeOk_servald rhizome add file '' file1 file1.manifest
+   extract_stdout_rowid rowid
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=0 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
+   extract_manifest_date date file1.manifest
 }
 test_ExtractManifestAfterAddNoAuthor() {
    executeOk_servald rhizome export manifest $manifestid file1x.manifest
    assert diff file1.manifest file1x.manifest
-   assertStdoutLineCount '==' 7
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date\$"
    assertStdoutGrep --matches=1 "^\.readonly:1\$"
 }
 
@@ -430,24 +454,28 @@ setup_ExtractFileAfterAdd() {
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
    tfw_cat --stderr
+   extract_stdout_rowid rowid
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
+   extract_manifest_date date file1.manifest
 }
 test_ExtractFileAfterAdd() {
    executeOk_servald rhizome extract file $manifestid file1x
    tfw_cat --stderr
    assert diff file1 file1x
    local size=$(( $(cat file1 | wc -c) + 0 ))
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
@@ -1013,9 +1041,11 @@ setup_ImportOwnBundle() {
    echo "Hello from B" >fileB
    executeOk_servald rhizome add file $SIDB2 fileB fileB.manifest
    assert_stdout_add_file fileB
+   extract_stdout_rowid rowid
    extract_manifest_id manifestid fileB.manifest
    extract_manifest_version version fileB.manifest
    extract_manifest_filehash filehash fileB.manifest
+   extract_manifest_date date fileB.manifest
    rm -f $SERVALINSTANCE_PATH/rhizome.db
    executeOk_servald rhizome list
    assert_rhizome_list
@@ -1031,14 +1061,16 @@ test_ImportOwnBundle() {
    tfw_cat --stderr
    assert cmp fileB.manifest fileBx.manifest
    assert cmp fileB fileBx
-   assertStdoutLineCount '==' 9
+   assertStdoutLineCount '==' 11
    local size=$(( $(cat fileB | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
-   assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
+   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --matches=1 "^date:$date\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
    assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"

From ba0ab14c6922369deb551a71705a06728002f49f Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 12 Nov 2013 11:39:06 +1030
Subject: [PATCH 15/21] Clean up inclusion of <netinet/in.h>

Make all #include <netinet/in.h> conditional upon HAVE_NETINET_IN_H

Remove unnecessary #include <netinet/in.h> from source files

Reformat include block in "serval.h" for readability
---
 lsif.c                  |  1 -
 net.c                   |  1 -
 net.h                   |  2 ++
 overlay_interface.c     |  1 -
 rhizome_packetformats.c |  1 -
 serval.h                | 69 ++++++++++++++++++++---------------------
 server.c                |  1 -
 7 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/lsif.c b/lsif.c
index 52e80c91..10065455 100644
--- a/lsif.c
+++ b/lsif.c
@@ -46,7 +46,6 @@
 #ifndef SIOCGIFCONF
 #include <sys/sockio.h>
 #endif
-#include <netinet/in.h>
 #include <netinet/if_ether.h>
 #if __MACH__
 #include <net/if_dl.h>
diff --git a/net.c b/net.c
index 6025757a..e5e2da00 100644
--- a/net.c
+++ b/net.c
@@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <fcntl.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
-#include <netinet/in.h>
 #include <time.h>
 
 #include "serval.h"
diff --git a/net.h b/net.h
index ab0cf6f7..108b3f56 100644
--- a/net.h
+++ b/net.h
@@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include <sys/types.h> // for size_t, ssize_t
 #include <sys/socket.h> // for struct sockaddr, socklen_t
+#ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h> // for struct in_addr
+#endif
 #include <arpa/inet.h> // for in_addr_t
 #include "log.h" // for __WHENCE__ and struct __sourceloc
 
diff --git a/overlay_interface.c b/overlay_interface.c
index e67570ff..172f4621 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -19,7 +19,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include <sys/types.h>
 #include <sys/socket.h>
-#include <netinet/in.h>
 #include <arpa/inet.h>
 #include <assert.h>
 #include <time.h>
diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c
index 11d52a84..092976f4 100644
--- a/rhizome_packetformats.c
+++ b/rhizome_packetformats.c
@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <stdlib.h>
 #include <sys/types.h>
 #include <sys/socket.h>
-#include <netinet/in.h>
 #include <arpa/inet.h>
 
 /* Android doesn't have log2(), and we don't really need to do floating point
diff --git a/serval.h b/serval.h
index d9d314b8..9301f25f 100644
--- a/serval.h
+++ b/serval.h
@@ -33,42 +33,41 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <sys/types.h>
 
 #ifdef WIN32
-#include "win32/win32.h"
+#   include "win32/win32.h"
 #else
-#include <unistd.h>
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_NET_ROUTE_H
-     #include <net/route.h>
-#endif
-#ifdef HAVE_LINUX_IF_H
-#include <linux/if.h>
-#else
-#ifdef HAVE_NET_IF_H
-#include <net/if.h>
-#endif
-#endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_LINUX_NETLINK_H
-#include <linux/netlink.h>
-#endif
-#ifdef HAVE_LINUX_RTNETLINK_H
-#include <linux/rtnetlink.h>
-#endif
-#ifdef HAVE_IFADDRS_H
-#include <ifaddrs.h>
-#endif
-#ifdef HAVE_SYS_SOCKIO_H
-#include <sys/sockio.h>
-#endif
-
-#ifdef HAVE_SYS_UCRED_H
-#include <sys/ucred.h>
-#endif
-#endif
+#   include <unistd.h>
+#   ifdef HAVE_SYS_SOCKET_H
+#     include <sys/socket.h>
+#   endif
+#   ifdef HAVE_NET_ROUTE_H
+#     include <net/route.h>
+#   endif
+#   ifdef HAVE_LINUX_IF_H
+#     include <linux/if.h>
+#   else
+#     ifdef HAVE_NET_IF_H
+#       include <net/if.h>
+#     endif
+#   endif
+#   ifdef HAVE_NETINET_IN_H
+#     include <netinet/in.h>
+#   endif
+#   ifdef HAVE_LINUX_NETLINK_H
+#     include <linux/netlink.h>
+#   endif
+#   ifdef HAVE_LINUX_RTNETLINK_H
+#     include <linux/rtnetlink.h>
+#   endif
+#   ifdef HAVE_IFADDRS_H
+#     include <ifaddrs.h>
+#   endif
+#   ifdef HAVE_SYS_SOCKIO_H
+#     include <sys/sockio.h>
+#   endif
+#   ifdef HAVE_SYS_UCRED_H
+#     include <sys/ucred.h>
+#   endif
+#endif //!WIN32
 
 #if !defined(FORASTERISK) && !defined(s_addr)
 #ifdef HAVE_ARPA_INET_H
diff --git a/server.c b/server.c
index 402263ab..7be22dfe 100644
--- a/server.c
+++ b/server.c
@@ -21,7 +21,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <unistd.h>
 #include <time.h>
 #include <sys/socket.h>
-#include <netinet/in.h>
 #include <sys/stat.h>
 
 #include "serval.h"

From b44046d6121167688892def3163a53f7d84126a2 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 12 Nov 2013 11:40:47 +1030
Subject: [PATCH 16/21] Forbid HTTP /restful/rhizome/bundlelist.json except
 from loopback

---
 rhizome_http.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/rhizome_http.c b/rhizome_http.c
index 8f26957c..807093aa 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -1,6 +1,6 @@
 /*
-Serval Distributed Numbering Architecture (DNA)
-Copyright (C) 2010 Paul Gardner-Stephen
+Serval DNA Rhizome HTTP external interface
+Copyright (C) 2013 Serval Project, Inc.
  
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
@@ -21,7 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <sys/socket.h>
 #include <signal.h>
 #ifdef HAVE_SYS_FILIO_H
-#include <sys/filio.h>
+# include <sys/filio.h>
 #endif
 #include <assert.h>
 
@@ -252,7 +252,7 @@ void rhizome_server_poll(struct sched_ent *alarm)
     } else {
       struct sockaddr_in *peerip=NULL;
       if (addr.sa_family == AF_INET) {
-	peerip = (struct sockaddr_in *)&addr;
+	peerip = (struct sockaddr_in *)&addr; // network order
 	INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
 	    addr_len, peerip->sin_family, peerip->sin_port,
 	    ((unsigned char*)&peerip->sin_addr.s_addr)[0],
@@ -318,10 +318,16 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_
   OUT();
 }
 
+static int is_from_loopback(const struct http_request *r)
+{
+  return   r->client_sockaddr_in.sin_family == AF_INET
+	&& ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET;
+}
+
 /* Return 1 if the given authorization credentials are acceptable.
  * Return 0 if not.
  */
-static int is_authorized(struct http_client_authorization *auth)
+static int is_authorized(const struct http_client_authorization *auth)
 {
   if (auth->scheme != BASIC)
     return 0;
@@ -335,6 +341,21 @@ static int is_authorized(struct http_client_authorization *auth)
   return 0;
 }
 
+static int authorize(struct http_request *r)
+{
+  if (!is_from_loopback(r)) {
+    http_request_simple_response(r, 403, NULL);
+    return 0;
+  }
+  if (!is_authorized(&r->request_header.authorization)) {
+    r->response.header.www_authenticate.scheme = BASIC;
+    r->response.header.www_authenticate.realm = "Serval Rhizome";
+    http_request_simple_response(r, 401, NULL);
+    return 0;
+  }
+  return 1;
+}
+
 static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
 
 static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
@@ -347,12 +368,8 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
     http_request_simple_response(&r->http, 405, NULL);
     return 0;
   }
-  if (!is_authorized(&r->http.request_header.authorization)) {
-    r->http.response.header.www_authenticate.scheme = BASIC;
-    r->http.response.header.www_authenticate.realm = "Serval Rhizome";
-    http_request_simple_response(&r->http, 401, NULL);
+  if (!authorize(&r->http))
     return 0;
-  }
   r->u.list.phase = LIST_HEADER;
   r->u.list.rowcount = 0;
   bzero(&r->u.list.cursor, sizeof r->u.list.cursor);

From 50bbb722d0ec0581cc5790e5a42c18cbd0f3531c Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 12 Nov 2013 17:44:03 +1030
Subject: [PATCH 17/21] Add uuid_t and UUID primitive functions

---
 headerfiles.mk |   1 +
 sourcefiles.mk |   1 +
 uuid.c         | 107 ++++++++++++++++++++++++++++++++++++++++++++++
 uuid.h         | 113 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 222 insertions(+)
 create mode 100644 uuid.c
 create mode 100644 uuid.h

diff --git a/headerfiles.mk b/headerfiles.mk
index 20fae731..ac76fd82 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -10,6 +10,7 @@ HDRS=	fifo.h \
 	rotbuf.h \
 	mem.h \
 	os.h \
+	uuid.h \
 	strbuf.h \
 	strbuf_helpers.h \
 	sha2.h \
diff --git a/sourcefiles.mk b/sourcefiles.mk
index 1b4516ac..f4954e22 100644
--- a/sourcefiles.mk
+++ b/sourcefiles.mk
@@ -69,6 +69,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)strbuf.c \
 	$(SERVAL_BASE)strbuf_helpers.c \
 	$(SERVAL_BASE)strlcpy.c \
+	$(SERVAL_BASE)uuid.c \
 	$(SERVAL_BASE)vomp.c \
 	$(SERVAL_BASE)vomp_console.c \
 	$(SERVAL_BASE)xprintf.c \
diff --git a/uuid.c b/uuid.c
new file mode 100644
index 00000000..3cf4cf3d
--- /dev/null
+++ b/uuid.c
@@ -0,0 +1,107 @@
+/*
+Serval DNA Universally Unique Identifier support
+Copyright (C) 2013 Serval Project Inc.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#define __SERVALDNA_UUID_H_INLINE
+#include "uuid.h"
+#include "os.h"
+#include "str.h"
+
+#include <assert.h>
+#ifdef HAVE_ARPA_INET_H
+# include <arpa/inet.h>
+#endif
+
+enum uuid_version uuid_get_version(const uuid_t *uuid)
+{
+  assert(uuid_is_valid(uuid));
+  switch (ntohs(uuid->u.record.time_hi_and_version) & 0xf000) {
+    case 0x1000: return UUID_VERSION_TIME_BASED;
+    case 0x2000: return UUID_VERSION_DCE_SECURITY;
+    case 0x3000: return UUID_VERSION_NAME_MD5;
+    case 0x4000: return UUID_VERSION_RANDOM;
+    case 0x5000: return UUID_VERSION_NAME_SHA1;
+  }
+  return UUID_VERSION_UNSUPPORTED;
+}
+
+void uuid_set_version(uuid_t *uuid, enum uuid_version version)
+{
+  uint16_t version_bits;
+  switch (version) {
+    case UUID_VERSION_TIME_BASED:   version_bits = 0x1000; break;
+    case UUID_VERSION_DCE_SECURITY: version_bits = 0x2000; break;
+    case UUID_VERSION_NAME_MD5:	    version_bits = 0x3000; break;
+    case UUID_VERSION_RANDOM:	    version_bits = 0x4000; break;
+    case UUID_VERSION_NAME_SHA1:    version_bits = 0x5000; break;
+    default: abort();
+  }
+  assert(uuid_is_valid(uuid));
+  uuid->u.record.time_hi_and_version = htons((ntohs(uuid->u.record.time_hi_and_version) & 0xfff) | version_bits);
+}
+
+int uuid_generate_random(uuid_t *uuid)
+{
+  if (urandombytes(uuid->u.binary, sizeof uuid->u.binary) == -1)
+    return -1;
+  // The following discards 6 random bits.
+  uuid->u.record.clock_seq_hi_and_reserved &= 0x3f;
+  uuid->u.record.clock_seq_hi_and_reserved |= 0x80;
+  uuid_set_version(uuid, UUID_VERSION_RANDOM);
+  return 0;
+}
+
+void uuid_to_str(const uuid_t *uuid, char *dst)
+{
+  assert(uuid_is_valid(uuid));
+  unsigned i;
+  for (i = 0; i != sizeof uuid->u.binary; ++i) {
+    switch (i) {
+      case 4: case 6: case 8: case 10:
+	*dst++ = '-';
+      default:
+	*dst++ = hexdigit[uuid->u.binary[i] >> 4];
+	*dst++ = hexdigit[uuid->u.binary[i] & 0xf];
+    }
+  }
+  *dst = '\0';
+}
+
+int str_to_uuid(const char *str, uuid_t *uuid, const char **afterp)
+{
+  const char *end = str;
+  int ret = 0;
+  if (	 strn_fromhex(uuid->u.binary, 4, end, &end) == 4
+      && *end == '-'
+      && strn_fromhex(uuid->u.binary + 4, 2, end + 1, &end) == 2
+      && *end == '-'
+      && strn_fromhex(uuid->u.binary + 6, 2, end + 1, &end) == 2
+      && *end == '-'
+      && strn_fromhex(uuid->u.binary + 8, 2, end + 1, &end) == 2
+      && *end == '-'
+      && strn_fromhex(uuid->u.binary + 10, 6, end + 1, &end) == 6
+  ) {
+    assert(end == str + 36);
+    ret = uuid_is_valid(uuid);
+  }
+  if (afterp)
+    *afterp = end;
+  if (ret == 0 || (!afterp && *end))
+    return 0;
+  return 1;
+}
diff --git a/uuid.h b/uuid.h
new file mode 100644
index 00000000..daa77d38
--- /dev/null
+++ b/uuid.h
@@ -0,0 +1,113 @@
+/*
+Serval DNA Universally Unique Identifier support
+Copyright (C) 2013 Serval Project Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#ifndef __SERVALDNA_UUID_H
+#define __SERVALDNA_UUID_H
+
+#include <stdint.h>
+
+#ifndef __SERVALDNA_UUID_H_INLINE
+# if __GNUC__ && !__GNUC_STDC_INLINE__
+#  define __SERVALDNA_UUID_H_INLINE extern inline
+# else
+#  define __SERVALDNA_UUID_H_INLINE inline
+# endif
+#endif
+
+/* A UUID is defined by RFC-4122 as a 128-bit identifier with the two most
+ * significant bits of the ninth byte being 1 and 0, which indicates the
+ * "variant" that is described by the RFC.  Other variants exist, but are not
+ * supported here, and are treated as INVALID by the functions defined below.
+ * Any attempt to pass an invalid UUID to a function that requires a valid UUID
+ * as input will probably result in the calling process being aborted (see
+ * SIGABRT, abort(3)).
+ *
+ * In a valid UUID, the four lowest significant bits of the seventh byte define
+ * the "version" of the UUID, which essentially indicates how it was generated.
+ * The RFC defines five SUPPORTED versions.  Any other version is UNSUPPORTED.
+ *
+ * The fields in the UUID 'record' structure are stored in network byte order,
+ * so code wishing to make use of the record structure must use ntohl(3) and
+ * ntohs(3) to read values, and htonl(3) and htons(3) to assign values.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+typedef struct uuid {
+  union {
+    struct {
+      uint32_t time_low;
+      uint16_t time_mid;
+      uint16_t time_hi_and_version;
+      uint8_t clock_seq_hi_and_reserved;
+      uint8_t clock_seq_low;
+      unsigned char node[6]; // uint48_t
+    } record;
+    unsigned char binary[16];
+  } u;
+} uuid_t;
+
+enum uuid_version {
+  UUID_VERSION_UNSUPPORTED = 0,
+  UUID_VERSION_TIME_BASED = 1,
+  UUID_VERSION_DCE_SECURITY = 2,
+  UUID_VERSION_NAME_MD5 = 3,
+  UUID_VERSION_RANDOM = 4,
+  UUID_VERSION_NAME_SHA1 = 5
+};
+
+__SERVALDNA_UUID_H_INLINE int uuid_is_valid(const uuid_t *any_uuid) {
+  return (any_uuid->u.record.clock_seq_hi_and_reserved & 0xc0) == 0x80;
+}
+
+enum uuid_version uuid_get_version(const uuid_t *valid_uuid);
+void uuid_set_version(uuid_t *valid_uuid, enum uuid_version);
+
+/* Returns -1 if error (eg, cannot open /dev/urandom), 0 if successful.
+ */
+int uuid_generate_random(uuid_t *dest_uuid);
+
+/* Formats the given valid UUID in its canonical string representation:
+ *    XXXXXXXX-VXXX-MXXX-XXXXXXXXXXXX
+ * where X is any hex digit
+ *       V is 1, 2, 3, 4 or 5 (supported versions) or any other hex digit
+ *            (unsupported versions)
+ *       M is 8, 9, A or B (high two bits are variant 01)
+ *
+ * The 'dst' argument must point to a buffer of 37 bytes.  The first 36 bytes
+ * are filled with the representation shown above, and the 37th byte dst[36] is
+ * set to NUL '\0'.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+void uuid_to_str(const uuid_t *valid_uuid, char *dst);
+
+/* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
+ * UUID, which may or not be supported.
+ *
+ * Returns 1 if a valid UUID is parsed, storing the value in *result (unless result is NULL) and
+ * storing a pointer to the immediately succeeding character in *afterp.  If afterp is NULL then
+ * returns 0 unless the immediately succeeding character is a NUL '\0'.  If no UUID is parsed or
+ * if the UUID is not valid, then returns 0, leaving *result unchanged and
+ * setting setting *afterp to point to the character where parsing failed.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+int str_to_uuid(const char *str, uuid_t *result, const char **afterp);
+
+#endif //__SERVALDNA_OS_H

From 64db53a092874ff63ad1a0c9ad70e4bac71aab47 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 12 Nov 2013 18:14:14 +1030
Subject: [PATCH 18/21] Add random UUID to Rhizome database

---
 rhizome.h          |  3 +++
 rhizome_database.c | 59 +++++++++++++++++++++++++++++++++++++++-------
 uuid.c             | 17 +++++++------
 uuid.h             |  7 +++++-
 4 files changed, 69 insertions(+), 17 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index 8f5c2055..37c75059 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <sqlite3.h>
 #include <limits.h>
 #include "sha2.h"
+#include "uuid.h"
 #include "str.h"
 #include "strbuf.h"
 #include "http_server.h"
@@ -405,6 +406,7 @@ int create_rhizome_datastore_dir();
 #define FORM_RHIZOME_IMPORT_PATH(buf,fmt,...) (form_rhizome_import_path((buf), sizeof(buf), (fmt), ##__VA_ARGS__))
 
 extern sqlite3 *rhizome_db;
+uuid_t rhizome_db_uuid;
 
 int rhizome_opendb();
 int rhizome_close_db();
@@ -517,6 +519,7 @@ enum sqlbind_type {
   TOHEX,              // const unsigned char *binary, unsigned bytes
   TEXT_TOUPPER,       // const char *text,
   TEXT_LEN_TOUPPER,   // const char *text, unsigned bytes
+  UUID_T,	      // const uuid_t *uuidp
   NUL = 1 << 15,      // NUL (no arg) ; NUL|INT, ...
   INDEX = 0xfade0000, // INDEX|INT, int index, ...
   NAMED = 0xdead0000  // NAMED|INT, const char *label, ...
diff --git a/rhizome_database.c b/rhizome_database.c
index bb2b6a0a..40865f77 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -75,7 +75,8 @@ int create_rhizome_datastore_dir()
   return emkdirs(rhizome_datastore_path(), 0700);
 }
 
-sqlite3 *rhizome_db=NULL;
+sqlite3 *rhizome_db = NULL;
+uuid_t rhizome_db_uuid;
 
 /* XXX Requires a messy join that might be slow. */
 int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
@@ -207,7 +208,10 @@ void verify_bundles(){
 
 int rhizome_opendb()
 {
-  if (rhizome_db) return 0;
+  if (rhizome_db) {
+    assert(uuid_is_valid(&rhizome_db_uuid));
+    return 0;
+  }
 
   IN();
   
@@ -257,11 +261,9 @@ int rhizome_opendb()
     ) {
       RETURN(WHY("Failed to create schema"));
     }
-
     /* Create indexes if they don't already exist */
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
-    
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
   }
   if (version<2){
@@ -273,26 +275,52 @@ int rhizome_opendb()
     verify_bundles();
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
   }
-  
   if (version<3){
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
   }
-  
   if (version<4){
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
   }
-  
+  if (version<5){
+    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
+    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
+  }
+
+  char buf[UUID_STRLEN + 1];
+  int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
+  if (r == -1)
+    RETURN(-1);
+  if (r) {
+    if (!str_to_uuid(buf, &rhizome_db_uuid, NULL)) {
+      WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf));
+      if (uuid_generate_random(&rhizome_db_uuid) == -1)
+	RETURN(WHY("Cannot generate new UUID for Rhizome database"));
+      if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", UUID_T, &rhizome_db_uuid, END) == -1)
+	RETURN(WHY("Failed to update new UUID in Rhizome database"));
+      if (config.debug.rhizome)
+	DEBUGF("Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
+    }
+  } else if (r == 0) {
+    if (uuid_generate_random(&rhizome_db_uuid) == -1)
+      RETURN(WHY("Cannot generate UUID for Rhizome database"));
+    if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", UUID_T, &rhizome_db_uuid, END) == -1)
+      RETURN(WHY("Failed to insert UUID into Rhizome database"));
+    if (config.debug.rhizome)
+      DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
+  }
+
   // TODO recreate tables with collate nocase on hex columns
-  
+
   /* Future schema updates should be performed here. 
    The above schema can be assumed to exist.
    All changes should attempt to preserve any existing data */
-  
+
   // We can't delete a file that is being transferred in another process at this very moment...
   if (config.rhizome.clean_on_open)
     rhizome_cleanup(NULL);
+  INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_db_uuid));
   RETURN(0);
   OUT();
 }
@@ -740,6 +768,19 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
 	      }
 	    }
 	    break;
+	  case UUID_T: {
+	      const uuid_t *uuidp = va_arg(ap, const uuid_t *);
+	      ++argnum;
+	      if (uuidp == NULL) {
+		BIND_NULL(UUID_T);
+	      } else {
+		char uuid_str[UUID_STRLEN + 1];
+		uuid_to_str(uuidp, uuid_str);
+		BIND_DEBUG(UUID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", uuid_str, UUID_STRLEN);
+		BIND_RETRY(sqlite3_bind_text, uuid_str, UUID_STRLEN, SQLITE_TRANSIENT);
+	      }
+	    }
+	    break;
 #undef BIND_RETRY
 	  default:
 	    FATALF("at bind arg %u, unsupported bind code typ=0x%08x: %s", argnum, typ, sqlite3_sql(statement));
diff --git a/uuid.c b/uuid.c
index 3cf4cf3d..8666fbbe 100644
--- a/uuid.c
+++ b/uuid.c
@@ -66,23 +66,26 @@ int uuid_generate_random(uuid_t *uuid)
   return 0;
 }
 
-void uuid_to_str(const uuid_t *uuid, char *dst)
+char *uuid_to_str(const uuid_t *uuid, char *const dst)
 {
+  char *p = dst;
   assert(uuid_is_valid(uuid));
   unsigned i;
   for (i = 0; i != sizeof uuid->u.binary; ++i) {
     switch (i) {
       case 4: case 6: case 8: case 10:
-	*dst++ = '-';
+	*p++ = '-';
       default:
-	*dst++ = hexdigit[uuid->u.binary[i] >> 4];
-	*dst++ = hexdigit[uuid->u.binary[i] & 0xf];
+	*p++ = hexdigit[uuid->u.binary[i] >> 4];
+	*p++ = hexdigit[uuid->u.binary[i] & 0xf];
     }
   }
-  *dst = '\0';
+  *p = '\0';
+  assert(p == dst + UUID_STRLEN);
+  return dst;
 }
 
-int str_to_uuid(const char *str, uuid_t *uuid, const char **afterp)
+int str_to_uuid(const char *const str, uuid_t *uuid, const char **afterp)
 {
   const char *end = str;
   int ret = 0;
@@ -96,7 +99,7 @@ int str_to_uuid(const char *str, uuid_t *uuid, const char **afterp)
       && *end == '-'
       && strn_fromhex(uuid->u.binary + 10, 6, end + 1, &end) == 6
   ) {
-    assert(end == str + 36);
+    assert(end == str + UUID_STRLEN);
     ret = uuid_is_valid(uuid);
   }
   if (afterp)
diff --git a/uuid.h b/uuid.h
index daa77d38..7a55bfc2 100644
--- a/uuid.h
+++ b/uuid.h
@@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #define __SERVALDNA_UUID_H
 
 #include <stdint.h>
+#include <alloca.h>
 
 #ifndef __SERVALDNA_UUID_H_INLINE
 # if __GNUC__ && !__GNUC_STDC_INLINE__
@@ -93,9 +94,13 @@ int uuid_generate_random(uuid_t *dest_uuid);
  * are filled with the representation shown above, and the 37th byte dst[36] is
  * set to NUL '\0'.
  *
+ * Returns the 'dst' argument.
+ *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-void uuid_to_str(const uuid_t *valid_uuid, char *dst);
+char *uuid_to_str(const uuid_t *valid_uuid, char *dst);
+#define UUID_STRLEN 36
+#define alloca_uuid_str(uuid) uuid_to_str(&(uuid), alloca(UUID_STRLEN + 1))
 
 /* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
  * UUID, which may or not be supported.

From 1634d68dd0761c285516a7db67934d89cd722930 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 13 Nov 2013 12:45:02 +1030
Subject: [PATCH 19/21] Format UUID strings as lower case hex

---
 str.c            | 5 +++--
 str.h            | 3 ++-
 strbuf.c         | 4 ++--
 strbuf_helpers.c | 4 ++--
 uuid.c           | 4 ++--
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/str.c b/str.c
index 182a7842..d5af0349 100644
--- a/str.c
+++ b/str.c
@@ -30,14 +30,15 @@
 #include <limits.h>
 #include <errno.h>
 
-const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
+const char hexdigit_upper[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
+const char hexdigit_lower[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
 
 char *tohex(char *dstHex, size_t dstStrLen, const unsigned char *srcBinary)
 {
   char *p;
   size_t i;
   for (p = dstHex, i = 0; i < dstStrLen; ++i)
-    *p++ = (i & 1) ? hexdigit[*srcBinary++ & 0xf] : hexdigit[*srcBinary >> 4];
+    *p++ = (i & 1) ? hexdigit_upper[*srcBinary++ & 0xf] : hexdigit_upper[*srcBinary >> 4];
   *p = '\0';
   return dstHex;
 }
diff --git a/str.h b/str.h
index 7f9827be..e84a8649 100644
--- a/str.h
+++ b/str.h
@@ -60,7 +60,8 @@ __STR_INLINE int is_xstring(const char *text, int len)
   return *text == '\0';
 }
 
-extern const char hexdigit[16];
+extern const char hexdigit_upper[16];
+extern const char hexdigit_lower[16];
 char *tohex(char *dstHex, size_t dstStrlen, const unsigned char *srcBinary);
 size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
 int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
diff --git a/strbuf.c b/strbuf.c
index df37a0a1..23efa133 100644
--- a/strbuf.c
+++ b/strbuf.c
@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #define __STRBUF_INLINE
 #include "strbuf.h"
+#include "str.h"
 
 static inline size_t min(size_t a, size_t b) {
   return a < b ? a : b;
@@ -76,7 +77,6 @@ strbuf strbuf_puts(strbuf sb, const char *text)
 
 strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
 {
-  static char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
   char *p = sb->current;
   sb->current += strlen;
   if (sb->start) {
@@ -84,7 +84,7 @@ strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
     // The following loop could overwrite the '\0' at *sp->end.
     size_t i;
     for (i = 0; i < strlen && p < e; ++i)
-      *p++ = (i & 1) ? hexdigit[*data++ & 0xf] : hexdigit[*data >> 4];
+      *p++ = (i & 1) ? hexdigit_upper[*data++ & 0xf] : hexdigit_upper[*data >> 4];
     // This will restore the '\0' at *sp->end if it was overwritten.
     *e = '\0';
   }
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 530be30d..093c5cc8 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -457,8 +457,8 @@ strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
     strbuf_putc(sb, '"');
     size_t i;
     for (i = 0; i != len; ++i) {
-      strbuf_putc(sb, hexdigit[*buf >> 4]);
-      strbuf_putc(sb, hexdigit[*buf++ & 0xf]);
+      strbuf_putc(sb, hexdigit_upper[*buf >> 4]);
+      strbuf_putc(sb, hexdigit_upper[*buf++ & 0xf]);
     }
     strbuf_putc(sb, '"');
   } else
diff --git a/uuid.c b/uuid.c
index 8666fbbe..8b629591 100644
--- a/uuid.c
+++ b/uuid.c
@@ -76,8 +76,8 @@ char *uuid_to_str(const uuid_t *uuid, char *const dst)
       case 4: case 6: case 8: case 10:
 	*p++ = '-';
       default:
-	*p++ = hexdigit[uuid->u.binary[i] >> 4];
-	*p++ = hexdigit[uuid->u.binary[i] & 0xf];
+	*p++ = hexdigit_lower[uuid->u.binary[i] >> 4];
+	*p++ = hexdigit_lower[uuid->u.binary[i] & 0xf];
     }
   }
   *p = '\0';

From a14326deeb31b5bd2c90b1310b33c55a27dcad9b Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 13 Nov 2013 12:45:32 +1030
Subject: [PATCH 20/21] Log SQLite PRAGMA statements as well

---
 rhizome_database.c | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/rhizome_database.c b/rhizome_database.c
index 40865f77..6f9ce6c4 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -107,11 +107,32 @@ int is_debug_rhizome_ads()
 
 static int (*sqlite_trace_func)() = is_debug_rhizome;
 const struct __sourceloc *sqlite_trace_whence = NULL;
+static int sqlite_trace_done;
 
+/* This is called by SQLite when executing a statement using sqlite3_step().  Unfortunately, it is
+ * not called on PRAGMA statements, and possibly others.  Hence the use of the profile callback (see
+ * below).
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
 static void sqlite_trace_callback(void *context, const char *rendered_sql)
 {
   if (sqlite_trace_func())
     logMessage(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql);
+  ++sqlite_trace_done;
+}
+
+/* This is called by SQLite when an executed statement finishes.  We use it to log rendered SQL
+ * statements when the trace callback (above) has not been called, eg, for PRAGMA statements.  This
+ * requires that the 'sqlite_trace_done' static be reset to zero whenever a new prepared statement
+ * is about to be stepped.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static void sqlite_profile_callback(void *context, const char *rendered_sql, sqlite_uint64 nanosec)
+{
+  if (!sqlite_trace_done)
+    sqlite_trace_callback(context, rendered_sql);
 }
 
 /* This function allows code like:
@@ -153,7 +174,6 @@ void verify_bundles(){
     sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
     const void *manifest = sqlite3_column_blob(statement, 1);
     size_t manifest_length = sqlite3_column_bytes(statement, 1);
-    
     rhizome_manifest *m=rhizome_new_manifest();
     int ret=0;
     ret = rhizome_read_manifest_file(m, manifest, manifest_length);
@@ -236,6 +256,7 @@ int rhizome_opendb()
     RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
   }
   sqlite3_trace(rhizome_db, sqlite_trace_callback, NULL);
+  sqlite3_profile(rhizome_db, sqlite_profile_callback, NULL);
   int loglevel = (config.debug.rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;
 
   /* Read Rhizome configuration */
@@ -826,6 +847,7 @@ int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state
   IN();
   int ret = -1;
   sqlite_trace_whence = &__whence;
+  sqlite_trace_done = 0;
   while (statement) {
     int stepcode = sqlite3_step(statement);
     switch (stepcode) {

From 4380fdcccd5111648092b5262d9d80244216832d Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 13 Nov 2013 16:58:28 +1030
Subject: [PATCH 21/21] Token in /restful/rhizome/bundlelist.json output

---
 rhizome.h          |  7 ++--
 rhizome_database.c | 52 ++++++++++++++++++---------
 rhizome_http.c     | 36 ++++++++++++++-----
 tests/rhizomehttp  | 89 ++++++++++++++++++++++++++--------------------
 uuid.c             | 19 ++++++----
 uuid.h             |  5 +++
 6 files changed, 136 insertions(+), 72 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index 37c75059..370a279d 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -631,12 +631,13 @@ struct rhizome_list_cursor {
   bool_t is_recipient_set;
   sid_t sender;
   sid_t recipient;
+  uint64_t rowid_since;
   // Set by calling the next() function.
   rhizome_manifest *manifest;
   // Private state.
   sqlite3_stmt *_statement;
-  int64_t _rowid_first;
-  int64_t _rowid_last;
+  uint64_t _rowid_current;
+  uint64_t _rowid_last; // for re-opening query
 };
 
 int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
@@ -745,7 +746,7 @@ typedef struct rhizome_http_request
     /* For responses that list manifests.
     */
     struct {
-        enum { LIST_HEADER = 0, LIST_BODY, LIST_DONE } phase;
+        enum { LIST_HEADER = 0, LIST_TOKEN, LIST_ROWS, LIST_DONE } phase;
         size_t rowcount;
         struct rhizome_list_cursor cursor;
     } list;
diff --git a/rhizome_database.c b/rhizome_database.c
index 6f9ce6c4..60c17ae1 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1493,11 +1493,10 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     strbuf_puts(b, " AND sender = @sender");
   if (c->is_recipient_set)
     strbuf_puts(b, " AND recipient = @recipient");
-  if (c->_rowid_first) {
-    assert(c->_rowid_last);
-    assert(c->_rowid_last <= c->_rowid_first);
-    strbuf_puts(b, " AND (rowid > @first OR rowid < @last)");
-  }
+  if (c->rowid_since)
+    strbuf_puts(b, " AND rowid > @since");
+  if (c->_rowid_last)
+    strbuf_puts(b, " AND rowid < @last");
   strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first
   if (strbuf_overrun(b))
     RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
@@ -1512,12 +1511,12 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     goto failure;
   if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
     goto failure;
-  if (   c->_rowid_first
-      && sqlite_bind(retry, c->_statement, NAMED|INT64, "@first", c->_rowid_first,
-					   NAMED|INT64, "@last", c->_rowid_last, END) == -1
-  )
+  if (c->rowid_since && sqlite_bind(retry, c->_statement, NAMED|INT64, "@since", c->rowid_since, END) == -1)
+    goto failure;
+  if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
     goto failure;
   c->manifest = NULL;
+  c->_rowid_current = 0;
   RETURN(0);
   OUT();
 failure:
@@ -1527,16 +1526,27 @@ failure:
   OUT();
 }
 
+/* Guaranteed to return manifests with monotonically descending rowid.  The first manifest will have
+ * the greatest rowid.
+ *
+ * Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest'
+ * field points to the fetched manifest.  Returns 0 if there are no more manifests in the list.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
 int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
 {
   IN();
   if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
     RETURN(-1);
-  while (sqlite_step_retry(retry, c->_statement) == SQLITE_ROW) {
+  while (1) {
     if (c->manifest) {
       rhizome_manifest_free(c->manifest);
+      c->_rowid_current = 0;
       c->manifest = NULL;
     }
+    if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW)
+      break;
     assert(sqlite3_column_count(c->_statement) == 6);
     assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
     assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
@@ -1544,13 +1554,22 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
     assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
     assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
+    uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
+    if (c->_rowid_current && q_rowid >= c->_rowid_current) {
+      WHYF("Query returned rowid=%"PRIu64" out of order (last was %"PRIu64") -- skipped", q_rowid, c->_rowid_current);
+      continue;
+    }
+    c->_rowid_current = q_rowid;
+    if (q_rowid <= c->rowid_since) {
+      WHYF("Query returned rowid=%"PRIu64" <= rowid_since=%"PRIu64" -- skipped", q_rowid, c->rowid_since);
+      continue;
+    }
     const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
     const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
     size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
     int64_t q_version = sqlite3_column_int64(c->_statement, 2);
     int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
     const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
-    uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
     sid_t *author = NULL;
     if (q_author) {
       author = alloca(sizeof *author);
@@ -1581,27 +1600,28 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
       continue;
     if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0))
       continue;
+    assert(c->_rowid_current != 0);
     // Don't do rhizome_verify_author(m); too CPU expensive for a listing.  Save that for when
     // the bundle is extracted or exported.
     RETURN(1);
   }
+  assert(c->_rowid_current == 0);
   RETURN(0);
   OUT();
 }
 
 void rhizome_list_commit(struct rhizome_list_cursor *c)
 {
-  assert(c->manifest->rowid != 0);
-  if (c->manifest->rowid > c->_rowid_first)
-    c->_rowid_first = c->manifest->rowid;
-  if (c->_rowid_last == 0 || c->manifest->rowid < c->_rowid_last)
-    c->_rowid_last = c->manifest->rowid;
+  assert(c->_rowid_current != 0);
+  if (c->_rowid_last == 0 || c->_rowid_current < c->_rowid_last)
+    c->_rowid_last = c->_rowid_current;
 }
 
 void rhizome_list_release(struct rhizome_list_cursor *c)
 {
   if (c->manifest) {
     rhizome_manifest_free(c->manifest);
+    c->_rowid_current = 0;
     c->manifest = NULL;
   }
   if (c->_statement) {
diff --git a/rhizome_http.c b/rhizome_http.c
index 807093aa..7897610e 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -377,6 +377,17 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
   return 0;
 }
 
+#define LIST_TOKEN_STRLEN_MAX (UUID_STRLEN + 30)
+#define alloca_list_token(rowid) list_token(alloca(LIST_TOKEN_STRLEN_MAX), (rowid))
+
+static char *list_token(char *buf, uint64_t rowid)
+{
+  strbuf b = strbuf_local(buf, LIST_TOKEN_STRLEN_MAX);
+  strbuf_uuid(b, &rhizome_db_uuid);
+  strbuf_sprintf(b, "-%"PRIu64, rowid);
+  return buf;
+}
+
 static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
 {
   const char *headers[] = {
@@ -396,7 +407,7 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
   };
   switch (r->u.list.phase) {
     case LIST_HEADER:
-      strbuf_puts(b, "[[");
+      strbuf_puts(b, "{\n\"header\":[");
       unsigned i;
       for (i = 0; i != NELS(headers); ++i) {
 	if (i)
@@ -405,24 +416,32 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
       }
       strbuf_puts(b, "]");
       if (!strbuf_overrun(b))
-	r->u.list.phase = LIST_BODY;
+	r->u.list.phase = LIST_TOKEN;
       return 1;
-    case LIST_BODY:
+    case LIST_TOKEN:
+    case LIST_ROWS:
       {
 	int ret = rhizome_list_next(retry, &r->u.list.cursor);
 	if (ret == -1)
 	  return -1;
+	rhizome_manifest *m = r->u.list.cursor.manifest;
+	if (r->u.list.phase == LIST_TOKEN) {
+	  strbuf_puts(b, ",\n\"token\":");
+	  strbuf_json_string(b, alloca_list_token(ret ? m->rowid : 0));
+	  strbuf_puts(b, ",\n\"rows\":[");
+	}
 	if (ret == 0) {
-	  strbuf_puts(b, "\n]\n");
+	  strbuf_puts(b, "\n]\n}\n");
 	  if (strbuf_overrun(b))
 	    return 0;
 	  r->u.list.phase = LIST_DONE;
 	  return 0;
 	}
-	rhizome_manifest *m = r->u.list.cursor.manifest;
 	assert(m->filesize != RHIZOME_SIZE_UNSET);
 	rhizome_lookup_author(m);
-	strbuf_puts(b, ",\n [");
+	if (r->u.list.phase != LIST_TOKEN)
+	  strbuf_putc(b, ',');
+	strbuf_puts(b, "\n[");
 	strbuf_sprintf(b, "%"PRIu64, m->rowid);
 	strbuf_putc(b, ',');
 	strbuf_json_string(b, m->service);
@@ -462,12 +481,13 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	  rhizome_list_commit(&r->u.list.cursor);
 	  ++r->u.list.rowcount;
 	}
+	r->u.list.phase = LIST_ROWS;
 	return 1;
       }
     case LIST_DONE:
-      break;
+      return 0;
   }
-  return 0;
+  abort();
 }
 
 static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
diff --git a/tests/rhizomehttp b/tests/rhizomehttp
index a9df4d62..de09eb17 100755
--- a/tests/rhizomehttp
+++ b/tests/rhizomehttp
@@ -123,53 +123,66 @@ test_RhizomeList() {
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
    tfw_cat http.headers bundlelist.json
    tfw_preserve bundlelist.json
-   assert [ "$(jq 'length' bundlelist.json)" = $((NBUNDLES+1)) ]
+   assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ]
    # The following jq(1) incantation transforms the JSON array in
    # bundlelist.json from the following form (which is optimised for
    # transmission size):
-   #     [
-   #        [  "_id", "service", "id", "version","date",".inserttime",
-   #           ".author",".fromhere","filesize","filehash","sender","recipient",
-   #           "name"
-   #        ],
-   #        [  rowid1, "service1", bundleid1, version1, .... ],
-   #        [  rowid2, "service2", bundleid2, version2, .... ],
-   #           ...
-   #        [  rowidN, "serviceN", bundleidN, versionN, .... ]
-   #     ]
+   #     {
+   #        "header":[  "_id", "service", "id", "version","date",".inserttime",
+   #              ".author",".fromhere","filesize","filehash","sender","recipient",
+   #              "name"
+   #           ],
+   #        "token":"xxx",
+   #        "rows":[
+   #              [  rowid1, "service1", bundleid1, version1, .... ],
+   #              [  rowid2, "service2", bundleid2, version2, .... ],
+   #                 ...
+   #              [  rowidN, "serviceN", bundleidN, versionN, .... ]
+   #           ]
+   #     }
    #
    # into an array of JSON objects:
-   #     [
-   #        {
-   #           "_id": rowid1,
-   #           "service": service1,
-   #           "id": bundleid1,
-   #           "version": version1,
+   #     {
+   #        "token":"xxx",
+   #        "bundles":[
+   #           {
+   #              "_id": rowid1,
+   #              "service": service1,
+   #              "id": bundleid1,
+   #              "version": version1,
+   #              ...
+   #           },
+   #           {
+   #              "_id": rowid2,
+   #              "service": service2,
+   #              "id": bundleid2,
+   #              "version": version2,
+   #              ...
+   #           },
    #           ...
-   #        },
-   #        {
-   #           "_id": rowid2,
-   #           "service": service2,
-   #           "id": bundleid2,
-   #           "version": version2,
-   #           ...
-   #        },
-   #        ...
-   #
-   #        {
-   #           "_id": rowidN,
-   #           "service": serviceN,
-   #           "id": bundleidN,
-   #           "version": versionN,
-   #           ...
-   #        }
-   #     ]
+   #           {
+   #              "_id": rowidN,
+   #              "service": serviceN,
+   #              "id": bundleidN,
+   #              "version": versionN,
+   #              ...
+   #           }
+   #        ]
+   #     }
    # which is much easier to test with jq(1) expressions.
-   jq '[ .[0] as $h | .[1:][] as $d | [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] | from_entries ]' \
+   jq \
+      '[
+         .header as $h |
+         .rows[] as $d |
+         [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] |
+         from_entries
+      ] as $bundles |
+      .token as $token |
+      { token:$token, bundles:$bundles }' \
       bundlelist.json >array_of_objects.json
-   #tfw_cat array_of_objects.json
+   tfw_preserve array_of_objects.json
    for ((n = 0; n != NBUNDLES; ++n)); do
-      jqscript="contains([
+      jqscript=".bundles | contains([
                   {  name:\"file$n\",
                      service:\"file\",
                      id:\"${BID[$n]}\",
diff --git a/uuid.c b/uuid.c
index 8b629591..d77f369b 100644
--- a/uuid.c
+++ b/uuid.c
@@ -66,22 +66,27 @@ int uuid_generate_random(uuid_t *uuid)
   return 0;
 }
 
-char *uuid_to_str(const uuid_t *uuid, char *const dst)
+strbuf strbuf_uuid(strbuf sb, const uuid_t *uuid)
 {
-  char *p = dst;
   assert(uuid_is_valid(uuid));
   unsigned i;
   for (i = 0; i != sizeof uuid->u.binary; ++i) {
     switch (i) {
       case 4: case 6: case 8: case 10:
-	*p++ = '-';
+	strbuf_putc(sb, '-');
       default:
-	*p++ = hexdigit_lower[uuid->u.binary[i] >> 4];
-	*p++ = hexdigit_lower[uuid->u.binary[i] & 0xf];
+	strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] >> 4]);
+	strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] & 0xf]);
     }
   }
-  *p = '\0';
-  assert(p == dst + UUID_STRLEN);
+  return sb;
+}
+
+char *uuid_to_str(const uuid_t *uuid, char *const dst)
+{
+  strbuf b = strbuf_local(dst, UUID_STRLEN + 1);
+  strbuf_uuid(b, uuid);
+  assert(!strbuf_overrun(b));
   return dst;
 }
 
diff --git a/uuid.h b/uuid.h
index 7a55bfc2..7609edcf 100644
--- a/uuid.h
+++ b/uuid.h
@@ -22,6 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include <stdint.h>
 #include <alloca.h>
+#include "strbuf.h"
 
 #ifndef __SERVALDNA_UUID_H_INLINE
 # if __GNUC__ && !__GNUC_STDC_INLINE__
@@ -102,6 +103,10 @@ char *uuid_to_str(const uuid_t *valid_uuid, char *dst);
 #define UUID_STRLEN 36
 #define alloca_uuid_str(uuid) uuid_to_str(&(uuid), alloca(UUID_STRLEN + 1))
 
+/* Append a UUID to the given strbuf, formatted as per the uuid_to_str() function.
+ */
+strbuf strbuf_uuid(strbuf, const uuid_t *valid_uuid);
+
 /* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
  * UUID, which may or not be supported.
  *