Fix HTTP /restful/rhizome/bundlelist.json

Write test case assertions using jq(1) utility, increase from four
bundles to 100.

Fix bugs in HTTP server content generation logic.

Make payload content generator read payload 4KiB at a time, to
always read on filesystem block boundaries for performance.  Increase
size of payload in relevant test case.

http_server.c

@@ -1656,6 +1656,8 @@ static void http_request_send_response(struct http_request *r)
     assert(r->response_buffer_sent <= r->response_buffer_length);
     uint64_t remaining = CONTENT_LENGTH_UNKNOWN;
     size_t unsent = r->response_buffer_length - r->response_buffer_sent;
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("HTTP response buffer contains %zu bytes unsent", unsent);
     if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
       remaining = r->response_length - r->response_sent;
       assert(unsent <= remaining);
@@ -1703,17 +1705,17 @@ static void http_request_send_response(struct http_request *r)
 	  return;
 	}
 	if (result.generated == 0 && result.need <= unfilled) {
-	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t, r->response_sent);
+	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t" (ret=%d)", r->response_sent, ret);
 	  http_request_finalise(r);
 	  return;
 	}
 	if (r->debug_flag && *r->debug_flag)
-	  DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer", result.generated, result.need);
+	  DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer (ret=%d)", result.generated, result.need, ret);
 	assert(result.generated <= unfilled);
 	r->response_buffer_length += result.generated;
 	r->response_buffer_need = result.need;
 	if (ret == 0)
-	  r->response.content_generator = NULL;
+	  r->response.content_generator = NULL; // ensure we never invoke again
 	continue;
       }
     } else if (remaining != CONTENT_LENGTH_UNKNOWN && unsent < remaining) {

rhizome_http.c

@@ -387,9 +387,8 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	strbuf_json_string(b, headers[i]);
       }
       strbuf_puts(b, "]");
-      if (strbuf_overrun(b))
-	return 0;
-      r->u.list.phase = LIST_BODY;
+      if (!strbuf_overrun(b))
+	r->u.list.phase = LIST_BODY;
       return 1;
     case LIST_BODY:
       {
@@ -442,10 +441,10 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	strbuf_putc(b, ',');
 	strbuf_json_string(b, m->name);
 	strbuf_puts(b, "]");
-	if (strbuf_overrun(b))
-	  return 0;
-	rhizome_list_commit(&r->u.list.cursor);
-	++r->u.list.rowcount;
+	if (!strbuf_overrun(b)) {
+	  rhizome_list_commit(&r->u.list.cursor);
+	  ++r->u.list.rowcount;
+	}
 	return 1;
       }
     case LIST_DONE:
@@ -465,6 +464,8 @@ static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsi
   strbuf b = strbuf_local((char *)buf, bufsz);
   while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) {
     if (strbuf_overrun(b)) {
+      if (config.debug.rhizome)
+	DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b));
       result->need = strbuf_count(b) + 1 - result->generated;
       break;
     }
@@ -544,20 +545,27 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
 
 static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
 {
+  // Only read multiples of 4k from disk.
+  const size_t blocksz = 1 << 12;
+  // Ask for a large buffer for all future reads.
+  const size_t preferred_bufsz = 16 * blocksz;
   // Reads the next part of the payload into the supplied buffer.
   rhizome_http_request *r = (rhizome_http_request *) hr;
   assert(r->u.read_state.offset < r->u.read_state.length);
-  size_t readlen = r->u.read_state.length - r->u.read_state.offset;
-  if (readlen > bufsz)
-    readlen = bufsz;
-  ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
-  if (n == -1)
-    return -1;
-  result->generated = (size_t) n;
-  // Ask for a large buffer for all future reads.
-  const size_t preferred_bufsz = 64 * 1024;
-  assert(r->u.read_state.offset < r->u.read_state.length);
-  size_t remain = r->u.read_state.length - r->u.read_state.offset;
+  uint64_t remain = r->u.read_state.length - r->u.read_state.offset;
+  size_t readlen = bufsz;
+  if (remain < bufsz)
+    readlen = remain;
+  else
+    readlen &= ~(blocksz - 1);
+  if (readlen > 0) {
+    ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
+    if (n == -1)
+      return -1;
+    result->generated = (size_t) n;
+  }
+  assert(r->u.read_state.offset <= r->u.read_state.length);
+  remain = r->u.read_state.length - r->u.read_state.offset;
   result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
   return remain ? 1 : 0;
 }

testdefs_rhizome.sh

@@ -306,6 +306,18 @@ extract_stdout_BK() {
    extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
 }
 
+extract_stdout_date() {
+   extract_stdout_keyvalue "$1" date "$rexp_date"
+}
+
+extract_stdout_filesize() {
+   extract_stdout_keyvalue "$1" filesize "$rexp_filesize"
+}
+
+extract_stdout_filehash() {
+   extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
+}
+
 extract_manifest() {
    local _var="$1"
    local _manifestfile="$2"
@@ -346,6 +358,10 @@ extract_manifest_version() {
    extract_manifest "$1" "$2" version "$rexp_version"
 }
 
+extract_manifest_date() {
+   extract_manifest "$1" "$2" date "$rexp_date"
+}
+
 extract_manifest_crypt() {
    extract_manifest "$1" "$2" crypt "$rexp_crypt"
 }

tests/rhizomehttp

@@ -99,20 +99,89 @@ teardown_AuthBasicWrong() {
    teardown
 }
 
-doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
+doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
 setup_RhizomeList() {
-   for n in 1 2 3 4; do
-      create_file file$n ${n}k
+   setup
+   NBUNDLES=100
+   for ((n = 0; n != NBUNDLES; ++n)); do
+      create_file file$n $((1000 + $n))
       executeOk_servald rhizome add file $SIDA file$n file$n.manifest
+      extract_stdout_manifestid BID[$n]
+      extract_stdout_version VERSION[$n]
+      extract_stdout_filesize SIZE[$n]
+      extract_stdout_filehash HASH[$n]
+      extract_stdout_date DATE[$n]
    done
 }
 test_RhizomeList() {
    executeOk curl \
          --silent --fail --show-error \
-         --output http.output \
+         --output bundlelist.json \
          --dump-header http.headers \
          --basic --user harry:potter \
          "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+   tfw_cat http.headers bundlelist.json
+   tfw_preserve bundlelist.json
+   assert [ "$(jq 'length' bundlelist.json)" = $((NBUNDLES+1)) ]
+   # The following jq(1) incantation transforms the JSON array in
+   # bundlelist.json from the following form (which is optimised for
+   # transmission size):
+   #     [
+   #        [  "_id", "service", "id", "version","date",".inserttime",
+   #           ".author",".fromhere","filesize","filehash","sender","recipient",
+   #           "name"
+   #        ],
+   #        [  rowid1, "service1", bundleid1, version1, .... ],
+   #        [  rowid2, "service2", bundleid2, version2, .... ],
+   #           ...
+   #        [  rowidN, "serviceN", bundleidN, versionN, .... ]
+   #     ]
+   #
+   # into an array of JSON objects:
+   #     [
+   #        {
+   #           "_id": rowid1,
+   #           "service": service1,
+   #           "id": bundleid1,
+   #           "version": version1,
+   #           ...
+   #        },
+   #        {
+   #           "_id": rowid2,
+   #           "service": service2,
+   #           "id": bundleid2,
+   #           "version": version2,
+   #           ...
+   #        },
+   #        ...
+   #
+   #        {
+   #           "_id": rowidN,
+   #           "service": serviceN,
+   #           "id": bundleidN,
+   #           "version": versionN,
+   #           ...
+   #        }
+   #     ]
+   # which is much easier to test with jq(1) expressions.
+   jq '[ .[0] as $h | .[1:][] as $d | [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] | from_entries ]' \
+      bundlelist.json >array_of_objects.json
+   #tfw_cat array_of_objects.json
+   for ((n = 0; n != NBUNDLES; ++n)); do
+      jqscript="contains([
+                  {  name:\"file$n\",
+                     service:\"file\",
+                     id:\"${BID[$n]}\",
+                     version:${VERSION[$n]},
+                     filesize:${SIZE[$n]},
+                     filehash:\"${HASH[$n]}\",
+                     date:${DATE[$n]},
+                     \".fromhere\":1,
+                     \".author\":\"$SIDA\",
+                  }
+               ])"
+      assert [ "$(jq "$jqscript" array_of_objects.json)" = true ]
+   done
 }
 
 doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"

tests/rhizomeprotocol

@@ -75,7 +75,7 @@ doc_FileTransfer="New bundle and update transfer to one node"
 setup_FileTransfer() {
    setup_common
    set_instance +A
-   rhizome_add_file file1 2048
+   rhizome_add_file file1 250000
    start_servald_instances +A +B
    foreach_instance +A assert_peers_are_instances +B
    foreach_instance +B assert_peers_are_instances +A