Improve HTTP /restful/rhizome/bundlelist.json token

Now tokens are included in each row instead of a single token in the
enclosing JSON object.  Only puts tokens on the first row, and
thereafter all rows with a rowid higher than for the previous row that
had a token.

rhizome.h

@@ -746,7 +746,8 @@ typedef struct rhizome_http_request
     /* For responses that list manifests.
     */
     struct {
-        enum { LIST_HEADER = 0, LIST_TOKEN, LIST_ROWS, LIST_DONE } phase;
+        enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase;
+        uint64_t rowid_highest;
         size_t rowcount;
         struct rhizome_list_cursor cursor;
     } list;

rhizome_http.c

@@ -397,6 +397,7 @@ static char *list_token(char *buf, uint64_t rowid)
 static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
 {
   const char *headers[] = {
+    ".token",
     "_id",
     "service",
     "id",
@@ -420,22 +421,16 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	  strbuf_putc(b, ',');
 	strbuf_json_string(b, headers[i]);
       }
-      strbuf_puts(b, "]");
+      strbuf_puts(b, "],\n\"rows\":[");
       if (!strbuf_overrun(b))
-	r->u.list.phase = LIST_TOKEN;
+	r->u.list.phase = LIST_ROWS;
       return 1;
-    case LIST_TOKEN:
     case LIST_ROWS:
       {
 	int ret = rhizome_list_next(retry, &r->u.list.cursor);
 	if (ret == -1)
 	  return -1;
 	rhizome_manifest *m = r->u.list.cursor.manifest;
-	if (r->u.list.phase == LIST_TOKEN) {
-	  strbuf_puts(b, ",\n\"token\":");
-	  strbuf_json_string(b, alloca_list_token(ret ? m->rowid : 0));
-	  strbuf_puts(b, ",\n\"rows\":[");
-	}
 	if (ret == 0) {
 	  strbuf_puts(b, "\n]\n}\n");
 	  if (strbuf_overrun(b))
@@ -445,9 +440,15 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
 	}
 	assert(m->filesize != RHIZOME_SIZE_UNSET);
 	rhizome_lookup_author(m);
-	if (r->u.list.phase != LIST_TOKEN)
+	if (r->u.list.rowcount != 0)
 	  strbuf_putc(b, ',');
 	strbuf_puts(b, "\n[");
+	if (m->rowid > r->u.list.rowid_highest) {
+	  strbuf_json_string(b, alloca_list_token(m->rowid));
+	  r->u.list.rowid_highest = m->rowid;
+	} else
+	  strbuf_json_null(b);
+	strbuf_putc(b, ',');
 	strbuf_sprintf(b, "%"PRIu64, m->rowid);
 	strbuf_putc(b, ',');
 	strbuf_json_string(b, m->service);

tests/rhizomehttp

@@ -103,6 +103,7 @@ doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
 setup_RhizomeList() {
    setup
    NBUNDLES=100
+   ROWID_MAX=0
    for ((n = 0; n != NBUNDLES; ++n)); do
       create_file file$n $((1000 + $n))
       executeOk_servald rhizome add file $SIDA file$n file$n.manifest
@@ -112,7 +113,9 @@ setup_RhizomeList() {
       extract_stdout_filehash HASH[$n]
       extract_stdout_date DATE[$n]
       extract_stdout_rowid ROWID[$n]
+      [ "${ROWID[$n]}" -gt "$ROWID_MAX" ] && ROWID_MAX=${ROWID[$n]}
    done
+   assert [ "$ROWID_MAX" -ge "$NBUNDLES" ]
 }
 test_RhizomeList() {
    executeOk curl \
@@ -128,24 +131,25 @@ test_RhizomeList() {
    # bundlelist.json from the following form (which is optimised for
    # transmission size):
    #     {
-   #        "header":[  "_id", "service", "id", "version","date",".inserttime",
+   #        "header":[
+   #              ".token", "_id", "service", "id", "version","date",".inserttime",
    #              ".author",".fromhere","filesize","filehash","sender","recipient",
    #              "name"
    #           ],
-   #        "token":"xxx",
    #        "rows":[
-   #              [  rowid1, "service1", bundleid1, version1, .... ],
+   #              [  "xx", rowid1, "service1", bundleid1, version1, .... ],
-   #              [  rowid2, "service2", bundleid2, version2, .... ],
+   #              [  null, rowid2, "service2", bundleid2, version2, .... ],
    #                 ...
-   #              [  rowidN, "serviceN", bundleidN, versionN, .... ]
+   #              [  null, rowidN, "serviceN", bundleidN, versionN, .... ]
    #           ]
    #     }
    #
    # into an array of JSON objects:
    #     {
-   #        "token":"xxx",
    #        "bundles":[
    #           {
+   #              "__index": 0,
+   #              ".token": "xx",
    #              "_id": rowid1,
    #              "service": service1,
    #              "id": bundleid1,
@@ -153,6 +157,8 @@ test_RhizomeList() {
    #              ...
    #           },
    #           {
+   #              "__index": 1,
+   #              ".token": null,
    #              "_id": rowid2,
    #              "service": service2,
    #              "id": bundleid2,
@@ -161,6 +167,8 @@ test_RhizomeList() {
    #           },
    #           ...
    #           {
+   #              "__index": 2,
+   #              ".token": null,
    #              "_id": rowidN,
    #              "service": serviceN,
    #              "id": bundleidN,
@@ -170,19 +178,25 @@ test_RhizomeList() {
    #        ]
    #     }
    # which is much easier to test with jq(1) expressions.
-   jq \
+   jq '
-      '[
+         [
-         .header as $h |
+            .header as $header |
-         .rows[] as $d |
+            .rows as $rows |
-         [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] |
+            $rows | keys | .[] as $index |
-         from_entries
+            [ $rows[$index] as $d | $d | keys | .[] as $i | {key:$header[$i], value:$d[$i]} ] |
-      ] as $bundles |
+            from_entries |
-      .token as $token |
+            .["__index"] = $index
-      { token:$token, bundles:$bundles }' \
+         ]
-      bundlelist.json >array_of_objects.json
+      ' bundlelist.json >array_of_objects.json
    tfw_preserve array_of_objects.json
    for ((n = 0; n != NBUNDLES; ++n)); do
-      jqscript=".bundles | contains([
+      if [ "${ROWID[$n]}" -eq "$ROWID_MAX" ]; then
+         # The first row must contain a non-null token string.
+         token=',".token":"","__index":0,'
+      else
+         token=',".token":null,'
+      fi
+      jqscript="contains([
                   {  name:\"file$n\",
                      service:\"file\",
                      id:\"${BID[$n]}\",
@@ -192,10 +206,11 @@ test_RhizomeList() {
                      date:${DATE[$n]},
                      _id:${ROWID[$n]},
                      \".fromhere\":1,
-                     \".author\":\"$SIDA\",
+                     \".author\":\"$SIDA\"
+                     $token
                   }
                ])"
-      assert [ "$(jq "$jqscript" array_of_objects.json)" = true ]
+      assert --message="$jqscript" [ "$(jq "$jqscript" array_of_objects.json)" = true ]
    done
 }