ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Token in /restful/rhizome/bundlelist.json output

Browse Source
This commit is contained in:
Andrew Bettison 2013-11-13 16:58:28 +10:30
parent a14326deeb
commit 4380fdcccd
6 changed files with 136 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
rhizome.h
View File

@ -631,12 +631,13 @@ struct rhizome_list_cursor {
bool_t is_recipient_set;
sid_t sender;
sid_t recipient;
uint64_t rowid_since;
// Set by calling the next() function.
rhizome_manifest *manifest;
// Private state.
sqlite3_stmt *_statement;
int64_t _rowid_first;
int64_t _rowid_last;
uint64_t _rowid_current;
uint64_t _rowid_last; // for re-opening query
};
int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
@ -745,7 +746,7 @@ typedef struct rhizome_http_request
/* For responses that list manifests.
*/
struct {
enum { LIST_HEADER = 0, LIST_BODY, LIST_DONE } phase;
enum { LIST_HEADER = 0, LIST_TOKEN, LIST_ROWS, LIST_DONE } phase;
size_t rowcount;
struct rhizome_list_cursor cursor;
} list;

52
rhizome_database.c
View File

@ -1493,11 +1493,10 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
strbuf_puts(b, " AND sender = @sender");
if (c->is_recipient_set)
strbuf_puts(b, " AND recipient = @recipient");
if (c->_rowid_first) {
assert(c->_rowid_last);
assert(c->_rowid_last <= c->_rowid_first);
strbuf_puts(b, " AND (rowid > @first OR rowid < @last)");
}
if (c->rowid_since)
strbuf_puts(b, " AND rowid > @since");
if (c->_rowid_last)
strbuf_puts(b, " AND rowid < @last");
strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first
if (strbuf_overrun(b))
RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
@ -1512,12 +1511,12 @@ int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
goto failure;
if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
goto failure;
if ( c->_rowid_first
&& sqlite_bind(retry, c->_statement, NAMED|INT64, "@first", c->_rowid_first,
NAMED|INT64, "@last", c->_rowid_last, END) == -1
)
if (c->rowid_since && sqlite_bind(retry, c->_statement, NAMED|INT64, "@since", c->rowid_since, END) == -1)
goto failure;
if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
goto failure;
c->manifest = NULL;
c->_rowid_current = 0;
RETURN(0);
OUT();
failure:
@ -1527,16 +1526,27 @@ failure:
OUT();
}
/* Guaranteed to return manifests with monotonically descending rowid. The first manifest will have
* the greatest rowid.
*
* Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest'
* field points to the fetched manifest. Returns 0 if there are no more manifests in the list.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
{
IN();
if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
RETURN(-1);
while (sqlite_step_retry(retry, c->_statement) == SQLITE_ROW) {
while (1) {
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW)
break;
assert(sqlite3_column_count(c->_statement) == 6);
assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
@ -1544,13 +1554,22 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
if (c->_rowid_current && q_rowid >= c->_rowid_current) {
WHYF("Query returned rowid=%"PRIu64" out of order (last was %"PRIu64") -- skipped", q_rowid, c->_rowid_current);
continue;
}
c->_rowid_current = q_rowid;
if (q_rowid <= c->rowid_since) {
WHYF("Query returned rowid=%"PRIu64" <= rowid_since=%"PRIu64" -- skipped", q_rowid, c->rowid_since);
continue;
}
const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
int64_t q_version = sqlite3_column_int64(c->_statement, 2);
int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
sid_t *author = NULL;
if (q_author) {
author = alloca(sizeof *author);
@ -1581,27 +1600,28 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
continue;
if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0))
continue;
assert(c->_rowid_current != 0);
// Don't do rhizome_verify_author(m); too CPU expensive for a listing. Save that for when
// the bundle is extracted or exported.
RETURN(1);
}
assert(c->_rowid_current == 0);
RETURN(0);
OUT();
}
void rhizome_list_commit(struct rhizome_list_cursor *c)
{
assert(c->manifest->rowid != 0);
if (c->manifest->rowid > c->_rowid_first)
c->_rowid_first = c->manifest->rowid;
if (c->_rowid_last == 0 || c->manifest->rowid < c->_rowid_last)
c->_rowid_last = c->manifest->rowid;
assert(c->_rowid_current != 0);
if (c->_rowid_last == 0 || c->_rowid_current < c->_rowid_last)
c->_rowid_last = c->_rowid_current;
}
void rhizome_list_release(struct rhizome_list_cursor *c)
{
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
if (c->_statement) {

36
rhizome_http.c
View File

@ -377,6 +377,17 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
return 0;
}
#define LIST_TOKEN_STRLEN_MAX (UUID_STRLEN + 30)
#define alloca_list_token(rowid) list_token(alloca(LIST_TOKEN_STRLEN_MAX), (rowid))
static char *list_token(char *buf, uint64_t rowid)
{
strbuf b = strbuf_local(buf, LIST_TOKEN_STRLEN_MAX);
strbuf_uuid(b, &rhizome_db_uuid);
strbuf_sprintf(b, "-%"PRIu64, rowid);
return buf;
}
static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
{
const char *headers[] = {
@ -396,7 +407,7 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
};
switch (r->u.list.phase) {
case LIST_HEADER:
strbuf_puts(b, "[[");
strbuf_puts(b, "{\n\"header\":[");
unsigned i;
for (i = 0; i != NELS(headers); ++i) {
if (i)
@ -405,24 +416,32 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
}
strbuf_puts(b, "]");
if (!strbuf_overrun(b))
r->u.list.phase = LIST_BODY;
r->u.list.phase = LIST_TOKEN;
return 1;
case LIST_BODY:
case LIST_TOKEN:
case LIST_ROWS:
{
int ret = rhizome_list_next(retry, &r->u.list.cursor);
if (ret == -1)
return -1;
rhizome_manifest *m = r->u.list.cursor.manifest;
if (r->u.list.phase == LIST_TOKEN) {
strbuf_puts(b, ",\n\"token\":");
strbuf_json_string(b, alloca_list_token(ret ? m->rowid : 0));
strbuf_puts(b, ",\n\"rows\":[");
}
if (ret == 0) {
strbuf_puts(b, "\n]\n");
strbuf_puts(b, "\n]\n}\n");
if (strbuf_overrun(b))
return 0;
r->u.list.phase = LIST_DONE;
return 0;
}
rhizome_manifest *m = r->u.list.cursor.manifest;
assert(m->filesize != RHIZOME_SIZE_UNSET);
rhizome_lookup_author(m);
strbuf_puts(b, ",\n [");
if (r->u.list.phase != LIST_TOKEN)
strbuf_putc(b, ',');
strbuf_puts(b, "\n[");
strbuf_sprintf(b, "%"PRIu64, m->rowid);
strbuf_putc(b, ',');
strbuf_json_string(b, m->service);
@ -462,12 +481,13 @@ static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *ret
rhizome_list_commit(&r->u.list.cursor);
++r->u.list.rowcount;
}
r->u.list.phase = LIST_ROWS;
return 1;
}
case LIST_DONE:
break;
return 0;
}
return 0;
abort();
}
static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)

89
tests/rhizomehttp
View File

@ -123,53 +123,66 @@ test_RhizomeList() {
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
tfw_cat http.headers bundlelist.json
tfw_preserve bundlelist.json
assert [ "$(jq 'length' bundlelist.json)" = $((NBUNDLES+1)) ]
assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ]
# The following jq(1) incantation transforms the JSON array in
# bundlelist.json from the following form (which is optimised for
# transmission size):
# [
# [ "_id", "service", "id", "version","date",".inserttime",
# ".author",".fromhere","filesize","filehash","sender","recipient",
# "name"
# ],
# [ rowid1, "service1", bundleid1, version1, .... ],
# [ rowid2, "service2", bundleid2, version2, .... ],
# ...
# [ rowidN, "serviceN", bundleidN, versionN, .... ]
# ]
# {
# "header":[ "_id", "service", "id", "version","date",".inserttime",
# ".author",".fromhere","filesize","filehash","sender","recipient",
# "name"
# ],
# "token":"xxx",
# "rows":[
# [ rowid1, "service1", bundleid1, version1, .... ],
# [ rowid2, "service2", bundleid2, version2, .... ],
# ...
# [ rowidN, "serviceN", bundleidN, versionN, .... ]
# ]
# }
#
# into an array of JSON objects:
# [
# {
# "_id": rowid1,
# "service": service1,
# "id": bundleid1,
# "version": version1,
# {
# "token":"xxx",
# "bundles":[
# {
# "_id": rowid1,
# "service": service1,
# "id": bundleid1,
# "version": version1,
# ...
# },
# {
# "_id": rowid2,
# "service": service2,
# "id": bundleid2,
# "version": version2,
# ...
# },
# ...
# },
# {
# "_id": rowid2,
# "service": service2,
# "id": bundleid2,
# "version": version2,
# ...
# },
# ...
#
# {
# "_id": rowidN,
# "service": serviceN,
# "id": bundleidN,
# "version": versionN,
# ...
# }
# ]
# {
# "_id": rowidN,
# "service": serviceN,
# "id": bundleidN,
# "version": versionN,
# ...
# }
# ]
# }
# which is much easier to test with jq(1) expressions.
jq '[ .[0] as $h | .[1:][] as $d | [ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] | from_entries ]' \
jq \
'[
.header as $h |
.rows[] as $d |
[ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] |
from_entries
] as $bundles |
.token as $token |
{ token:$token, bundles:$bundles }' \
bundlelist.json >array_of_objects.json
#tfw_cat array_of_objects.json
tfw_preserve array_of_objects.json
for ((n = 0; n != NBUNDLES; ++n)); do
jqscript="contains([
jqscript=".bundles | contains([
{ name:\"file$n\",
service:\"file\",
id:\"${BID[$n]}\",

19
uuid.c
View File

@ -66,22 +66,27 @@ int uuid_generate_random(uuid_t *uuid)
return 0;
}
char *uuid_to_str(const uuid_t *uuid, char *const dst)
strbuf strbuf_uuid(strbuf sb, const uuid_t *uuid)
{
char *p = dst;
assert(uuid_is_valid(uuid));
unsigned i;
for (i = 0; i != sizeof uuid->u.binary; ++i) {
switch (i) {
case 4: case 6: case 8: case 10:
*p++ = '-';
strbuf_putc(sb, '-');
default:
*p++ = hexdigit_lower[uuid->u.binary[i] >> 4];
*p++ = hexdigit_lower[uuid->u.binary[i] & 0xf];
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] >> 4]);
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] & 0xf]);
}
}
*p = '\0';
assert(p == dst + UUID_STRLEN);
return sb;
}
char *uuid_to_str(const uuid_t *uuid, char *const dst)
{
strbuf b = strbuf_local(dst, UUID_STRLEN + 1);
strbuf_uuid(b, uuid);
assert(!strbuf_overrun(b));
return dst;
}

5
uuid.h
View File

@ -22,6 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <stdint.h>
#include <alloca.h>
#include "strbuf.h"
#ifndef __SERVALDNA_UUID_H_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
@ -102,6 +103,10 @@ char *uuid_to_str(const uuid_t *valid_uuid, char *dst);
#define UUID_STRLEN 36
#define alloca_uuid_str(uuid) uuid_to_str(&(uuid), alloca(UUID_STRLEN + 1))
/* Append a UUID to the given strbuf, formatted as per the uuid_to_str() function.
*/
strbuf strbuf_uuid(strbuf, const uuid_t *valid_uuid);
/* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
* UUID, which may or not be supported.
*