ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-04-13 05:43:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'naf4' into 'development'

Browse Source
This commit is contained in:
Andrew Bettison 2013-11-13 17:59:01 +10:30
commit 973bb9c897
29 changed files with 1226 additions and 379 deletions

132
commandline.c

@ -1439,10 +1439,18 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
cli_field_name(context, ".author", ":");
cli_put_string(context, alloca_tohex_sid_t(mout->author), "\n");
}
cli_field_name(context, ".rowid", ":");
cli_put_long(context, m->rowid, "\n");
cli_field_name(context, ".inserttime", ":");
cli_put_long(context, m->inserttime, "\n");
if (mout->has_bundle_key) {
cli_field_name(context, "BK", ":");
cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n");
}
if (mout->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, mout->date, "\n");
}
cli_field_name(context, "version", ":");
cli_put_long(context, mout->version, "\n");
cli_field_name(context, "filesize", ":");
@ -1546,6 +1554,10 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
}
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
if (m->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, m->date, "\n");
}
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
assert(m->filesize != RHIZOME_SIZE_UNSET);
@ -1720,6 +1732,19 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
}
cli_field_name(context, "manifestid", ":");
cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
if (m->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, m->date, "\n");
}
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
}
if (m->haveSecret) {
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
@ -1731,19 +1756,12 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
cli_field_name(context, ".author", ":");
cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
}
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
cli_field_name(context, "inserttime", ":");
cli_field_name(context, ".rowid", ":");
cli_put_long(context, m->rowid, "\n");
cli_field_name(context, ".inserttime", ":");
cli_put_long(context, m->inserttime, "\n");
cli_field_name(context, ".readonly", ":");
cli_put_long(context, m->haveSecret?0:1, "\n");
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
}
}
int retfile=0;
@ -1819,25 +1837,99 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *service, *name, *sender_sid, *recipient_sid, *offset, *limit;
const char *service = NULL, *name = NULL, *sender_hex = NULL, *recipient_hex = NULL, *offset_ascii = NULL, *limit_ascii = NULL;
cli_arg(parsed, "service", &service, NULL, "");
cli_arg(parsed, "name", &name, NULL, "");
cli_arg(parsed, "sender_sid", &sender_sid, cli_optional_sid, "");
cli_arg(parsed, "recipient_sid", &recipient_sid, cli_optional_sid, "");
cli_arg(parsed, "offset", &offset, cli_uint, "0");
cli_arg(parsed, "limit", &limit, cli_uint, "0");
cli_arg(parsed, "sender_sid", &sender_hex, cli_optional_sid, "");
cli_arg(parsed, "recipient_sid", &recipient_hex, cli_optional_sid, "");
cli_arg(parsed, "offset", &offset_ascii, cli_uint, "0");
cli_arg(parsed, "limit", &limit_ascii, cli_uint, "0");
/* Create the instance directory if it does not yet exist */
if (create_serval_instance_dir() == -1)
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
int r=-1;
if (rhizome_opendb() != -1){
r=rhizome_list_manifests(context, service, name, sender_sid, recipient_sid, atoi(offset), atoi(limit), 0);
if (rhizome_opendb() == -1) {
keyring_free(keyring);
return -1;
}
size_t rowlimit = atoi(limit_ascii);
size_t rowoffset = atoi(offset_ascii);
struct rhizome_list_cursor cursor;
bzero(&cursor, sizeof cursor);
cursor.service = service && service[0] ? service : NULL;
cursor.name = name && name[0] ? name : NULL;
if (sender_hex && sender_hex[0]) {
if (str_to_sid_t(&cursor.sender, sender_hex) == -1)
return WHYF("Invalid <sender>: %s", sender_hex);
cursor.is_sender_set = 1;
}
if (recipient_hex && recipient_hex[0]) {
if (str_to_sid_t(&cursor.recipient, recipient_hex) == -1)
return WHYF("Invalid <recipient: %s", recipient_hex);
cursor.is_recipient_set = 1;
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_list_open(&retry, &cursor) == -1) {
keyring_free(keyring);
return -1;
}
const char *headers[]={
"_id",
"service",
"id",
"version",
"date",
".inserttime",
".author",
".fromhere",
"filesize",
"filehash",
"sender",
"recipient",
"name"
};
cli_columns(context, NELS(headers), headers);
size_t rowcount = 0;
int n;
while ((n = rhizome_list_next(&retry, &cursor)) == 1) {
++rowcount;
if (rowcount <= rowoffset)
continue;
if (rowlimit == 0 || rowcount <= rowoffset + rowlimit) {
rhizome_manifest *m = cursor.manifest;
assert(m->filesize != RHIZOME_SIZE_UNSET);
rhizome_lookup_author(m);
cli_put_long(context, m->rowid, ":");
cli_put_string(context, m->service, ":");
cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
cli_put_long(context, m->version, ":");
cli_put_long(context, m->has_date ? m->date : 0, ":");
cli_put_long(context, m->inserttime, ":");
switch (m->authorship) {
case AUTHOR_LOCAL:
case AUTHOR_AUTHENTIC:
cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
cli_put_long(context, 1, ":");
break;
default:
cli_put_string(context, NULL, ":");
cli_put_long(context, 0, ":");
break;
}
cli_put_long(context, m->filesize, ":");
cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
cli_put_string(context, m->name, "\n");
}
}
rhizome_list_release(&cursor);
keyring_free(keyring);
return r;
if (n == -1)
return -1;
cli_row_count(context, rowcount);
return 0;
}
int app_keyring_create(const struct cli_parsed *parsed, struct cli_context *context)

1
headerfiles.mk

@ -11,6 +11,7 @@ HDRS= fifo.h \
rotbuf.h \
mem.h \
os.h \
uuid.h \
strbuf.h \
strbuf_helpers.h \
sha2.h \

207
http_server.c

@ -118,6 +118,8 @@ void http_request_init(struct http_request *r, int sockfd)
void http_request_free_response_buffer(struct http_request *r)
{
if (r->response_free_buffer) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("Free response buffer of %zu bytes", r->response_buffer_size);
r->response_free_buffer(r->response_buffer);
r->response_free_buffer = NULL;
}
@ -127,6 +129,8 @@ void http_request_free_response_buffer(struct http_request *r)
int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
{
// Don't allocate a new buffer if the existing one contains content.
assert(r->response_buffer_sent == r->response_buffer_length);
const char *const bufe = r->buffer + sizeof r->buffer;
assert(r->received < bufe);
size_t rbufsiz = bufe - r->received;
@ -134,6 +138,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
http_request_free_response_buffer(r);
r->response_buffer = (char *) r->received;
r->response_buffer_size = rbufsiz;
if (r->debug_flag && *r->debug_flag)
DEBUGF("Static response buffer %zu bytes", r->response_buffer_size);
return 0;
}
if (bufsiz != r->response_buffer_size) {
@ -142,6 +148,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
return -1;
r->response_free_buffer = free;
r->response_buffer_size = bufsiz;
if (r->debug_flag && *r->debug_flag)
DEBUGF("Allocated response buffer %zu bytes", r->response_buffer_size);
}
assert(r->response_buffer_size >= bufsiz);
assert(r->response_buffer != NULL);
@ -1642,45 +1650,89 @@ static void http_request_receive(struct http_request *r)
*/
static void http_request_send_response(struct http_request *r)
{
assert(r->response_sent <= r->response_length);
while (r->response_sent < r->response_length) {
while (1) {
if (r->response_length != CONTENT_LENGTH_UNKNOWN)
assert(r->response_sent <= r->response_length);
assert(r->response_buffer_sent <= r->response_buffer_length);
if (r->response_buffer_sent == r->response_buffer_length) {
if (r->response.content_generator) {
// Content generator must fill or partly fill response_buffer and set response_buffer_sent
// and response_buffer_length. May also malloc() a bigger buffer and set response_buffer to
// point to it.
r->response_buffer_sent = r->response_buffer_length = 0;
if (r->response.content_generator(r) == -1) {
if (r->debug_flag && *r->debug_flag)
DEBUG("Content generation error, closing connection");
http_request_finalise(r);
return;
}
assert(r->response_buffer_sent <= r->response_buffer_length);
if (r->response_buffer_sent == r->response_buffer_length) {
WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
r->response_sent, r->response_length, r->response_length - r->response_sent);
http_request_finalise(r);
return;
}
} else {
WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes",
r->response_length, r->response_length - r->response_sent);
http_request_finalise(r);
return;
}
uint64_t remaining = CONTENT_LENGTH_UNKNOWN;
size_t unsent = r->response_buffer_length - r->response_buffer_sent;
if (r->debug_flag && *r->debug_flag)
DEBUGF("HTTP response buffer contains %zu bytes unsent", unsent);
if (r->response_length != CONTENT_LENGTH_UNKNOWN) {
remaining = r->response_length - r->response_sent;
assert(unsent <= remaining);
assert(r->response_buffer_need <= remaining);
if (remaining == 0)
break; // no more to generate
}
assert(r->response_buffer_sent < r->response_buffer_length);
size_t bytes = r->response_buffer_length - r->response_buffer_sent;
if (r->response_sent + bytes > r->response_length) {
WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating",
r->response_length,
r->response_sent + bytes - r->response_length);
bytes = r->response_length - r->response_sent;
if (unsent == 0)
r->response_buffer_sent = r->response_buffer_length = 0;
if (r->response.content_generator) {
// If the buffer is smaller than the content generator needs, and it contains no unsent
// content, then allocate a larger buffer.
if (r->response_buffer_need > r->response_buffer_size && unsent == 0) {
if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1) {
WHYF("HTTP response truncated at offset=%"PRIhttp_size_t" due to insufficient buffer space",
r->response_sent);
http_request_finalise(r);
return;
}
}
// If there are some sent bytes at the start of the buffer and only a few unsent bytes, then
// move the unsent content to the start of the buffer to make more room.
if (r->response_buffer_sent > 0 && unsent < 128) {
memmove(r->response_buffer, r->response_buffer + r->response_buffer_sent, unsent);
r->response_buffer_length -= r->response_buffer_sent;
r->response_buffer_sent = 0;
}
// If there is enough unfilled room at the end of the buffer, then fill the buffer with some
// more content.
assert(r->response_buffer_length <= r->response_buffer_size);
size_t unfilled = r->response_buffer_size - r->response_buffer_length;
if (unfilled > 0 && unfilled >= r->response_buffer_need) {
// The content generator must fill or partly fill the part of the buffer we indicate and
// return the number of bytes appended. If it returns zero, it means it has no more
// content (EOF), and must not be called again. If the return value exceeds the buffer size
// we supply, it gives the amount of free space the generator needs in order to append; the
// generator will not append any bytes until that much free space is available. If returns
// -1, it means an unrecoverable error occurred, and the generator must not be called again.
struct http_content_generator_result result;
bzero(&result, sizeof result);
int ret = r->response.content_generator(r, (unsigned char *) r->response_buffer + r->response_buffer_length, unfilled, &result);
if (ret == -1) {
WHY("Content generation error, closing connection");
http_request_finalise(r);
return;
}
if (result.generated == 0 && result.need <= unfilled) {
WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t" (ret=%d)", r->response_sent, ret);
http_request_finalise(r);
return;
}
if (r->debug_flag && *r->debug_flag)
DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer (ret=%d)", result.generated, result.need, ret);
assert(result.generated <= unfilled);
r->response_buffer_length += result.generated;
r->response_buffer_need = result.need;
if (ret == 0)
r->response.content_generator = NULL; // ensure we never invoke again
continue;
}
} else if (remaining != CONTENT_LENGTH_UNKNOWN && unsent < remaining) {
WHYF("HTTP response generator finished prematurely at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
r->response_sent, r->response_length, remaining);
http_request_finalise(r);
return;
} else if (unsent == 0)
break;
assert(unsent > 0);
if (remaining != CONTENT_LENGTH_UNKNOWN && unsent > remaining) {
WHYF("HTTP response overruns Content-Length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating",
r->response_length, unsent - remaining);
unsent = remaining;
}
sigPipeFlag = 0;
ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes);
ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, unsent);
if (written == -1) {
if (r->debug_flag && *r->debug_flag)
DEBUG("HTTP socket write error, closing connection");
@ -1708,8 +1760,8 @@ static void http_request_send_response(struct http_request *r)
r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
unschedule(&r->alarm);
schedule(&r->alarm);
// If we wrote less than we tried, then go back to polling.
if (written < (size_t) bytes)
// If we wrote less than we tried, then go back to polling, otherwise keep generating content.
if (written < (size_t) unsent)
return;
}
if (r->debug_flag && *r->debug_flag)
@ -1825,30 +1877,54 @@ static int _render_response(struct http_request *r)
struct http_response hr = r->response;
assert(hr.result_code >= 100);
assert(hr.result_code < 600);
// Status code 401 must be accompanied by a WWW-Authenticate header.
if (hr.result_code == 401)
assert(hr.header.www_authenticate.scheme != NOAUTH);
const char *result_string = httpResultString(hr.result_code);
strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size);
if (hr.content == NULL && hr.content_generator == NULL) {
// Cannot specify both static (pre-rendered) content AND generated content.
assert(!(hr.content && hr.content_generator));
if (hr.content || hr.content_generator) {
// With static (pre-rendered) content, the content length is mandatory (so we know how much data
// follows the 'hr.content' pointer. Generated content will generally not send a Content-Length
// header, nor send partial content, but they might.
if (hr.content)
assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
// Ensure that all partial content fields are consistent. If content length or resource length
// are unknown, there can be no range field.
if ( hr.header.content_length != CONTENT_LENGTH_UNKNOWN
&& hr.header.resource_length != CONTENT_LENGTH_UNKNOWN
) {
assert(hr.header.content_length <= hr.header.resource_length);
assert(hr.header.content_range_start + hr.header.content_length <= hr.header.resource_length);
} else {
assert(hr.header.content_range_start == 0);
}
// Convert a 200 status code into 206 if only partial content is being sent. This saves page
// handlers having to decide between 200 (OK) and 206 (Partial Content), they can just set the
// content and resource length fields and pass 200 to http_request_response_static(), and this
// logic will change it to 206 if appropriate.
if ( hr.header.content_length != CONTENT_LENGTH_UNKNOWN
&& hr.header.resource_length != CONTENT_LENGTH_UNKNOWN
&& hr.header.content_length > 0
&& hr.header.content_length < hr.header.resource_length
) {
if (hr.result_code == 200)
hr.result_code = 206; // Partial Content
}
} else {
// If no content is supplied at all, then render a standard, short body based solely on result
// code.
assert(hr.header.content_length == CONTENT_LENGTH_UNKNOWN);
assert(hr.header.resource_length == CONTENT_LENGTH_UNKNOWN);
assert(hr.header.content_range_start == 0);
assert(hr.result_code != 206);
strbuf cb = strbuf_alloca(100 + strlen(result_string));
strbuf_sprintf(cb, "<html><h1>%03u %s</h1></html>", hr.result_code, result_string);
hr.content = strbuf_str(cb);
hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
hr.header.content_type = "text/html";
hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
hr.header.content_range_start = 0;
} else {
assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
assert(hr.header.resource_length != CONTENT_LENGTH_UNKNOWN);
assert(hr.header.content_length <= hr.header.resource_length);
assert(hr.header.content_range_start + hr.header.content_length <= hr.header.resource_length);
// To save page handlers having to decide between 200 (OK) and 206 (Partial Content), they can
// just set the content range fields and pass 200 to http_request_response_static(), and this
// logic will change it to 206 if appropriate.
if (hr.header.content_length > 0 && hr.header.content_length < hr.header.resource_length && hr.result_code == 200)
hr.result_code = 206; // Partial Content
}
assert(hr.header.content_type != NULL);
assert(hr.header.content_type[0]);
@ -1865,6 +1941,8 @@ static int _render_response(struct http_request *r)
if (hr.result_code == 206) {
// Must only use result code 206 (Partial Content) if the content is in fact less than the whole
// resource length.
assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN);
assert(hr.header.resource_length != CONTENT_LENGTH_UNKNOWN);
assert(hr.header.content_length > 0);
assert(hr.header.content_length < hr.header.resource_length);
strbuf_sprintf(sb,
@ -1874,7 +1952,8 @@ static int _render_response(struct http_request *r)
hr.header.resource_length
);
}
strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN)
strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
const char *scheme = NULL;
switch (hr.header.www_authenticate.scheme) {
case NOAUTH: break;
@ -1887,16 +1966,25 @@ static int _render_response(struct http_request *r)
strbuf_puts(sb, "\r\n");
}
strbuf_puts(sb, "\r\n");
if (strbuf_overrun(sb))
return 0;
r->response_length = strbuf_len(sb) + hr.header.content_length;
if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN)
r->response_length = strbuf_count(sb) + hr.header.content_length;
else
r->response_length = CONTENT_LENGTH_UNKNOWN;
r->response_buffer_need = strbuf_count(sb) + 1; // the header and the strbuf terminating NUL
if (hr.content) {
assert(r->response_length != CONTENT_LENGTH_UNKNOWN);
if (r->response_buffer_need < r->response_length)
r->response_buffer_need = r->response_length;
} else
assert(hr.content_generator);
if (r->response_buffer_size < r->response_buffer_need)
return 0; // doesn't fit
assert(!strbuf_overrun(sb));
if (hr.content) {
if (r->response_buffer_size < r->response_length)
return 0;
bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
r->response_buffer_length = r->response_length;
} else {
r->response_buffer_length = strbuf_len(sb);
r->response_buffer_length = strbuf_count(sb);
}
r->response_buffer_sent = 0;
return 1;
@ -1916,9 +2004,9 @@ static void http_request_render_response(struct http_request *r)
// rendered headers, so after this step, whether or not the buffer was overrun, we know the total
// length of the response.
if (!_render_response(r)) {
// If the response did not fit into the existing buffer, then allocate a large buffer from the
// heap and try rendering again.
if (http_request_set_response_bufsize(r, r->response_length + 1) == -1)
// If the static response did not fit into the existing buffer, then allocate a large buffer
// from the heap and try rendering again.
if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1)
WHY("Cannot render HTTP response, out of memory");
else if (!_render_response(r))
FATAL("Re-render of HTTP response overflowed buffer");
@ -1978,6 +2066,7 @@ static void http_request_start_response(struct http_request *r)
return;
}
}
r->response_buffer_need = 0;
r->response_sent = 0;
if (r->debug_flag && *r->debug_flag)
DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_buffer_length));

12
http_server.h

@ -96,13 +96,18 @@ struct http_response_headers {
struct http_www_authenticate www_authenticate;
};
typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *);
struct http_content_generator_result {
size_t generated;
size_t need;
};
typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, size_t, struct http_content_generator_result *);
struct http_response {
uint16_t result_code;
struct http_response_headers header;
const char *content;
HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content
HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content
};
#define MIME_FILENAME_MAXLEN 127
@ -139,7 +144,7 @@ void http_request_free_response_buffer(struct http_request *r);
int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
void http_request_finalise(struct http_request *r);
void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR);
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
@ -192,6 +197,7 @@ struct http_request {
http_size_t response_length; // total response bytes (header + content)
http_size_t response_sent; // for counting up to response_length
char *response_buffer;
size_t response_buffer_need;
size_t response_buffer_size;
size_t response_buffer_length;
size_t response_buffer_sent;

1
lsif.c

@ -46,7 +46,6 @@
#ifndef SIOCGIFCONF
#include <sys/sockio.h>
#endif
#include <netinet/in.h>
#include <netinet/if_ether.h>
#if __MACH__
#include <net/if_dl.h>

1
net.c

@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <time.h>
#include "serval.h"

2
net.h

@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h> // for size_t, ssize_t
#include <sys/socket.h> // for struct sockaddr, socklen_t
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h> // for struct in_addr
#endif
#include <arpa/inet.h> // for in_addr_t
#include "log.h" // for __WHENCE__ and struct __sourceloc

1
overlay_interface.c

@ -19,7 +19,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <time.h>

83
rhizome.h

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sqlite3.h>
#include <limits.h>
#include "sha2.h"
#include "uuid.h"
#include "str.h"
#include "strbuf.h"
#include "http_server.h"
@ -291,6 +292,12 @@ typedef struct rhizome_manifest
sid_t sender;
sid_t recipient;
/* Local data, not encapsulated in the bundle. The ROWID of the SQLite
* MANIFESTS table row in which this manifest is stored. Zero if the
* manifest has not been stored yet.
*/
uint64_t rowid;
/* Local data, not encapsulated in the bundle. The system time of the most
* recent INSERT or UPDATE of the manifest into the store. Zero if the manifest
* has not been stored yet.
@ -342,6 +349,7 @@ typedef struct rhizome_manifest
#define rhizome_manifest_set_recipient(m,v) _rhizome_manifest_set_recipient(__WHENCE__,(m),(v))
#define rhizome_manifest_del_recipient(m) _rhizome_manifest_del_recipient(__WHENCE__,(m))
#define rhizome_manifest_set_crypt(m,v) _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
#define rhizome_manifest_set_rowid(m,v) _rhizome_manifest_set_rowid(__WHENCE__,(m),(v))
#define rhizome_manifest_set_inserttime(m,v) _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v))
#define rhizome_manifest_set_author(m,v) _rhizome_manifest_set_author(__WHENCE__,(m),(v))
#define rhizome_manifest_del_author(m) _rhizome_manifest_del_author(__WHENCE__,(m))
@ -364,6 +372,7 @@ void _rhizome_manifest_del_sender(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t);
void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
@ -397,6 +406,7 @@ int create_rhizome_datastore_dir();
#define FORM_RHIZOME_IMPORT_PATH(buf,fmt,...) (form_rhizome_import_path((buf), sizeof(buf), (fmt), ##__VA_ARGS__))
extern sqlite3 *rhizome_db;
uuid_t rhizome_db_uuid;
int rhizome_opendb();
int rhizome_close_db();
@ -509,6 +519,7 @@ enum sqlbind_type {
TOHEX, // const unsigned char *binary, unsigned bytes
TEXT_TOUPPER, // const char *text,
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
UUID_T, // const uuid_t *uuidp
NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ...
INDEX = 0xfade0000, // INDEX|INT, int index, ...
NAMED = 0xdead0000 // NAMED|INT, const char *label, ...
@ -564,9 +575,6 @@ int64_t rhizome_bar_version(const unsigned char *bar);
uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar);
int rhizome_is_bar_interesting(unsigned char *bar);
int rhizome_is_manifest_interesting(rhizome_manifest *m);
int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
const char *sender_sid, const char *recipient_sid,
size_t rowlimit, size_t rowoffset, char count_rows);
int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m);
int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m);
int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
@ -613,6 +621,30 @@ int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned cha
int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int timeout);
int rhizome_ignore_manifest_check(unsigned char *bid_prefix, int prefix_len);
/* Rhizome list cursor for iterating over all or a subset of manifests in the store.
*/
struct rhizome_list_cursor {
// Query parameters that narrow the set of listed bundles.
const char *service;
const char *name;
bool_t is_sender_set;
bool_t is_recipient_set;
sid_t sender;
sid_t recipient;
uint64_t rowid_since;
// Set by calling the next() function.
rhizome_manifest *manifest;
// Private state.
sqlite3_stmt *_statement;
uint64_t _rowid_current;
uint64_t _rowid_last; // for re-opening query
};
int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *);
void rhizome_list_commit(struct rhizome_list_cursor *);
void rhizome_list_release(struct rhizome_list_cursor *);
/* one manifest is required per candidate, plus a few spare.
so MAX_RHIZOME_MANIFESTS must be > MAX_CANDIDATES.
*/
@ -687,35 +719,38 @@ typedef struct rhizome_http_request
{
struct http_request http; // MUST BE FIRST ELEMENT
/* Identify request from others being run.
Monotonic counter feeds it. Only used for debugging when we write
post-<uuid>.log files for multi-part form requests. */
/* Identify request from others being run. Monotonic counter feeds it. Only
* used for debugging when we write post-<uuid>.log files for multi-part form
* requests.
*/
unsigned int uuid;
struct rhizome_read read_state;
/* File currently being written to while decoding POST multipart form */
/* For receiving a POST multipart form:
*/
// Which part is currently being received
enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
// Temporary file currently current part is being written to
int part_fd;
/* Which parts have been received in POST multipart form */
// Which parts have already been received
bool_t received_manifest;
bool_t received_data;
/* Name of data file supplied */
// Name of data file supplied in part's Content-Disposition header, filename
// parameter (if any)
char data_file_name[MIME_FILENAME_MAXLEN + 1];
/* The source specification data which are used in different ways by different
request types */
char source[1024];
int64_t source_index;
int64_t source_count;
int source_record_size;
unsigned int source_flags;
const char *sql_table;
const char *sql_row;
int64_t rowid;
/* source_index used for offset in blob */
int64_t blob_end;
union {
/* For responses that send part or all of a payload.
*/
struct rhizome_read read_state;
/* For responses that list manifests.
*/
struct {
enum { LIST_HEADER = 0, LIST_TOKEN, LIST_ROWS, LIST_DONE } phase;
size_t rowcount;
struct rhizome_list_cursor cursor;
} list;
} u;
} rhizome_http_request;

5
rhizome_bundle.c

@ -311,6 +311,11 @@ void _rhizome_manifest_set_crypt(struct __sourceloc __whence, rhizome_manifest *
m->payloadEncryption = flag;
}
void _rhizome_manifest_set_rowid(struct __sourceloc __whence, rhizome_manifest *m, uint64_t rowid)
{
m->rowid = rowid;
}
void _rhizome_manifest_set_inserttime(struct __sourceloc __whence, rhizome_manifest *m, time_ms_t time)
{
m->inserttime = time;

6
rhizome_crypto.c

@ -384,11 +384,11 @@ void rhizome_find_bundle_author_and_secret(rhizome_manifest *m)
rhizome_manifest_set_author(m, authorSidp);
m->authorship = AUTHOR_AUTHENTIC;
// if this bundle is already in the database, update the author.
if (m->inserttime)
if (m->rowid)
sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
"UPDATE MANIFESTS SET author = ? WHERE id = ?;",
"UPDATE MANIFESTS SET author = ? WHERE rowid = ?;",
SID_T, &m->author,
RHIZOME_BID_T, &m->cryptoSignPublic,
INT64, m->rowid,
END);
RETURNVOID; // bingo
}

309
rhizome_database.c

@ -76,7 +76,8 @@ int create_rhizome_datastore_dir()
return emkdirs(rhizome_datastore_path(), 0700);
}
sqlite3 *rhizome_db=NULL;
sqlite3 *rhizome_db = NULL;
uuid_t rhizome_db_uuid;
/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
@ -107,11 +108,32 @@ int is_debug_rhizome_ads()
static int (*sqlite_trace_func)() = is_debug_rhizome;
const struct __sourceloc *sqlite_trace_whence = NULL;
static int sqlite_trace_done;
/* This is called by SQLite when executing a statement using sqlite3_step(). Unfortunately, it is
* not called on PRAGMA statements, and possibly others. Hence the use of the profile callback (see
* below).
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void sqlite_trace_callback(void *context, const char *rendered_sql)
{
if (sqlite_trace_func())
logMessage(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql);
++sqlite_trace_done;
}
/* This is called by SQLite when an executed statement finishes. We use it to log rendered SQL
* statements when the trace callback (above) has not been called, eg, for PRAGMA statements. This
* requires that the 'sqlite_trace_done' static be reset to zero whenever a new prepared statement
* is about to be stepped.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void sqlite_profile_callback(void *context, const char *rendered_sql, sqlite_uint64 nanosec)
{
if (!sqlite_trace_done)
sqlite_trace_callback(context, rendered_sql);
}
/* This function allows code like:
@ -153,7 +175,6 @@ void verify_bundles(){
sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
const void *manifest = sqlite3_column_blob(statement, 1);
size_t manifest_length = sqlite3_column_bytes(statement, 1);
rhizome_manifest *m=rhizome_new_manifest();
int ret=0;
ret = rhizome_read_manifest_file(m, manifest, manifest_length);
@ -208,7 +229,10 @@ void verify_bundles(){
int rhizome_opendb()
{
if (rhizome_db) return 0;
if (rhizome_db) {
assert(uuid_is_valid(&rhizome_db_uuid));
return 0;
}
IN();
@ -233,6 +257,7 @@ int rhizome_opendb()
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
}
sqlite3_trace(rhizome_db, sqlite_trace_callback, NULL);
sqlite3_profile(rhizome_db, sqlite_profile_callback, NULL);
int loglevel = (config.debug.rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;
/* Read Rhizome configuration */
@ -258,11 +283,9 @@ int rhizome_opendb()
) {
RETURN(WHY("Failed to create schema"));
}
/* Create indexes if they don't already exist */
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
}
if (version<2){
@ -274,26 +297,52 @@ int rhizome_opendb()
verify_bundles();
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
}
if (version<3){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
}
if (version<4){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
}
if (version<5){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
}
char buf[UUID_STRLEN + 1];
int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
if (r == -1)
RETURN(-1);
if (r) {
if (!str_to_uuid(buf, &rhizome_db_uuid, NULL)) {
WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf));
if (uuid_generate_random(&rhizome_db_uuid) == -1)
RETURN(WHY("Cannot generate new UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", UUID_T, &rhizome_db_uuid, END) == -1)
RETURN(WHY("Failed to update new UUID in Rhizome database"));
if (config.debug.rhizome)
DEBUGF("Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
}
} else if (r == 0) {
if (uuid_generate_random(&rhizome_db_uuid) == -1)
RETURN(WHY("Cannot generate UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", UUID_T, &rhizome_db_uuid, END) == -1)
RETURN(WHY("Failed to insert UUID into Rhizome database"));
if (config.debug.rhizome)
DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
}
// TODO recreate tables with collate nocase on hex columns
/* Future schema updates should be performed here.
The above schema can be assumed to exist.
All changes should attempt to preserve any existing data */
// We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open)
rhizome_cleanup(NULL);
INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_db_uuid));
RETURN(0);
OUT();
}
@ -741,6 +790,19 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
}
}
break;
case UUID_T: {
const uuid_t *uuidp = va_arg(ap, const uuid_t *);
++argnum;
if (uuidp == NULL) {
BIND_NULL(UUID_T);
} else {
char uuid_str[UUID_STRLEN + 1];
uuid_to_str(uuidp, uuid_str);
BIND_DEBUG(UUID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", uuid_str, UUID_STRLEN);
BIND_RETRY(sqlite3_bind_text, uuid_str, UUID_STRLEN, SQLITE_TRANSIENT);
}
}
break;
#undef BIND_RETRY
default:
FATALF("at bind arg %u, unsupported bind code typ=0x%08x: %s", argnum, typ, sqlite3_sql(statement));
@ -786,6 +848,7 @@ int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state
IN();
int ret = -1;
sqlite_trace_whence = &__whence;
sqlite_trace_done = 0;
while (statement) {
int stepcode = sqlite3_step(statement);
switch (stepcode) {
@ -1299,6 +1362,10 @@ int rhizome_store_bundle(rhizome_manifest *m)
time_ms_t now = gettime_ms();
// The INSERT OR REPLACE statement will delete a row with the same ID (primary key) if it exists,
// so a new autoincremented ROWID will be allocated whether or not the manifest with this ID is
// already in the table. Other code depends on this property: that ROWID is monotonically
// increasing with time and unique.
sqlite3_stmt *stmt;
if ((stmt = sqlite_prepare_bind(&retry,
"INSERT OR REPLACE INTO MANIFESTS("
@ -1340,6 +1407,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
goto rollback;
sqlite3_finalize(stmt);
stmt = NULL;
rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db));
rhizome_manifest_set_inserttime(m, now);
// if (serverMode)
@ -1408,90 +1476,101 @@ rollback:
return -1;
}
struct rhizome_list_cursor {
// Query parameters that narrow the set of listed bundles.
const char *service;
const char *name;
sid_t sender;
sid_t recipient;
// Set by calling the next() function.
int64_t rowid;
rhizome_manifest *manifest;
size_t rowcount;
// Private state.
sqlite3_stmt *_statement;
unsigned _offset;
};
/* The cursor struct must be zerofilled and the query parameters optionally filled in prior to
* calling this function.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
{
IN();
strbuf b = strbuf_alloca(1024);
strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
if (cursor->service)
if (c->service)
strbuf_puts(b, " AND service = @service");
if (cursor->name)
if (c->name)
strbuf_puts(b, " AND name like @name");
if (!is_sid_t_any(cursor->sender))
if (c->is_sender_set)
strbuf_puts(b, " AND sender = @sender");
if (!is_sid_t_any(cursor->recipient))
if (c->is_recipient_set)
strbuf_puts(b, " AND recipient = @recipient");
strbuf_puts(b, " ORDER BY inserttime DESC LIMIT -1 OFFSET @offset");
if (c->rowid_since)
strbuf_puts(b, " AND rowid > @since");
if (c->_rowid_last)
strbuf_puts(b, " AND rowid < @last");
strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first
if (strbuf_overrun(b))
RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
cursor->_statement = sqlite_prepare(retry, strbuf_str(b));
if (cursor->_statement == NULL)
c->_statement = sqlite_prepare(retry, strbuf_str(b));
if (c->_statement == NULL)
RETURN(-1);
if (sqlite_bind(retry, cursor->_statement, NAMED|INT, "@offset", cursor->_offset, END) == -1)
if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
goto failure;
if (cursor->service && sqlite_bind(retry, cursor->_statement, NAMED|STATIC_TEXT, "@service", cursor->service, END) == -1)
if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
goto failure;
if (cursor->name && sqlite_bind(retry, cursor->_statement, NAMED|STATIC_TEXT, "@name", cursor->name, END) == -1)
if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
goto failure;
if (!is_sid_t_any(cursor->sender) && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@sender", &cursor->sender, END) == -1)
if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
goto failure;
if (!is_sid_t_any(cursor->recipient) && sqlite_bind(retry, cursor->_statement, NAMED|SID_T, "@recipient", &cursor->recipient, END) == -1)
if (c->rowid_since && sqlite_bind(retry, c->_statement, NAMED|INT64, "@since", c->rowid_since, END) == -1)
goto failure;
cursor->manifest = NULL;
if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
goto failure;
c->manifest = NULL;
c->_rowid_current = 0;
RETURN(0);
OUT();
failure:
sqlite3_finalize(cursor->_statement);
cursor->_statement = NULL;
sqlite3_finalize(c->_statement);
c->_statement = NULL;
RETURN(-1);
OUT();
}
static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *cursor)
/* Guaranteed to return manifests with monotonically descending rowid. The first manifest will have
* the greatest rowid.
*
* Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest'
* field points to the fetched manifest. Returns 0 if there are no more manifests in the list.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
{
IN();
if (cursor->_statement == NULL && rhizome_list_open(retry, cursor) == -1)
if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
RETURN(-1);
while (sqlite_step_retry(retry, cursor->_statement) == SQLITE_ROW) {
++cursor->_offset;
if (cursor->manifest) {
rhizome_manifest_free(cursor->manifest);
cursor->manifest = NULL;
while (1) {
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
assert(sqlite3_column_count(cursor->_statement) == 6);
assert(sqlite3_column_type(cursor->_statement, 0) == SQLITE_TEXT);
assert(sqlite3_column_type(cursor->_statement, 1) == SQLITE_BLOB);
assert(sqlite3_column_type(cursor->_statement, 2) == SQLITE_INTEGER);
assert(sqlite3_column_type(cursor->_statement, 3) == SQLITE_INTEGER);
assert(sqlite3_column_type(cursor->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(cursor->_statement, 4) == SQLITE_NULL);
assert(sqlite3_column_type(cursor->_statement, 5) == SQLITE_INTEGER);
const char *q_manifestid = (const char *) sqlite3_column_text(cursor->_statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(cursor->_statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(cursor->_statement, 1); // must call after sqlite3_column_blob()
int64_t q_version = sqlite3_column_int64(cursor->_statement, 2);
int64_t q_inserttime = sqlite3_column_int64(cursor->_statement, 3);
const char *q_author = (const char *) sqlite3_column_text(cursor->_statement, 4);
cursor->rowid = sqlite3_column_int64(cursor->_statement, 5);
if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW)
break;
assert(sqlite3_column_count(c->_statement) == 6);
assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
assert(sqlite3_column_type(c->_statement, 2) == SQLITE_INTEGER);
assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
if (c->_rowid_current && q_rowid >= c->_rowid_current) {
WHYF("Query returned rowid=%"PRIu64" out of order (last was %"PRIu64") -- skipped", q_rowid, c->_rowid_current);
continue;
}
c->_rowid_current = q_rowid;
if (q_rowid <= c->rowid_since) {
WHYF("Query returned rowid=%"PRIu64" <= rowid_since=%"PRIu64" -- skipped", q_rowid, c->rowid_since);
continue;
}
const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
int64_t q_version = sqlite3_column_int64(c->_statement, 2);
int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
sid_t *author = NULL;
if (q_author) {
author = alloca(sizeof *author);
@ -1500,7 +1579,7 @@ static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_curs
continue;
}
}
rhizome_manifest *m = cursor->manifest = rhizome_new_manifest();
rhizome_manifest *m = c->manifest = rhizome_new_manifest();
if (m == NULL)
RETURN(-1);
if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
@ -1514,102 +1593,42 @@ static int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_curs
}
if (author)
rhizome_manifest_set_author(m, author);
rhizome_manifest_set_rowid(m, q_rowid);
rhizome_manifest_set_inserttime(m, q_inserttime);
if (cursor->service && !(m->service && strcasecmp(cursor->service, m->service) == 0))
if (c->service && !(m->service && strcasecmp(c->service, m->service) == 0))
continue;
if (!is_sid_t_any(cursor->sender) && !(m->has_sender && cmp_sid_t(&cursor->sender, &m->sender) == 0))
if (c->is_sender_set && !(m->has_sender && cmp_sid_t(&c->sender, &m->sender) == 0))
continue;
if (!is_sid_t_any(cursor->recipient) && !(m->has_recipient && cmp_sid_t(&cursor->recipient, &m->recipient) == 0))
if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0))
continue;
assert(c->_rowid_current != 0);
// Don't do rhizome_verify_author(m); too CPU expensive for a listing. Save that for when
// the bundle is extracted or exported.
++cursor->rowcount;
RETURN(1);
}
assert(c->_rowid_current == 0);
RETURN(0);
OUT();
}
static void rhizome_list_release(struct rhizome_list_cursor *cursor)
void rhizome_list_commit(struct rhizome_list_cursor *c)
{
if (cursor->manifest) {
rhizome_manifest_free(cursor->manifest);
cursor->manifest = NULL;
}
if (cursor->_statement) {
sqlite3_finalize(cursor->_statement);
cursor->_statement = NULL;
}
assert(c->_rowid_current != 0);
if (c->_rowid_last == 0 || c->_rowid_current < c->_rowid_last)
c->_rowid_last = c->_rowid_current;
}
int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
const char *sender_hex, const char *recipient_hex,
size_t rowlimit, size_t rowoffset, char count_rows)
void rhizome_list_release(struct rhizome_list_cursor *c)
{
IN();
struct rhizome_list_cursor cursor;
bzero(&cursor, sizeof cursor);
cursor.service = service && service[0] ? service : NULL;
cursor.name = name && name[0] ? name : NULL;
if (sender_hex && *sender_hex && str_to_sid_t(&cursor.sender, sender_hex) == -1)
RETURN(WHYF("Invalid sender SID: %s", sender_hex));
if (recipient_hex && *recipient_hex && str_to_sid_t(&cursor.recipient, recipient_hex) == -1)
RETURN(WHYF("Invalid recipient SID: %s", recipient_hex));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_list_open(&retry, &cursor) == -1)
RETURN(-1);
const char *names[]={
"_id",
"service",
"id",
"version",
"date",
".inserttime",
".author",
".fromhere",
"filesize",
"filehash",
"sender",
"recipient",
"name"
};
cli_columns(context, NELS(names), names);
while (rhizome_list_next(&retry, &cursor) == 1) {
rhizome_manifest *m = cursor.manifest;
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (cursor.rowcount < rowoffset)
continue;
if (rowlimit == 0 || cursor.rowcount <= rowlimit) {
rhizome_lookup_author(m);
cli_put_long(context, cursor.rowid, ":");
cli_put_string(context, m->service, ":");
cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
cli_put_long(context, m->version, ":");
cli_put_long(context, m->has_date ? m->date : 0, ":");
cli_put_long(context, m->inserttime, ":");
switch (m->authorship) {
case AUTHOR_LOCAL:
case AUTHOR_AUTHENTIC:
cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":");
cli_put_long(context, 1, ":");
break;
default:
cli_put_string(context, NULL, ":");
cli_put_long(context, 0, ":");
break;
}
cli_put_long(context, m->filesize, ":");
cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
cli_put_string(context, m->name, "\n");
} else if (!count_rows)
break;
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
if (c->_statement) {
sqlite3_finalize(c->_statement);
c->_statement = NULL;
}
rhizome_list_release(&cursor);
cli_row_count(context, cursor.rowcount);
RETURN(0);
OUT();
}
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
@ -1734,6 +1753,7 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
int64_t q_inserttime = sqlite3_column_int64(statement, 3);
const char *q_author = (const char *) sqlite3_column_text(statement, 4);
size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
uint64_t q_rowid = sqlite3_column_int64(statement, 5);
if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1)
return WHYF("Manifest %s exists but is invalid", q_id);
if (q_author) {
@ -1745,6 +1765,7 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
}
if (m->version != q_version)
WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version);
rhizome_manifest_set_rowid(m, q_rowid);
rhizome_manifest_set_inserttime(m, q_inserttime);
return 0;
}
@ -1760,7 +1781,7 @@ int rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, manifest, version, inserttime, author FROM manifests WHERE id = ?",
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id = ?",
RHIZOME_BID_T, bidp,
END);
if (!statement)
@ -1790,7 +1811,7 @@ int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned pr
like[prefix_strlen] = '%';
like[prefix_strlen + 1] = '\0';
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, manifest, version, inserttime, author FROM manifests WHERE id like ?",
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id like ?",
TEXT, like,
END);
if (!statement)

251
rhizome_http.c

@ -1,6 +1,6 @@
/*
Serval Distributed Numbering Architecture (DNA)
Copyright (C) 2010 Paul Gardner-Stephen
Serval DNA Rhizome HTTP external interface
Copyright (C) 2013 Serval Project, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -21,7 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/socket.h>
#include <signal.h>
#ifdef HAVE_SYS_FILIO_H
#include <sys/filio.h>
# include <sys/filio.h>
#endif
#include <assert.h>
@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "overlay_address.h"
#include "conf.h"
#include "str.h"
#include "strbuf_helpers.h"
#include "rhizome.h"
#include "http_server.h"
@ -231,7 +232,7 @@ success:
static void rhizome_server_finalise_http_request(struct http_request *_r)
{
rhizome_http_request *r = (rhizome_http_request *) _r;
rhizome_read_close(&r->read_state);
rhizome_read_close(&r->u.read_state);
request_count--;
}
@ -251,7 +252,7 @@ void rhizome_server_poll(struct sched_ent *alarm)
} else {
struct sockaddr_in *peerip=NULL;
if (addr.sa_family == AF_INET) {
peerip = (struct sockaddr_in *)&addr;
peerip = (struct sockaddr_in *)&addr; // network order
INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
addr_len, peerip->sin_family, peerip->sin_port,
((unsigned char*)&peerip->sin_addr.s_addr)[0],
@ -272,8 +273,8 @@ void rhizome_server_poll(struct sched_ent *alarm)
request_count++;
request->uuid = rhizome_http_request_uuid_counter++;
request->data_file_name[0] = '\0';
request->read_state.blob_fd = -1;
request->read_state.blob_rowid = -1;
request->u.read_state.blob_fd = -1;
request->u.read_state.blob_rowid = -1;
if (peerip)
request->http.client_sockaddr_in = *peerip;
request->http.handle_headers = rhizome_dispatch;
@ -317,10 +318,16 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_
OUT();
}
static int is_from_loopback(const struct http_request *r)
{
return r->client_sockaddr_in.sin_family == AF_INET
&& ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET;
}
/* Return 1 if the given authorization credentials are acceptable.
* Return 0 if not.
*/
static int is_authorized(struct http_client_authorization *auth)
static int is_authorized(const struct http_client_authorization *auth)
{
if (auth->scheme != BASIC)
return 0;
@ -334,6 +341,23 @@ static int is_authorized(struct http_client_authorization *auth)
return 0;
}
static int authorize(struct http_request *r)
{
if (!is_from_loopback(r)) {
http_request_simple_response(r, 403, NULL);
return 0;
}
if (!is_authorized(&r->request_header.authorization)) {
r->response.header.www_authenticate.scheme = BASIC;
r->response.header.www_authenticate.realm = "Serval Rhizome";
http_request_simple_response(r, 401, NULL);
return 0;
}
return 1;
}
static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
@ -344,16 +368,152 @@ static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (!is_authorized(&r->http.request_header.authorization)) {
r->http.response.header.www_authenticate.scheme = BASIC;
r->http.response.header.www_authenticate.realm = "Serval Rhizome";
http_request_simple_response(&r->http, 401, NULL);
if (!authorize(&r->http))
return 0;
}
http_request_simple_response(&r->http, 200, NULL);
r->u.list.phase = LIST_HEADER;
r->u.list.rowcount = 0;
bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
return 0;
}
#define LIST_TOKEN_STRLEN_MAX (UUID_STRLEN + 30)
#define alloca_list_token(rowid) list_token(alloca(LIST_TOKEN_STRLEN_MAX), (rowid))
static char *list_token(char *buf, uint64_t rowid)
{
strbuf b = strbuf_local(buf, LIST_TOKEN_STRLEN_MAX);
strbuf_uuid(b, &rhizome_db_uuid);
strbuf_sprintf(b, "-%"PRIu64, rowid);
return buf;
}
static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
{
const char *headers[] = {
"_id",
"service",
"id",
"version",
"date",
".inserttime",
".author",
".fromhere",
"filesize",
"filehash",
"sender",
"recipient",
"name"
};
switch (r->u.list.phase) {
case LIST_HEADER:
strbuf_puts(b, "{\n\"header\":[");
unsigned i;
for (i = 0; i != NELS(headers); ++i) {
if (i)
strbuf_putc(b, ',');
strbuf_json_string(b, headers[i]);
}
strbuf_puts(b, "]");
if (!strbuf_overrun(b))
r->u.list.phase = LIST_TOKEN;
return 1;
case LIST_TOKEN:
case LIST_ROWS:
{
int ret = rhizome_list_next(retry, &r->u.list.cursor);
if (ret == -1)
return -1;
rhizome_manifest *m = r->u.list.cursor.manifest;
if (r->u.list.phase == LIST_TOKEN) {
strbuf_puts(b, ",\n\"token\":");
strbuf_json_string(b, alloca_list_token(ret ? m->rowid : 0));
strbuf_puts(b, ",\n\"rows\":[");
}
if (ret == 0) {
strbuf_puts(b, "\n]\n}\n");
if (strbuf_overrun(b))
return 0;
r->u.list.phase = LIST_DONE;
return 0;
}
assert(m->filesize != RHIZOME_SIZE_UNSET);
rhizome_lookup_author(m);
if (r->u.list.phase != LIST_TOKEN)
strbuf_putc(b, ',');
strbuf_puts(b, "\n[");
strbuf_sprintf(b, "%"PRIu64, m->rowid);
strbuf_putc(b, ',');
strbuf_json_string(b, m->service);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
strbuf_putc(b, ',');
strbuf_sprintf(b, "%"PRId64, m->version);
strbuf_putc(b, ',');
if (m->has_date)
strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
else
strbuf_json_null(b);
strbuf_putc(b, ',');
strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
switch (m->authorship) {
case AUTHOR_LOCAL:
case AUTHOR_AUTHENTIC:
strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
strbuf_puts(b, ",1,");
break;
default:
strbuf_json_null(b);
strbuf_puts(b, ",1,");
break;
}
strbuf_sprintf(b, "%"PRIu64, m->filesize);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
strbuf_putc(b, ',');
strbuf_json_string(b, m->name);
strbuf_puts(b, "]");
if (!strbuf_overrun(b)) {
rhizome_list_commit(&r->u.list.cursor);
++r->u.list.rowcount;
}
r->u.list.phase = LIST_ROWS;
return 1;
}
case LIST_DONE:
return 0;
}
abort();
}
static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(bufsz > 0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
int ret = rhizome_list_open(&retry, &r->u.list.cursor);
if (ret == -1)
return -1;
strbuf b = strbuf_local((char *)buf, bufsz);
while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) {
if (strbuf_overrun(b)) {
if (config.debug.rhizome)
DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b));
result->need = strbuf_count(b) + 1 - result->generated;
break;
}
result->generated = strbuf_len(b);
if (ret == 0)
break;
}
rhizome_list_release(&r->u.list.cursor);
return ret;
}
static int neighbour_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
@ -420,30 +580,31 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
return 0;
}
static int rhizome_file_content(struct http_request *hr)
static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
{
// Only read multiples of 4k from disk.
const size_t blocksz = 1 << 12;
// Ask for a large buffer for all future reads.
const size_t preferred_bufsz = 16 * blocksz;
// Reads the next part of the payload into the supplied buffer.
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->http.response_buffer_sent == 0);
assert(r->http.response_buffer_length == 0);
assert(r->read_state.offset < r->read_state.length);
uint64_t readlen = r->read_state.length - r->read_state.offset;
size_t suggested_size = 64 * 1024;
if (suggested_size > readlen)
suggested_size = readlen;
if (r->http.response_buffer_size < suggested_size)
http_request_set_response_bufsize(&r->http, suggested_size);
if (r->http.response_buffer == NULL)
http_request_set_response_bufsize(&r->http, 1);
if (r->http.response_buffer == NULL)
return -1;
ssize_t len = rhizome_read(&r->read_state,
(unsigned char *)r->http.response_buffer,
r->http.response_buffer_size);
if (len == -1)
return -1;
assert((size_t) len <= r->http.response_buffer_size);
r->http.response_buffer_length += (size_t) len;
return 0;
assert(r->u.read_state.offset < r->u.read_state.length);
uint64_t remain = r->u.read_state.length - r->u.read_state.offset;
size_t readlen = bufsz;
if (remain < bufsz)
readlen = remain;
else
readlen &= ~(blocksz - 1);
if (readlen > 0) {
ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
if (n == -1)
return -1;
result->generated = (size_t) n;
}
assert(r->u.read_state.offset <= r->u.read_state.length);
remain = r->u.read_state.length - r->u.read_state.offset;
result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
return remain ? 1 : 0;
}
static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
@ -464,35 +625,35 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
rhizome_filehash_t filehash;
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
return 1;
bzero(&r->read_state, sizeof r->read_state);
int n = rhizome_open_read(&r->read_state, &filehash);
bzero(&r->u.read_state, sizeof r->u.read_state);
int n = rhizome_open_read(&r->u.read_state, &filehash);
if (n == -1) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (n != 0)
return 1;
if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) {
rhizome_read_close(&r->read_state);
if (r->u.read_state.length == -1 && rhizome_read(&r->u.read_state, NULL, 0)) {
rhizome_read_close(&r->u.read_state);
return 1;
}
assert(r->read_state.length != -1);
r->http.response.header.resource_length = r->read_state.length;
assert(r->u.read_state.length != -1);
r->http.response.header.resource_length = r->u.read_state.length;
if (r->http.request_header.content_range_count > 0) {
assert(r->http.request_header.content_range_count == 1);
struct http_range closed;
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length);
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
if (n == 0 || http_range_bytes(&closed, 1) == 0) {
http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
return 0;
}
r->http.response.header.content_range_start = closed.first;
r->http.response.header.content_length = closed.last - closed.first + 1;
r->read_state.offset = closed.first;
r->u.read_state.offset = closed.first;
} else {
r->http.response.header.content_range_start = 0;
r->http.response.header.content_length = r->http.response.header.resource_length;
r->read_state.offset = 0;
r->u.read_state.offset = 0;
}
http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
return 0;

1
rhizome_packetformats.c

@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
/* Android doesn't have log2(), and we don't really need to do floating point

69
serval.h

@ -33,42 +33,41 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h>
#ifdef WIN32
#include "win32/win32.h"
# include "win32/win32.h"
#else
#include <unistd.h>
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_NET_ROUTE_H
#include <net/route.h>
#endif
#ifdef HAVE_LINUX_IF_H
#include <linux/if.h>
#else
#ifdef HAVE_NET_IF_H
#include <net/if.h>
#endif
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_LINUX_NETLINK_H
#include <linux/netlink.h>
#endif
#ifdef HAVE_LINUX_RTNETLINK_H
#include <linux/rtnetlink.h>
#endif
#ifdef HAVE_IFADDRS_H
#include <ifaddrs.h>
#endif
#ifdef HAVE_SYS_SOCKIO_H
#include <sys/sockio.h>
#endif
#ifdef HAVE_SYS_UCRED_H
#include <sys/ucred.h>
#endif
#endif
# include <unistd.h>
# ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
# endif
# ifdef HAVE_NET_ROUTE_H
# include <net/route.h>
# endif
# ifdef HAVE_LINUX_IF_H
# include <linux/if.h>
# else
# ifdef HAVE_NET_IF_H
# include <net/if.h>
# endif
# endif
# ifdef HAVE_NETINET_IN_H
# include <netinet/in.h>
# endif
# ifdef HAVE_LINUX_NETLINK_H
# include <linux/netlink.h>
# endif
# ifdef HAVE_LINUX_RTNETLINK_H
# include <linux/rtnetlink.h>
# endif
# ifdef HAVE_IFADDRS_H
# include <ifaddrs.h>
# endif
# ifdef HAVE_SYS_SOCKIO_H
# include <sys/sockio.h>
# endif
# ifdef HAVE_SYS_UCRED_H
# include <sys/ucred.h>
# endif
#endif //!WIN32
#if !defined(FORASTERISK) && !defined(s_addr)
#ifdef HAVE_ARPA_INET_H

3
server.c

@ -21,7 +21,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <unistd.h>
#include <time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include "serval.h"
@ -108,11 +107,9 @@ int server(const struct cli_parsed *parsed)
sigemptyset(&sig.sa_mask); // Block the same signals during handler
sigaddset(&sig.sa_mask, SIGHUP);
sigaddset(&sig.sa_mask, SIGINT);
sigaddset(&sig.sa_mask, SIGQUIT);
sig.sa_flags = 0;
sigaction(SIGHUP, &sig, NULL);
sigaction(SIGINT, &sig, NULL);
sigaction(SIGQUIT, &sig, NULL);
/* Record PID to advertise that the server is now running */
char filename[1024];

1
sourcefiles.mk

@ -69,6 +69,7 @@ SERVAL_SOURCES = \
$(SERVAL_BASE)strbuf.c \
$(SERVAL_BASE)strbuf_helpers.c \
$(SERVAL_BASE)strlcpy.c \
$(SERVAL_BASE)uuid.c \
$(SERVAL_BASE)vomp.c \
$(SERVAL_BASE)vomp_console.c \
$(SERVAL_BASE)xprintf.c \

5
str.c

@ -30,14 +30,15 @@
#include <limits.h>
#include <errno.h>
const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
const char hexdigit_upper[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
const char hexdigit_lower[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
char *tohex(char *dstHex, size_t dstStrLen, const unsigned char *srcBinary)
{
char *p;
size_t i;
for (p = dstHex, i = 0; i < dstStrLen; ++i)
*p++ = (i & 1) ? hexdigit[*srcBinary++ & 0xf] : hexdigit[*srcBinary >> 4];
*p++ = (i & 1) ? hexdigit_upper[*srcBinary++ & 0xf] : hexdigit_upper[*srcBinary >> 4];
*p = '\0';
return dstHex;
}

3
str.h

@ -60,7 +60,8 @@ __STR_INLINE int is_xstring(const char *text, int len)
return *text == '\0';
}
extern const char hexdigit[16];
extern const char hexdigit_upper[16];
extern const char hexdigit_lower[16];
char *tohex(char *dstHex, size_t dstStrlen, const unsigned char *srcBinary);
size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary);

4
strbuf.c

@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define __STRBUF_INLINE
#include "strbuf.h"
#include "str.h"
static inline size_t min(size_t a, size_t b) {
return a < b ? a : b;
@ -76,7 +77,6 @@ strbuf strbuf_puts(strbuf sb, const char *text)
strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
{
static char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
char *p = sb->current;
sb->current += strlen;
if (sb->start) {
@ -84,7 +84,7 @@ strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
// The following loop could overwrite the '\0' at *sp->end.
size_t i;
for (i = 0; i < strlen && p < e; ++i)
*p++ = (i & 1) ? hexdigit[*data++ & 0xf] : hexdigit[*data >> 4];
*p++ = (i & 1) ? hexdigit_upper[*data++ & 0xf] : hexdigit_upper[*data >> 4];
// This will restore the '\0' at *sp->end if it was overwritten.
*e = '\0';
}

52
strbuf_helpers.c

@ -36,6 +36,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/uio.h>
#include "http_server.h"
#include "strbuf_helpers.h"
#include "str.h"
static inline strbuf _toprint(strbuf sb, char c)
{
@ -414,6 +415,57 @@ strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
return sb;
}
strbuf strbuf_json_null(strbuf sb)
{
strbuf_puts(sb, "null");
return sb;
}
strbuf strbuf_json_string(strbuf sb, const char *str)
{
if (str) {
strbuf_putc(sb, '"');
for (; *str; ++str) {
if (*str == '"' || *str == '\\') {
strbuf_putc(sb, '\\');
strbuf_putc(sb, *str);
}
else if (*str == '\b')
strbuf_puts(sb, "\\b");
else if (*str == '\f')
strbuf_puts(sb, "\\f");
else if (*str == '\n')
strbuf_puts(sb, "\\n");
else if (*str == '\r')
strbuf_puts(sb, "\\r");
else if (*str == '\t')
strbuf_puts(sb, "\\t");
else if (iscntrl(*str))
strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str);
else
strbuf_putc(sb, *str);
}
strbuf_putc(sb, '"');
} else
strbuf_json_null(sb);
return sb;
}
strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
{
if (buf) {
strbuf_putc(sb, '"');
size_t i;
for (i = 0; i != len; ++i) {
strbuf_putc(sb, hexdigit_upper[*buf >> 4]);
strbuf_putc(sb, hexdigit_upper[*buf++ & 0xf]);
}
strbuf_putc(sb, '"');
} else
strbuf_json_null(sb);
return sb;
}
strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
{
unsigned i;

7
strbuf_helpers.h

@ -151,6 +151,13 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
*/
strbuf strbuf_append_quoted_string(strbuf sb, const char *str);
/* Append various JSON elements.
* @author Andrew Bettison <andrew@servalproject.com>
*/
strbuf strbuf_json_null(strbuf sb);
strbuf strbuf_json_string(strbuf sb, const char *str);
strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len);
/* Append a representation of a struct http_range[] array.
* @author Andrew Bettison <andrew@servalproject.com>
*/

7
testdefs.sh

@ -57,14 +57,17 @@ extract_stdout_keyvalue_optional() {
esac
local _label_re=$(escape_grep_basic "$_label")
local _delim_re=$(escape_grep_basic "$_delim")
local _line=$(replayStdout | $GREP "^$_label_re$_delim_re")
local _line=$($GREP "^$_label_re$_delim_re" "$TFWSTDOUT")
local _value=
local _return=1
if [ -n "$_line" ]; then
_value="${_line#*$_delim}"
_return=0
fi
[ -n "$_var" ] && eval $_var="\$_value"
if [ -n "$_var" ]; then
eval $_var="\$_value"
eval tfw_log "$_var=\$(shellarg "\${$_var}")"
fi
return $_return
}

24
testdefs_rhizome.sh

@ -302,10 +302,30 @@ extract_stdout_secret() {
extract_stdout_keyvalue "$1" .secret "$rexp_bundlesecret"
}
extract_stdout_rowid() {
extract_stdout_keyvalue "$1" .rowid "$rexp_rowid"
}
extract_stdout_inserttime() {
extract_stdout_keyvalue "$1" .inserttime "$rexp_date"
}
extract_stdout_BK() {
extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
}
extract_stdout_date() {
extract_stdout_keyvalue "$1" date "$rexp_date"
}
extract_stdout_filesize() {
extract_stdout_keyvalue "$1" filesize "$rexp_filesize"
}
extract_stdout_filehash() {
extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
}
extract_manifest() {
local _var="$1"
local _manifestfile="$2"
@ -346,6 +366,10 @@ extract_manifest_version() {
extract_manifest "$1" "$2" version "$rexp_version"
}
extract_manifest_date() {
extract_manifest "$1" "$2" date "$rexp_date"
}
extract_manifest_crypt() {
extract_manifest "$1" "$2" crypt "$rexp_crypt"
}

92
tests/rhizomehttp

@ -99,20 +99,104 @@ teardown_AuthBasicWrong() {
teardown
}
doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
setup_RhizomeList() {
for n in 1 2 3 4; do
create_file file$n ${n}k
setup
NBUNDLES=100
for ((n = 0; n != NBUNDLES; ++n)); do
create_file file$n $((1000 + $n))
executeOk_servald rhizome add file $SIDA file$n file$n.manifest
extract_stdout_manifestid BID[$n]
extract_stdout_version VERSION[$n]
extract_stdout_filesize SIZE[$n]
extract_stdout_filehash HASH[$n]
extract_stdout_date DATE[$n]
extract_stdout_rowid ROWID[$n]
done
}
test_RhizomeList() {
executeOk curl \
--silent --fail --show-error \
--output http.output \
--output bundlelist.json \
--dump-header http.headers \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
tfw_cat http.headers bundlelist.json
tfw_preserve bundlelist.json
assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ]
# The following jq(1) incantation transforms the JSON array in
# bundlelist.json from the following form (which is optimised for
# transmission size):
# {
# "header":[ "_id", "service", "id", "version","date",".inserttime",
# ".author",".fromhere","filesize","filehash","sender","recipient",
# "name"
# ],
# "token":"xxx",
# "rows":[
# [ rowid1, "service1", bundleid1, version1, .... ],
# [ rowid2, "service2", bundleid2, version2, .... ],
# ...
# [ rowidN, "serviceN", bundleidN, versionN, .... ]
# ]
# }
#
# into an array of JSON objects:
# {
# "token":"xxx",
# "bundles":[
# {
# "_id": rowid1,
# "service": service1,
# "id": bundleid1,
# "version": version1,
# ...
# },
# {
# "_id": rowid2,
# "service": service2,
# "id": bundleid2,
# "version": version2,
# ...
# },
# ...
# {
# "_id": rowidN,
# "service": serviceN,
# "id": bundleidN,
# "version": versionN,
# ...
# }
# ]
# }
# which is much easier to test with jq(1) expressions.
jq \
'[
.header as $h |
.rows[] as $d |
[ $d | keys | .[] as $i | {key:$h[$i], value:$d[$i]} ] |
from_entries
] as $bundles |
.token as $token |
{ token:$token, bundles:$bundles }' \
bundlelist.json >array_of_objects.json
tfw_preserve array_of_objects.json
for ((n = 0; n != NBUNDLES; ++n)); do
jqscript=".bundles | contains([
{ name:\"file$n\",
service:\"file\",
id:\"${BID[$n]}\",
version:${VERSION[$n]},
filesize:${SIZE[$n]},
filehash:\"${HASH[$n]}\",
date:${DATE[$n]},
_id:${ROWID[$n]},
\".fromhere\":1,
\".author\":\"$SIDA\",
}
])"
assert [ "$(jq "$jqscript" array_of_objects.json)" = true ]
done
}
doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"

86
tests/rhizomeops

@ -208,23 +208,27 @@ setup_ExtractManifestAfterAdd() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAdd() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
@ -238,23 +242,27 @@ setup_ExtractManifestFileAfterAdd() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestFileAfterAdd() {
executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
@ -271,29 +279,35 @@ setup_ExtractManifestFileFromExtBlob() {
executeOk_servald config set rhizome.external_blobs 1
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid1
executeOk_servald config set rhizome.external_blobs 0
echo "Another test file" >file2
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
extract_stdout_rowid rowid2
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
extract_manifest_id manifestid1 file1.manifest
extract_manifest_version version1 file1.manifest
extract_manifest_filehash filehash1 file1.manifest
extract_manifest_date date1 file1.manifest
extract_manifest_id manifestid2 file2.manifest
extract_manifest_version version2 file2.manifest
extract_manifest_filehash filehash2 file2.manifest
extract_manifest_date date2 file2.manifest
}
test_ExtractManifestFileFromExtBlob() {
executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
assertStdoutGrep --matches=1 "^version:$version1\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
@ -303,14 +317,16 @@ test_ExtractManifestFileFromExtBlob() {
assert diff file1 file1x
executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
local size=$(( $(cat file2 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
assertStdoutGrep --matches=1 "^version:$version2\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date2\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
@ -342,26 +358,30 @@ setup_ExtractManifestToStdout() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestToStdout() {
executeOk_servald rhizome export manifest $manifestid -
assertStdoutLineCount '>=' 9
assertStdoutLineCount '>=' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --line=..9 --matches=1 "^service:file$"
assertStdoutGrep --line=..9 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..9 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..9 --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --line=..9 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..9 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..9 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..9 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..9 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=10 --matches=1 "^manifest:"
replayStdout | $SED -n '10s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '11,$p' >>file1x.manifest
assertStdoutGrep --line=..11 --matches=1 "^service:file$"
assertStdoutGrep --line=..11 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..11 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..11 --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --line=..11 --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --line=..11 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..11 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..11 --matches=1 "^date:$date\$"
assertStdoutGrep --line=..11 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..11 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..11 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=12 --matches=1 "^manifest:"
replayStdout | $SED -n '12s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '13,$p' >>file1x.manifest
cat file1.manifest >file1n.manifest
echo >>file1n.manifest
tfw_cat file1n.manifest file1x.manifest
@ -374,23 +394,27 @@ setup_ExtractManifestAfterAddNoAuthor() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file '' file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAddNoAuthor() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
assert diff file1.manifest file1x.manifest
assertStdoutLineCount '==' 7
assertStdoutLineCount '==' 9
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.readonly:1\$"
}
@ -430,24 +454,28 @@ setup_ExtractFileAfterAdd() {
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
tfw_cat --stderr
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractFileAfterAdd() {
executeOk_servald rhizome extract file $manifestid file1x
tfw_cat --stderr
assert diff file1 file1x
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
@ -1013,9 +1041,11 @@ setup_ImportOwnBundle() {
echo "Hello from B" >fileB
executeOk_servald rhizome add file $SIDB2 fileB fileB.manifest
assert_stdout_add_file fileB
extract_stdout_rowid rowid
extract_manifest_id manifestid fileB.manifest
extract_manifest_version version fileB.manifest
extract_manifest_filehash filehash fileB.manifest
extract_manifest_date date fileB.manifest
rm -f $SERVALINSTANCE_PATH/rhizome.db
executeOk_servald rhizome list
assert_rhizome_list
@ -1031,14 +1061,16 @@ test_ImportOwnBundle() {
tfw_cat --stderr
assert cmp fileB.manifest fileBx.manifest
assert cmp fileB fileBx
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 11
local size=$(( $(cat fileB | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"

2
tests/rhizomeprotocol

@ -75,7 +75,7 @@ doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
setup_common
set_instance +A
rhizome_add_file file1 2048
rhizome_add_file file1 250000
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A

115
uuid.c Normal file

@ -0,0 +1,115 @@
/*
Serval DNA Universally Unique Identifier support
Copyright (C) 2013 Serval Project Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#define __SERVALDNA_UUID_H_INLINE
#include "uuid.h"
#include "os.h"
#include "str.h"
#include <assert.h>
#ifdef HAVE_ARPA_INET_H
# include <arpa/inet.h>
#endif
enum uuid_version uuid_get_version(const uuid_t *uuid)
{
assert(uuid_is_valid(uuid));
switch (ntohs(uuid->u.record.time_hi_and_version) & 0xf000) {
case 0x1000: return UUID_VERSION_TIME_BASED;
case 0x2000: return UUID_VERSION_DCE_SECURITY;
case 0x3000: return UUID_VERSION_NAME_MD5;
case 0x4000: return UUID_VERSION_RANDOM;
case 0x5000: return UUID_VERSION_NAME_SHA1;
}
return UUID_VERSION_UNSUPPORTED;
}
void uuid_set_version(uuid_t *uuid, enum uuid_version version)
{
uint16_t version_bits;
switch (version) {
case UUID_VERSION_TIME_BASED: version_bits = 0x1000; break;
case UUID_VERSION_DCE_SECURITY: version_bits = 0x2000; break;
case UUID_VERSION_NAME_MD5: version_bits = 0x3000; break;
case UUID_VERSION_RANDOM: version_bits = 0x4000; break;
case UUID_VERSION_NAME_SHA1: version_bits = 0x5000; break;
default: abort();
}
assert(uuid_is_valid(uuid));
uuid->u.record.time_hi_and_version = htons((ntohs(uuid->u.record.time_hi_and_version) & 0xfff) | version_bits);
}
int uuid_generate_random(uuid_t *uuid)
{
if (urandombytes(uuid->u.binary, sizeof uuid->u.binary) == -1)
return -1;
// The following discards 6 random bits.
uuid->u.record.clock_seq_hi_and_reserved &= 0x3f;
uuid->u.record.clock_seq_hi_and_reserved |= 0x80;
uuid_set_version(uuid, UUID_VERSION_RANDOM);
return 0;
}
strbuf strbuf_uuid(strbuf sb, const uuid_t *uuid)
{
assert(uuid_is_valid(uuid));
unsigned i;
for (i = 0; i != sizeof uuid->u.binary; ++i) {
switch (i) {
case 4: case 6: case 8: case 10:
strbuf_putc(sb, '-');
default:
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] >> 4]);
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] & 0xf]);
}
}
return sb;
}
char *uuid_to_str(const uuid_t *uuid, char *const dst)
{
strbuf b = strbuf_local(dst, UUID_STRLEN + 1);
strbuf_uuid(b, uuid);
assert(!strbuf_overrun(b));
return dst;
}
int str_to_uuid(const char *const str, uuid_t *uuid, const char **afterp)
{
const char *end = str;
int ret = 0;
if ( strn_fromhex(uuid->u.binary, 4, end, &end) == 4
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 4, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 6, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 8, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 10, 6, end + 1, &end) == 6
) {
assert(end == str + UUID_STRLEN);
ret = uuid_is_valid(uuid);
}
if (afterp)
*afterp = end;
if (ret == 0 || (!afterp && *end))
return 0;
return 1;
}

123
uuid.h Normal file

@ -0,0 +1,123 @@
/*
Serval DNA Universally Unique Identifier support
Copyright (C) 2013 Serval Project Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef __SERVALDNA_UUID_H
#define __SERVALDNA_UUID_H
#include <stdint.h>
#include <alloca.h>
#include "strbuf.h"
#ifndef __SERVALDNA_UUID_H_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
# define __SERVALDNA_UUID_H_INLINE extern inline
# else
# define __SERVALDNA_UUID_H_INLINE inline
# endif
#endif
/* A UUID is defined by RFC-4122 as a 128-bit identifier with the two most
* significant bits of the ninth byte being 1 and 0, which indicates the
* "variant" that is described by the RFC. Other variants exist, but are not
* supported here, and are treated as INVALID by the functions defined below.
* Any attempt to pass an invalid UUID to a function that requires a valid UUID
* as input will probably result in the calling process being aborted (see
* SIGABRT, abort(3)).
*
* In a valid UUID, the four lowest significant bits of the seventh byte define
* the "version" of the UUID, which essentially indicates how it was generated.
* The RFC defines five SUPPORTED versions. Any other version is UNSUPPORTED.
*
* The fields in the UUID 'record' structure are stored in network byte order,
* so code wishing to make use of the record structure must use ntohl(3) and
* ntohs(3) to read values, and htonl(3) and htons(3) to assign values.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
typedef struct uuid {
union {
struct {
uint32_t time_low;
uint16_t time_mid;
uint16_t time_hi_and_version;
uint8_t clock_seq_hi_and_reserved;
uint8_t clock_seq_low;
unsigned char node[6]; // uint48_t
} record;
unsigned char binary[16];
} u;
} uuid_t;
enum uuid_version {
UUID_VERSION_UNSUPPORTED = 0,
UUID_VERSION_TIME_BASED = 1,
UUID_VERSION_DCE_SECURITY = 2,
UUID_VERSION_NAME_MD5 = 3,
UUID_VERSION_RANDOM = 4,
UUID_VERSION_NAME_SHA1 = 5
};
__SERVALDNA_UUID_H_INLINE int uuid_is_valid(const uuid_t *any_uuid) {
return (any_uuid->u.record.clock_seq_hi_and_reserved & 0xc0) == 0x80;
}
enum uuid_version uuid_get_version(const uuid_t *valid_uuid);
void uuid_set_version(uuid_t *valid_uuid, enum uuid_version);
/* Returns -1 if error (eg, cannot open /dev/urandom), 0 if successful.
*/
int uuid_generate_random(uuid_t *dest_uuid);
/* Formats the given valid UUID in its canonical string representation:
* XXXXXXXX-VXXX-MXXX-XXXXXXXXXXXX
* where X is any hex digit
* V is 1, 2, 3, 4 or 5 (supported versions) or any other hex digit
* (unsupported versions)
* M is 8, 9, A or B (high two bits are variant 01)
*
* The 'dst' argument must point to a buffer of 37 bytes. The first 36 bytes
* are filled with the representation shown above, and the 37th byte dst[36] is
* set to NUL '\0'.
*
* Returns the 'dst' argument.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
char *uuid_to_str(const uuid_t *valid_uuid, char *dst);
#define UUID_STRLEN 36
#define alloca_uuid_str(uuid) uuid_to_str(&(uuid), alloca(UUID_STRLEN + 1))
/* Append a UUID to the given strbuf, formatted as per the uuid_to_str() function.
*/
strbuf strbuf_uuid(strbuf, const uuid_t *valid_uuid);
/* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
* UUID, which may or not be supported.
*
* Returns 1 if a valid UUID is parsed, storing the value in *result (unless result is NULL) and
* storing a pointer to the immediately succeeding character in *afterp. If afterp is NULL then
* returns 0 unless the immediately succeeding character is a NUL '\0'. If no UUID is parsed or
* if the UUID is not valid, then returns 0, leaving *result unchanged and
* setting setting *afterp to point to the character where parsing failed.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int str_to_uuid(const char *str, uuid_t *result, const char **afterp);
#endif //__SERVALDNA_OS_H