ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-21 09:51:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'naf4' into 'development'

Browse Source 
New HTTP RESTful Rhizome API, plus tests, and lots of Rhizome
improvements and refactoring
This commit is contained in:
Andrew Bettison 2013-12-30 18:14:34 +10:30
commit d4320f20da
32 changed files with 3209 additions and 1402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

358
commandline.c
View File

@ -527,6 +527,65 @@ void cli_flush(struct cli_context *context)
fflush(stdout);
}
static void cli_put_manifest(struct cli_context *context, const rhizome_manifest *m)
{
assert(m->filesize != RHIZOME_SIZE_UNSET);
cli_field_name(context, "manifestid", ":"); // TODO rename to "bundleid" or "bid"
cli_put_string(context, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), "\n");
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
if (m->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
}
if (m->has_bundle_key) {
cli_field_name(context, "BK", ":");
cli_put_string(context, alloca_tohex_rhizome_bk_t(m->bundle_key), "\n");
}
if (m->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, m->date, "\n");
}
switch (m->payloadEncryption) {
case PAYLOAD_CRYPT_UNKNOWN:
break;
case PAYLOAD_CLEAR:
cli_field_name(context, "crypt", ":");
cli_put_long(context, 0, "\n");
break;
case PAYLOAD_ENCRYPTED:
cli_field_name(context, "crypt", ":");
cli_put_long(context, 1, "\n");
break;
}
if (m->service) {
cli_field_name(context, "service", ":");
cli_put_string(context, m->service, "\n");
}
if (m->name) {
cli_field_name(context, "name", ":");
cli_put_string(context, m->name, "\n");
}
cli_field_name(context, ".readonly", ":");
cli_put_long(context, m->haveSecret ? 0 : 1, "\n");
if (m->haveSecret) {
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_field_name(context, ".secret", ":");
cli_put_string(context, secret, "\n");
}
if (m->authorship == AUTHOR_AUTHENTIC) {
cli_field_name(context, ".author", ":");
cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
}
cli_field_name(context, ".rowid", ":");
cli_put_long(context, m->rowid, "\n");
cli_field_name(context, ".inserttime", ":");
cli_put_long(context, m->inserttime, "\n");
}
int app_echo(const struct cli_parsed *parsed, struct cli_context *context)
{
if (config.debug.verbose)
@ -1372,7 +1431,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
if (bskhex && !*bskhex)
bskhex=NULL;
if (bskhex && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
if (bskhex && str_to_rhizome_bk_t(&bsk, bskhex) == -1)
return WHYF("invalid bsk: \"%s\"", bskhex);
int journal = strcasecmp(parsed->args[1], "journal")==0;
@ -1401,8 +1460,8 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
/* Don't verify the manifest, because it will fail if it is incomplete.
This is okay, because we fill in any missing bits and sanity check before
trying to write it out. However, we do insist that whatever we load is
valid and not malformed. */
if (rhizome_read_manifest_file(m, manifestpath, 0) == -1 || m->malformed) {
parsed okay and not malformed. */
if (rhizome_read_manifest_from_file(m, manifestpath) || m->malformed) {
rhizome_manifest_free(m);
keyring_free(keyring);
return WHY("Manifest file could not be loaded -- not added to rhizome");
@ -1451,87 +1510,78 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
return -1;
}
enum rhizome_bundle_status status = RHIZOME_BUNDLE_STATUS_NEW;
enum rhizome_payload_status pstatus;
if (journal){
if (rhizome_append_journal_file(m, 0, filepath)){
rhizome_manifest_free(m);
keyring_free(keyring);
return -1;
pstatus = rhizome_append_journal_file(m, 0, filepath);
} else {
pstatus = rhizome_stat_payload_file(m, filepath);
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) {
assert(m->filesize > 0);
pstatus = rhizome_store_payload_file(m, filepath);
}
}else{
if (rhizome_stat_file(m, filepath) == -1) {
rhizome_manifest_free(m);
keyring_free(keyring);
return -1;
}
if (m->filesize) {
if (rhizome_add_file(m, filepath) == -1) {
rhizome_manifest_free(m);
keyring_free(keyring);
return -1;
}
switch (pstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
status = RHIZOME_BUNDLE_STATUS_ERROR;
break;
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
status = RHIZOME_BUNDLE_STATUS_INCONSISTENT;
break;
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
status = RHIZOME_BUNDLE_STATUS_FAKE;
break;
default:
FATALF("pstatus = %d", pstatus);
}
rhizome_manifest *mout = m;
if (status == RHIZOME_BUNDLE_STATUS_NEW) {
if (!rhizome_manifest_validate(m) || m->malformed)
status = RHIZOME_BUNDLE_STATUS_INVALID;
else {
status = rhizome_manifest_finalise(m, &mout, !force_new);
if (mout && mout != m && !rhizome_manifest_validate(mout)) {
WHYF("Stored manifest id=%s is invalid -- overwriting", alloca_tohex_rhizome_bid_t(mout->cryptoSignPublic));
status = RHIZOME_BUNDLE_STATUS_NEW;
}
}
}
rhizome_manifest *mout = NULL;
int ret = rhizome_manifest_finalise(m, &mout, !force_new);
if (ret<0){
rhizome_manifest_free(m);
keyring_free(keyring);
return -1;
switch (status) {
case RHIZOME_BUNDLE_STATUS_NEW:
if (mout && mout != m)
rhizome_manifest_free(mout);
mout = m;
// fall through
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
cli_put_manifest(context, mout);
if ( manifestpath && *manifestpath
&& rhizome_write_manifest_file(mout, manifestpath, 0) == -1
)
WHYF("Could not write manifest to %s", alloca_str_toprint(manifestpath));
break;
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
case RHIZOME_BUNDLE_STATUS_ERROR:
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_FAKE:
break;
default:
FATALF("status=%d", status);
}
if (manifestpath && *manifestpath
&& rhizome_write_manifest_file(mout, manifestpath, 0) == -1)
ret = WHY("Could not overwrite manifest file.");
if (mout->service) {
cli_field_name(context, "service", ":");
cli_put_string(context, mout->service, "\n");
}
{
cli_field_name(context, "manifestid", ":");
cli_put_string(context, alloca_tohex_rhizome_bid_t(mout->cryptoSignPublic), "\n");
}
assert(m->haveSecret);
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(mout->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_field_name(context, ".secret", ":");
cli_put_string(context, secret, "\n");
}
assert(mout->authorship != AUTHOR_LOCAL);
if (mout->authorship == AUTHOR_AUTHENTIC) {
cli_field_name(context, ".author", ":");
cli_put_string(context, alloca_tohex_sid_t(mout->author), "\n");
}
cli_field_name(context, ".rowid", ":");
cli_put_long(context, m->rowid, "\n");
cli_field_name(context, ".inserttime", ":");
cli_put_long(context, m->inserttime, "\n");
if (mout->has_bundle_key) {
cli_field_name(context, "BK", ":");
cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n");
}
if (mout->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, mout->date, "\n");
}
cli_field_name(context, "version", ":");
cli_put_long(context, mout->version, "\n");
cli_field_name(context, "filesize", ":");
cli_put_long(context, mout->filesize, "\n");
if (mout->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n");
}
if (mout->name) {
cli_field_name(context, "name", ":");
cli_put_string(context, mout->name, "\n");
}
if (mout != m)
if (mout && mout != m) {
rhizome_manifest_free(mout);
m = NULL;
}
rhizome_manifest_free(m);
keyring_free(keyring);
return ret;
return status;
}
int app_slip_test(const struct cli_parsed *parsed, struct cli_context *context)
@ -1586,55 +1636,27 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
cli_arg(parsed, "manifestpath", &manifestpath, NULL, "");
if (rhizome_opendb() == -1)
return -1;
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
return WHY("Out of manifests.");
int status=rhizome_bundle_import_files(m, manifestpath, filepath);
if (status<0)
goto cleanup;
// TODO generalise the way we dump manifest details from add, import & export
// so callers can also generalise their parsing
if (m->service) {
cli_field_name(context, "service", ":");
cli_put_string(context, m->service, "\n");
rhizome_manifest *m_out = NULL;
enum rhizome_bundle_status status = rhizome_bundle_import_files(m, &m_out, manifestpath, filepath);
switch (status) {
case RHIZOME_BUNDLE_STATUS_NEW:
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
cli_put_manifest(context, m_out);
break;
case RHIZOME_BUNDLE_STATUS_ERROR:
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
break;
default:
FATALF("rhizome_bundle_import_files() returned %d", status);
}
{
cli_field_name(context, "manifestid", ":");
cli_put_string(context, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), "\n");
}
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_field_name(context, ".secret", ":");
cli_put_string(context, secret, "\n");
}
if (m->has_bundle_key) {
cli_field_name(context, "BK", ":");
cli_put_string(context, alloca_tohex_rhizome_bk_t(m->bundle_key), "\n");
}
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
if (m->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, m->date, "\n");
}
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
}
if (m->name) {
cli_field_name(context, "name", ":");
cli_put_string(context, m->name, "\n");
}
cleanup:
if (m_out && m_out != m)
rhizome_manifest_free(m_out);
rhizome_manifest_free(m);
return status;
}
@ -1651,11 +1673,10 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, struct cli_cont
if (!m)
return WHY("Out of manifests.");
int ret = -1;
if ( rhizome_read_manifest_file(m, manifestpath, 0) != -1
if ( rhizome_read_manifest_from_file(m, manifestpath) != -1
&& rhizome_manifest_validate(m)
&& rhizome_manifest_verify(m)
) {
assert(m->finalised);
if (rhizome_write_manifest_file(m, filepath, 1) != -1)
ret = 0;
}
@ -1781,60 +1802,28 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
return WHY("Out of manifests");
}
ret = rhizome_retrieve_manifest(&bid, m);
if (ret==0){
assert(m->finalised);
if (bskhex)
rhizome_apply_bundle_secret(m, &bsk);
rhizome_authenticate_author(m);
if (m->service) {
cli_field_name(context, "service", ":");
cli_put_string(context, m->service, "\n");
}
cli_field_name(context, "manifestid", ":");
cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
cli_field_name(context, "version", ":");
cli_put_long(context, m->version, "\n");
if (m->has_date) {
cli_field_name(context, "date", ":");
cli_put_long(context, m->date, "\n");
}
cli_field_name(context, "filesize", ":");
cli_put_long(context, m->filesize, "\n");
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize != 0) {
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
}
if (m->haveSecret) {
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_field_name(context, ".secret", ":");
cli_put_string(context, secret, "\n");
}
assert(m->authorship != AUTHOR_LOCAL);
if (m->authorship == AUTHOR_AUTHENTIC) {
cli_field_name(context, ".author", ":");
cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
}
cli_field_name(context, ".rowid", ":");
cli_put_long(context, m->rowid, "\n");
cli_field_name(context, ".inserttime", ":");
cli_put_long(context, m->inserttime, "\n");
cli_field_name(context, ".readonly", ":");
cli_put_long(context, m->haveSecret?0:1, "\n");
cli_put_manifest(context, m);
}
int retfile=0;
enum rhizome_payload_status pstatus = RHIZOME_PAYLOAD_STATUS_EMPTY;
if (ret==0 && m->filesize != 0 && filepath && *filepath){
if (extract){
// Save the file, implicitly decrypting if required.
retfile = rhizome_extract_file(m, filepath);
pstatus = rhizome_extract_file(m, filepath);
if (pstatus != RHIZOME_PAYLOAD_STATUS_EMPTY && pstatus != RHIZOME_PAYLOAD_STATUS_STORED)
WHYF("rhizome_extract_file() returned %d", pstatus);
}else{
// Save the file without attempting to decrypt
uint64_t length;
retfile = rhizome_dump_file(&m->filehash, filepath, &length);
pstatus = rhizome_dump_file(&m->filehash, filepath, &length);
if (pstatus != RHIZOME_PAYLOAD_STATUS_EMPTY && pstatus != RHIZOME_PAYLOAD_STATUS_STORED)
WHYF("rhizome_dump_file() returned %d", pstatus);
}
}
@ -1847,19 +1836,28 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
} else {
int append = (strcmp(manifestpath, filepath)==0)?1:0;
// don't write out the manifest if we were asked to append it and writing the file failed.
if ((!append) || retfile==0){
/* If the manifest has been read in from database, the blob is there,
and we can lie and say we are finalised and just want to write it
out. TODO: really should have a dirty/clean flag, so that write
works if clean but not finalised. */
m->finalised=1;
if (!append || (pstatus == RHIZOME_PAYLOAD_STATUS_EMPTY || pstatus == RHIZOME_PAYLOAD_STATUS_STORED)) {
if (rhizome_write_manifest_file(m, manifestpath, append) == -1)
ret = -1;
}
}
}
if (retfile)
ret = retfile == -1 ? -1 : 1;
switch (pstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
ret = 1; // payload not found
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
ret = -1;
break;
default:
FATALF("pstatus = %d", pstatus);
}
if (m)
rhizome_manifest_free(m);
keyring_free(keyring);
@ -1884,9 +1882,21 @@ int app_rhizome_export_file(const struct cli_parsed *parsed, struct cli_context
if (!rhizome_exists(&hash))
return 1;
uint64_t length;
int ret = rhizome_dump_file(&hash, filepath, &length);
if (ret)
return ret == -1 ? -1 : 1;
enum rhizome_payload_status pstatus = rhizome_dump_file(&hash, filepath, &length);
switch (pstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
return 1; // payload not found
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
return -1;
default:
FATALF("pstatus = %d", pstatus);
}
cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(hash), "\n");
cli_field_name(context, "filesize", ":");

3
conf.h
View File

@ -657,6 +657,9 @@ int cf_fmt_int32_nonneg(const char **, const int32_t *intp);
int cf_opt_uint32_nonzero(uint32_t *intp, const char *text);
int cf_fmt_uint32_nonzero(const char **, const uint32_t *intp);
int cf_opt_uint32_scaled(uint32_t *intp, const char *text);
int cf_fmt_uint32_scaled(const char **, const uint32_t *intp);
int cf_opt_uint64_scaled(uint64_t *intp, const char *text);
int cf_fmt_uint64_scaled(const char **, const uint64_t *intp);

24
conf_schema.c
View File

@ -428,6 +428,30 @@ int cf_cmp_uint32_time_interval(const uint32_t *a, const uint32_t *b)
return cf_cmp_uint32(a, b);
}
int cf_opt_uint32_scaled(uint32_t *intp, const char *text)
{
uint32_t result;
const char *end;
if (!str_to_uint32_scaled(text, 10, &result, &end) || *end)
return CFINVALID;
*intp = result;
return CFOK;
}
int cf_fmt_uint32_scaled(const char **textp, const uint32_t *uintp)
{
char buf[25];
int n = uint32_scaled_to_str(buf, sizeof buf, *uintp);
assert(n != 0);
*textp = str_edup(buf);
return CFOK;
}
int cf_cmp_uint32_scaled(const uint32_t *a, const uint32_t *b)
{
return *a < *b ? -1 : *a > *b ? 1 : 0;
}
int cf_opt_uint64_scaled(uint64_t *intp, const char *text)
{
uint64_t result;

2
conf_schema.h
View File

@ -426,7 +426,7 @@ ATOM(bool_t, clean_on_open, 0, boolean,, "If true, Rhizome datab
ATOM(bool_t, clean_on_start, 1, boolean,, "If true, Rhizome database is cleaned at start of daemon")
STRING(256, datastore_path, "", absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, uint64_scaled,, "Size of database in bytes")
ATOM(bool_t, external_blobs, 0, boolean,, "Store rhizome bundles as separate files.")
ATOM(uint32_t, max_blob_size, 128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
ATOM(uint64_t, rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")

25
dataformats.c
View File

@ -40,7 +40,7 @@ int str_to_sid_t(sid_t *sid, const char *hex)
int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp)
{
if (str_startswith(hex, "broadcast", endp) == 0) {
if (str_startswith(hex, "broadcast", endp)) {
if (sid)
*sid = SID_BROADCAST;
return 0;
@ -92,9 +92,10 @@ int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
*bid = tmp;
if (bid)
*bid = tmp;
if (endp)
*endp = hex + sizeof bid->binary * 2;
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
@ -114,9 +115,10 @@ int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
*hashp = tmp;
if (hashp)
*hashp = tmp;
if (endp)
*endp = hex + sizeof hashp->binary * 2;
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
@ -125,6 +127,19 @@ int str_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex)
return bkp ? fromhexstr(bkp->binary, hex, sizeof bkp->binary) : is_xstring(hex, RHIZOME_BUNDLE_KEY_STRLEN) ? 0 : -1;
}
int strn_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex, const char **endp)
{
rhizome_bk_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
if (bkp)
*bkp = tmp;
if (endp)
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
int rhizome_strn_is_bundle_crypt_key(const char *key)
{
return is_xsubstring(key, RHIZOME_CRYPT_KEY_STRLEN);

266
http_server.c
View File

@ -108,7 +108,9 @@ void http_request_init(struct http_request *r, int sockfd)
r->alarm.poll.fd = sockfd;
r->alarm.poll.events = POLLIN;
r->phase = RECEIVE;
r->received = r->end = r->parsed = r->cursor = r->buffer;
r->reserved = r->buffer;
// Put aside a few bytes for reserving strings, so that the path can be reserved ok.
r->received = r->end = r->parsed = r->cursor = r->buffer + 32;
r->parser = http_request_parse_verb;
watch(&r->alarm);
http_request_set_idle_timeout(r);
@ -140,11 +142,11 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
// Don't allocate a new buffer if the existing one contains content.
assert(r->response_buffer_sent == r->response_buffer_length);
const char *const bufe = r->buffer + sizeof r->buffer;
assert(r->received < bufe);
size_t rbufsiz = bufe - r->received;
assert(r->reserved < bufe);
size_t rbufsiz = bufe - r->reserved;
if (bufsiz <= rbufsiz) {
http_request_free_response_buffer(r);
r->response_buffer = (char *) r->received;
r->response_buffer = (char *) r->reserved;
r->response_buffer_size = rbufsiz;
if (r->debug_flag && *r->debug_flag)
DEBUGF("Static response buffer %zu bytes", r->response_buffer_size);
@ -197,37 +199,110 @@ static int _matches(struct substring str, const char *text)
}
#endif
static const char * _reserve(struct http_request *r, struct substring str)
void write_pointer(unsigned char *mem, void *v)
{
char *reslim = r->buffer + sizeof r->buffer - 1024; // always leave this much unreserved space
assert(r->received <= reslim);
uintptr_t n = (uintptr_t) v;
unsigned i;
for (i = 0; i != sizeof v; ++i)
mem[i] = n >> (8 * i);
}
void *read_pointer(const unsigned char *mem)
{
uintptr_t n = 0;
unsigned i;
for (i = 0; i != sizeof(void*); ++i)
n |= mem[i] << (8 * i);
return (void *) n;
}
/* Allocate space from the start of the request buffer to hold the given substring plus a
* terminating NUL. Enough bytes must have already been marked as parsed in order to make room,
* otherwise the reservation fails and returns 0. If successful, copies the substring plus a
* terminating NUL into the reserved space, places a pointer to the reserved area into '*resp', and
* returns 1.
*
* Keeps a copy to the pointer 'resp', so that when the reserved area is released, all pointers into
* it can be set to NULL automatically. This provides some safety: if the pointer is accidentally
* dereferenced after the release it will cause a SEGV instead of using a string that has been
* overwritten. It does not protect from using copies of '*resp', which of course will not be have
* been set to NULL by the release.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int _reserve(struct http_request *r, const char **resp, struct substring str)
{
// Reserved string pointer must lie within this http_request struct.
assert((char*)resp >= (char*)r);
assert((char*)resp < (char*)(r + 1));
size_t len = str.end - str.start;
size_t siz = len + 1;
if (r->received + siz > reslim) {
// Substring must contain no NUL chars.
assert(strnchr(str.start, len, '\0') == NULL);
char *reslim = r->buffer + sizeof r->buffer - 1024; // always leave this much unreserved space
assert(r->reserved <= reslim);
size_t siz = sizeof(char**) + len + 1;
if (r->reserved + siz > reslim) {
r->response.result_code = 414;
return NULL;
return 0;
}
if (r->received + siz > r->parsed) {
if (r->reserved + siz > r->parsed) {
WARNF("Error during HTTP parsing, unparsed content %s would be overwritten by reserving %s",
alloca_toprint(30, r->parsed, r->end - r->parsed),
alloca_substring_toprint(str)
);
r->response.result_code = 500;
return NULL;
return 0;
}
char *ret = (char *) r->received;
if (ret != str.start)
memmove(ret, str.start, len);
ret[len] = '\0';
r->received += siz;
const char ***respp = (const char ***) r->reserved;
char *restr = (char *)(respp + 1);
write_pointer((unsigned char*)respp, resp); // can't use *respp = resp; could cause SIGBUS if not aligned
if (restr != str.start)
memmove(restr, str.start, len);
restr[len] = '\0';
r->reserved += siz;
if (r->reserved > r->received)
r->received = r->reserved;
assert(r->received <= r->parsed);
return ret;
*resp = restr;
return 1;
}
static const char * _reserve_str(struct http_request *r, const char *str)
/* The same as _reserve(), but takes a NUL-terminated string as a source argument instead of a
* substring.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int _reserve_str(struct http_request *r, const char **resp, const char *str)
{
struct substring sub = { .start = str, .end = str + strlen(str) };
return _reserve(r, sub);
return _reserve(r, resp, sub);
}
/* Release all the strings reserved by _reserve(), returning the space to the request buffer, and
* resetting to NULL all the pointers to reserved strings that were set by _reserve().
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void _release_reserved(struct http_request *r)
{
char *res = r->buffer;
while (res < r->reserved) {
assert(res + sizeof(char**) + 1 <= r->reserved);
const char ***respp = (const char ***) res;
char *restr = (char *)(respp + 1);
const char **resp = read_pointer((const unsigned char*)respp); // can't use resp = *respp; could cause SIGBUS if not aligned
assert((const char*)resp >= (const char*)r);
assert((const char*)resp < (const char*)(r + 1));
assert(*resp == restr);
*resp = NULL;
for (res = restr; res < r->reserved && *res; ++res)
;
assert(res < r->reserved);
assert(*res == '\0');
++res;
}
assert(res == r->reserved);
r->reserved = r->buffer;
}
static inline int _end_of_content(struct http_request *r)
@ -463,12 +538,12 @@ static size_t _parse_token_or_quoted_string(struct http_request *r, char *dst, s
static inline int _parse_http_size_t(struct http_request *r, http_size_t *szp)
{
return !_run_out(r) && isdigit(*r->cursor) && str_to_uint64(r->cursor, 10, szp, &r->cursor);
return !_run_out(r) && isdigit(*r->cursor) && str_to_uint64(r->cursor, 10, szp, (const char **)&r->cursor);
}
static inline int _parse_uint(struct http_request *r, unsigned int *uintp)
static inline int _parse_uint32(struct http_request *r, uint32_t *uint32p)
{
return !_run_out(r) && isdigit(*r->cursor) && str_to_uint(r->cursor, 10, uintp, &r->cursor);
return !_run_out(r) && isdigit(*r->cursor) && str_to_uint32(r->cursor, 10, uint32p, (const char **)&r->cursor);
}
static unsigned _parse_ranges(struct http_request *r, struct http_range *range, unsigned nrange)
@ -523,7 +598,7 @@ static int _parse_content_type(struct http_request *r, struct mime_content_type
return 0;
}
while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) {
const char *start = r->cursor;
char *start = r->cursor;
if (_skip_literal(r, "charset=")) {
size_t n = _parse_token_or_quoted_string(r, ct->charset, sizeof ct->charset);
if (n == 0)
@ -560,7 +635,7 @@ static int _parse_content_type(struct http_request *r, struct mime_content_type
static size_t _parse_base64(struct http_request *r, char *bin, size_t binsize)
{
return base64_decode((unsigned char *)bin, binsize, r->cursor, r->end - r->cursor, &r->cursor, B64_CONSUME_ALL, is_http_space);
return base64_decode((unsigned char *)bin, binsize, r->cursor, r->end - r->cursor, (const char **)&r->cursor, B64_CONSUME_ALL, is_http_space);
}
static int _parse_authorization_credentials_basic(struct http_request *r, struct http_client_credentials_basic *cred, char *buf, size_t bufsz)
@ -579,14 +654,14 @@ static int _parse_authorization_credentials_basic(struct http_request *r, struct
static int _parse_authorization(struct http_request *r, struct http_client_authorization *auth, size_t header_bytes)
{
const char *start = r->cursor;
char *start = r->cursor;
if (_skip_literal(r, "Basic") && _skip_space(r)) {
size_t bufsz = 5 + header_bytes * 3 / 4; // enough for base64 decoding
char buf[bufsz];
if (_parse_authorization_credentials_basic(r, &auth->credentials.basic, buf, bufsz)) {
auth->scheme = BASIC;
if ( (auth->credentials.basic.user = _reserve_str(r, auth->credentials.basic.user)) == NULL
|| (auth->credentials.basic.password = _reserve_str(r, auth->credentials.basic.password)) == NULL
if ( !_reserve_str(r, &auth->credentials.basic.user, auth->credentials.basic.user)
|| !_reserve_str(r, &auth->credentials.basic.password, auth->credentials.basic.password)
)
return 0; // error
return 1;
@ -699,7 +774,7 @@ static int http_request_parse_path(struct http_request *r)
return 400;
}
_commit(r);
if ((r->path = _reserve(r, path)) == NULL)
if (!_reserve(r, &r->path, path))
return 0; // error
r->parser = http_request_parse_http_version;
return 0;
@ -718,12 +793,12 @@ static int http_request_parse_http_version(struct http_request *r)
// Parse HTTP version: HTTP/m.n followed by CRLF.
assert(r->version_major == 0);
assert(r->version_minor == 0);
unsigned major, minor;
uint32_t major, minor;
if (!( _skip_literal(r, "HTTP/")
&& _parse_uint(r, &major)
&& _parse_uint32(r, &major)
&& major > 0 && major < UINT8_MAX
&& _skip_literal(r, ".")
&& _parse_uint(r, &minor)
&& _parse_uint32(r, &minor)
&& minor < UINT8_MAX
&& _skip_eol(r)
)
@ -795,7 +870,7 @@ static int http_request_parse_header(struct http_request *r)
return r->handle_headers(r);
return 0;
}
const char *const nextline = r->cursor;
char *const nextline = r->cursor;
_rewind(r);
const char *const sol = r->cursor;
if (_skip_literal_nocase(r, "Content-Length:")) {
@ -927,7 +1002,7 @@ static int http_request_start_body(struct http_request *r)
DEBUGF("Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb);
return 400;
}
if (r->request_header.content_type.type) {
if (r->request_header.content_type.type[0]) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("Malformed HTTP %s request: Content-Type not allowed", r->verb);
return 400;
@ -940,7 +1015,7 @@ static int http_request_start_body(struct http_request *r)
DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb);
return 411;
}
if (r->request_header.content_type.type == NULL) {
if (r->request_header.content_type.type[0] == '\0') {
if (r->debug_flag && *r->debug_flag)
DEBUGF("Malformed HTTP %s request: missing Content-Type header", r->verb);
return 400;
@ -999,7 +1074,7 @@ static int _parse_content_disposition(struct http_request *r, struct mime_conten
return 0;
}
while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) {
const char *start = r->cursor;
char *start = r->cursor;
if (_skip_literal(r, "filename=")) {
size_t n = _parse_token_or_quoted_string(r, cd->filename, sizeof cd->filename);
if (n == 0)
@ -1059,7 +1134,33 @@ malformed:
return 1;
}
static void http_request_form_data_start_part(struct http_request *r, int b)
#define _HANDLER_RESULT(result) do { \
if (r->phase != RECEIVE) \
return 1; \
if (result) { \
assert((result) >= 400); \
assert((result) < 600); \
return (result); \
} \
} while (0)
#define _INVOKE_HANDLER_VOID(FUNC) do { \
if (r->form_data.FUNC) { \
if (r->debug_flag && *r->debug_flag) \
DEBUGF(#FUNC "()"); \
int result = r->form_data.FUNC(r); \
_HANDLER_RESULT(result); \
} \
} while (0)
#define _INVOKE_HANDLER_BUF_LEN(FUNC, START, END) do { \
if (r->form_data.FUNC && (START) != (END)) { \
if (r->debug_flag && *r->debug_flag) \
DEBUGF(#FUNC "(%s length=%zu)", alloca_toprint(50, (START), (END) - (START)), (END) - (START)); \
int result = r->form_data.FUNC(r, (START), (END) - (START)); \
_HANDLER_RESULT(result); \
} \
} while (0)
static int http_request_form_data_start_part(struct http_request *r, int b)
{
switch (r->form_data_state) {
case BODY:
@ -1073,11 +1174,7 @@ static void http_request_form_data_start_part(struct http_request *r, int b)
}
// fall through...
case HEADER:
if (r->form_data.handle_mime_part_end) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_part_end()");
r->form_data.handle_mime_part_end(r);
}
_INVOKE_HANDLER_VOID(handle_mime_part_end);
break;
default:
break;
@ -1087,13 +1184,10 @@ static void http_request_form_data_start_part(struct http_request *r, int b)
bzero(&r->part_header, sizeof r->part_header);
r->part_body_length = 0;
r->part_header.content_length = CONTENT_LENGTH_UNKNOWN;
if (r->form_data.handle_mime_part_start) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_part_start()");
r->form_data.handle_mime_part_start(r);
}
_INVOKE_HANDLER_VOID(handle_mime_part_start);
} else
r->form_data_state = EPILOGUE;
return 0;
}
/* If parsing completes (ie, parsed to end of epilogue), then sets r->parser to NULL and returns 0,
@ -1123,22 +1217,16 @@ static int http_request_parse_body_form_data(struct http_request *r)
case PREAMBLE: {
if (config.debug.httpd)
DEBUGF("PREAMBLE");
const char *start = r->parsed;
char *start = r->parsed;
for (; at_start || _skip_to_crlf(r); at_start = 0) {
const char *end_preamble = r->cursor;
int b;
if ((b = _skip_mime_boundary(r))) {
assert(end_preamble >= r->parsed);
if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_preamble(%s length=%zu)",
alloca_toprint(50, r->parsed, end_preamble - r->parsed), end_preamble - r->parsed);
r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
}
_INVOKE_HANDLER_BUF_LEN(handle_mime_preamble, r->parsed, end_preamble);
_rewind_crlf(r);
_commit(r);
http_request_form_data_start_part(r, b);
return 0;
return http_request_form_data_start_part(r, b);
}
}
if (_end_of_content(r)) {
@ -1148,12 +1236,8 @@ static int http_request_parse_body_form_data(struct http_request *r)
}
_rewind_optional_cr(r);
_commit(r);
if (r->parsed > start && r->form_data.handle_mime_preamble) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_preamble(%s length=%zu)",
alloca_toprint(50, start, r->parsed - start), r->parsed - start);
r->form_data.handle_mime_preamble(r, start, r->parsed - start);
}
assert(r->parsed >= start);
_INVOKE_HANDLER_BUF_LEN(handle_mime_preamble, start, r->parsed);
}
return 100; // need more data
case HEADER: {
@ -1170,7 +1254,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
_commit(r);
return 0;
}
const char *sol = r->cursor;
char *const sol = r->cursor;
// A blank line finishes the headers. The CRLF does not form part of the body.
if (_skip_crlf(r)) {
_commit(r);
@ -1181,9 +1265,9 @@ static int http_request_parse_body_form_data(struct http_request *r)
alloca_mime_content_type(&r->part_header.content_type),
alloca_mime_content_disposition(&r->part_header.content_disposition)
);
r->form_data.handle_mime_part_header(r, &r->part_header);
int result = r->form_data.handle_mime_part_header(r, &r->part_header);
_HANDLER_RESULT(result); \
}
r->form_data_state = BODY;
return 0;
}
@ -1198,8 +1282,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
_commit(r);
// A boundary in the middle of headers finishes the current part and starts a new part.
// An end boundary terminates the current part and starts the epilogue.
http_request_form_data_start_part(r, b);
return 0;
return http_request_form_data_start_part(r, b);
}
if (_run_out(r))
return 100; // read more and try again
@ -1284,23 +1367,20 @@ static int http_request_parse_body_form_data(struct http_request *r)
case BODY:
if (config.debug.httpd)
DEBUGF("BODY");
const char *start = r->parsed;
char *start = r->parsed;
while (_skip_to_crlf(r)) {
int b;
const char *end_body = r->cursor;
char *end_body = r->cursor;
_skip_crlf(r);
if ((b = _skip_mime_boundary(r))) {
_rewind_crlf(r);
_commit(r);
assert(end_body >= start);
r->part_body_length += end_body - start;
if (end_body > start && r->form_data.handle_mime_body) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, end_body - start), end_body - start);
r->form_data.handle_mime_body(r, start, end_body - start); // excluding CRLF at end
}
http_request_form_data_start_part(r, b);
return 0;
// Note: the handler function may modify the data in-place (eg, Rhizome does encryption
// that way).
_INVOKE_HANDLER_BUF_LEN(handle_mime_body, start, end_body); // excluding CRLF at end
return http_request_form_data_start_part(r, b);
}
}
if (_end_of_content(r)) {
@ -1312,27 +1392,22 @@ static int http_request_parse_body_form_data(struct http_request *r)
_commit(r);
assert(r->parsed >= start);
r->part_body_length += r->parsed - start;
if (r->parsed > start && r->form_data.handle_mime_body) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, r->parsed - start), r->parsed - start);
r->form_data.handle_mime_body(r, start, r->parsed - start);
}
// Note: the handler function may modify the data in-place
_INVOKE_HANDLER_BUF_LEN(handle_mime_body, start, r->parsed);
return 100; // need more data
case EPILOGUE:
if (config.debug.httpd)
DEBUGF("EPILOGUE");
r->cursor = r->end;
if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("handle_mime_epilogue(%s length=%zu)",
alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed);
}
assert(r->cursor >= r->parsed);
_INVOKE_HANDLER_BUF_LEN(handle_mime_epilogue, r->parsed, r->cursor);
_commit(r);
assert(_run_out(r));
if (_end_of_content(r))
return 0; // done
return 100; // need more data
default:
FATALF("form_data_state = %d", r->form_data_state);
}
abort(); // not reached
}
@ -1423,7 +1498,7 @@ static void http_request_receive(struct http_request *r)
result = 500;
}
} else {
HTTP_REQUEST_PARSER oldparser = r->parser;
HTTP_REQUEST_PARSER *oldparser = r->parser;
const char *oldparsed = r->parsed;
if (r->parser == NULL) {
if (r->debug_flag && *r->debug_flag)
@ -1446,9 +1521,10 @@ static void http_request_receive(struct http_request *r)
result = 500;
}
}
if (result >= 300)
if (result >= 200 && result < 600) {
assert(r->response.result_code == 0 || r->response.result_code == result);
r->response.result_code = result;
else if (result) {
} else if (result) {
if (r->debug_flag && *r->debug_flag)
DEBUGF("Internal failure parsing HTTP request: invalid result=%d", result);
r->response.result_code = 500;
@ -1462,8 +1538,10 @@ static void http_request_receive(struct http_request *r)
return;
}
}
if (r->phase != RECEIVE)
if (r->phase != RECEIVE) {
assert(r->response.result_code != 0);
return;
}
if (r->response.result_code == 0) {
WHY("No HTTP response set, using 500 Server Error");
r->response.result_code = 500;
@ -1694,6 +1772,8 @@ static const char *httpResultString(int response_code)
case 403: return "Forbidden";
case 404: return "Not Found";
case 405: return "Method Not Allowed";
case 408: return "Request Timeout";
case 409: return "Conflict";
case 411: return "Length Required";
case 414: return "Request-URI Too Long";
case 415: return "Unsupported Media Type";
@ -1804,6 +1884,9 @@ static int _render_response(struct http_request *r)
strbuf_append_quoted_string(sb, hr.header.www_authenticate.realm);
strbuf_puts(sb, "\r\n");
}
if (r->render_extra_headers)
r->render_extra_headers(r, sb);
assert(strcmp(strbuf_substr(sb, -2), "\r\n") == 0);
strbuf_puts(sb, "\r\n");
if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN)
r->response_length = strbuf_count(sb) + hr.header.content_length;
@ -1866,6 +1949,7 @@ static size_t http_request_drain(struct http_request *r)
static void http_request_start_response(struct http_request *r)
{
assert(r->phase == RECEIVE);
_release_reserved(r);
if (r->response.content || r->response.content_generator) {
assert(r->response.header.content_type != NULL);
assert(r->response.header.content_type[0]);

37
http_server.h
View File

@ -129,12 +129,16 @@ struct mime_part_headers {
};
struct http_mime_handler {
void (*handle_mime_preamble)(struct http_request *, const char *, size_t);
void (*handle_mime_part_start)(struct http_request *);
void (*handle_mime_part_header)(struct http_request *, const struct mime_part_headers *);
void (*handle_mime_body)(struct http_request *, const char *, size_t);
void (*handle_mime_part_end)(struct http_request *);
void (*handle_mime_epilogue)(struct http_request *, const char *, size_t);
// All these functions may abort the request processing by returning an HTTP
// filure status code in the range 400-599 or by initiating an HTTP response
// directly (changing the phase from RECEIVE to TRANSMIT). They can return
// zero to indicate that parsing should proceed.
int (*handle_mime_preamble)(struct http_request *, char *, size_t);
int (*handle_mime_part_start)(struct http_request *);
int (*handle_mime_part_header)(struct http_request *, const struct mime_part_headers *);
int (*handle_mime_body)(struct http_request *, char *, size_t);
int (*handle_mime_part_end)(struct http_request *);
int (*handle_mime_epilogue)(struct http_request *, char *, size_t);
};
struct http_request;
@ -148,7 +152,8 @@ void http_request_response_static(struct http_request *r, int result, const char
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
typedef int HTTP_REQUEST_PARSER(struct http_request *);
typedef void HTTP_RENDERER(struct http_request *, strbuf);
struct http_request {
struct sched_ent alarm; // MUST BE FIRST ELEMENT
@ -173,17 +178,18 @@ struct http_request {
struct http_request_headers request_header;
// Parsing is done by setting 'parser' to point to a series of parsing
// functions as the parsing state progresses.
HTTP_REQUEST_PARSER parser; // current parser function
HTTP_REQUEST_PARSER *parser; // current parser function
// The caller may set these up, and they are invoked by the parser as request
// parsing reaches different stages.
HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
HTTP_REQUEST_PARSER handle_headers; // called after all HTTP headers are parsed
HTTP_REQUEST_PARSER handle_content_end; // called after all content is received
HTTP_REQUEST_PARSER *handle_first_line; // called after first line is parsed
HTTP_REQUEST_PARSER *handle_headers; // called after all HTTP headers are parsed
HTTP_REQUEST_PARSER *handle_content_end; // called after all content is received
// The following are used for managing the buffer during RECEIVE phase.
const char *received; // start of received data in buffer[]
const char *end; // end of received data in buffer[]
const char *parsed; // start of unparsed data in buffer[]
const char *cursor; // for parsing
char *reserved; // end of reserved data in buffer[]
char *received; // start of received data in buffer[]
char *end; // end of received data in buffer[]
char *parsed; // start of unparsed data in buffer[]
char *cursor; // for parsing
http_size_t request_content_remaining;
// The following are used for parsing a multipart body.
enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
@ -193,6 +199,7 @@ struct http_request {
// The following are used for constructing the response that will be sent in
// TRANSMIT phase.
struct http_response response;
HTTP_RENDERER *render_extra_headers;
// The following are used during TRANSMIT phase to control buffering and
// sending.
http_size_t response_length; // total response bytes (header + content)

4
keyring.c
View File

@ -2083,9 +2083,7 @@ struct nm_record nm_cache[NM_CACHE_SLOTS];
unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp)
{
IN();
if (!known_sidp) { RETURNNULL(WHYNULL("known pub key is null")); }
if (!unknown_sidp) { RETURNNULL(WHYNULL("unknown pub key is null")); }
if (!keyring) { RETURNNULL(WHYNULL("keyring is null")); }
assert(keyring != NULL);
/* See if we have it cached already */
unsigned i;

127
meshms.c
View File

@ -108,6 +108,7 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
assert(m->haveSecret);
if (m->haveSecret == NEW_BUNDLE_ID) {
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
rhizome_manifest_set_name(m, "");
if (rhizome_fill_manifest(m, NULL, my_sidp) == -1)
return WHY("Invalid manifest");
if (config.debug.meshms) {
@ -261,17 +262,17 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome
DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid));
if (rhizome_retrieve_manifest(bid, m))
return -1;
int ret = rhizome_open_decrypt_read(m, &ply->read);
if (ret == 1)
enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &ply->read);
if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW)
WARNF("Payload was not found for manifest %s, %"PRIu64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (ret != 0)
return ret;
if (pstatus != RHIZOME_PAYLOAD_STATUS_STORED && pstatus != RHIZOME_PAYLOAD_STATUS_EMPTY)
return -1;
assert(m->filesize != RHIZOME_SIZE_UNSET);
ply->read.offset = ply->read.length = m->filesize;
return 0;
}
static int ply_read_close(struct ply_read *ply)
static void ply_read_close(struct ply_read *ply)
{
if (ply->buffer){
free(ply->buffer);
@ -279,7 +280,7 @@ static int ply_read_close(struct ply_read *ply)
}
ply->buffer_size=0;
ply->buff.len=0;
return rhizome_read_close(&ply->read);
rhizome_read_close(&ply->read);
}
// read the next record from the ply (backwards)
@ -366,14 +367,36 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv,
assert(m->haveSecret);
assert(m->authorship == AUTHOR_AUTHENTIC);
if (rhizome_append_journal_buffer(m, 0, buffer, len))
enum rhizome_payload_status pstatus = rhizome_append_journal_buffer(m, 0, buffer, len);
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
goto end;
if (rhizome_manifest_finalise(m, &mout, 1))
goto end;
ret=0;
enum rhizome_bundle_status status = rhizome_manifest_finalise(m, &mout, 1);
switch (status) {
case RHIZOME_BUNDLE_STATUS_ERROR:
// error is already logged
break;
case RHIZOME_BUNDLE_STATUS_NEW:
ret = 0;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
WHYF("MeshMS ply manifest (version=%"PRIu64") gazumped by Rhizome store (version=%"PRIu64")",
m->version, mout->version);
break;
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
WHYF("MeshMS ply manifest not consistent with payload");
break;
case RHIZOME_BUNDLE_STATUS_FAKE:
WHYF("MeshMS ply manifest is not signed");
break;
case RHIZOME_BUNDLE_STATUS_INVALID:
WHYF("MeshMS ply manifest is invalid");
break;
default:
FATALF("status=%d", status);
}
end:
if (mout && mout!=m)
rhizome_manifest_free(mout);
@ -384,7 +407,8 @@ end:
// update if any conversations are unread or need to be acked.
// return -1 for failure, 1 if the conversation index needs to be saved.
static int update_conversation(const sid_t *my_sid, struct conversations *conv){
static int update_conversation(const sid_t *my_sid, struct conversations *conv)
{
if (config.debug.meshms)
DEBUG("Checking if conversation needs to be acked");
@ -488,7 +512,8 @@ end:
}
// update conversations, and return 1 if the conversation index should be saved
static int update_conversations(const sid_t *my_sid, struct conversations *conv){
static int update_conversations(const sid_t *my_sid, struct conversations *conv)
{
if (!conv)
return 0;
int ret = 0;
@ -518,13 +543,13 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
struct rhizome_read_buffer buff;
bzero(&buff, sizeof(buff));
int ret = rhizome_open_decrypt_read(m, &read);
if (ret == -1)
int ret = -1;
enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &read);
if (pstatus != RHIZOME_PAYLOAD_STATUS_STORED)
goto end;
unsigned char version=0xFF;
ssize_t r = rhizome_read_buffered(&read, &buff, &version, 1);
ret = -1;
if (r == -1)
goto end;
if (version != 1) {
@ -592,12 +617,13 @@ static ssize_t write_conversation(struct rhizome_write *write, struct conversati
len+=measure_packed_uint(conv->read_offset);
len+=measure_packed_uint(conv->their_size);
}
DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %zu",
alloca_tohex_sid_t(conv->them),
conv->their_last_message,
conv->read_offset,
conv->their_size,
len);
if (config.debug.meshms)
DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %zu",
alloca_tohex_sid_t(conv->them),
conv->their_last_message,
conv->read_offset,
conv->their_size,
len);
}
// write the two child nodes
ssize_t ret = write_conversation(write, conv->_left);
@ -629,21 +655,46 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
// then write it
rhizome_manifest_set_version(m, m->version + 1);
rhizome_manifest_set_filesize(m, (size_t)len + 1);
if (rhizome_write_open_manifest(&write, m) == -1)
goto end;
unsigned char version=1;
if (rhizome_write_buffer(&write, &version, 1) == -1)
goto end;
if (write_conversation(&write, conv) == -1)
goto end;
if (rhizome_finish_write(&write))
goto end;
rhizome_manifest_set_filehash(m, &write.id);
if (rhizome_manifest_finalise(m, &mout, 1))
goto end;
ret=0;
rhizome_manifest_set_filehash(m, NULL);
enum rhizome_payload_status pstatus = rhizome_write_open_manifest(&write, m);
if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) {
unsigned char version=1;
if (rhizome_write_buffer(&write, &version, 1) == -1)
goto end;
if (write_conversation(&write, conv) == -1)
goto end;
pstatus = rhizome_finish_write(&write);
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
goto end;
rhizome_manifest_set_filehash(m, &write.id);
}
enum rhizome_bundle_status status = rhizome_manifest_finalise(m, &mout, 1);
switch (status) {
case RHIZOME_BUNDLE_STATUS_ERROR:
// error is already logged
break;
case RHIZOME_BUNDLE_STATUS_NEW:
ret = 0;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
WHYF("MeshMS conversation manifest (version=%"PRIu64") gazumped by Rhizome store (version=%"PRIu64")",
m->version, mout->version);
break;
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
WHY("MeshMS conversation manifest not consistent with payload");
break;
case RHIZOME_BUNDLE_STATUS_FAKE:
WHY("MeshMS conversation manifest is not signed");
break;
case RHIZOME_BUNDLE_STATUS_INVALID:
WHY("MeshMS conversation manifest is invalid");
break;
default:
FATALF("status=%d", status);
}
end:
if (ret)
rhizome_fail_write(&write);

21
os.c
View File

@ -162,3 +162,24 @@ ssize_t read_symlink(const char *path, char *buf, size_t len)
buf[nr] = '\0';
return nr;
}
ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_size)
{
int fd = open(path, O_RDONLY);
if (fd == -1)
return WHYF_perror("open(%d,%s,O_RDONLY)", fd, alloca_str_toprint(path));
ssize_t ret;
struct stat stat;
if (fstat(fd, &stat) == -1)
ret = WHYF_perror("fstat(%d)", fd);
else if ((size_t)stat.st_size > buffer_size)
ret = WHYF("file %s (size %zu) is larger than available buffer (%zu)", alloca_str_toprint(path), (size_t)stat.st_size, buffer_size);
else {
ret = read(fd, buffer, buffer_size);
if (ret == -1)
ret = WHYF_perror("read(%d,%s,%zu)", fd, alloca_str_toprint(path), buffer_size);
}
if (close(fd) == -1)
ret = WHYF_perror("close(%d)", fd);
return ret;
}

8
os.h
View File

@ -125,4 +125,12 @@ int urandombytes(unsigned char *buf, size_t len);
*/
ssize_t read_symlink(const char *path, char *buf, size_t len);
/* Read the whole file into the given buffer. If the file will not fit into
* the buffer or if there is an error opening or reading the file, logs an
* error and returns -1. Otherwise, returns the number of bytes read.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_size);
#endif //__SERVAL_DNA__OS_H

6
overlay_mdp_services.c
View File

@ -90,14 +90,14 @@ int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, ui
write_uint64(&reply.out.payload[1+16+8], offset);
int bytes_read = rhizome_read_cached(bid, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength);
ssize_t bytes_read = rhizome_read_cached(bid, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength);
if (bytes_read<=0)
break;
reply.out.payload_length=1+16+8+8+bytes_read;
reply.out.payload_length=1+16+8+8+(size_t)bytes_read;
// Mark the last block of the file, if required
if (bytes_read < blockLength)
if ((size_t)bytes_read < blockLength)
reply.out.payload[0]='T';
// send packet

228
rhizome.c
View File

@ -93,32 +93,28 @@ int rhizome_fetch_delay_ms()
}
/* Import a bundle from a pair of files, one containing the manifest and the optional other
containing the payload. The logic is all in rhizome_bundle_import(). This function just wraps
that function and manages file and object buffers and lifetimes.
*/
int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath)
* containing the payload. The work is all done by rhizome_bundle_import() and
* rhizome_store_manifest().
*/
enum rhizome_bundle_status rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **mout, const char *manifest_path, const char *filepath)
{
if (config.debug.rhizome)
DEBUGF("(manifest_path=%s, filepath=%s)",
manifest_path ? alloca_str_toprint(manifest_path) : "NULL",
filepath ? alloca_str_toprint(filepath) : "NULL");
unsigned char buffer[MAX_MANIFEST_BYTES];
size_t buffer_len = 0;
int ret = 0;
// manifest has been appended to the end of the file.
if (strcmp(manifest_path, filepath)==0){
unsigned char marker[4];
int ret=0;
FILE *f = fopen(filepath, "r");
if (f == NULL)
return WHYF_perror("Could not open manifest file %s for reading.", filepath);
if (fseek(f, -sizeof(marker), SEEK_END))
ret=WHY_perror("Unable to seek to end of file");
if (ret==0){
ret = fread(marker, 1, sizeof(marker), f);
if (ret==sizeof(marker))
@ -126,86 +122,62 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
else
ret=WHY_perror("Unable to read end of manifest marker");
}
if (ret==0){
if (marker[2]!=0x41 || marker[3]!=0x10)
ret=WHYF("Expected 0x4110 marker at end of file");
}
if (ret==0){
buffer_len = read_uint16(marker);
if (buffer_len < 1 || buffer_len > MAX_MANIFEST_BYTES)
ret=WHYF("Invalid manifest length %zu", buffer_len);
}
if (ret==0){
if (fseek(f, -(buffer_len+sizeof(marker)), SEEK_END))
ret=WHY_perror("Unable to seek to end of file");
}
if (ret==0){
ssize_t nread = fread(buffer, 1, buffer_len, f);
if ((size_t)nread != buffer_len)
ret=WHY_perror("Unable to read manifest contents");
if (ret == 0 && fread(m->manifestdata, buffer_len, 1, f) != 1) {
if (ferror(f))
ret = WHYF("fread(%p,%zu,1,%s) error", m->manifestdata, buffer_len, alloca_str_toprint(filepath));
else if (feof(f))
ret = WHYF("fread(%p,%zu,1,%s) hit end of file", m->manifestdata, buffer_len, alloca_str_toprint(filepath));
}
fclose(f);
if (ret)
return ret;
manifest_path=(char*)buffer;
} else {
ssize_t size = read_whole_file(manifest_path, m->manifestdata, sizeof m->manifestdata);
if (size == -1)
ret = -1;
buffer_len = (size_t) size;
}
if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1)
return WHY("could not read manifest file");
if (!rhizome_manifest_validate(m))
return WHY("manifest is invalid");
if (!rhizome_manifest_verify(m))
return WHY("could not verify manifest");
/* Do we already have this manifest or newer? */
uint64_t dbVersion = 0;
if (sqlite_exec_uint64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1)
return WHY("Select failure");
if (dbVersion >= m->version)
return 2;
int status = rhizome_import_file(m, filepath);
if (status<0)
return status;
return rhizome_add_manifest(m, 1);
}
int rhizome_manifest_check_sanity(rhizome_manifest *m)
{
/* Ensure manifest meets basic sanity checks. */
int ret = 0;
if (m->version == 0)
ret = WHY("Manifest must have a version number");
if (m->filesize == RHIZOME_SIZE_UNSET)
ret = WHY("Manifest missing 'filesize' field");
else if (m->filesize && rhizome_filehash_t_is_zero(m->filehash))
ret = WHY("Manifest 'filehash' field has not been set");
if (m->service == NULL)
ret = WHY("Manifest missing 'service' field");
else if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
if (m->name == NULL)
ret = WHY("Manifest with service='" RHIZOME_SERVICE_FILE "' missing 'name' field");
} else if (strcasecmp(m->service, RHIZOME_SERVICE_MESHMS) == 0
|| strcasecmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) {
if (!m->has_sender)
ret = WHYF("Manifest with service='%s' missing 'sender' field", m->service);
if (!m->has_recipient)
ret = WHYF("Manifest with service='%s' missing 'recipient' field", m->service);
if (ret)
return ret;
m->manifest_all_bytes = buffer_len;
if ( rhizome_manifest_parse(m) == -1
|| !rhizome_manifest_validate(m)
|| !rhizome_manifest_verify(m)
)
return RHIZOME_BUNDLE_STATUS_INVALID;
enum rhizome_bundle_status status = rhizome_manifest_check_stored(m, mout);
if (status == RHIZOME_BUNDLE_STATUS_NEW) {
enum rhizome_payload_status pstatus = rhizome_import_payload_from_file(m, filepath);
switch (pstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_NEW:
if (rhizome_store_manifest(m) == -1)
return -1;
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
return -1;
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
status = RHIZOME_BUNDLE_STATUS_INCONSISTENT;
break;
default:
FATALF("pstatus = %d", pstatus);
}
}
else if (!rhizome_str_is_manifest_service(m->service))
ret = WHYF("Manifest invalid 'service' field %s", alloca_str_toprint(m->service));
if (!m->has_date)
ret = WHY("Manifest missing 'date' field");
return ret;
return status;
}
/* Sets the bundle key "BK" field of a manifest. Returns 1 if the field was set, 0 if not.
@ -278,50 +250,90 @@ int rhizome_manifest_add_bundle_key(rhizome_manifest *m)
RETURN(0);
}
int rhizome_add_manifest(rhizome_manifest *m, int ttl)
/* Test the status of a given manifest 'm' (id, version) with respect to the Rhizome store, and
* return a code which indicates whether 'm' should be stored or not, setting *mout to 'm' or
* to point to a newly allocated manifest. The caller is responsible for freeing *mout if *mout !=
* m. If the caller passes mout==NULL then no new manifest is allocated.
*
* - If the store contains no manifest with the given id, sets *mout = m and returns
* RHIZOME_BUNDLE_STATUS_NEW, ie, the manifest 'm' should be stored.
*
* - If the store contains a manifest with the same id and an older version, sets *mout to the
* stored manifest and returns RHIZOME_BUNDLE_STATUS_NEW, ie, the manifest 'm' should be
* stored.
*
* - If the store contains a manifest with the same id and version, sets *mout to the stored
* manifest and returns RHIZOME_BUNDLE_STATUS_SAME. The caller must compare *m and *mout, and
* if they are not identical, must decide what to do.
*
* - If the store contains a manifest with the same id and a later version, sets *mout to the
* stored manifest and returns RHIZOME_BUNDLE_STATUS_OLD, ie, the manifest 'm' should NOT be
* stored.
*
* - If there is an error querying the Rhizome store or allocating a new manifest structure, logs
* an error and returns -1.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
enum rhizome_bundle_status rhizome_manifest_check_stored(rhizome_manifest *m, rhizome_manifest **mout)
{
if (config.debug.rhizome)
DEBUGF("rhizome_add_manifest(m=%p, ttl=%d)",m, ttl);
if (m->finalised==0)
return WHY("Manifest must be finalised before being stored");
/* Store time to live, clamped to within legal range */
m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
if (rhizome_manifest_check_sanity(m))
assert(m->has_id);
assert(m->version != 0);
rhizome_manifest *stored_m = rhizome_new_manifest();
if (stored_m == NULL)
return -1;
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0 && !rhizome_exists(&m->filehash))
return WHY("File has not been imported");
/* If the manifest already has an ID */
if (rhizome_bid_t_is_zero(m->cryptoSignPublic))
return WHY("Manifest does not have an ID");
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
uint64_t storedversion = -1;
switch (sqlite_exec_uint64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END)) {
int n = rhizome_retrieve_manifest(&m->cryptoSignPublic, stored_m);
switch (n) {
case -1:
return WHY("Select failed");
case 0:
if (config.debug.rhizome) DEBUG("No existing manifest");
break;
rhizome_manifest_free(stored_m);
return -1;
case 1:
if (config.debug.rhizome)
DEBUGF("Found existing version=%"PRIu64", new version=%"PRIu64, storedversion, m->version);
if (m->version < storedversion)
return WHY("Newer version exists");
if (m->version == storedversion)
return WHYF("Already have %s:%"PRIu64", not adding", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (config.debug.rhizome)
DEBUGF("No stored manifest with id=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
rhizome_manifest_free(stored_m);
if (mout)
*mout = m;
return RHIZOME_BUNDLE_STATUS_NEW;
case 0:
break;
default:
return WHY("Select found too many rows!");
FATALF("rhizome_retrieve_manifest() returned %d", n);
}
if (mout)
*mout = stored_m;
else
rhizome_manifest_free(stored_m);
enum rhizome_bundle_status result = RHIZOME_BUNDLE_STATUS_NEW;
const char *what = "newer than";
if (m->version < stored_m->version) {
result = RHIZOME_BUNDLE_STATUS_OLD;
what = "older than";
}
if (m->version == stored_m->version) {
return RHIZOME_BUNDLE_STATUS_SAME;
what = "same as";
}
if (config.debug.rhizome)
DEBUGF("Bundle %s:%"PRIu64" is %s stored version %"PRIu64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, what, stored_m->version);
return result;
}
/* Okay, it is written, and can be put directly into the rhizome database now */
return rhizome_store_bundle(m);
enum rhizome_bundle_status rhizome_add_manifest(rhizome_manifest *m, rhizome_manifest **mout)
{
if (config.debug.rhizome)
DEBUGF("rhizome_add_manifest(m=manifest[%d](%p), mout=%p)", m->manifest_record_number, m, mout);
if (!m->finalised && !rhizome_manifest_validate(m))
return RHIZOME_BUNDLE_STATUS_INVALID;
assert(m->finalised);
if (!m->selfSigned && !rhizome_manifest_verify(m))
return RHIZOME_BUNDLE_STATUS_FAKE;
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0 && !rhizome_exists(&m->filehash))
return WHY("Payload has not been stored");
enum rhizome_bundle_status status = rhizome_manifest_check_stored(m, mout);
if (status == RHIZOME_BUNDLE_STATUS_NEW && rhizome_store_manifest(m) == -1)
return -1;
return status;
}
/* When voice traffic is being carried, we need to throttle Rhizome down

196
rhizome.h
View File

@ -111,6 +111,7 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) {
#define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary)
int cmp_rhizome_bk_t(const rhizome_bk_t *a, const rhizome_bk_t *b);
int str_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex);
int strn_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex, const char **endp);
extern time_ms_t rhizome_voice_timeout;
@ -223,6 +224,10 @@ typedef struct rhizome_manifest
*/
bool_t has_id;
/* Set if the filehash field contains a file hash.
*/
bool_t has_filehash;
/* Set if the tail field is valid, ie, the bundle is a journal.
*/
bool_t is_journal;
@ -256,9 +261,6 @@ typedef struct rhizome_manifest
AUTHOR_AUTHENTIC // a local identity is the verified author
} authorship;
/* time-to-live in hops of this manifest. */
int ttl;
int fileHighestPriority;
/* Absolute path of the file associated with the manifest */
@ -450,12 +452,36 @@ struct rhizome_manifest_summary {
int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ);
enum rhizome_bundle_status {
RHIZOME_BUNDLE_STATUS_ERROR = -1,
RHIZOME_BUNDLE_STATUS_NEW = 0, // bundle is newer than store
RHIZOME_BUNDLE_STATUS_SAME = 1, // same version already in store
RHIZOME_BUNDLE_STATUS_DUPLICATE = 2, // equivalent bundle already in store
RHIZOME_BUNDLE_STATUS_OLD = 3, // newer version already in store
RHIZOME_BUNDLE_STATUS_INVALID = 4, // manifest is invalid
RHIZOME_BUNDLE_STATUS_FAKE = 5, // manifest signature not valid
RHIZOME_BUNDLE_STATUS_INCONSISTENT = 6, // manifest filesize/filehash does not match supplied payload
};
enum rhizome_payload_status {
RHIZOME_PAYLOAD_STATUS_ERROR = -1,
RHIZOME_PAYLOAD_STATUS_EMPTY = 0, // payload is empty (zero length)
RHIZOME_PAYLOAD_STATUS_NEW = 1, // payload is not yet in store
RHIZOME_PAYLOAD_STATUS_STORED = 2, // payload is already in store
RHIZOME_PAYLOAD_STATUS_WRONG_SIZE = 3, // payload's size does not match manifest
RHIZOME_PAYLOAD_STATUS_WRONG_HASH = 4, // payload's hash does not match manifest
RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL = 5, // cannot encrypt/decrypt (payload key unknown)
};
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize);
int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_validate(rhizome_manifest *m);
int rhizome_manifest_parse(rhizome_manifest *m);
int rhizome_manifest_verify(rhizome_manifest *m);
int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m);
@ -463,12 +489,12 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m);
rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence);
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
int rhizome_manifest_pack_variables(rhizome_manifest *m);
int rhizome_store_bundle(rhizome_manifest *m);
int rhizome_store_manifest(rhizome_manifest *m);
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath);
int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath);
int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath);
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp);
int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *);
@ -478,11 +504,9 @@ void rhizome_find_bundle_author_and_secret(rhizome_manifest *m);
int rhizome_lookup_author(rhizome_manifest *m);
void rhizome_authenticate_author(rhizome_manifest *m);
int rhizome_manifest_verify(rhizome_manifest *m);
int rhizome_manifest_check_sanity(rhizome_manifest *m_in);
int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate);
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl);
enum rhizome_bundle_status rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **m_out, int deduplicate);
enum rhizome_bundle_status rhizome_manifest_check_stored(rhizome_manifest *m, rhizome_manifest **m_out);
enum rhizome_bundle_status rhizome_add_manifest(rhizome_manifest *m_in, rhizome_manifest **m_out);
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount);
int rhizome_find_privatekey(rhizome_manifest *m);
@ -542,6 +566,27 @@ int _sqlite_exec_uint64_retry(struct __sourceloc, sqlite_retry_state *retry, uin
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqltext, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap);
int _sqlite_blob_open_retry(
struct __sourceloc,
int log_level,
sqlite_retry_state *retry,
const char *dbname,
const char *tablename,
const char *colname,
sqlite3_int64 rowid,
int flags,
sqlite3_blob **blobp
);
int _sqlite_blob_write_retry(
struct __sourceloc,
int log_level,
sqlite_retry_state *retry,
sqlite3_blob *blob,
const void *buf,
int len,
int offset
);
int _sqlite_blob_close(struct __sourceloc, int log_level, sqlite3_blob *blob);
// The 'arg' arguments in the following macros appear to be unnecessary, but
// they serve a very useful purpose, so don't remove them! They ensure that
@ -568,11 +613,15 @@ int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, st
#define sqlite_exec_uint64_retry(rs,res,sql,arg,...) _sqlite_exec_uint64_retry(__WHENCE__, (rs), (res), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,sql,arg,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__)
#define sqlite_blob_open_retry(rs,db,table,col,row,flags,blobp) \
_sqlite_blob_open_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (db), (table), (col), (row), (flags), (blobp))
#define sqlite_blob_close(blob) _sqlite_blob_close(__WHENCE__, LOG_LEVEL_ERROR, (blob));
#define sqlite_blob_write_retry(rs,blob,buf,siz,off) _sqlite_blob_write_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (blob), (buf), (siz), (off))
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
int rhizome_update_file_priority(const char *fileid);
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
uint64_t rhizome_bar_version(const unsigned char *bar);
uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar);
@ -731,20 +780,67 @@ typedef struct rhizome_http_request
*/
unsigned int uuid;
/* For receiving a POST multipart form:
/* For requests/responses that pertain to a single manifest.
*/
// Which part is currently being received
enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
// Temporary file currently current part is being written to
int part_fd;
// Which parts have already been received
bool_t received_manifest;
bool_t received_data;
// Name of data file supplied in part's Content-Disposition header, filename
// parameter (if any)
char data_file_name[MIME_FILENAME_MAXLEN + 1];
rhizome_manifest *manifest;
/* Finaliser for union contents (below).
*/
void (*finalise_union)(struct rhizome_http_request *);
/* Mutually exclusive response arguments.
*/
union {
/* For receiving Rhizome Direct import request
*/
struct {
// Which part is currently being received
const char *current_part;
// Temporary file currently current part is being written to
int part_fd;
// Which parts have already been received
bool_t received_manifest;
bool_t received_data;
// Name of data file supplied in part's Content-Disposition header, filename
// parameter (if any)
char data_file_name[MIME_FILENAME_MAXLEN + 1];
}
direct_import;
/* For receiving RESTful Rhizome insert request
*/
struct {
// Which part is currently being received
const char *current_part;
// Which parts have already been received
bool_t received_author;
bool_t received_secret;
bool_t received_manifest;
bool_t received_payload;
// For storing the "bundle-author" hex SID as we receive it
char author_hex[SID_STRLEN];
size_t author_hex_len;
sid_t author;
// For storing the "bundle-secret" hex as we receive it
char secret_hex[RHIZOME_BUNDLE_KEY_STRLEN];
size_t secret_hex_len;
rhizome_bk_t bundle_secret;
// The "force-new" parameter
char force_new_text[5]; // enough for "false"
size_t force_new_text_len;
bool_t force_new;
// For storing the manifest text (malloc/realloc) as we receive it
char *manifest_text;
size_t manifest_text_size;
size_t manifest_len;
// For receiving the payload
enum rhizome_payload_status payload_status;
uint64_t payload_size;
struct rhizome_write write;
}
insert;
/* For responses that send part or all of a payload.
*/
struct rhizome_read read_state;
@ -752,14 +848,16 @@ typedef struct rhizome_http_request
/* For responses that list manifests.
*/
struct {
enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase;
uint64_t rowid_highest;
size_t rowcount;
time_ms_t end_time;
struct rhizome_list_cursor cursor;
} list;
enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase;
uint64_t rowid_highest;
size_t rowcount;
time_ms_t end_time;
struct rhizome_list_cursor cursor;
}
list;
} u;
} rhizome_http_request;
int rhizome_received_content(const unsigned char *bidprefix,uint64_t version,
@ -917,34 +1015,34 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
/* rhizome storage methods */
int rhizome_exists(const rhizome_filehash_t *hashp);
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size);
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
int rhizome_write_file(struct rhizome_write *write, const char *filename);
int rhizome_fail_write(struct rhizome_write *write);
int rhizome_finish_write(struct rhizome_write *write);
int rhizome_import_file(rhizome_manifest *m, const char *filepath);
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length);
int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
void rhizome_fail_write(struct rhizome_write *write);
enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write);
enum rhizome_payload_status rhizome_import_payload_from_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length);
enum rhizome_payload_status rhizome_stat_payload_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, const char *filepath);
int rhizome_derive_payload_key(rhizome_manifest *m);
int rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len);
int rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename);
int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
enum rhizome_payload_status rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len);
enum rhizome_payload_status rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename);
enum rhizome_payload_status rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset,
const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
enum rhizome_payload_status rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
ssize_t rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
int rhizome_read_close(struct rhizome_read *read);
int rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath);
int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, uint64_t *lengthp);
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, size_t length);
void rhizome_read_close(struct rhizome_read *read);
enum rhizome_payload_status rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state);
enum rhizome_payload_status rhizome_extract_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, uint64_t *lengthp);
ssize_t rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, size_t length);
int rhizome_cache_close();
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);

251
rhizome_bundle.c
View File

@ -63,7 +63,6 @@ static int _rhizome_manifest_del(struct __sourceloc __whence, rhizome_manifest *
free((char *) m->vars[i]);
free((char *) m->values[i]);
--m->var_count;
m->finalised = 0;
ret = 1;
break;
}
@ -90,7 +89,6 @@ static const char *_rhizome_manifest_set(struct __sourceloc __whence, rhizome_ma
return NULL;
free((char *)m->values[i]);
m->values[i] = ret;
m->finalised = 0;
return ret;
}
if (m->var_count >= NELS(m->vars))
@ -104,7 +102,6 @@ static const char *_rhizome_manifest_set(struct __sourceloc __whence, rhizome_ma
return NULL;
}
m->var_count++;
m->finalised = 0;
return ret;
}
@ -119,9 +116,8 @@ void _rhizome_manifest_set_id(struct __sourceloc __whence, rhizome_manifest *m,
{
const char *v = rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(*bidp));
assert(v); // TODO: remove known manifest fields from vars[]
if (bidp != &m->cryptoSignPublic && cmp_rhizome_bid_t(&m->cryptoSignPublic, bidp) != 0) {
m->cryptoSignPublic = *bidp;
// The BID just changed, so the secret key and bundle key are no longer valid.
// If the BID is changed, the secret key and bundle key are no longer valid.
if (m->has_id && bidp != &m->cryptoSignPublic && cmp_rhizome_bid_t(&m->cryptoSignPublic, bidp) != 0) {
if (m->haveSecret) {
m->haveSecret = SECRET_UNKNOWN;
bzero(m->cryptoSignSecret, sizeof m->cryptoSignSecret); // not strictly necessary but aids debugging
@ -134,6 +130,9 @@ void _rhizome_manifest_set_id(struct __sourceloc __whence, rhizome_manifest *m,
if (m->authorship == AUTHOR_AUTHENTIC)
m->authorship = AUTHOR_LOCAL;
}
m->cryptoSignPublic = *bidp;
m->has_id = 1;
m->finalised = 0;
}
void _rhizome_manifest_set_version(struct __sourceloc __whence, rhizome_manifest *m, uint64_t version)
@ -141,6 +140,7 @@ void _rhizome_manifest_set_version(struct __sourceloc __whence, rhizome_manifest
const char *v = rhizome_manifest_set_ui64(m, "version", version);
assert(v); // TODO: remove known manifest fields from vars[]
m->version = version;
m->finalised = 0;
}
void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifest *m, uint64_t size)
@ -148,8 +148,7 @@ void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifes
const char *v = rhizome_manifest_set_ui64(m, "filesize", size);
assert(v); // TODO: remove known manifest fields from vars[]
m->filesize = size;
if (m->filesize == 0)
rhizome_manifest_set_filehash(m, NULL);
m->finalised = 0;
}
/* Must always set file size before setting the file hash, to avoid assertion failures.
@ -158,15 +157,16 @@ void _rhizome_manifest_set_filehash(struct __sourceloc __whence, rhizome_manifes
{
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (hash) {
assert(m->filesize > 0);
const char *v = rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(*hash));
assert(v); // TODO: remove known manifest fields from vars[]
m->filehash = *hash;
m->has_filehash = 1;
} else {
assert(m->filesize == 0);
rhizome_manifest_del(m, "filehash");
m->filehash = RHIZOME_FILEHASH_NONE;
m->has_filehash = 0;
}
m->finalised = 0;
}
void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m, uint64_t tail)
@ -175,6 +175,7 @@ void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m
assert(v); // TODO: remove known manifest fields from vars[]
m->tail = tail;
m->is_journal = (tail != RHIZOME_SIZE_UNSET);
m->finalised = 0;
}
void _rhizome_manifest_set_bundle_key(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_bk_t *bkp)
@ -184,6 +185,7 @@ void _rhizome_manifest_set_bundle_key(struct __sourceloc __whence, rhizome_manif
assert(v); // TODO: remove known manifest fields from vars[]
m->bundle_key = *bkp;
m->has_bundle_key = 1;
m->finalised = 0;
} else
_rhizome_manifest_del_bundle_key(__whence, m);
}
@ -194,6 +196,7 @@ void _rhizome_manifest_del_bundle_key(struct __sourceloc __whence, rhizome_manif
rhizome_manifest_del(m, "BK");
m->has_bundle_key = 0;
m->bundle_key = RHIZOME_BK_NONE; // not strictly necessary, but aids debugging
m->finalised = 0;
} else
assert(rhizome_manifest_get(m, "BK") == NULL);
// Once there is no BK field, any authenticated authorship is no longer.
@ -208,6 +211,7 @@ void _rhizome_manifest_set_service(struct __sourceloc __whence, rhizome_manifest
const char *v = rhizome_manifest_set(m, "service", service);
assert(v); // TODO: remove known manifest fields from vars[]
m->service = v;
m->finalised = 0;
} else
_rhizome_manifest_del_service(__whence, m);
}
@ -216,6 +220,7 @@ void _rhizome_manifest_del_service(struct __sourceloc __whence, rhizome_manifest
{
if (m->service) {
m->service = NULL;
m->finalised = 0;
rhizome_manifest_del(m, "service");
} else
assert(rhizome_manifest_get(m, "service") == NULL);
@ -223,6 +228,7 @@ void _rhizome_manifest_del_service(struct __sourceloc __whence, rhizome_manifest
void _rhizome_manifest_set_name(struct __sourceloc __whence, rhizome_manifest *m, const char *name)
{
m->finalised = 0;
if (name) {
assert(rhizome_str_is_manifest_name(name));
const char *v = rhizome_manifest_set(m, "name", name);
@ -238,6 +244,7 @@ void _rhizome_manifest_del_name(struct __sourceloc __whence, rhizome_manifest *m
{
if (m->name) {
m->name = NULL;
m->finalised = 0;
rhizome_manifest_del(m, "name");
} else
assert(rhizome_manifest_get(m, "name") == NULL);
@ -249,6 +256,7 @@ void _rhizome_manifest_set_date(struct __sourceloc __whence, rhizome_manifest *m
assert(v); // TODO: remove known manifest fields from vars[]
m->date = date;
m->has_date = 1;
m->finalised = 0;
}
void _rhizome_manifest_set_sender(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp)
@ -258,6 +266,7 @@ void _rhizome_manifest_set_sender(struct __sourceloc __whence, rhizome_manifest
assert(v); // TODO: remove known manifest fields from vars[]
m->sender = *sidp;
m->has_sender = 1;
m->finalised = 0;
} else
_rhizome_manifest_del_sender(__whence, m);
}
@ -268,6 +277,7 @@ void _rhizome_manifest_del_sender(struct __sourceloc __whence, rhizome_manifest
rhizome_manifest_del(m, "sender");
m->sender = SID_ANY;
m->has_sender = 0;
m->finalised = 0;
} else
assert(rhizome_manifest_get(m, "sender") == NULL);
}
@ -279,6 +289,7 @@ void _rhizome_manifest_set_recipient(struct __sourceloc __whence, rhizome_manife
assert(v); // TODO: remove known manifest fields from vars[]
m->recipient = *sidp;
m->has_recipient = 1;
m->finalised = 0;
} else
_rhizome_manifest_del_recipient(__whence, m);
}
@ -289,6 +300,7 @@ void _rhizome_manifest_del_recipient(struct __sourceloc __whence, rhizome_manife
rhizome_manifest_del(m, "recipient");
m->recipient = SID_ANY;
m->has_recipient = 0;
m->finalised = 0;
} else
assert(rhizome_manifest_get(m, "recipient") == NULL);
}
@ -312,6 +324,7 @@ void _rhizome_manifest_set_crypt(struct __sourceloc __whence, rhizome_manifest *
default: abort();
}
m->payloadEncryption = flag;
m->finalised = 0;
}
void _rhizome_manifest_set_rowid(struct __sourceloc __whence, rhizome_manifest *m, uint64_t rowid)
@ -354,15 +367,18 @@ void _rhizome_manifest_del_author(struct __sourceloc __whence, rhizome_manifest
/* Compute the hash of the manifest's body, including the NUL byte that separates the body from
* the signature block, and verify that a signature is present and is correct.
*
* Returns 1 if the manifest signature is valid, ie, the signature is a self-signature using the
* manifest's own private key. Sets the m->finalised flag to 1.
* If the manifest signature is valid, ie, the signature is a self-signature using the
* manifest's own private key, then sets the m->selfSigned flag and returns 1.
*
* Returns 0 if there are no signatures or if the signature block does not verify.
* If there are no signatures or if the signature block does not verify, then clears the
* m->selfSigned flag and returns 0.
*
* Only call this function on manifests for which rhizome_manifest_validate(m) has returned true.
* Only call this function on manifests for which rhizome_manifest_validate(m) has returned true
* (ie, m->finalised is set).
*/
int rhizome_manifest_verify(rhizome_manifest *m)
{
assert(m->finalised);
assert(m->manifest_body_bytes > 0);
assert(m->manifest_all_bytes > 0);
assert(m->manifest_body_bytes <= m->manifest_all_bytes);
@ -377,7 +393,6 @@ int rhizome_manifest_verify(rhizome_manifest *m)
break;
}
assert(ofs <= m->manifest_all_bytes);
// Make sure the first signatory's public key is the bundle ID
assert(m->has_id);
if (m->sig_count == 0) {
@ -395,23 +410,9 @@ int rhizome_manifest_verify(rhizome_manifest *m)
return 0;
}
m->selfSigned = 1;
m->finalised = 1;
return 1;
}
ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_size)
{
int fd = open(path, O_RDONLY);
if (fd == -1)
return WHYF_perror("open(%s,O_RDONLY)", alloca_str_toprint(path));
ssize_t ret = read(fd, buffer, buffer_size);
if (ret == -1)
ret = WHYF_perror("read(%s,%zu)", alloca_str_toprint(path), buffer_size);
if (close(fd) == -1)
ret = WHY_perror("close");
return ret;
}
static void rhizome_manifest_clear(rhizome_manifest *m)
{
while (m->var_count) {
@ -427,6 +428,7 @@ static void rhizome_manifest_clear(rhizome_manifest *m)
}
m->malformed = 0;
m->has_id = 0;
m->has_filehash = 0;
m->is_journal = 0;
m->filesize = RHIZOME_SIZE_UNSET;
m->tail = RHIZOME_SIZE_UNSET;
@ -532,18 +534,24 @@ int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifes
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int rhizome_manifest_parse(rhizome_manifest *m)
int rhizome_manifest_parse(rhizome_manifest *m)
{
IN();
assert(m->manifest_all_bytes <= sizeof m->manifestdata);
assert(m->manifest_body_bytes == 0);
assert(m->var_count == 0);
assert(!m->finalised);
assert(!m->malformed);
assert(!m->has_id);
assert(!m->has_filehash);
assert(!m->is_journal);
assert(m->filesize == RHIZOME_SIZE_UNSET);
assert(m->tail == RHIZOME_SIZE_UNSET);
assert(m->version == 0);
assert(!m->has_date);
assert(!m->has_sender);
assert(!m->has_recipient);
assert(m->payloadEncryption == PAYLOAD_CRYPT_UNKNOWN);
unsigned has_invalid_essential = 0;
unsigned has_duplicate = 0;
@ -593,6 +601,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
status = FIELD_DUPLICATE;
else if (strcasecmp(label, "id") == 0) {
if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) != -1) {
assert(!m->has_id);
status = FIELD_OK;
m->has_id = 1;
if (config.debug.rhizome_manifest)
@ -603,6 +612,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
else if (strcasecmp(label, "version") == 0) {
uint64_t version;
if (str_to_uint64(value, 10, &version, NULL) && version != 0) {
assert(m->version == 0);
status = FIELD_OK;
m->version = version;
if (config.debug.rhizome_manifest)
@ -611,8 +621,10 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
status = FIELD_INVALID;
}
else if (strcasecmp(label, "filehash") == 0) {
if (str_to_rhizome_filehash_t(&m->filehash, value) != -1 && !rhizome_filehash_t_is_zero(m->filehash)) {
if (str_to_rhizome_filehash_t(&m->filehash, value) != -1) {
assert(!m->has_filehash);
status = FIELD_OK;
m->has_filehash = 1;
if (config.debug.rhizome_manifest)
DEBUGF("PARSE manifest[%d].filehash = %s", m->manifest_record_number, alloca_tohex_rhizome_filehash_t(m->filehash));
} else
@ -621,6 +633,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
else if (strcasecmp(label, "filesize") == 0) {
uint64_t filesize;
if (str_to_uint64(value, 10, &filesize, NULL) && filesize != RHIZOME_SIZE_UNSET) {
assert(m->filesize == RHIZOME_SIZE_UNSET);
status = FIELD_OK;
m->filesize = filesize;
if (config.debug.rhizome_manifest)
@ -631,6 +644,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
else if (strcasecmp(label, "tail") == 0) {
uint64_t tail;
if (str_to_uint64(value, 10, &tail, NULL) && tail != RHIZOME_SIZE_UNSET) {
assert(m->tail == RHIZOME_SIZE_UNSET);
status = FIELD_OK;
m->tail = tail;
m->is_journal = 1;
@ -645,6 +659,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
// layer can refuse to add (or export?) the bundle.
else if (strcasecmp(label, "BK") == 0) {
if (str_to_rhizome_bk_t(&m->bundle_key, value) != -1) {
assert(!m->has_bundle_key);
status = FIELD_OK;
m->has_bundle_key = 1;
if (config.debug.rhizome_manifest)
@ -654,6 +669,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
}
else if (strcasecmp(label, "service") == 0) {
if (rhizome_str_is_manifest_service(value)) {
assert(m->service == NULL);
status = FIELD_OK;
m->service = value; // will be free()d when vars[] and values[] are free()d
if (config.debug.rhizome_manifest)
@ -664,6 +680,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
else if (strcasecmp(label, "date") == 0) {
int64_t date;
if (str_to_int64(value, 10, &date, NULL)) {
assert(!m->has_date);
status = FIELD_OK;
m->date = date;
m->has_date = 1;
@ -674,6 +691,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
}
else if (strcasecmp(label, "sender") == 0) {
if (str_to_sid_t(&m->sender, value) != -1) {
assert(!m->has_sender);
status = FIELD_OK;
m->has_sender = 1;
if (config.debug.rhizome_manifest)
@ -683,6 +701,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
}
else if (strcasecmp(label, "recipient") == 0) {
if (str_to_sid_t(&m->recipient, value) != -1) {
assert(!m->has_recipient);
status = FIELD_OK;
m->has_recipient = 1;
if (config.debug.rhizome_manifest)
@ -698,6 +717,7 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
}
else if (strcasecmp(label, "crypt") == 0) {
if (strcmp(value, "0") == 0 || strcmp(value, "1") == 0) {
assert(m->payloadEncryption == PAYLOAD_CRYPT_UNKNOWN);
status = FIELD_OK;
m->payloadEncryption = (value[0] == '1') ? PAYLOAD_ENCRYPTED : PAYLOAD_CLEAR;
if (config.debug.rhizome_manifest)
@ -754,8 +774,13 @@ static int rhizome_manifest_parse(rhizome_manifest *m)
OUT();
}
/* Return 1 if all necessary fields are present, 0 if not. Increment m->malformed if any
* unnecessary fields are missing.
/* If all essential (transport) fields are present and well formed then sets the m->finalised field
* and returns 1, otherwise returns 0.
*
* Increments m->malformed if any non-essential fields are missing or invalid. It is up to the
* caller to check the m->malformed field and decide whether or not to process a malformed manifest.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_manifest_validate(rhizome_manifest *m)
{
@ -774,55 +799,62 @@ int rhizome_manifest_validate(rhizome_manifest *m)
if (config.debug.rhizome_manifest)
DEBUG("Missing 'filesize' field");
ret = 0;
} else if (m->filesize == 0 && m->has_filehash) {
if (config.debug.rhizome_manifest)
DEBUG("Spurious 'filehash' field");
ret = 0;
} else if (m->filesize != 0 && !m->has_filehash) {
if (config.debug.rhizome_manifest)
DEBUG("Missing 'filehash' field");
ret = 0;
}
if (rhizome_filehash_t_is_zero(m->filehash)) {
if (m->filesize > 0) {
if (config.debug.rhizome_manifest)
DEBUG("Missing 'filehash' field");
ret = 0;
}
} else {
if (m->filesize == 0) {
if (config.debug.rhizome_manifest)
DEBUG("Spurious 'filehash' field");
ret = 0;
}
}
// warn if expected fields are missing
// Warn if expected fields are missing or invalid
if (m->service == NULL) {
if (config.debug.rhizome_manifest)
DEBUG("Missing 'service' field");
++m->malformed;
}
else if (strcmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
if (m->name == NULL) {
if (config.debug.rhizome_manifest)
DEBUG("Manifest with service='" RHIZOME_SERVICE_FILE "' missing 'name' field");
++m->malformed;
}
} else if (strcmp(m->service, RHIZOME_SERVICE_MESHMS) == 0
|| strcmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0
) {
if (!m->has_sender) {
if (config.debug.rhizome_manifest)
DEBUGF("Manifest with service='%s' missing 'sender' field", m->service);
++m->malformed;
}
if (!m->has_recipient) {
if (config.debug.rhizome_manifest)
DEBUGF("Manifest with service='%s' missing 'recipient' field", m->service);
++m->malformed;
}
}
else if (!rhizome_str_is_manifest_service(m->service)) {
if (config.debug.rhizome_manifest)
DEBUGF("Manifest invalid 'service' field %s", alloca_str_toprint(m->service));
++m->malformed;
}
if (!m->has_date) {
if (config.debug.rhizome_manifest)
DEBUG("Missing 'date' field");
++m->malformed;
}
m->finalised = ret;
return ret;
}
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferP)
int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename)
{
if (!m)
return WHY("Null manifest");
if (bufferP>sizeof(m->manifestdata))
return WHY("Buffer too big");
if (bufferP) {
m->manifest_all_bytes=bufferP;
memcpy(m->manifestdata, filename, m->manifest_all_bytes);
} else {
ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata);
if (bytes == -1)
return -1;
m->manifest_all_bytes = (size_t) bytes;
}
switch (rhizome_manifest_parse(m)) {
case 0: return 0;
case -1: return -1;
default: return WHY("Invalid manifest");
}
ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata);
if (bytes == -1)
return -1;
m->manifest_all_bytes = (size_t) bytes;
return rhizome_manifest_parse(m);
}
int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out)
@ -975,7 +1007,7 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
/* Convert variable list into manifest text body and compute the hash. Do not sign.
*/
int rhizome_manifest_pack_variables(rhizome_manifest *m)
static int rhizome_manifest_pack_variables(rhizome_manifest *m)
{
assert(m->var_count <= NELS(m->vars));
strbuf sb = strbuf_local((char*)m->manifestdata, sizeof m->manifestdata);
@ -1021,6 +1053,7 @@ int rhizome_manifest_selfsign(rhizome_manifest *m)
);
bcopy(sig.signature, m->manifestdata + m->manifest_body_bytes, sig.signatureLength);
m->manifest_all_bytes = m->manifest_body_bytes + sig.signatureLength;
m->selfSigned = 1;
return 0;
}
@ -1028,10 +1061,7 @@ int rhizome_write_manifest_file(rhizome_manifest *m, const char *path, char appe
{
if (config.debug.rhizome)
DEBUGF("write manifest (%zd bytes) to %s", m->manifest_all_bytes, path);
if (!m)
return WHY("Manifest is null.");
if (!m->finalised)
return WHY("Manifest must be finalised before it can be written.");
assert(m->finalised);
int fd = open(path, O_WRONLY | O_CREAT | (append ? O_APPEND : 0), 0666);
if (fd == -1)
return WHYF_perror("open(%s,O_WRONLY|O_CREAT%s,0666)", alloca_str_toprint(path), append ? "|O_APPEND" : "");
@ -1065,44 +1095,68 @@ int rhizome_manifest_dump(rhizome_manifest *m, const char *msg)
return 0;
}
int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate)
enum rhizome_bundle_status rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate)
{
IN();
int ret=0;
if (m->filesize == RHIZOME_SIZE_UNSET)
RETURN(WHY("Manifest filesize unknown"));
if (!m->finalised && !rhizome_manifest_validate(m))
RETURN(RHIZOME_BUNDLE_STATUS_INVALID);
// if a manifest was supplied with an ID, don't bother to check for a duplicate.
// we only want to filter out added files with no existing manifest.
if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID && rhizome_find_duplicate(m, mout) == 1)
RETURN(2);
*mout=m;
if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID) {
enum rhizome_bundle_status status = rhizome_find_duplicate(m, mout);
switch (status) {
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_ERROR:
RETURN(status);
case RHIZOME_BUNDLE_STATUS_NEW:
break;
default:
FATALF("rhizome_find_duplicate() returned %d", status);
}
}
*mout = m;
/* Convert to final form for signing and writing to disk */
if (rhizome_manifest_pack_variables(m))
RETURN(WHY("Could not convert manifest to wire format"));
/* Sign it */
assert(!m->selfSigned);
if (rhizome_manifest_selfsign(m))
RETURN(WHY("Could not sign manifest"));
assert(m->selfSigned);
/* mark manifest as finalised */
m->finalised=1;
ret = rhizome_add_manifest(m, 255 /* TTL */);
RETURN(ret);
enum rhizome_bundle_status status = rhizome_add_manifest(m, mout);
RETURN(status);
OUT();
}
/* Returns 1 if the name was successfully set, 0 if not.
*/
int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath)
{
const char *name = strrchr(filepath, '/');
if (!name)
name = filepath;
if (!rhizome_str_is_manifest_name(name)) {
WARNF("invalid rhizome name %s -- not used", alloca_str_toprint(name));
return 0;
}
rhizome_manifest_set_name(m, name);
return 1;
}
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
* - use the current time for "date" and "version"
* - use the given author SID, or the 'sender' if present, as the author
* - create an ID if there is none, otherwise authenticate the existing one
* - if service is file, then use the payload file's basename for "name"
*/
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp)
{
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
- use the current time for "date" and "version"
- if service is file, then use the payload file's basename for "name"
*/
/* Set version of manifest from current time if not already set. */
if (m->version == 0)
rhizome_manifest_set_version(m, gettime_ms());
@ -1117,7 +1171,7 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
/* Set the bundle ID (public key) and secret key.
*/
if (!m->haveSecret && rhizome_bid_t_is_zero(m->cryptoSignPublic)) {
if (!m->haveSecret && !m->has_id) {
if (config.debug.rhizome)
DEBUG("creating new bundle");
if (rhizome_manifest_createid(m) == -1)
@ -1144,18 +1198,13 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
}
if (strcasecmp(RHIZOME_SERVICE_FILE, m->service) == 0) {
if (m->name == NULL) {
if (filepath && *filepath) {
const char *name = strrchr(filepath, '/');
rhizome_manifest_set_name(m, name ? name + 1 : filepath);
} else
rhizome_manifest_set_name(m, "");
if (m->name) {
if (config.debug.rhizome)
DEBUGF("missing 'name', set default name=%s", alloca_str_toprint(m->name));
} else {
if (config.debug.rhizome)
DEBUGF("manifest contains name=%s", alloca_str_toprint(m->name));
}
DEBUGF("manifest already contains name=%s", alloca_str_toprint(m->name));
} else if (filepath)
rhizome_manifest_set_name_from_path(m, filepath);
else if (config.debug.rhizome)
DEBUGF("manifest missing 'name'");
}
// Anything sent from one person to another should be considered private and encrypted by default.

74
rhizome_crypto.c
View File

@ -300,7 +300,7 @@ int rhizome_apply_bundle_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
DEBUGF("manifest[%d] bsk=%s", m->manifest_record_number, bsk ? alloca_tohex_rhizome_bk_t(*bsk) : "NULL");
assert(m->haveSecret == SECRET_UNKNOWN);
assert(is_all_matching(m->cryptoSignSecret, sizeof m->cryptoSignSecret, 0));
assert(!rhizome_bid_t_is_zero(m->cryptoSignPublic));
assert(m->has_id);
assert(bsk != NULL);
assert(!rhizome_is_bk_none(bsk));
if (rhizome_verify_bundle_privatekey(bsk->binary, m->cryptoSignPublic.binary)) {
@ -450,7 +450,7 @@ typedef struct manifest_signature_block_cache {
#define SIG_CACHE_SIZE 1024
manifest_signature_block_cache sig_cache[SIG_CACHE_SIZE];
int rhizome_manifest_lookup_signature_validity(const unsigned char *hash, const unsigned char *sig, int sig_len)
static int rhizome_manifest_lookup_signature_validity(const unsigned char *hash, const unsigned char *sig, int sig_len)
{
IN();
unsigned int slot=0;
@ -552,8 +552,8 @@ static void add_nonce(unsigned char *nonce, uint64_t value)
}
}
/* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries
* for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE)
/* Encrypt a block of a stream in-place, allowing for offsets that don't align perfectly to block
* boundaries for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE).
*/
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset,
const unsigned char *key, const unsigned char *nonce)
@ -594,57 +594,71 @@ int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t
return 0;
}
/* If payload key is known, sets m->payloadKey and m->payloadNonce and returns 1.
* Otherwise, returns 0;
*/
int rhizome_derive_payload_key(rhizome_manifest *m)
{
// don't do anything if the manifest isn't flagged as being encrypted
if (m->payloadEncryption != PAYLOAD_ENCRYPTED)
return 0;
if (m->has_sender && m->has_recipient){
assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
unsigned char hash[crypto_hash_sha512_BYTES];
if (m->has_sender && m->has_recipient) {
unsigned char *nm_bytes=NULL;
unsigned cn=0, in=0, kp=0;
if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->sender)){
cn=in=kp=0;
if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->recipient)){
return WHYF("Neither the sender %s nor the recipient %s appears in our keyring",
WARNF("Neither sender=%s nor recipient=%s is in keyring",
alloca_tohex_sid_t(m->sender),
alloca_tohex_sid_t(m->recipient));
return 0;
}
nm_bytes=keyring_get_nm_bytes(&m->recipient, &m->sender);
nm_bytes = keyring_get_nm_bytes(&m->recipient, &m->sender);
if (config.debug.rhizome)
DEBUGF("derived payload key from recipient=%s* to sender=%s*",
alloca_tohex_sid_t_trunc(m->recipient, 7),
alloca_tohex_sid_t_trunc(m->sender, 7)
);
}else{
nm_bytes=keyring_get_nm_bytes(&m->sender, &m->recipient);
nm_bytes = keyring_get_nm_bytes(&m->sender, &m->recipient);
if (config.debug.rhizome)
DEBUGF("derived payload key from sender=%s* to recipient=%s*",
alloca_tohex_sid_t_trunc(m->sender, 7),
alloca_tohex_sid_t_trunc(m->recipient, 7)
);
}
if (!nm_bytes)
return -1;
unsigned char hash[crypto_hash_sha512_BYTES];
assert(nm_bytes != NULL);
crypto_hash_sha512(hash, nm_bytes, crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES);
bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
}else{
if (!m->haveSecret)
return WHY("Cannot derive payload key because bundle secret is unknown");
if (!m->haveSecret) {
WHY("Cannot derive payload key because bundle secret is unknown");
return 0;
}
if (config.debug.rhizome)
DEBUGF("derived payload key from bundle secret bsk=%s", alloca_tohex(m->cryptoSignSecret, sizeof m->cryptoSignSecret));
unsigned char raw_key[9+crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]="sasquatch";
bcopy(m->cryptoSignSecret, &raw_key[9], crypto_sign_edwards25519sha512batch_SECRETKEYBYTES);
unsigned char hash[crypto_hash_sha512_BYTES];
crypto_hash_sha512(hash, raw_key, sizeof(raw_key));
bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
}
bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
if (config.debug.rhizome_manifest)
DEBUGF("SET manifest[%d].payloadKey = %s", m->manifest_record_number, alloca_tohex(m->payloadKey, sizeof m->payloadKey));
// journal bundles must always have the same nonce, regardless of version.
// otherwise, generate nonce from version#bundle id#version;
unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary];
write_uint64(&raw_nonce[0], m->is_journal ? 0 : m->version);
uint64_t nonce_version = m->is_journal ? 0 : m->version;
write_uint64(&raw_nonce[0], nonce_version);
bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary);
write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->is_journal ? 0 : m->version);
unsigned char hash[crypto_hash_sha512_BYTES];
write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], nonce_version);
if (config.debug.rhizome)
DEBUGF("derived payload nonce from bid=%s version=%"PRIu64, alloca_tohex_sid_t(m->cryptoSignPublic), nonce_version);
crypto_hash_sha512(hash, raw_nonce, sizeof(raw_nonce));
bcopy(hash, m->payloadNonce, sizeof(m->payloadNonce));
return 0;
if (config.debug.rhizome_manifest)
DEBUGF("SET manifest[%d].payloadNonce = %s", m->manifest_record_number, alloca_tohex(m->payloadNonce, sizeof m->payloadNonce));
return 1;
}

160
rhizome_database.c
View File

@ -174,24 +174,28 @@ void verify_bundles()
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;");
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
const void *manifest = sqlite3_column_blob(statement, 1);
size_t manifest_length = sqlite3_column_bytes(statement, 1);
rhizome_manifest *m=rhizome_new_manifest();
int ret = -1;
if ( rhizome_read_manifest_file(m, manifest, manifest_length) == 0
&& rhizome_manifest_validate(m)
&& rhizome_manifest_verify(m)
) {
assert(m->finalised);
// Store it again, to ensure that MANIFESTS columns are up to date.
ret = rhizome_store_bundle(m);
const void *blob = sqlite3_column_blob(statement, 1);
size_t blob_length = sqlite3_column_bytes(statement, 1);
rhizome_manifest *m = rhizome_new_manifest();
if (m) {
memcpy(m->manifestdata, blob, blob_length);
m->manifest_all_bytes = blob_length;
int ret = -1;
if ( rhizome_manifest_parse(m) != -1
&& rhizome_manifest_validate(m)
&& rhizome_manifest_verify(m)
) {
assert(m->finalised);
// Store it again, to ensure that MANIFESTS columns are up to date.
ret = rhizome_store_manifest(m);
}
if (ret) {
if (config.debug.rhizome)
DEBUGF("Removing invalid manifest entry @%lld", rowid);
sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
}
rhizome_manifest_free(m);
}
if (ret) {
if (config.debug.rhizome)
DEBUGF("Removing invalid manifest entry @%lld", rowid);
sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
}
rhizome_manifest_free(m);
}
sqlite3_finalize(statement);
}
@ -1071,6 +1075,88 @@ int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
}
int _sqlite_blob_open_retry(
struct __sourceloc __whence,
int log_level,
sqlite_retry_state *retry,
const char *dbname,
const char *tablename,
const char *colname,
sqlite3_int64 rowid,
int flags,
sqlite3_blob **blobp
)
{
IN();
while (1) {
int code = sqlite3_blob_open(rhizome_db, dbname, tablename, colname, rowid, flags, blobp);
switch (code) {
case SQLITE_OK:
if (retry)
_sqlite_retry_done(__whence, retry, "sqlite3_blob_open()");
RETURN(code);
case SQLITE_DONE:
case SQLITE_ROW:
LOGF(log_level, "sqlite3_blob_open() returned unexpected code (%d)", code);
RETURN(-1);
case SQLITE_BUSY:
case SQLITE_LOCKED:
if (retry && _sqlite_retry(__whence, retry, "sqlite3_blob_open()"))
break; // back to sqlite3_blob_open()
// fall through...
default:
LOGF(log_level, "sqlite3_blob_open() failed (%d), %s", code, sqlite3_errmsg(rhizome_db));
RETURN(-1);
}
}
FATAL("not reached");
OUT();
}
int _sqlite_blob_write_retry(
struct __sourceloc __whence,
int log_level,
sqlite_retry_state *retry,
sqlite3_blob *blob,
const void *buf,
int len,
int offset
)
{
IN();
while (1) {
int code = sqlite3_blob_write(blob, buf, len, offset);
switch (code) {
case SQLITE_OK:
if (retry)
_sqlite_retry_done(__whence, retry, "sqlite3_blob_write()");
RETURN(code);
case SQLITE_DONE:
case SQLITE_ROW:
LOGF(log_level, "sqlite3_blob_write() returned unexpected code (%d)", code);
RETURN(-1);
case SQLITE_BUSY:
case SQLITE_LOCKED:
if (retry && _sqlite_retry(__whence, retry, "sqlite3_blob_write()"))
break; // back to sqlite3_blob_open()
// fall through...
default:
LOGF(log_level, "sqlite3_blob_write() failed (%d), %s", code, sqlite3_errmsg(rhizome_db));
RETURN(-1);
}
}
FATAL("not reached");
OUT();
}
int _sqlite_blob_close(struct __sourceloc __whence, int log_level, sqlite3_blob *blob)
{
int code = sqlite3_blob_close(blob);
if (code != SQLITE_OK)
LOGF(log_level, "sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db));
return 0;
}
static uint64_t rhizome_database_used_bytes()
{
uint64_t db_page_size;
@ -1105,7 +1191,14 @@ static int rhizome_delete_external(const rhizome_filehash_t *hashp)
char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(*hashp)))
return -1;
return unlink(blob_path);
if (unlink(blob_path) == -1) {
if (errno != ENOENT)
return WHYF_perror("unlink(%s)", alloca_str_toprint(blob_path));
return 1;
}
if (config.debug.externalblobs)
DEBUGF("Deleted blob file %s", blob_path);
return 0;
}
static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
@ -1340,10 +1433,9 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES,
and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
*/
int rhizome_store_bundle(rhizome_manifest *m)
int rhizome_store_manifest(rhizome_manifest *m)
{
if (!m->finalised)
return WHY("Manifest was not finalised");
assert(m->finalised);
// If we don't have the secret for this manifest, only store it if its self-signature is valid
if (!m->haveSecret && !m->selfSigned)
@ -1606,7 +1698,9 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
rhizome_manifest *m = c->manifest = rhizome_new_manifest();
if (m == NULL)
RETURN(-1);
if ( rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1
memcpy(m->manifestdata, manifestblob, manifestblobsize);
m->manifest_all_bytes = manifestblobsize;
if ( rhizome_manifest_parse(m) == -1
|| !rhizome_manifest_validate(m)
) {
WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
@ -1691,13 +1785,13 @@ int rhizome_update_file_priority(const char *fileid)
}
/* Search the database for a manifest having the same name and payload content, and if the version
* is known, having the same version. Returns 1 if a duplicate is found (setting *found to point to
* the duplicate's manifest), returns 0 if no duplicate is found (leaving *found unchanged).
* Returns -1 on error (leaving *found undefined).
* is known, having the same version. Returns RHIZOME_BUNDLE_STATUS_DUPLICATE if a duplicate is found
* (setting *found to point to the duplicate's manifest), returns RHIZOME_BUNDLE_STATUS_NEW if no
* duplicate is found (leaving *found unchanged). Returns -1 on error (leaving *found undefined).
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
{
if (m->service == NULL)
return WHY("Manifest has no service");
@ -1715,7 +1809,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
strbuf_puts(b, " AND recipient = ?");
if (strbuf_overrun(b))
return WHYF("SQL command too long: %s", strbuf_str(b));
int ret = 0;
int ret = RHIZOME_BUNDLE_STATUS_NEW;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END);
if (!statement)
@ -1743,9 +1837,11 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
if ( rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1
memcpy(blob_m->manifestdata, manifestblob, manifestblobsize);
blob_m->manifest_all_bytes = manifestblobsize;
if ( rhizome_manifest_parse(blob_m) == -1
|| !rhizome_manifest_validate(blob_m)
) {
) {
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
goto next;
}
@ -1768,7 +1864,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
*found = blob_m;
if (config.debug.rhizome)
DEBUGF("Found duplicate payload, %s", q_manifestid);
ret = 1;
ret = RHIZOME_BUNDLE_STATUS_DUPLICATE;
break;
next:
if (blob_m)
@ -1787,7 +1883,9 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
const char *q_author = (const char *) sqlite3_column_text(statement, 4);
size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
uint64_t q_rowid = sqlite3_column_int64(statement, 5);
if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1 || !rhizome_manifest_validate(m))
memcpy(m->manifestdata, q_blob, q_blobsize);
m->manifest_all_bytes = q_blobsize;
if (rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m))
return WHYF("Manifest bid=%s in database but invalid", q_id);
if (q_author) {
sid_t author;

11
rhizome_direct.c
View File

@ -438,10 +438,15 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix, size_t
if (manifestblobsize<1||manifestblobsize>1024) goto error;
const char *manifestblob = (char *) sqlite3_column_blob(statement, 0);
if (!manifestblob) goto error;
if (!manifestblob)
goto error;
rhizome_manifest *m=rhizome_new_manifest();
if ( rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
goto error;
memcpy(m->manifestdata, manifestblob, manifestblobsize);
m->manifest_all_bytes = manifestblobsize;
if ( rhizome_manifest_parse(m) == -1
|| !rhizome_manifest_validate(m)
) {
rhizome_manifest_free(m);

188
rhizome_direct_http.c
View File

@ -57,11 +57,11 @@ static void rhizome_direct_clear_temporary_files(rhizome_http_request *r)
static int rhizome_direct_import_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (!r->received_manifest) {
if (!r->u.direct_import.received_manifest) {
http_request_simple_response(&r->http, 400, "Missing 'manifest' part");
return 0;
}
if (!r->received_data) {
if (!r->u.direct_import.received_data) {
http_request_simple_response(&r->http, 400, "Missing 'data' part");
return 0;
}
@ -79,12 +79,12 @@ static int rhizome_direct_import_end(struct http_request *hr)
alloca_str_toprint(manifest_path),
alloca_str_toprint(payload_path)
);
int ret = 0;
enum rhizome_bundle_status status = 0;
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
ret = WHY("Out of manifests");
status = WHY("Out of manifests");
else {
ret = rhizome_bundle_import_files(m, manifest_path, payload_path);
status = rhizome_bundle_import_files(m, NULL, manifest_path, payload_path);
rhizome_manifest_free(m);
}
rhizome_direct_clear_temporary_files(r);
@ -96,13 +96,28 @@ static int rhizome_direct_import_end(struct http_request *hr)
the import fails due to malformed data etc.
(should probably also indicate if we have a newer version if possible)
*/
switch (ret) {
case 0:
switch (status) {
case RHIZOME_BUNDLE_STATUS_NEW:
http_request_simple_response(&r->http, 201, "Bundle succesfully imported");
return 0;
case 2:
case RHIZOME_BUNDLE_STATUS_SAME:
http_request_simple_response(&r->http, 200, "Bundle already imported");
return 0;
case RHIZOME_BUNDLE_STATUS_OLD:
http_request_simple_response(&r->http, 403, "Newer bundle already stored");
return 0;
case RHIZOME_BUNDLE_STATUS_INVALID:
http_request_simple_response(&r->http, 403, "Manifest is invalid");
return 0;
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
http_request_simple_response(&r->http, 403, "Manifest is inconsistent with file");
return 0;
case RHIZOME_BUNDLE_STATUS_FAKE:
http_request_simple_response(&r->http, 403, "Manifest not signed");
return 0;
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_ERROR:
break;
}
http_request_simple_response(&r->http, 500, "Internal Error: Rhizome import failed");
return 0;
@ -111,7 +126,7 @@ static int rhizome_direct_import_end(struct http_request *hr)
int rhizome_direct_enquiry_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (!r->received_data) {
if (!r->u.direct_import.received_data) {
http_request_simple_response(&r->http, 400, "Missing 'data' part");
return 0;
}
@ -174,14 +189,14 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
// If given a file without a manifest, we should only accept if it we are configured to do so, and
// the connection is from localhost. Otherwise people could cause your servald to create
// arbitrary bundles, which would be bad.
if (!r->received_manifest) {
if (!r->u.direct_import.received_manifest) {
char payload_path[512];
if (form_temporary_file_path(r, payload_path, "data") == -1) {
http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
return 0;
}
if (config.debug.rhizome)
DEBUGF("Call rhizome_add_file(%s)", alloca_str_toprint(payload_path));
DEBUGF("Call rhizome_store_payload_file(%s)", alloca_str_toprint(payload_path));
char manifestTemplate[1024];
manifestTemplate[0] = '\0';
if (config.rhizome.api.addfile.manifest_template_file[0]) {
@ -207,14 +222,14 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
rhizome_direct_clear_temporary_files(r);
return 0;
}
if (manifestTemplate[0] && rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) {
if (manifestTemplate[0] && rhizome_read_manifest_from_file(m, manifestTemplate) == -1) {
WHY("Manifest template read failed");
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 500, "Internal Error: Malformed manifest template");
return 0;
}
if (rhizome_stat_file(m, payload_path)) {
if (rhizome_stat_payload_file(m, payload_path) != RHIZOME_PAYLOAD_STATUS_NEW) {
WHY("Payload file stat failed");
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
@ -227,7 +242,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
if (m->service == NULL)
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
const sid_t *author = is_sid_t_any(config.rhizome.api.addfile.default_author) ? NULL : &config.rhizome.api.addfile.default_author;
if (rhizome_fill_manifest(m, r->data_file_name, author)) {
if (rhizome_fill_manifest(m, r->u.direct_import.data_file_name, author)) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest");
@ -238,7 +253,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
// TODO, stream file into database
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0) {
if (rhizome_add_file(m, payload_path)) {
if (rhizome_store_payload_file(m, payload_path) != RHIZOME_PAYLOAD_STATUS_NEW) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 500, "Internal Error: Could not store file");
@ -246,7 +261,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
}
}
rhizome_manifest *mout = NULL;
if (rhizome_manifest_finalise(m, &mout, 1)) {
if (rhizome_manifest_finalise(m, &mout, 1) == -1) {
if (mout && mout != m)
rhizome_manifest_free(mout);
rhizome_manifest_free(m);
@ -270,107 +285,107 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
}
}
static void rhizome_direct_process_mime_start(struct http_request *hr)
static char PART_MANIFEST[] = "manifest";
static char PART_DATA[] = "data";
static int rhizome_direct_process_mime_start(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->current_part == NONE);
assert(r->part_fd == -1);
assert(r->u.direct_import.current_part == NULL);
assert(r->u.direct_import.part_fd == -1);
return 0;
}
static void rhizome_direct_process_mime_end(struct http_request *hr)
static int rhizome_direct_process_mime_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->part_fd != -1) {
if (close(r->part_fd) == -1) {
WHYF_perror("close(%d)", r->part_fd);
if (r->u.direct_import.part_fd != -1) {
if (close(r->u.direct_import.part_fd) == -1) {
WHYF_perror("close(%d)", r->u.direct_import.part_fd);
http_request_simple_response(&r->http, 500, "Internal Error: Close temporary file failed");
return;
return 500;
}
r->part_fd = -1;
r->u.direct_import.part_fd = -1;
}
switch (r->current_part) {
case MANIFEST:
r->received_manifest = 1;
break;
case DATA:
r->received_data = 1;
break;
case NONE:
break;
}
r->current_part = NONE;
if (r->u.direct_import.current_part == PART_MANIFEST)
r->u.direct_import.received_manifest = 1;
else if (r->u.direct_import.current_part == PART_DATA)
r->u.direct_import.received_data = 1;
r->u.direct_import.current_part = NULL;
return 0;
}
static void rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
static int rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (strcmp(h->content_disposition.name, "data") == 0) {
r->current_part = DATA;
strncpy(r->data_file_name, h->content_disposition.filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0';
if (strcmp(h->content_disposition.name, PART_DATA) == 0) {
r->u.direct_import.current_part = PART_DATA;
strncpy(r->u.direct_import.data_file_name,
h->content_disposition.filename,
sizeof r->u.direct_import.data_file_name)
[sizeof r->u.direct_import.data_file_name - 1] = '\0';
}
else if (strcmp(h->content_disposition.name, "manifest") == 0) {
r->current_part = MANIFEST;
else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) {
r->u.direct_import.current_part = PART_MANIFEST;
} else
return;
return 0;
char path[512];
if (form_temporary_file_path(r, path, h->content_disposition.name) == -1) {
http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
return;
return 0;
}
if ((r->part_fd = open(path, O_WRONLY | O_CREAT, 0666)) == -1) {
if ((r->u.direct_import.part_fd = open(path, O_WRONLY | O_CREAT, 0666)) == -1) {
WHYF_perror("open(%s,O_WRONLY|O_CREAT,0666)", alloca_str_toprint(path));
http_request_simple_response(&r->http, 500, "Internal Error: Create temporary file failed");
return;
return 0;
}
return 0;
}
static void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len)
static int rhizome_direct_process_mime_body(struct http_request *hr, char *buf, size_t len)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->part_fd != -1) {
if (write_all(r->part_fd, buf, len) == -1) {
if (r->u.direct_import.part_fd != -1) {
if (write_all(r->u.direct_import.part_fd, buf, len) == -1) {
http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed");
return;
return 500;
}
}
return 0;
}
int rhizome_direct_import(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_POST) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 404;
if (r->http.verb != HTTP_VERB_POST)
return 405;
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_import_end;
r->current_part = NONE;
r->part_fd = -1;
r->data_file_name[0] = '\0';
return 0;
r->u.direct_import.current_part = NULL;
r->u.direct_import.part_fd = -1;
r->u.direct_import.data_file_name[0] = '\0';
return 1;
}
int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_POST) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 404;
if (r->http.verb != HTTP_VERB_POST)
return 405;
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_enquiry_end;
r->current_part = NONE;
r->part_fd = -1;
r->data_file_name[0] = '\0';
return 0;
r->u.direct_import.current_part = NULL;
r->u.direct_import.part_fd = -1;
r->u.direct_import.data_file_name[0] = '\0';
return 1;
}
/* Servald can be configured to accept files without manifests via HTTP from localhost, so that
@ -381,11 +396,9 @@ int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_POST) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 404;
if (r->http.verb != HTTP_VERB_POST)
return 405;
if ( r->http.client_sockaddr_in.sin_family != AF_INET
|| r->http.client_sockaddr_in.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr
) {
@ -394,18 +407,17 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
alloca_in_addr(&config.rhizome.api.addfile.allow_host)
);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 404, "<html><h1>Not available from here</h1></html>");
return 0;
return 403; // Forbidden
}
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_addfile_end;
r->current_part = NONE;
r->part_fd = -1;
r->data_file_name[0] = '\0';
return 0;
r->u.direct_import.current_part = NULL;
r->u.direct_import.part_fd = -1;
r->u.direct_import.data_file_name[0] = '\0';
return 1;
}
int rhizome_direct_dispatch(rhizome_http_request *r, const char *UNUSED(remainder))
@ -414,7 +426,7 @@ int rhizome_direct_dispatch(rhizome_http_request *r, const char *UNUSED(remainde
&& strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0
)
return rhizome_direct_addfile(r, "");
return 1;
return 404;
}
static int receive_http_response(int sock, char *buffer, size_t buffer_len, struct http_response_parts *parts)
@ -714,8 +726,20 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
struct rhizome_read read;
bzero(&read, sizeof read);
if (rhizome_open_read(&read, &filehash))
goto closeit;
enum rhizome_bundle_status pstatus = rhizome_open_read(&read, &filehash);
switch (pstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
goto closeit;
default:
FATALF("pstatus = %d", pstatus);
}
uint64_t read_ofs;
for(read_ofs=0;read_ofs<m->filesize;){

113
rhizome_fetch.c
View File

@ -89,7 +89,7 @@ struct rhizome_fetch_slot {
unsigned char mdpRXWindow[32*200];
};
static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot);
static enum rhizome_start_fetch_result rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot);
static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot);
/* Represents a queue of fetch candidates and a single active fetch for bundle payloads whose size
@ -475,17 +475,35 @@ int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int
static int rhizome_import_received_bundle(struct rhizome_manifest *m)
{
m->finalised = 1;
if (!rhizome_manifest_validate(m))
return 0;
if (config.debug.rhizome_rx) {
DEBUGF("manifest len=%zu has %u signatories. Associated filesize=%"PRIu64" bytes",
m->manifest_all_bytes, m->sig_count, m->filesize);
dump("manifest", m->manifestdata, m->manifest_all_bytes);
}
return rhizome_add_manifest(m, m->ttl - 1 /* TTL */);
enum rhizome_bundle_status status = rhizome_add_manifest(m, NULL);
switch (status) {
case RHIZOME_BUNDLE_STATUS_NEW:
return 0;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
return 1;
case RHIZOME_BUNDLE_STATUS_ERROR:
case RHIZOME_BUNDLE_STATUS_INVALID:
return -1;
default:
FATALF("rhizome_add_manifest() returned %d", status);
}
}
// begin fetching a bundle
static int schedule_fetch(struct rhizome_fetch_slot *slot)
/* Returns STARTED (0) if the fetch was started.
* Returns IMPORTED if the payload is already in the store.
* Returns -1 on error.
*/
static enum rhizome_start_fetch_result
schedule_fetch(struct rhizome_fetch_slot *slot)
{
IN();
int sock = -1;
@ -536,9 +554,27 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
if (strbuf_overrun(r))
RETURN(WHY("request overrun"));
slot->request_len = strbuf_len(r);
if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->filesize, RHIZOME_PRIORITY_DEFAULT))
RETURN(-1);
enum rhizome_payload_status status = rhizome_open_write(&slot->write_state,
&slot->manifest->filehash,
slot->manifest->filesize,
RHIZOME_PRIORITY_DEFAULT);
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
RETURN(IMPORTED);
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
RETURN(WHY("error writing new payload"));
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
RETURN(WHY("payload size does not match"));
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
RETURN(WHY("payload hash does not match"));
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
RETURN(WHY("payload cannot be encrypted"));
default:
FATALF("status = %d", status);
}
} else {
strbuf r = strbuf_local(slot->request, sizeof slot->request);
strbuf_sprintf(r, "GET /rhizome/manifestbyprefix/%s HTTP/1.0\r\n\r\n", alloca_tohex(slot->bid.binary, slot->prefix_length));
@ -596,14 +632,15 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
slot->alarm.alarm = gettime_ms() + config.rhizome.idle_timeout;
slot->alarm.deadline = slot->alarm.alarm + config.rhizome.idle_timeout;
schedule(&slot->alarm);
RETURN(0);
RETURN(STARTED);
}
enum rhizome_start_fetch_result result;
bail_http:
/* Fetch via overlay, either because no IP address was provided, or because
the connection/attempt to fetch via HTTP failed. */
rhizome_fetch_switch_to_mdp(slot);
RETURN(0);
result = rhizome_fetch_switch_to_mdp(slot);
RETURN(result);
OUT();
}
@ -729,23 +766,20 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
if (config.debug.rhizome_rx)
DEBUGF(" is new");
// If the payload is already available, no need to fetch, so import now.
if (rhizome_exists(&m->filehash)){
if (config.debug.rhizome_rx)
DEBUGF(" fetch not started - payload already present, so importing instead");
if (rhizome_add_manifest(m, m->ttl-1) == -1)
RETURN(WHY("add manifest failed"));
RETURN(IMPORTED);
}
/* Prepare for fetching */
slot->peer_ipandport = *peerip;
slot->peer_sid = *peersidp;
slot->manifest = m;
if (schedule_fetch(slot) == -1)
RETURN(-1);
RETURN(STARTED);
enum rhizome_start_fetch_result result = schedule_fetch(slot);
// If the payload is already available, no need to fetch, so import now.
if (result == IMPORTED) {
if (config.debug.rhizome_rx)
DEBUGF(" fetch not started - payload already present, so importing instead");
if (rhizome_add_manifest(m, NULL) == -1)
RETURN(WHY("add manifest failed"));
}
RETURN(result);
}
/* Returns STARTED (0) if the fetch was started.
@ -774,10 +808,7 @@ rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip,
for inserting into the database, but we can avoid the temporary file in
the process. */
if (schedule_fetch(slot) == -1) {
return -1;
}
return STARTED;
return schedule_fetch(slot);
}
/* Activate the next fetch for the given slot. This takes the next job from the head of the slot's
@ -1003,7 +1034,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
OUT();
}
static int rhizome_fetch_close(struct rhizome_fetch_slot *slot)
static void rhizome_fetch_close(struct rhizome_fetch_slot *slot)
{
if (config.debug.rhizome_rx)
DEBUGF("close Rhizome fetch slot=%d", slotno(slot));
@ -1035,8 +1066,6 @@ static int rhizome_fetch_close(struct rhizome_fetch_slot *slot)
// Activate the next queued fetch that is eligible for this slot. Try starting candidates from
// all queues with the same or smaller size thresholds until the slot is taken.
rhizome_start_next_queued_fetch(slot);
return 0;
}
static void rhizome_fetch_mdp_slot_callback(struct sched_ent *alarm)
@ -1174,7 +1203,7 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){
return 0;
}
static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
static enum rhizome_start_fetch_result rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
{
/* In Rhizome Direct we use the same fetch slot system, but we aren't actually
a running servald instance, so we cannot fall back to MDP. This is detected
@ -1187,11 +1216,13 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
*/
IN();
if (!is_rhizome_mdp_enabled()){
RETURN(rhizome_fetch_close(slot));
rhizome_fetch_close(slot);
RETURN(-1);
}
if (!my_subscriber) {
DEBUGF("I don't have an identity, so we cannot fall back to MDP");
RETURN(rhizome_fetch_close(slot));
rhizome_fetch_close(slot);
RETURN(-1);
}
if (config.debug.rhizome_rx)
@ -1241,7 +1272,7 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
slot->mdpRXBlockLength = config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
rhizome_fetch_mdp_requestblocks(slot);
RETURN(0);
RETURN(STARTED);
OUT();
}
@ -1277,7 +1308,7 @@ void rhizome_fetch_write(struct rhizome_fetch_slot *slot)
return;
}
int rhizome_write_complete(struct rhizome_fetch_slot *slot)
static int rhizome_write_complete(struct rhizome_fetch_slot *slot)
{
IN();
@ -1289,12 +1320,13 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
if (config.debug.rhizome_rx)
DEBUGF("Received all of file via rhizome -- now to import it");
if (rhizome_finish_write(&slot->write_state)){
enum rhizome_payload_status status = rhizome_finish_write(&slot->write_state);
if (status != RHIZOME_PAYLOAD_STATUS_EMPTY && status != RHIZOME_PAYLOAD_STATUS_NEW) {
rhizome_fetch_close(slot);
RETURN(-1);
}
if (rhizome_import_received_bundle(slot->manifest)){
if (rhizome_import_received_bundle(slot->manifest) == -1){
rhizome_fetch_close(slot);
RETURN(-1);
}
@ -1321,7 +1353,9 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
call schedule queued items. */
rhizome_manifest *m = rhizome_new_manifest();
if (m) {
if ( rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1
memcpy(m->manifestdata, slot->manifest_buffer, (size_t)slot->manifest_bytes);
m->manifest_all_bytes = (size_t)slot->manifest_bytes;
if ( rhizome_manifest_parse(m) == -1
|| !rhizome_manifest_validate(m)
) {
DEBUGF("Couldn't read manifest");
@ -1445,9 +1479,10 @@ int rhizome_received_content(const unsigned char *bidprefix,
}
if (m){
if (rhizome_import_buffer(m, bytes, count) >= 0 && !rhizome_import_received_bundle(m)){
if (rhizome_import_buffer(m, bytes, count) == RHIZOME_PAYLOAD_STATUS_NEW) {
INFOF("Completed MDP transfer in one hit for file %s",
alloca_tohex_rhizome_filehash_t(m->filehash));
rhizome_import_received_bundle(m);
if (c)
candidate_unqueue(c);
}

799
rhizome_http.c
View File

@ -30,8 +30,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "overlay_address.h"
#include "conf.h"
#include "str.h"
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "rhizome.h"
#include "dataformats.h"
#include "http_server.h"
#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
@ -45,6 +47,8 @@ struct http_handler{
static HTTP_HANDLER restful_rhizome_bundlelist_json;
static HTTP_HANDLER restful_rhizome_newsince;
static HTTP_HANDLER restful_rhizome_insert;
static HTTP_HANDLER restful_rhizome_;
static HTTP_HANDLER rhizome_status_page;
static HTTP_HANDLER rhizome_file_page;
@ -61,6 +65,8 @@ extern HTTP_HANDLER rhizome_direct_dispatch;
struct http_handler paths[]={
{"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json},
{"/restful/rhizome/newsince/", restful_rhizome_newsince},
{"/restful/rhizome/insert", restful_rhizome_insert},
{"/restful/rhizome/", restful_rhizome_},
{"/rhizome/status", rhizome_status_page},
{"/rhizome/file/", rhizome_file_page},
{"/rhizome/import", rhizome_direct_import},
@ -82,19 +88,20 @@ static int rhizome_dispatch(struct http_request *hr)
for (i = 0; i < NELS(paths); ++i) {
const char *remainder;
if (str_startswith(r->http.path, paths[i].path, &remainder)){
int ret = paths[i].parser(r, remainder);
if (ret < 0) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (ret == 0)
int result = paths[i].parser(r, remainder);
if (result == -1 || (result >= 200 && result < 600))
return result;
if (result == 1)
return 0;
if (result)
return WHYF("dispatch function for %s returned invalid result %d", paths[i].path, result);
}
}
http_request_simple_response(&r->http, 404, NULL);
return 0;
return 404;
}
static HTTP_RENDERER render_manifest_headers;
struct sched_ent server_alarm;
struct profile_total server_stats = {
.name = "rhizome_server_poll",
@ -103,7 +110,7 @@ struct profile_total server_stats = {
/*
HTTP server and client code for rhizome transfers and rhizome direct.
Selection of either use is made when starting the HTTP server and
specifying the call-back function to use on client connections.
specifying the call-back function to use on client connections.
*/
uint16_t rhizome_http_server_port = 0;
@ -232,10 +239,32 @@ success:
}
static void rhizome_server_finalise_http_request(struct http_request *_r)
static void finalise_union_read_state(rhizome_http_request *r)
{
rhizome_http_request *r = (rhizome_http_request *) _r;
rhizome_read_close(&r->u.read_state);
}
static void finalise_union_rhizome_insert(rhizome_http_request *r)
{
if (r->u.insert.manifest_text) {
free(r->u.insert.manifest_text);
r->u.insert.manifest_text = NULL;
}
if (r->u.insert.write.blob_fd != -1)
rhizome_fail_write(&r->u.insert.write);
}
static void rhizome_server_finalise_http_request(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = NULL;
}
if (r->finalise_union) {
r->finalise_union(r);
r->finalise_union = NULL;
}
request_count--;
}
@ -275,9 +304,6 @@ void rhizome_server_poll(struct sched_ent *alarm)
} else {
request_count++;
request->uuid = rhizome_http_request_uuid_counter++;
request->data_file_name[0] = '\0';
request->u.read_state.blob_fd = -1;
request->u.read_state.blob_rowid = 0;
if (peerip)
request->http.client_sockaddr_in = *peerip;
request->http.handle_headers = rhizome_dispatch;
@ -306,14 +332,14 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_
int count = 0;
for (; p != bufend; ++p) {
switch (*p) {
case '\n':
case '\n':
if (++count==2)
RETURN(p - buf);
case '\r': // ignore CR
case '\0': // ignore NUL (telnet inserts them)
break;
default:
count = 0;
default:
count = 0;
break;
}
}
@ -346,17 +372,14 @@ static int is_authorized(const struct http_client_authorization *auth)
static int authorize(struct http_request *r)
{
if (!is_from_loopback(r)) {
http_request_simple_response(r, 403, NULL);
return 0;
}
if (!is_from_loopback(r))
return 403;
if (!is_authorized(&r->request_header.authorization)) {
r->response.header.www_authenticate.scheme = BASIC;
r->response.header.www_authenticate.realm = "Serval Rhizome";
http_request_simple_response(r, 401, NULL);
return 0;
return 401;
}
return 1;
return 0;
}
#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8))
@ -391,43 +414,41 @@ static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
return 403;
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (!authorize(&r->http))
return 0;
return 404;
if (r->http.verb != HTTP_VERB_GET)
return 405;
int ret = authorize(&r->http);
if (ret)
return ret;
r->u.list.phase = LIST_HEADER;
r->u.list.rowcount = 0;
bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
return 0;
return 1;
}
static int restful_rhizome_newsince(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
return 403;
uint64_t rowid;
const char *end = NULL;
if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (!authorize(&r->http))
return 0;
return 404;
if (r->http.verb != HTTP_VERB_GET)
return 405;
int ret = authorize(&r->http);
if (ret)
return ret;
r->u.list.phase = LIST_HEADER;
r->u.list.rowcount = 0;
bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
r->u.list.cursor.rowid_since = rowid;
r->u.list.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000;
http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
return 0;
return 1;
}
static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
@ -563,59 +584,514 @@ static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsi
return ret;
}
static HTTP_REQUEST_PARSER restful_rhizome_insert_end;
static int insert_mime_part_start(struct http_request *);
static int insert_mime_part_end(struct http_request *);
static int insert_mime_part_header(struct http_request *, const struct mime_part_headers *);
static int insert_mime_part_body(struct http_request *, char *, size_t);
static int restful_rhizome_insert(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 404;
if (!is_rhizome_http_enabled())
return 403;
if (r->http.verb != HTTP_VERB_POST)
return 405;
int ret = authorize(&r->http);
if (ret)
return ret;
// Parse the request body as multipart/form-data.
assert(r->u.insert.current_part == NULL);
assert(!r->u.insert.received_author);
assert(!r->u.insert.received_secret);
assert(!r->u.insert.received_manifest);
assert(!r->u.insert.received_payload);
bzero(&r->u.insert.write, sizeof r->u.insert.write);
r->u.insert.write.blob_fd = -1;
r->finalise_union = finalise_union_rhizome_insert;
r->http.form_data.handle_mime_part_start = insert_mime_part_start;
r->http.form_data.handle_mime_part_end = insert_mime_part_end;
r->http.form_data.handle_mime_part_header = insert_mime_part_header;
r->http.form_data.handle_mime_body = insert_mime_part_body;
// Perform the insert once the body has arrived.
r->http.handle_content_end = restful_rhizome_insert_end;
return 1;
}
static char PART_MANIFEST[] = "manifest";
static char PART_PAYLOAD[] = "payload";
static char PART_AUTHOR[] = "bundle-author";
static char PART_SECRET[] = "bundle-secret";
static int insert_mime_part_start(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->u.insert.current_part == NULL);
return 0;
}
static int http_response_form_part(rhizome_http_request *r, const char *what, const char *partname, const char *text, size_t textlen)
{
if (config.debug.rhizome)
DEBUGF("%s \"%s\" form part %s", what, partname, text ? alloca_toprint(-1, text, textlen) : "");
strbuf msg = strbuf_alloca(100);
strbuf_sprintf(msg, "%s \"%s\" form part", what, partname);
http_request_simple_response(&r->http, 403, strbuf_str(msg));
return 403;
}
static int insert_make_manifest(rhizome_http_request *r)
{
if (!r->u.insert.received_manifest)
return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
if ((r->manifest = rhizome_new_manifest())) {
if (r->u.insert.manifest_len == 0)
return 0;
assert(r->u.insert.manifest_len <= sizeof r->manifest->manifestdata);
memcpy(r->manifest->manifestdata, r->u.insert.manifest_text, r->u.insert.manifest_len);
r->manifest->manifest_all_bytes = r->u.insert.manifest_len;
int n = rhizome_manifest_parse(r->manifest);
switch (n) {
case -1:
break;
case 0:
if (!r->manifest->malformed)
return 0;
// fall through
case 1:
http_request_simple_response(&r->http, 403, "Malformed manifest");
return 403;
default:
WHYF("rhizome_manifest_parse() returned %d", n);
break;
}
}
return 500;
}
static int insert_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (strcmp(h->content_disposition.name, PART_AUTHOR) == 0) {
if (r->u.insert.received_author)
return http_response_form_part(r, "Duplicate", PART_AUTHOR, NULL, 0);
r->u.insert.current_part = PART_AUTHOR;
}
else if (strcmp(h->content_disposition.name, PART_SECRET) == 0) {
if (r->u.insert.received_secret)
return http_response_form_part(r, "Duplicate", PART_SECRET, NULL, 0);
r->u.insert.current_part = PART_SECRET;
}
else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) {
// Reject a request if it has a repeated manifest part.
if (r->u.insert.received_manifest)
return http_response_form_part(r, "Duplicate", PART_MANIFEST, NULL, 0);
assert(r->u.insert.manifest_text == NULL);
assert(r->u.insert.manifest_text_size == 0);
assert(r->u.insert.manifest_len == 0);
if ( strcmp(h->content_type.type, "rhizome-manifest") != 0
|| strcmp(h->content_type.subtype, "text") != 0
)
return http_response_form_part(r, "Unsupported Content-Type in", PART_MANIFEST, NULL, 0);
r->u.insert.current_part = PART_MANIFEST;
}
else if (strcmp(h->content_disposition.name, PART_PAYLOAD) == 0) {
// Reject a request if it has a repeated payload part.
if (r->u.insert.received_payload)
return http_response_form_part(r, "Duplicate", PART_PAYLOAD, NULL, 0);
// Reject a request if it has a missing manifest part preceding the payload part.
if (!r->u.insert.received_manifest)
return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
assert(r->manifest != NULL);
r->u.insert.current_part = PART_PAYLOAD;
// If the manifest does not contain a 'name' field, then assign it from the payload filename.
if ( strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0
&& r->manifest->name == NULL
&& *h->content_disposition.filename
)
rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
// Start writing the payload content into the Rhizome store. Note: r->manifest->filesize can be
// RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field.
r->u.insert.payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
r->u.insert.payload_size = 0;
switch (r->u.insert.payload_status) {
case RHIZOME_PAYLOAD_STATUS_ERROR:
WHYF("rhizome_write_open_manifest() returned %d", r->u.insert.payload_status);
return 500;
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: initialise payload hash so it can be compared with stored payload
break;
default:
break;
}
}
else
return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0);
return 0;
}
static int accumulate_text(rhizome_http_request *r, const char *partname, char *textbuf, size_t textsiz, size_t *textlenp, const char *buf, size_t len)
{
if (len) {
size_t newlen = *textlenp + len;
if (newlen > textsiz) {
if (config.debug.rhizome)
DEBUGF("Form part \"%s\" too long, %zu bytes overflows maximum %zu by %zu",
partname, newlen, textsiz, (size_t)(newlen - textsiz)
);
strbuf msg = strbuf_alloca(100);
strbuf_sprintf(msg, "Overflow in \"%s\" form part", partname);
http_request_simple_response(&r->http, 403, strbuf_str(msg));
return 0;
}
memcpy(textbuf + *textlenp, buf, len);
*textlenp = newlen;
}
return 1;
}
static int insert_mime_part_body(struct http_request *hr, char *buf, size_t len)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->u.insert.current_part == PART_AUTHOR) {
accumulate_text(r, PART_AUTHOR,
r->u.insert.author_hex,
sizeof r->u.insert.author_hex,
&r->u.insert.author_hex_len,
buf, len);
}
else if (r->u.insert.current_part == PART_SECRET) {
accumulate_text(r, PART_SECRET,
r->u.insert.secret_hex,
sizeof r->u.insert.secret_hex,
&r->u.insert.secret_hex_len,
buf, len);
}
else if (r->u.insert.current_part == PART_MANIFEST) {
if (len == 0)
return 0;
size_t newlen = r->u.insert.manifest_len + len;
if (newlen > MAX_MANIFEST_BYTES) {
if (config.debug.rhizome)
DEBUGF("manifest too large, %zu bytes overflows maximum %zu by %zu",
newlen, MAX_MANIFEST_BYTES, (size_t)(newlen - MAX_MANIFEST_BYTES)
);
http_request_simple_response(&r->http, 403, "Manifest size overflow");
return 403;
}
if (newlen > r->u.insert.manifest_text_size) {
if ((r->u.insert.manifest_text = erealloc(r->u.insert.manifest_text, newlen)) == NULL)
return 500;
r->u.insert.manifest_text_size = newlen;
}
memcpy(r->u.insert.manifest_text + r->u.insert.manifest_len, buf, len);
r->u.insert.manifest_len = newlen;
}
else if (r->u.insert.current_part == PART_PAYLOAD) {
r->u.insert.payload_size += len;
switch (r->u.insert.payload_status) {
case RHIZOME_PAYLOAD_STATUS_NEW:
if (rhizome_write_buffer(&r->u.insert.write, (unsigned char *)buf, len) == -1)
return 500;
break;
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: calculate payload hash so it can be compared with stored payload
break;
default:
break;
}
} else
FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part));
return 0;
}
static int insert_mime_part_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->u.insert.current_part == PART_AUTHOR) {
if ( r->u.insert.author_hex_len != sizeof r->u.insert.author_hex
|| strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, NULL) == -1
)
return http_response_form_part(r, "Invalid", PART_AUTHOR, r->u.insert.author_hex, r->u.insert.author_hex_len);
r->u.insert.received_author = 1;
if (config.debug.rhizome)
DEBUGF("received %s = %s", PART_AUTHOR, alloca_tohex_sid_t(r->u.insert.author));
}
else if (r->u.insert.current_part == PART_SECRET) {
if ( r->u.insert.secret_hex_len != sizeof r->u.insert.secret_hex
|| strn_to_rhizome_bk_t(&r->u.insert.bundle_secret, r->u.insert.secret_hex, NULL) == -1
)
return http_response_form_part(r, "Invalid", PART_SECRET, r->u.insert.secret_hex, r->u.insert.secret_hex_len);
r->u.insert.received_secret = 1;
if (config.debug.rhizome)
DEBUGF("received %s = %s", PART_SECRET, alloca_tohex_rhizome_bk_t(r->u.insert.bundle_secret));
}
else if (r->u.insert.current_part == PART_MANIFEST) {
r->u.insert.received_manifest = 1;
int result = insert_make_manifest(r);
if (result)
return result;
if (r->manifest->has_id && r->u.insert.received_secret)
rhizome_apply_bundle_secret(r->manifest, &r->u.insert.bundle_secret);
if (r->manifest->service == NULL)
rhizome_manifest_set_service(r->manifest, RHIZOME_SERVICE_FILE);
if (rhizome_fill_manifest(r->manifest, NULL, r->u.insert.received_author ? &r->u.insert.author: NULL) == -1) {
WHY("rhizome_fill_manifest() failed");
return 500;
}
if (r->manifest->is_journal) {
http_request_simple_response(&r->http, 403, "Insert not supported for journals");
return 403;
}
assert(r->manifest != NULL);
}
else if (r->u.insert.current_part == PART_PAYLOAD) {
r->u.insert.received_payload = 1;
if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_NEW)
r->u.insert.payload_status = rhizome_finish_write(&r->u.insert.write);
if (r->u.insert.payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_finish_write() returned status = %d", r->u.insert.payload_status);
return 500;
}
} else
FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part));
r->u.insert.current_part = NULL;
return 0;
}
static int restful_rhizome_insert_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (!r->u.insert.received_manifest)
return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
if (!r->u.insert.received_payload)
return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0);
// Fill in the missing manifest fields and ensure payload and manifest are consistent.
assert(r->manifest != NULL);
assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET);
switch (r->u.insert.payload_status) {
case RHIZOME_PAYLOAD_STATUS_ERROR:
return 500;
case RHIZOME_PAYLOAD_STATUS_NEW:
if (r->manifest->filesize == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_filesize(r->manifest, r->u.insert.write.file_length);
// fall through
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: check that stored hash matches received payload's hash
// fall through
case RHIZOME_PAYLOAD_STATUS_EMPTY:
assert(r->manifest->filesize != RHIZOME_SIZE_UNSET);
if (r->u.insert.payload_size == r->manifest->filesize)
break;
// fall through
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
{
strbuf msg = strbuf_alloca(200);
strbuf_sprintf(msg, "Payload size (%"PRIu64") contradicts manifest (filesize=%"PRIu64")", r->u.insert.payload_size, r->manifest->filesize);
http_request_simple_response(&r->http, 403, strbuf_str(msg));
return 403;
}
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest");
return 403;
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
http_request_simple_response(&r->http, 403, "Missing bundle secret");
return 403;
default:
FATALF("payload_status = %d", r->u.insert.payload_status);
}
// Finalise the manifest and add it to the store.
if (r->manifest->filesize) {
if (!r->manifest->has_filehash)
rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id);
else
assert(cmp_rhizome_filehash_t(&r->u.insert.write.id, &r->manifest->filehash) == 0);
}
if (!rhizome_manifest_validate(r->manifest) || r->manifest->malformed) {
http_request_simple_response(&r->http, 403, "Manifest is malformed");
return 403;
}
if (!r->manifest->haveSecret) {
http_request_simple_response(&r->http, 403, "Missing bundle secret");
return 403;
}
rhizome_manifest *mout = NULL;
int result;
switch (rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new)) {
case RHIZOME_BUNDLE_STATUS_NEW:
result = 201;
if (mout && mout != r->manifest)
rhizome_manifest_free(mout);
mout = NULL;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_OLD:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
result = 200;
break;
case RHIZOME_BUNDLE_STATUS_INVALID:
result = 403;
break;
case RHIZOME_BUNDLE_STATUS_ERROR:
default:
result = 500;
break;
}
if (mout && mout != r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
if (result >= 400)
return result;
rhizome_authenticate_author(r->manifest);
r->http.render_extra_headers = render_manifest_headers;
http_request_response_static(&r->http, result, "rhizome-manifest/text",
(const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
);
return 0;
}
static int rhizome_response_content_init_filehash(rhizome_http_request *r, const rhizome_filehash_t *hash);
static int rhizome_response_content_init_payload(rhizome_http_request *r, rhizome_manifest *);
static HTTP_CONTENT_GENERATOR rhizome_payload_content;
static HTTP_HANDLER restful_rhizome_bid_rhm;
static HTTP_HANDLER restful_rhizome_bid_raw_bin;
static HTTP_HANDLER restful_rhizome_bid_decrypted_bin;
static int restful_rhizome_(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 403;
HTTP_HANDLER *handler = NULL;
rhizome_bid_t bid;
const char *end;
if (strn_to_rhizome_bid_t(&bid, remainder, &end) != -1) {
if (strcmp(end, ".rhm") == 0) {
handler = restful_rhizome_bid_rhm;
remainder = "";
} else if (strcmp(end, "/raw.bin") == 0) {
handler = restful_rhizome_bid_raw_bin;
remainder = "";
} else if (strcmp(end, "/decrypted.bin") == 0) {
handler = restful_rhizome_bid_decrypted_bin;
remainder = "";
}
}
if (handler == NULL)
return 404;
if (r->http.verb != HTTP_VERB_GET)
return 405;
int ret = authorize(&r->http);
if (ret)
return ret;
if ((r->manifest = rhizome_new_manifest()) == NULL)
return 500;
ret = rhizome_retrieve_manifest(&bid, r->manifest);
if (ret == -1)
return 500;
if (ret == 0) {
rhizome_authenticate_author(r->manifest);
r->http.render_extra_headers = render_manifest_headers;
} else {
assert(r->manifest == NULL);
assert(r->http.render_extra_headers == NULL);
}
ret = handler(r, remainder);
return ret;
}
static int restful_rhizome_bid_rhm(rhizome_http_request *r, const char *remainder)
{
if (*remainder || r->manifest == NULL)
return 404;
http_request_response_static(&r->http, 200, "rhizome-manifest/text",
(const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
);
return 1;
}
static int restful_rhizome_bid_raw_bin(rhizome_http_request *r, const char *remainder)
{
if (*remainder || r->manifest == NULL)
return 404;
if (r->manifest->filesize == 0) {
http_request_response_static(&r->http, 200, "application/binary", "", 0);
return 1;
}
int ret = rhizome_response_content_init_filehash(r, &r->manifest->filehash);
if (ret)
return ret;
http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content);
return 1;
}
static int restful_rhizome_bid_decrypted_bin(rhizome_http_request *r, const char *remainder)
{
if (*remainder || r->manifest == NULL)
return 404;
if (r->manifest->filesize == 0) {
// TODO use Content Type from manifest (once it is implemented)
http_request_response_static(&r->http, 200, "application/binary", "", 0);
return 1;
}
int ret = rhizome_response_content_init_payload(r, r->manifest);
if (ret)
return ret;
// TODO use Content Type from manifest (once it is implemented)
http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content);
return 1;
}
static int neighbour_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (r->http.verb != HTTP_VERB_GET)
return 405;
char buf[8*1024];
strbuf b = strbuf_local(buf, sizeof buf);
sid_t neighbour_sid;
if (str_to_sid_t(&neighbour_sid, remainder) == -1)
return 1;
return 404;
struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0);
if (!neighbour)
return 1;
return 404;
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
link_neighbour_status_html(b, neighbour);
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 0;
return 1;
}
static int interface_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (r->http.verb != HTTP_VERB_GET)
return 405;
char buf[8*1024];
strbuf b=strbuf_local(buf, sizeof buf);
int index=atoi(remainder);
if (index<0 || index>=OVERLAY_MAX_INTERFACES)
return 1;
return 404;
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
interface_state_html(b, &overlay_interfaces[index]);
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 0;
return 1;
}
static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
return 403;
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 404;
if (r->http.verb != HTTP_VERB_GET)
return 405;
char buf[32*1024];
strbuf b = strbuf_local(buf, sizeof buf);
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
@ -626,10 +1102,83 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
if (strbuf_overrun(b))
return -1;
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 1;
}
static int rhizome_response_content_init_read_state(rhizome_http_request *r)
{
if (r->u.read_state.length == RHIZOME_SIZE_UNSET && rhizome_read(&r->u.read_state, NULL, 0)) {
rhizome_read_close(&r->u.read_state);
return 404;
}
assert(r->u.read_state.length != RHIZOME_SIZE_UNSET);
r->http.response.header.resource_length = r->u.read_state.length;
if (r->http.request_header.content_range_count > 0) {
assert(r->http.request_header.content_range_count == 1);
struct http_range closed;
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
if (n == 0 || http_range_bytes(&closed, 1) == 0)
return 416; // Request Range Not Satisfiable
r->http.response.header.content_range_start = closed.first;
r->http.response.header.content_length = closed.last - closed.first + 1;
r->u.read_state.offset = closed.first;
} else {
r->http.response.header.content_range_start = 0;
r->http.response.header.content_length = r->http.response.header.resource_length;
r->u.read_state.offset = 0;
}
return 0;
}
static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
static int rhizome_response_content_init_filehash(rhizome_http_request *r, const rhizome_filehash_t *hash)
{
bzero(&r->u.read_state, sizeof r->u.read_state);
r->u.read_state.blob_fd = -1;
assert(r->finalise_union == NULL);
r->finalise_union = finalise_union_read_state;
enum rhizome_payload_status status = rhizome_open_read(&r->u.read_state, hash);
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
return 404;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
return -1;
default:
FATALF("status = %d", status);
}
return rhizome_response_content_init_read_state(r);
}
static int rhizome_response_content_init_payload(rhizome_http_request *r, rhizome_manifest *m)
{
bzero(&r->u.read_state, sizeof r->u.read_state);
r->u.read_state.blob_fd = -1;
assert(r->finalise_union == NULL);
r->finalise_union = finalise_union_read_state;
enum rhizome_payload_status status = rhizome_open_decrypt_read(m, &r->u.read_state);
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
return 404;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
return -1;
default:
FATALF("status = %d", status);
}
return rhizome_response_content_init_read_state(r);
}
static int rhizome_payload_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
{
// Only read multiples of 4k from disk.
const size_t blocksz = 1 << 12;
@ -661,93 +1210,95 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
{
/* Stream the specified payload */
if (!is_rhizome_http_enabled())
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 403;
if (r->http.verb != HTTP_VERB_GET)
return 405;
if (r->http.request_header.content_range_count > 1) {
// To support byte range sets, eg, Range: bytes=0-100,200-300,400- we would have
// to reply with a multipart/byteranges MIME content.
http_request_simple_response(&r->http, 501, "Not Implemented: Byte range sets");
return 0;
return 1;
}
rhizome_filehash_t filehash;
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
return 1;
bzero(&r->u.read_state, sizeof r->u.read_state);
int n = rhizome_open_read(&r->u.read_state, &filehash);
if (n == -1) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (n != 0)
return 1;
if (r->u.read_state.length == RHIZOME_SIZE_UNSET && rhizome_read(&r->u.read_state, NULL, 0)) {
rhizome_read_close(&r->u.read_state);
return 1;
}
assert(r->u.read_state.length != RHIZOME_SIZE_UNSET);
r->http.response.header.resource_length = r->u.read_state.length;
if (r->http.request_header.content_range_count > 0) {
assert(r->http.request_header.content_range_count == 1);
struct http_range closed;
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
if (n == 0 || http_range_bytes(&closed, 1) == 0) {
http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
return 0;
}
r->http.response.header.content_range_start = closed.first;
r->http.response.header.content_length = closed.last - closed.first + 1;
r->u.read_state.offset = closed.first;
} else {
r->http.response.header.content_range_start = 0;
r->http.response.header.content_length = r->http.response.header.resource_length;
r->u.read_state.offset = 0;
}
http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
return 0;
int ret = rhizome_response_content_init_filehash(r, &filehash);
if (ret)
return ret;
http_request_response_generated(&r->http, 200, "application/binary", rhizome_payload_content);
return 1;
}
static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 403;
if (r->http.verb != HTTP_VERB_GET)
return 405;
rhizome_bid_t prefix;
const char *endp = NULL;
unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp);
if (endp == NULL || *endp != '\0' || prefix_len < 1)
return 1; // not found
rhizome_manifest *m = rhizome_new_manifest();
int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m);
return 404; // not found
if ((r->manifest = rhizome_new_manifest()) == NULL)
return 500;
int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, r->manifest);
if (ret == -1)
http_request_simple_response(&r->http, 500, NULL);
else if (ret == 0)
http_request_response_static(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
rhizome_manifest_free(m);
return ret <= 0 ? 0 : 1;
return 500;
if (ret == 0) {
http_request_response_static(&r->http, 200, "application/binary", (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes);
return 1;
}
return 404;
}
static int fav_icon_header(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
return 404;
http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
return 0;
return 1;
}
static void render_manifest_headers(struct http_request *hr, strbuf sb)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
rhizome_manifest *m = r->manifest;
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Id: %s\r\n", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Version: %"PRIu64"\r\n", m->version);
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filesize: %"PRIu64"\r\n", m->filesize);
if (m->filesize != 0)
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Filehash: %s\r\n", alloca_tohex_rhizome_filehash_t(m->filehash));
if (m->has_bundle_key)
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-BK: %s\r\n", alloca_tohex_rhizome_bk_t(m->bundle_key));
if (m->has_date)
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Date: %"PRIu64"\r\n", m->date);
if (m->name) {
strbuf_puts(sb, "Serval-Rhizome-Bundle-Name: ");
strbuf_append_quoted_string(sb, m->name);
strbuf_puts(sb, "\r\n");
}
if (m->service)
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Service: %s\r\n", m->service);
assert(m->authorship != AUTHOR_LOCAL);
if (m->authorship == AUTHOR_AUTHENTIC)
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Author: %s\r\n", alloca_tohex_sid_t(m->author));
assert(m->haveSecret);
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Secret: %s\r\n", secret);
}
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Rowid: %"PRIu64"\r\n", m->rowid);
strbuf_sprintf(sb, "Serval-Rhizome-Bundle-Inserttime: %"PRIu64"\r\n", m->inserttime);
}
static int root_page(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
return 404;
if (r->http.verb != HTTP_VERB_GET)
return 405;
char temp[8192];
strbuf b = strbuf_local(temp, sizeof temp);
strbuf_sprintf(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>"
@ -768,8 +1319,8 @@ static int root_page(rhizome_http_request *r, const char *remainder)
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b)) {
WHY("HTTP Root page buffer overrun");
http_request_simple_response(&r->http, 500, NULL);
} else
http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b));
return 0;
return 500;
}
http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b));
return 1;
}

8
rhizome_packetformats.c
View File

@ -112,9 +112,7 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
v=(maxLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
v=(maxLong+180)*(65535/360); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
/* TTL */
if (m->ttl>0) bar[RHIZOME_BAR_TTL_OFFSET]=m->ttl-1;
else bar[RHIZOME_BAR_TTL_OFFSET]=0;
bar[RHIZOME_BAR_TTL_OFFSET]=0;
RETURN(0);
OUT();
@ -350,7 +348,9 @@ int overlay_rhizome_saw_advertisements(struct decode_context *context, struct ov
// The manifest looks potentially interesting, so now do a full parse and validation.
if ((m = rhizome_new_manifest()) == NULL)
goto next;
if ( rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1
memcpy(m->manifestdata, data, manifest_length);
m->manifest_all_bytes = manifest_length;
if ( rhizome_manifest_parse(m) == -1
|| !rhizome_manifest_validate(m)
) {
WARN("Malformed manifest");

828
rhizome_store.c
View File

File diff suppressed because it is too large Load Diff

44
str.c
View File

@ -645,7 +645,7 @@ char *str_str(char *haystack, const char *needle, size_t haystack_len)
return NULL;
}
int str_to_int(const char *str, int base, int *result, const char **afterp)
int str_to_int32(const char *str, int base, int32_t *result, const char **afterp)
{
if (isspace(*str))
return 0;
@ -654,14 +654,14 @@ int str_to_int(const char *str, int base, int *result, const char **afterp)
long value = strtol(str, (char**)&end, base);
if (afterp)
*afterp = end;
if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN || isdigit(*end) || (!afterp && *end))
if (errno == ERANGE || end == str || value > INT32_MAX || value < INT32_MIN || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
return 1;
}
int str_to_uint(const char *str, int base, unsigned *result, const char **afterp)
int str_to_uint32(const char *str, int base, uint32_t *result, const char **afterp)
{
if (isspace(*str))
return 0;
@ -670,7 +670,7 @@ int str_to_uint(const char *str, int base, unsigned *result, const char **afterp
unsigned long value = strtoul(str, (char**)&end, base);
if (afterp)
*afterp = end;
if (errno == ERANGE || end == str || value > UINT_MAX || isdigit(*end) || (!afterp && *end))
if (errno == ERANGE || end == str || value > UINT32_MAX || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
@ -758,6 +758,42 @@ int str_to_int64_scaled(const char *str, int base, int64_t *result, const char *
return 1;
}
int str_to_uint32_scaled(const char *str, int base, uint32_t *result, const char **afterp)
{
uint32_t value;
const char *end = str;
if (!str_to_uint32(str, base, &value, &end)) {
if (afterp)
*afterp = end;
return 0;
}
value *= scale_factor(end, &end);
if (afterp)
*afterp = end;
else if (*end)
return 0;
if (result)
*result = value;
return 1;
}
int uint32_scaled_to_str(char *str, size_t len, uint32_t value)
{
char symbol = '\0';
int i;
for (i = 0; i != NELS(scale_factors); ++i)
if (value % scale_factors[i].factor == 0) {
value /= scale_factors[i].factor;
symbol = scale_factors[i].symbol;
break;
}
strbuf b = strbuf_local(str, len);
strbuf_sprintf(b, "%lu", (unsigned long) value);
if (symbol)
strbuf_putc(b, symbol);
return strbuf_overrun(b) ? 0 : 1;
}
int str_to_uint64_scaled(const char *str, int base, uint64_t *result, const char **afterp)
{
uint64_t value;

7
str.h
View File

@ -391,8 +391,8 @@ char *str_str(char *haystack, const char *needle, size_t haystack_len);
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int str_to_int(const char *str, int base, int *result, const char **afterp);
int str_to_uint(const char *str, int base, unsigned *result, const char **afterp);
int str_to_int32(const char *str, int base, int32_t *result, const char **afterp);
int str_to_uint32(const char *str, int base, uint32_t *result, const char **afterp);
int str_to_int64(const char *str, int base, int64_t *result, const char **afterp);
int str_to_uint64(const char *str, int base, uint64_t *result, const char **afterp);
@ -410,6 +410,8 @@ int str_to_uint64(const char *str, int base, uint64_t *result, const char **afte
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int str_to_int32_scaled(const char *str, int base, int32_t *result, const char **afterp);
int str_to_uint32_scaled(const char *str, int base, uint32_t *result, const char **afterp);
int str_to_int64_scaled(const char *str, int base, int64_t *result, const char **afterp);
int str_to_uint64_scaled(const char *str, int base, uint64_t *result, const char **afterp);
uint64_t scale_factor(const char *str, const char **afterp);
@ -423,6 +425,7 @@ uint64_t scale_factor(const char *str, const char **afterp);
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int uint32_scaled_to_str(char *str, size_t len, uint32_t value);
int uint64_scaled_to_str(char *str, size_t len, uint64_t value);
/* Parse a string as a time interval (seconds) in millisecond resolution. Return the number of

10
testdefs_rhizome.sh
View File

@ -24,7 +24,7 @@ rexp_bundlesecret="$rexp_bundlekey"
rexp_filehash='[0-9a-fA-F]\{128\}'
rexp_filesize='[0-9]\{1,\}'
rexp_version='[0-9]\{1,\}'
rexp_crypt='[0-9]\{1,\}'
rexp_crypt='[01]'
rexp_date='[0-9]\{1,\}'
rexp_rowid='[0-9]\{1,\}'
@ -298,6 +298,10 @@ extract_stdout_version() {
extract_stdout_keyvalue "$1" version "$rexp_version"
}
extract_stdout_author() {
extract_stdout_keyvalue "$1" .author "$rexp_author"
}
extract_stdout_secret() {
extract_stdout_keyvalue "$1" .secret "$rexp_bundlesecret"
}
@ -326,6 +330,10 @@ extract_stdout_filehash() {
extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
}
extract_stdout_crypt() {
extract_stdout_keyvalue "$1" crypt "$rexp_crypt"
}
extract_manifest() {
local _var="$1"
local _manifestfile="$2"

1
testframework.sh
View File

@ -840,6 +840,7 @@ _tfw_getopts() {
*grep:--fixed-strings) _tfw_opt_grepopts+=(-F);;
assertcontentgrep:--matches=+([0-9])) _tfw_opt_matches="${1#*=}";;
assertcontentgrep:--matches=*) _tfw_error "invalid value: $1";;
assertcontentgrep:--ignore-case) _tfw_opt_grepopts+=(-i);;
assertcontent*:--line=+([0-9])) _tfw_opt_line="${1#*=}"; _tfw_opt_line_msg="line $_tfw_opt_line";;
assertcontent*:--line=+([0-9])..) _tfw_opt_line="${1#*=}\$"; _tfw_opt_line_msg="lines $_tfw_opt_line";;
assertcontent*:--line=..+([0-9])) _tfw_opt_line="1${1#*=}"; _tfw_opt_line_msg="lines $_tfw_opt_line";;

1
tests/all
View File

@ -29,6 +29,7 @@ includeTests dnahelper
includeTests dnaprotocol
includeTests rhizomeops
includeTests rhizomeprotocol
includeTests rhizomehttp
includeTests meshms
includeTests directory_service

8
tests/meshms
View File

@ -33,6 +33,7 @@ setup_logging() {
set debug.meshms on \
set debug.rhizome on \
set debug.rhizome_manifest on \
set debug.externalblobs on \
set debug.rejecteddata on \
set log.console.level debug \
set log.console.show_time on
@ -72,6 +73,7 @@ test_MessageDelivery() {
tfw_log "CONV_BID=$CONV_BID CONV_SECRET=$CONV_SECRET"
# 5. mark the first message as read
executeOk_servald meshms read messages $SIDA2 $SIDA1 5
tfw_cat --stderr
check_meshms_bundles
executeOk_servald meshms list messages $SIDA2 $SIDA1
assertStdoutGrep --stdout --matches=1 "^0:19:<:How are you\$"
@ -99,14 +101,14 @@ check_meshms_bundles() {
# The only "file" bundle should be the conversation list
executeOk_servald rhizome list file
rhizome_list_unpack X
assert [ $XNROWS -eq 1 ]
assert [ ${XBID[0]} = $CONV_BID ]
assert --stdout --stderr [ $XNROWS -eq 1 ]
assert --stdout --stderr [ ${XBID[0]} = $CONV_BID ]
executeOk_servald rhizome extract bundle $CONV_BID manifest.conv payload.conv $CONV_SECRET
tfw_cat -v manifest.conv --hexdump payload.conv
# The only "MeshMS2" bundles should be the two ply bundles
executeOk_servald rhizome list MeshMS2
rhizome_list_unpack X
assert [ $XNROWS -eq 2 ]
assert --stdout [ $XNROWS -eq 2 ]
local bid
for bid in ${XBID[*]}; do
executeOk_servald rhizome extract bundle $bid manifest.$bid payload.$bid

599
tests/rhizomehttp
View File

@ -1,6 +1,6 @@
#!/bin/bash
# Tests for Serval rhizome operations.
# Tests for Serval DNA HTTP RESTful interface
#
# Copyright 2013 Serval Project, Inc.
#
@ -26,6 +26,7 @@ shopt -s extglob
setup() {
CR=' '
VT=' '
setup_curl 7
setup_jq 1.2
setup_servald
@ -35,8 +36,11 @@ setup() {
set rhizome.api.restful.users.harry.password potter \
set rhizome.api.restful.users.ron.password weasley \
set rhizome.api.restful.users.hermione.password grainger
create_single_identity
echo "$SIDA1" >sids
if [ -z "$IDENTITY_COUNT" ]; then
create_single_identity
else
create_identities $IDENTITY_COUNT
fi
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
@ -56,12 +60,14 @@ set_rhizome_config() {
executeOk_servald config \
set debug.httpd on \
set debug.rhizome_httpd on \
set debug.rhizome_manifest on \
set debug.externalblobs on \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
}
doc_AuthBasicMissing="Basic Authentication credentials are required"
doc_AuthBasicMissing="HTTP RESTful missing Basic Authentication credentials"
test_AuthBasicMissing() {
executeOk curl \
--silent --show-error --write-out '%{http_code}' \
@ -69,14 +75,14 @@ test_AuthBasicMissing() {
--dump-header http.headers \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assertStdoutIs '401'
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR$"
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$"
}
teardown_AuthBasicMissing() {
tfw_cat http.headers http.output
teardown
}
doc_AuthBasicWrong="Basic Authentication credentials must be correct"
doc_AuthBasicWrong="HTTP RESTful incorrect Basic Authentication credentials"
test_AuthBasicWrong() {
executeOk curl \
--silent --show-error --write-out '%{http_code}' \
@ -85,7 +91,7 @@ test_AuthBasicWrong() {
--basic --user fred:nurks \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assertStdoutIs '401'
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR$"
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR\$"
executeOk curl \
--silent --fail --show-error --write-out '%{http_code}' \
--output http.output \
@ -100,16 +106,40 @@ teardown_AuthBasicWrong() {
}
add_bundles() {
local encrypted=false
case "$1" in
--encrypted) encrypted=true; shift;;
esac
local n
for ((n = $1; n <= $2; ++n)); do
create_file file$n $((1000 + $n))
if $encrypted; then
echo "crypt=1" >file$n.manifest
fi
executeOk_servald rhizome add file $SIDA file$n file$n.manifest
extract_stdout_manifestid BID[$n]
extract_stdout_version VERSION[$n]
extract_stdout_filesize SIZE[$n]
extract_stdout_filehash HASH[$n]
extract_stdout_date DATE[$n]
extract_stdout_BK BK[$n]
extract_stdout_rowid ROWID[$n]
extract_stdout_author AUTHOR[$n]
extract_stdout_secret SECRET[$n]
extract_stdout_inserttime INSERTTIME[$n]
NAME[$n]=file$n
if $encrypted; then
extract_stdout_crypt CRYPT[$n]
assert [ "${CRYPT[$n]}" = 1 ]
else
CRYPT[$n]=
fi
executeOk_servald rhizome export file ${HASH[$n]} raw$n
if $encrypted; then
assert ! cmp file$n raw$n
else
assert cmp file$n raw$n
fi
[ "${ROWID[$n]}" -gt "${ROWID_MAX:-0}" ] && ROWID_MAX=${ROWID[$n]}
done
}
@ -182,7 +212,7 @@ assertJq() {
assert --message="$jqscript" [ "$(jq "$jqscript" "$json")" = true ]
}
doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
doc_RhizomeList="HTTP RESTful list Rhizome bundles as JSON"
setup_RhizomeList() {
setup
NBUNDLES=100
@ -235,7 +265,7 @@ curl_newsince() {
"http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json"
}
doc_RhizomeNewSince="Fetch Rhizome bundle list since token in JSON format"
doc_RhizomeNewSince="HTTP RESTful list Rhizome bundles since token as JSON"
setup_RhizomeNewSince() {
setup
executeOk_servald config set rhizome.api.restful.newsince_timeout 60s
@ -293,43 +323,558 @@ test_RhizomeNewSince() {
done
}
doc_RhizomeManifest="Fetch Rhizome bundle manifest"
assert_http_response_headers() {
local n=$1
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Id: ${BID[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Version: ${VERSION[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Filesize: ${SIZE[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Filehash: ${HASH[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-BK: ${BK[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Date: ${DATE[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Name: \"${NAME[$n]}\"$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Service: file$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Author: ${AUTHOR[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Secret: ${SECRET[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Inserttime: ${INSERTTIME[$n]}$CR\$"
assertGrep --matches=1 http.headers$n "^Serval-Rhizome-Bundle-Rowid: ${ROWID[$n]}$CR\$"
}
doc_RhizomeManifest="HTTP RESTful fetch Rhizome manifest"
setup_RhizomeManifest() {
setup
add_bundles 0 2
}
test_RhizomeManifest() {
:
for n in 0 1 2; do
executeOk curl \
--silent --fail --show-error \
--output bundle$n.rhm \
--dump-header http.headers$n \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/${BID[$n]}.rhm"
tfw_cat http.headers$n bundle$n.rhm
tfw_preserve bundle$n.rhm
done
for n in 0 1 2; do
assert diff file$n.manifest bundle$n.rhm
assert_http_response_headers $n
done
}
doc_RhizomePayloadRaw="Fetch Rhizome raw payload"
doc_RhizomePayloadRaw="HTTP RESTful fetch Rhizome raw payload"
setup_RhizomePayloadRaw() {
setup
add_bundles 0 1
add_bundles --encrypted 2 3
}
test_RhizomePayloadRaw() {
:
for n in 0 1 2 3; do
executeOk curl \
--silent --fail --show-error \
--output raw.bin$n \
--dump-header http.headers$n \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/${BID[$n]}/raw.bin"
tfw_cat http.headers$n raw.bin$n
done
for n in 0 1 2 3; do
assert cmp raw$n raw.bin$n
assert_http_response_headers $n
done
}
doc_RhizomePayloadDecrypted="Fetch Rhizome decrypted payload"
doc_RhizomePayloadDecrypted="HTTP RESTful fetch Rhizome decrypted payload"
setup_RhizomePayloadDecrypted() {
setup
add_bundles 0 1
add_bundles --encrypted 2 3
}
test_RhizomePayloadDecrypted() {
:
for n in 0 1 2 3; do
executeOk curl \
--silent --fail --show-error \
--output decrypted.bin$n \
--dump-header http.headers$n \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/${BID[$n]}/decrypted.bin"
tfw_cat http.headers$n decrypted.bin$n
done
for n in 0 1 2 3; do
assert cmp file$n decrypted.bin$n
assert_http_response_headers $n
done
}
doc_RhizomeInsert="Insert new Rhizome bundle"
extract_http_header() {
local __var="$1"
local __headerfile="$2"
local __header="$3"
local __rexp="$4"
local __value=$($SED -n -e "/^$__header:[ $VT]*$__rexp$CR\$/s/^$__header:[ $VT]*\(.*\)$CR\$/\1/p" "$__headerfile")
assert --message="$__headerfile contains valid '$__header' header" \
--dump-on-fail="$__headerfile" \
[ -n "$__value" ]
[ -n "$__var" ] && eval $__var=\"\$__value\"
}
http_unquote_string() {
local __var="$1"
local __unq="$(eval echo '"${'$__var'}"' | sed -e 's/^"//' -e 's/"$//' -e 's/\\\(.\)/\1/g')"
eval $__var=\"\$__unq\"
}
doc_RhizomeInsert="HTTP RESTful insert new Rhizome bundles"
setup_RhizomeInsert() {
IDENTITY_COUNT=3
SIDA4=
setup
for n in 1 2 3 4; do
create_file file$n $((1000 + $n))
create_file nfile$n $((1100 + $n))
payload_filename[$n]=
eval author[$n]=\$SIDA$n
service[$n]=file
done
name[1]=blarg
echo "name=blarg" >manifest1
name[2]=file2
echo "crypt=1" >manifest2
name[3]=kibble
payload_filename[3]=kibble
>manifest3
name[4]=
service[4]=wah
echo -e "service=wah\ncrypt=0" >manifest4
}
test_RhizomeInsert() {
for n in 1 2 3 4; do
authorargs=()
[ -n "${author[$n]}" ] && authorargs=(--form "bundle-author=${author[$n]}")
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file$n.manifest \
--dump-header http.header$n \
--basic --user harry:potter \
"${authorargs[@]}" \
--form "manifest=@manifest$n;type=rhizome-manifest/text" \
--form "payload=@file$n${payload_filename[$n]:+;filename=\"${payload_filename[$n]}\"}" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header$n file$n.manifest
assertExitStatus == 0
assertStdoutIs 201
extract_http_header BID[$n] http.header$n Serval-Rhizome-Bundle-Id "$rexp_manifestid"
extract_http_header VERSION[$n] http.header$n Serval-Rhizome-Bundle-Version "$rexp_version"
extract_http_header SIZE[$n] http.header$n Serval-Rhizome-Bundle-Filesize "$rexp_filesize"
extract_http_header HASH[$n] http.header$n Serval-Rhizome-Bundle-Filehash "$rexp_filehash"
extract_http_header DATE[$n] http.header$n Serval-Rhizome-Bundle-Date "$rexp_date"
extract_http_header ROWID[$n] http.header$n Serval-Rhizome-Bundle-Rowid "$rexp_rowid"
extract_http_header INSERTTIME[$n] http.header$n Serval-Rhizome-Bundle-Inserttime "$rexp_date"
extract_http_header SECRET[$n] http.header$n Serval-Rhizome-Bundle-Secret "$rexp_bundlesecret"
extract_http_header SERVICE[$n] http.header$n Serval-Rhizome-Bundle-Service ".*"
assert [ ${SIZE[$n]} -eq $((1000 + $n)) ]
assert [ "${SERVICE[$n]}" = "${service[$n]}" ]
extract_manifest_id BID file$n.manifest
extract_manifest_version VERSION file$n.manifest
extract_manifest_filesize SIZE file$n.manifest
extract_manifest_filehash HASH file$n.manifest
extract_manifest_date DATE file$n.manifest
extract_manifest_service SERVICE file$n.manifest
assert [ "$BID" = "${BID[$n]}" ]
assert [ "$VERSION" = "${VERSION[$n]}" ]
assert [ "$SIZE" = "${SIZE[$n]}" ]
assert [ "$HASH" = "${HASH[$n]}" ]
assert [ "$DATE" = "${DATE[$n]}" ]
assert [ "$SERVICE" = "${SERVICE[$n]}" ]
if [ -n "${name[$n]}" ]; then
extract_http_header NAME[$n] http.header$n Serval-Rhizome-Bundle-Name ".*"
http_unquote_string NAME[$n]
assert [ "${NAME[$n]}" = "${name[$n]}" ]
extract_manifest_name NAME file$n.manifest
assert [ "$NAME" = "${NAME[$n]}" ]
fi
if [ -n "${author[$n]}" ]; then
extract_http_header AUTHOR[$n] http.header$n Serval-Rhizome-Bundle-Author "$rexp_sid"
assert [ "${AUTHOR[$n]}" = "${author[$n]}" ]
extract_http_header BK[$n] http.header$n Serval-Rhizome-Bundle-BK "$rexp_bundlekey"
extract_manifest_BK BK file$n.manifest
assert [ "$BK" = "${BK[$n]}" ]
fi
done
executeOk_servald rhizome list
assert_rhizome_list \
--fromhere=1 \
--author=${author[1]} file1 \
--author=${author[2]} file2 \
--author=${author[3]} file3 \
--fromhere=0 \
--author=${author[4]} file4
for n in 1 2 3 4; do
executeOk_servald rhizome extract bundle ${BID[$n]} xfile$n.manifest xfile$n
assert diff xfile$n.manifest file$n.manifest
assert diff file$n xfile$n
done
for n in 1 2 3 4; do
$SED -e '/^version=/d;/^date=/d;/^filehash=/d;/^filesize=/d' xfile$n.manifest >nmanifest$n
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output nfile$n.manifest \
--dump-header http.header$n \
--basic --user harry:potter \
--form "manifest=@nmanifest$n;type=rhizome-manifest/text" \
--form "payload=@nfile$n;filename=\"nfile$n\"" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header$n nfile$n.manifest
assertExitStatus == 0
if [ -n "${author[$n]}" ]; then
assertStdoutIs 201
else
assertStdoutIs 403
assertGrep --ignore-case nfile$n.manifest "missing bundle secret"
fi
done
}
doc_RhizomeInsertAnon="HTTP RESTful update anonymous Rhizome bundle"
setup_RhizomeInsertAnon() {
setup
create_file file1 1001
executeOk_servald rhizome add file '' file1 file1.manifest
extract_stdout_secret SECRET
assertGrep --matches=0 file1.manifest '^BK='
$SED -e '/^version=/d;/^date=/d;/^filehash=/d;/^filesize=/d' file1.manifest >file2.manifest
create_file file2 1002
}
test_RhizomeInsertAnon() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output ifile2.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "bundle-secret=$SECRET" \
--form "manifest=@file2.manifest;type=rhizome-manifest/text" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header ifile2.manifest
assertExitStatus == 0
assertStdoutIs 201
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 --manifest=ifile2.manifest file2
}
doc_RhizomeInsertEmpty="HTTP RESTful insert empty Rhizome bundle"
setup_RhizomeInsertEmpty() {
setup
>empty
assert [ ! -s empty ]
}
test_RhizomeInsertEmpty() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output empty.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=;type=rhizome-manifest/text" \
--form "payload=@empty;filename=\"lucky\"" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header empty.manifest
assertExitStatus == 0
assertStdoutIs 201
extract_manifest_id BID empty.manifest
executeOk_servald rhizome list
assert_rhizome_list empty
executeOk_servald rhizome extract bundle $BID xempty.manifest xempty
assert [ ! -e xempty ]
assert diff xempty.manifest empty.manifest
}
doc_RhizomeInsertLarge="HTTP RESTful insert 50 MiB Rhizome bundle"
setup_RhizomeInsertLarge() {
setup
create_file file1 50m
}
test_RhizomeInsertLarge() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file1.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=;type=rhizome-manifest/text" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header file1.manifest
assertExitStatus == 0
assertStdoutIs 201
extract_manifest_id BID file1.manifest
executeOk_servald rhizome list
assert_rhizome_list file1
executeOk_servald rhizome extract bundle $BID xfile1.manifest xfile1
assert diff xfile1.manifest file1.manifest
assert cmp file1 xfile1
}
doc_RhizomeInsertMissingManifest="HTTP RESTful insert missing 'manifest' form part"
setup_RhizomeInsertMissingManifest() {
setup
echo 'File one' >file1
}
test_RhizomeInsertMissingManifest() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'missing.*manifest.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertIncorrectManifestType="HTTP RESTful insert incorrect 'manifest' content type"
setup_RhizomeInsertIncorrectManifestType() {
setup
echo 'File one' >file1
}
test_RhizomeInsertIncorrectManifestType() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=;type=rhizome-manifest/something" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'unsupported content-type.*manifest.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertDuplicateManifest="HTTP RESTful insert duplicate 'manifest' form part"
setup_RhizomeInsertDuplicateManifest() {
setup
echo 'File one' >file1
echo 'name=wah' >file1.manifest
echo 'name=bee' >file2.manifest
}
test_RhizomeInsertDuplicateManifest() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "manifest=@file2.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'duplicate.*manifest.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertJournal="HTTP RESTful insert does not support journals"
setup_RhizomeInsertJournal() {
setup
echo 'File one' >file1
echo 'tail=0' >file1.manifest
}
test_RhizomeInsertJournal() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'not supported.*journal'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertMissingPayload="HTTP RESTful insert missing 'payload' form part"
setup_RhizomeInsertMissingPayload() {
setup
echo 'File one' >file1
echo 'name=wah' >file1.manifest
}
test_RhizomeInsertMissingPayload() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'missing.*payload.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertDuplicatePayload="HTTP RESTful insert duplicate 'payload' form part"
setup_RhizomeInsertDuplicatePayload() {
setup
echo 'File one' >file1
echo 'File two' >file2
echo 'name=wah' >file1.manifest
}
test_RhizomeInsertDuplicatePayload() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'duplicate.*payload.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertPartOrder="HTTP RESTful insert 'payload' form part before 'manifest'"
setup_RhizomeInsertPartOrder() {
setup
echo 'File one' >file1
echo 'name=wah' >file1.manifest
}
test_RhizomeInsertPartOrder() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "payload=@file1" \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'missing.*manifest.*form.*part'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertPartUnsupported="HTTP RESTful insert unsupported form part"
setup_RhizomeInsertPartUnsupported() {
setup
echo 'File one' >file1
echo 'name=wah' >file1.manifest
}
test_RhizomeInsertPartUnsupported() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
--form "happyhappy=joyjoy" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'unsupported.*form.*part'
assertGrep http.body 'happyhappy'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertIncorrectFilesize="HTTP RESTful insert with incorrect filesize"
setup_RhizomeInsertIncorrectFilesize() {
setup
echo 'File one' >file1
echo 'filesize=6' >file1.manifest
echo 'File two' >file2
echo 'filesize=100' >file2.manifest
}
test_RhizomeInsertIncorrectFilesize() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'payload size.*contradicts manifest'
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file2.manifest;type=rhizome-manifest/text" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'payload size.*contradicts manifest'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_RhizomeInsertIncorrectFilehash="HTTP RESTful insert with incorrect filehash"
setup_RhizomeInsertIncorrectFilehash() {
setup
echo 'File one' >file1
echo 'filehash=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF' >file1.manifest
}
test_RhizomeInsertIncorrectFilehash() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome-manifest/text" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/insert"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertGrep --ignore-case http.body 'payload hash.*contradicts manifest'
executeOk_servald rhizome list
assert_rhizome_list
}
doc_MeshmsListConversations="HTTP RESTful list MeshMS conversations as JSON"
Xtest_MeshmsListConversations() {
:
}
doc_MeshmsListConversations="List MeshMS conversations"
test_MeshmsListConversations() {
doc_MeshmsListMessages="HTTP RESTful list MeshMS messages in one conversation as JSON"
Xtest_MeshmsListMessages() {
:
}
doc_MeshmsListMessages="List all MeshMS messages in a single conversation"
test_MeshmsListMessages() {
doc_MeshmsListMessagesSince="HTTP RESTful list MeshMS messages in one conversation since token as JSON"
Xtest_MeshmsListMessagesSince() {
:
}
doc_MeshmsListMessagesSince="List MeshMS messages in a single conversation since token"
test_MeshmsListMessagesSince() {
:
}
doc_MeshmsSend="Send MeshMS message"
test_MeshmsSend() {
doc_MeshmsSend="HTTP RESTful send MeshMS message"
Xtest_MeshmsSend() {
:
}

194
tests/rhizomeops
View File

@ -214,24 +214,27 @@ setup_ExtractManifestAfterAdd() {
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_BK BK file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAdd() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 11
assertStdoutLineCount '==' 13
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^BK:$BK\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
}
@ -248,24 +251,27 @@ setup_ExtractManifestFileAfterAdd() {
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_BK BK file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestFileAfterAdd() {
executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 11
assertStdoutLineCount '==' 13
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^BK:$BK\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
assert [ -e file1x ]
@ -276,11 +282,11 @@ doc_ExtractManifestFileFromExtBlob="Export bundle from external blob"
setup_ExtractManifestFileFromExtBlob() {
setup_servald
setup_rhizome
executeOk_servald config set rhizome.external_blobs 1
executeOk_servald config set rhizome.max_blob_size 0
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid1
executeOk_servald config set rhizome.external_blobs 0
executeOk_servald config set rhizome.max_blob_size 1000
echo "Another test file" >file2
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
extract_stdout_rowid rowid2
@ -289,47 +295,53 @@ setup_ExtractManifestFileFromExtBlob() {
extract_manifest_id manifestid1 file1.manifest
extract_manifest_version version1 file1.manifest
extract_manifest_filehash filehash1 file1.manifest
extract_manifest_BK BK1 file1.manifest
extract_manifest_date date1 file1.manifest
extract_manifest_id manifestid2 file2.manifest
extract_manifest_version version2 file2.manifest
extract_manifest_filehash filehash2 file2.manifest
extract_manifest_BK BK2 file2.manifest
extract_manifest_date date2 file2.manifest
}
test_ExtractManifestFileFromExtBlob() {
executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 11
assertStdoutLineCount '==' 13
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
assertStdoutGrep --matches=1 "^version:$version1\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
assertStdoutGrep --matches=1 "^BK:$BK1\$"
assertStdoutGrep --matches=1 "^date:$date1\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
assert [ -e file1x ]
assert diff file1 file1x
executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 11
assertStdoutLineCount '==' 13
local size=$(( $(cat file2 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
assertStdoutGrep --matches=1 "^version:$version2\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
assertStdoutGrep --matches=1 "^BK:$BK2\$"
assertStdoutGrep --matches=1 "^date:$date2\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file2\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date2\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file2x.manifest ]
assert diff file2.manifest file2x.manifest
assert [ -e file2x ]
@ -340,7 +352,7 @@ doc_CorruptExternalBlob="A corrupted payload should fail to export"
setup_CorruptExternalBlob() {
setup_servald
setup_rhizome
executeOk_servald config set rhizome.external_blobs 1
executeOk_servald config set rhizome.max_blob_size 0
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_manifest_id manifestid file1.manifest
@ -362,26 +374,29 @@ setup_ExtractManifestToStdout() {
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_BK BK file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestToStdout() {
executeOk_servald rhizome export manifest $manifestid -
assertStdoutLineCount '>=' 11
assertStdoutLineCount '>=' 14
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --line=..11 --matches=1 "^service:file$"
assertStdoutGrep --line=..11 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..11 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..11 --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --line=..11 --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --line=..11 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..11 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..11 --matches=1 "^date:$date\$"
assertStdoutGrep --line=..11 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..11 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..11 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=12 --matches=1 "^manifest:"
replayStdout | $SED -n '12s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '13,$p' >>file1x.manifest
assertStdoutGrep --line=..13 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..13 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..13 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..13 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..13 --matches=1 "^BK:$BK\$"
assertStdoutGrep --line=..13 --matches=1 "^date:$date\$"
assertStdoutGrep --line=..13 --matches=1 "^service:file\$"
assertStdoutGrep --line=..13 --matches=1 "^name:file1\$"
assertStdoutGrep --line=..13 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=..13 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..13 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..13 --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --line=..13 --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --line=14 --matches=1 "^manifest:"
replayStdout | $SED -n '14s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '15,$p' >>file1x.manifest
cat file1.manifest >file1n.manifest
echo >>file1n.manifest
tfw_cat file1n.manifest file1x.manifest
@ -405,17 +420,18 @@ setup_ExtractManifestAfterAddNoAuthor() {
test_ExtractManifestAfterAddNoAuthor() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
assert diff file1.manifest file1x.manifest
assertStdoutLineCount '==' 9
assertStdoutLineCount '==' 10
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.readonly:1\$"
}
doc_ExtractManifestNonExistent="Export non-existent manifest"
@ -460,6 +476,7 @@ setup_ExtractFileAfterAdd() {
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_BK BK file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractFileAfterAdd() {
@ -467,18 +484,20 @@ test_ExtractFileAfterAdd() {
tfw_cat --stderr
assert diff file1 file1x
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutLineCount '==' 11
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutLineCount '==' 13
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^BK:$BK\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
}
doc_ExtractFileMissing="Extract and export non-existent file"
@ -547,7 +566,7 @@ setup_AddDeDuplicate() {
test_AddDeDuplicate() {
# Add first file again - should return a "duplicate" status code and nothing
# should change in its manifests.
execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file1 file1.manifestA
execute --exit-status=2 --stderr --core-backtrace $servald rhizome add file $SIDB1 file1 file1.manifestA
assert [ -s file1.manifestA ]
assert_stdout_add_file file1
extract_stdout_secret file1_dup_secret
@ -601,13 +620,14 @@ setup_AddMismatched() {
setup_AddDeDuplicate
}
test_AddMismatched() {
# Try to add another file using an existing manifest, should fail and leave
# the manifest file unchanged because the manifest's version field has not
# advanced.
# Try to add another file using an existing manifest, should fail with status
# code indicating inconsistency.
cp file1.manifest file1_2.manifest
execute $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
assertExitStatus '!=' 0
assert diff file1.manifest file1_2.manifest
# Exit status 6 means manifest and payload do not match (filesize/filehash).
execute --exit-status=6 --stderr --core-backtrace $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
tfw_cat file1.manifest file1_2.manifest
# Output manifest should be the same as the re-used manigfest
assert --stderr cmp file1.manifest file1_2.manifest
# And rhizome store should be unchanged.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
@ -626,11 +646,13 @@ setup_AddUpdateSameVersion() {
cp file1_2.manifest file1_2.manifest.orig
}
test_AddUpdateSameVersion() {
# Try to add another file using an existing manifest Id and Version, should
# fail and update the manifest file to show existing bundle's manifest.
tfw_cat -v file1_2.manifest
execute $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
execute --exit-status=1 $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
assertExitStatus --stderr '!=' 0
tfw_cat -v file1_2.manifest
assert cmp file1_2.manifest file1_2.manifest.orig
tfw_cat -v file1_2.manifest file1.manifest
assert cmp file1_2.manifest file1.manifest
# And rhizome store should be unchanged.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
@ -910,7 +932,7 @@ setup_MeshMSAddMissingSender() {
}
test_MeshMSAddMissingSender() {
execute $servald rhizome add file $SIDB1 file1 file1.manifest
assertExitStatus '!=' 0
assertExitStatus --stdout --stderr '!=' 0
}
doc_MeshMSAddMissingRecipient="Add MeshMS without recipient fails"
@ -1039,7 +1061,8 @@ setup_ImportForeignBundle() {
test_ImportForeignBundle() {
executeOk_servald rhizome import bundle fileA fileA.manifest
assert_stdout_import_bundle fileA
execute --exit-status=2 --stdout --stderr $servald rhizome import bundle fileA fileA.manifest
# Exit status 1 means bundle is already in store
execute --exit-status=1 --stdout --stderr $servald rhizome import bundle fileA fileA.manifest
assert_stdout_import_bundle fileA
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 fileA
@ -1056,6 +1079,7 @@ setup_ImportOwnBundle() {
extract_manifest_id manifestid fileB.manifest
extract_manifest_version version fileB.manifest
extract_manifest_filehash filehash fileB.manifest
extract_manifest_BK BK fileB.manifest
extract_manifest_date date fileB.manifest
rm -f $SERVALINSTANCE_PATH/rhizome.db
executeOk_servald rhizome list
@ -1072,19 +1096,21 @@ test_ImportOwnBundle() {
tfw_cat --stderr
assert cmp fileB.manifest fileBx.manifest
assert cmp fileB fileBx
assertStdoutLineCount '==' 11
assertStdoutLineCount '==' 13
local size=$(( $(cat fileB | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^BK:$BK\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^service:file\$"
assertStdoutGrep --matches=1 "^name:fileB\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
# Now bundle author should be known, so appears to be from here
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB

10
tests/rhizomeprotocol
View File

@ -326,7 +326,7 @@ setup_FileTransferBigMDPExtBlob() {
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0 \
set rhizome.external_blobs 1 \
set rhizome.max_blob_size 0 \
set debug.externalblobs 1
setup_bigfile_common
}
@ -340,7 +340,7 @@ setup_FileTransferBigHTTPExtBlob() {
foreach_instance +A +B \
executeOk_servald config \
set rhizome.mdp.enable 0 \
set rhizome.external_blobs 1 \
set rhizome.max_blob_size 0 \
set debug.externalblobs 1
setup_bigfile_common
}
@ -402,7 +402,7 @@ setup_FileTransferMultiMDPExtBlob() {
foreach_instance +A +B +C +D +E \
executeOk_servald config \
set rhizome.http.enable 0 \
set rhizome.external_blobs 1 \
set rhizome.max_blob_size 0 \
set debug.externalblobs 1
setup_multitransfer_common
}
@ -416,7 +416,7 @@ setup_FileTransferMultiHTTPExtBlob() {
foreach_instance +A +B +C +D +E \
executeOk_servald config \
set rhizome.mdp.enable 0 \
set rhizome.external_blobs 1 \
set rhizome.max_blob_size 0 \
set debug.externalblobs 1
setup_multitransfer_common
}
@ -449,7 +449,7 @@ doc_CorruptPayload="A corrupted payload should be re-fetched"
setup_CorruptPayload() {
setup_common
set_instance +A
executeOk_servald config set rhizome.external_blobs 1
executeOk_servald config set rhizome.max_blob_size 0
rhizome_add_file file1 1024
create_file file2 1024
start_servald_instances +A +B