Merge branch 'development' into 'naf4'

commandline.c

@@ -1350,20 +1350,20 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
   } else {
     if (config.debug.rhizome)
       DEBUGF("Creating new manifest");
-    if (journal){
+    if (journal) {
-      m->journalTail = 0;
+      rhizome_manifest_set_filesize(m, 0);
-      rhizome_manifest_set_ll(m,"tail",m->journalTail);
+      rhizome_manifest_set_tail(m, 0);
     }
   }
 
-  if (journal && m->journalTail==-1)
+  if (journal && !m->is_journal)
     return WHY("Existing manifest is not a journal");
 
-  if ((!journal) && m->journalTail>=0)
+  if (!journal && m->is_journal)
     return WHY("Existing manifest is a journal");
 
-  if (rhizome_manifest_get(m, "service", NULL, 0) == NULL)
+  if (m->service == NULL)
-    rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
+    rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
 
   if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL, bskhex ? &bsk : NULL)){
     rhizome_manifest_free(m);
@@ -1376,13 +1376,12 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
       return -1;
     }
   }else{
-    if (rhizome_stat_file(m, filepath)){
+    if (rhizome_stat_file(m, filepath) == -1) {
       rhizome_manifest_free(m);
       return -1;
     }
-  
+    if (m->filesize) {
-    if (m->fileLength){
+      if (rhizome_add_file(m, filepath) == -1) {
-      if (rhizome_add_file(m, filepath)){
         rhizome_manifest_free(m);
         return -1;
       }
@@ -1390,7 +1389,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
   }
   
   rhizome_manifest *mout = NULL;
-  int ret=rhizome_manifest_finalise(m, &mout, !force_new);
+  int ret = rhizome_manifest_finalise(m, &mout, !force_new);
   if (ret<0){
     rhizome_manifest_free(m);
     return -1;
@@ -1399,10 +1398,9 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
   if (manifestpath && *manifestpath
       && rhizome_write_manifest_file(mout, manifestpath, 0) == -1)
     ret = WHY("Could not overwrite manifest file.");
-  const char *service = rhizome_manifest_get(mout, "service", NULL, 0);
+  if (mout->service) {
-  if (service) {
     cli_field_name(context, "service", ":");
-    cli_put_string(context, service, "\n");
+    cli_put_string(context, mout->service, "\n");
   }
   {
     cli_field_name(context, "manifestid", ":");
@@ -1414,27 +1412,25 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     cli_field_name(context, ".secret", ":");
     cli_put_string(context, secret, "\n");
   }
-  if (*authorSidHex) {
+  if (m->has_author) {
     cli_field_name(context, ".author", ":");
-    cli_put_string(context, alloca_tohex_sid_t(authorSid), "\n");
+    cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
   }
-  const char *bk = rhizome_manifest_get(mout, "BK", NULL, 0);
+  if (mout->has_bundle_key) {
-  if (bk) {
     cli_field_name(context, "BK", ":");
-    cli_put_string(context, bk, "\n");
+    cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n");
   }
   cli_field_name(context, "version", ":");
-  cli_put_long(context, m->version, "\n");
+  cli_put_long(context, mout->version, "\n");
   cli_field_name(context, "filesize", ":");
-  cli_put_long(context, mout->fileLength, "\n");
+  cli_put_long(context, mout->filesize, "\n");
-  if (mout->fileLength != 0) {
+  if (mout->filesize != 0) {
     cli_field_name(context, "filehash", ":");
     cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n");
   }
-  const char *name = rhizome_manifest_get(mout, "name", NULL, 0);
+  if (mout->name) {
-  if (name) {
     cli_field_name(context, "name", ":");
-    cli_put_string(context, name, "\n");
+    cli_put_string(context, mout->name, "\n");
   }
   if (mout != m)
     rhizome_manifest_free(mout);
@@ -1506,10 +1502,9 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
   // TODO generalise the way we dump manifest details from add, import & export
   // so callers can also generalise their parsing
   
-  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
+  if (m->service) {
-  if (service) {
     cli_field_name(context, "service", ":");
-    cli_put_string(context, service, "\n");
+    cli_put_string(context, m->service, "\n");
   }
   {
     cli_field_name(context, "manifestid", ":");
@@ -1521,23 +1516,22 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
     cli_field_name(context, ".secret", ":");
     cli_put_string(context, secret, "\n");
   }
-  const char *bk = rhizome_manifest_get(m, "BK", NULL, 0);
+  if (m->has_bundle_key) {
-  if (bk) {
     cli_field_name(context, "BK", ":");
-    cli_put_string(context, bk, "\n");
+    cli_put_string(context, alloca_tohex_rhizome_bk_t(m->bundle_key), "\n");
   }
   cli_field_name(context, "version", ":");
   cli_put_long(context, m->version, "\n");
   cli_field_name(context, "filesize", ":");
-  cli_put_long(context, m->fileLength, "\n");
+  cli_put_long(context, m->filesize, "\n");
-  if (m->fileLength != 0) {
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize != 0) {
     cli_field_name(context, "filehash", ":");
     cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
   }
-  const char *name = rhizome_manifest_get(m, "name", NULL, 0);
+  if (m->name) {
-  if (name) {
     cli_field_name(context, "name", ":");
-    cli_put_string(context, name, "\n");
+    cli_put_string(context, m->name, "\n");
   }
   
 cleanup:
@@ -1666,7 +1660,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
     bskhex=NULL;
   
   rhizome_bk_t bsk;
-  if (bskhex && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
+  if (bskhex && str_to_rhizome_bk_t(&bsk, bskhex) == -1)
     return WHYF("invalid bsk: \"%s\"", bskhex);
 
   rhizome_manifest *m = rhizome_new_manifest();
@@ -1677,19 +1671,32 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
   
   if (ret==0){
     // ignore errors
-    rhizome_extract_privatekey(m, NULL);
+    rhizome_extract_privatekey(m, bskhex ? &bsk : NULL);
-    const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
 
-    cli_field_name(context, "service", ":");    cli_put_string(context, blob_service, "\n");
+    if (m->service) {
-    cli_field_name(context, "manifestid", ":"); cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
+      cli_field_name(context, "service", ":");
-    cli_field_name(context, "version", ":");    cli_put_long(context, m->version, "\n");
+      cli_put_string(context, m->service, "\n");
-    cli_field_name(context, "inserttime", ":"); cli_put_long(context, m->inserttime, "\n");
-    if (m->haveSecret) {
-      cli_field_name(context, ".author", ":");  cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
     }
-    cli_field_name(context, ".readonly", ":");  cli_put_long(context, m->haveSecret?0:1, "\n");
+    cli_field_name(context, "manifestid", ":");
-    cli_field_name(context, "filesize", ":");   cli_put_long(context, m->fileLength, "\n");
+    cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
-    if (m->fileLength != 0) {
+    if (m->haveSecret) {
+      char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
+      rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
+      cli_field_name(context, ".secret", ":");
+      cli_put_string(context, secret, "\n");
+      cli_field_name(context, ".author", ":");
+      cli_put_string(context, alloca_tohex_sid_t(m->author), "\n");
+    }
+    cli_field_name(context, "version", ":");
+    cli_put_long(context, m->version, "\n");
+    cli_field_name(context, "inserttime", ":");
+    cli_put_long(context, m->inserttime, "\n");
+    cli_field_name(context, ".readonly", ":");
+    cli_put_long(context, m->haveSecret?0:1, "\n");
+    cli_field_name(context, "filesize", ":");
+    cli_put_long(context, m->filesize, "\n");
+    assert(m->filesize != RHIZOME_SIZE_UNSET);
+    if (m->filesize != 0) {
       cli_field_name(context, "filehash", ":");
       cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
     }
@@ -1697,7 +1704,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
   
   int retfile=0;
   
-  if (ret==0 && m->fileLength != 0 && filepath && *filepath){
+  if (ret==0 && m->filesize != 0 && filepath && *filepath){
     if (extract){
       // Save the file, implicitly decrypting if required.
       // TODO, this may cause us to search for an author a second time if the above call to rhizome_extract_privatekey failed

conf_schema.h

@@ -261,7 +261,8 @@ ATOM(bool_t, slipdecode,                0, boolean,, "")
 ATOM(bool_t, slipbytestream,            0, boolean,, "")
 ATOM(bool_t, packetconstruction,        0, boolean,, "")
 ATOM(bool_t, rhizome,                   0, boolean,, "")
-ATOM(bool_t, rhizome_bind,              0, boolean,, "")
+ATOM(bool_t, rhizome_manifest,          0, boolean,, "")
+ATOM(bool_t, rhizome_sql_bind,          0, boolean,, "")
 ATOM(bool_t, rhizome_httpd,             0, boolean,, "")
 ATOM(bool_t, rhizome_tx,                0, boolean,, "")
 ATOM(bool_t, rhizome_rx,                0, boolean,, "")

dataformats.c

@@ -116,6 +116,11 @@ int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const
   return 0;
 }
 
+int str_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex)
+{
+  return fromhexstr(bkp->binary, hex, sizeof bkp->binary);
+}
+
 int rhizome_strn_is_manifest_id(const char *id)
 {
   return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN);
@@ -156,6 +161,15 @@ int rhizome_str_is_file_hash(const char *hash)
   return is_xstring(hash, RHIZOME_FILEHASH_STRLEN);
 }
 
+int rhizome_str_is_manifest_service(const char *text)
+{
+  if (text[0] == '\0')
+    return 0;
+  while (*text && (isalnum(*text) || *text == '_' || *text == '.'))
+    ++text;
+  return *text == '\0';
+}
+
 int str_is_did(const char *did)
 {
   size_t len = 0;

http_server.c

@@ -76,9 +76,9 @@ static struct profile_total http_server_stats = {
 #define DEBUG_DUMP_PARSER(r) do { \
       if (config.debug.httpd) \
 	DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \
-	    r->parsed - r->received, alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \
+	    (int)(r->parsed - r->received), alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \
-	    r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor), \
+	    (int)(r->cursor - r->received), alloca_toprint(50, r->cursor, r->end - r->cursor), \
-	    r->end - r->received, \
+	    (int)(r->end - r->received), \
 	    r->request_content_remaining \
 	  ); \
     } while (0)
@@ -1019,7 +1019,7 @@ static int http_request_parse_header(struct http_request *r)
       _commit(r);
       if (n > NELS(r->request_header.content_ranges)) {
 	if (r->debug_flag && *r->debug_flag)
-	  DEBUGF("HTTP request Range header overflow (%u ranges in set, can only handle %u): %s",
+	  DEBUGF("HTTP request Range header overflow (%u ranges in set, can only handle %zu): %s",
 	      n, NELS(r->request_header.content_ranges), alloca_toprint(-1, sol, eol - sol));
 	// In this case ignore the Range: header -- respond with the entire resource.
 	r->request_header.content_range_count = 0;

meshms.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include "serval.h"
 #include "rhizome.h"
 #include "log.h"
@@ -82,15 +83,24 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
     return -1;
   
   // always consider the content encrypted, we don't need to rely on the manifest itself.
-  m->payloadEncryption = 1;
+  rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED);
+  assert(m->haveSecret);
   if (m->haveSecret == NEW_BUNDLE_ID) {
-    rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
+    rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
-    if (rhizome_fill_manifest(m, NULL, NULL, NULL) == -1)
+    if (rhizome_fill_manifest(m, NULL, my_sidp, NULL) == -1)
       return WHY("Invalid manifest");
+    if (config.debug.meshms) {
+      char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
+      rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
+      // The 'meshms' automated test depends on this message; do not alter.
+      DEBUGF("MESHMS CONVERSATION BUNDLE bid=%s secret=%s",
+	    alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
+	    secret
+	  );
+    }
   } else {
-    const char *service = rhizome_manifest_get(m, "service", NULL, 0);
+    if (strcmp(m->service, RHIZOME_SERVICE_FILE) != 0)
-    if (strcmp(service, RHIZOME_SERVICE_FILE) != 0)
+      return WHYF("Invalid manifest, service=%s but should be %s", m->service, RHIZOME_SERVICE_MESHMS2);
-      return WHYF("Invalid manifest, service=%s but should be %s", service, RHIZOME_SERVICE_MESHMS2);
   }
   return 0;
 }
@@ -185,33 +195,39 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si
   return 0;
 }
 
-static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid){
+static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid)
+{
   struct conversations *conv=NULL;
   if (meshms_conversations_list(my_sid, their_sid, &conv))
     return NULL;
   if (!conv){
     conv = emalloc_zero(sizeof(struct conversations));
-    bcopy(their_sid->binary, conv->them.binary, sizeof(sid_t));
+    conv->them = *their_sid;
   }
   return conv;
 }
 
-static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m){
+static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m)
-  m->journalTail = 0;
+{
-  const char *my_sidhex = alloca_tohex_sid_t(*my_sid);
+  if (config.debug.meshms)
-  const char *their_sidhex = alloca_tohex_sid_t(conv->them);
+    DEBUGF("Creating ply for my_sid=%s them=%s",
-  rhizome_manifest_set(m, "service", RHIZOME_SERVICE_MESHMS2);
+	alloca_tohex_sid_t(conv->them),
-  rhizome_manifest_set(m, "sender", my_sidhex);
+	alloca_tohex_sid_t(*my_sid));
-  rhizome_manifest_set(m, "recipient", their_sidhex);
+  rhizome_manifest_set_service(m, RHIZOME_SERVICE_MESHMS2);
-  rhizome_manifest_set_ll(m, "tail", m->journalTail);
+  rhizome_manifest_set_sender(m, my_sid);
+  rhizome_manifest_set_recipient(m, &conv->them);
+  rhizome_manifest_set_filesize(m, 0);
+  rhizome_manifest_set_tail(m, 0);
   if (rhizome_fill_manifest(m, NULL, my_sid, NULL))
     return -1;
+  assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
   conv->my_ply.bundle_id = m->cryptoSignPublic;
   conv->found_my_ply = 1;
   return 0;
 }
 
-static int append_footer(unsigned char *buffer, char type, int payload_len){
+static int append_footer(unsigned char *buffer, char type, int payload_len)
+{
   payload_len = (payload_len << 4) | (type&0xF);
   write_uint16(buffer, payload_len);
   return 2;
@@ -228,11 +244,13 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome
     WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
   if (ret != 0)
     return ret;
-  ply->read.offset = ply->read.length = m->fileLength;
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  ply->read.offset = ply->read.length = m->filesize;
   return 0;
 }
 
-static int ply_read_close(struct ply_read *ply){
+static int ply_read_close(struct ply_read *ply)
+{
   if (ply->buffer){
     free(ply->buffer);
     ply->buffer=NULL;
@@ -244,16 +262,17 @@ static int ply_read_close(struct ply_read *ply){
 
 // read the next record from the ply (backwards)
 // returns 1 on EOF, -1 on failure
-static int ply_read_next(struct ply_read *ply){
+static int ply_read_next(struct ply_read *ply)
-  ply->record_end_offset=ply->read.offset;
+{
+  ply->record_end_offset = ply->read.offset;
   unsigned char footer[2];
-  ply->read.offset-=sizeof(footer);
+  if (ply->read.offset <= sizeof footer) {
-  if (ply->read.offset<=0){
     if (config.debug.meshms)
       DEBUGF("EOF");
     return 1;
   }
-  if (rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof(footer)) < sizeof(footer))
+  ply->read.offset -= sizeof footer;
+  if (rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer) != sizeof footer)
     return -1;
   // (rhizome_read automatically advances the offset by the number of bytes read)
   ply->record_length=read_uint16(footer);
@@ -264,7 +283,7 @@ static int ply_read_next(struct ply_read *ply){
     DEBUGF("Found record %d, length %d @%"PRId64, ply->type, ply->record_length, ply->record_end_offset);
   
   // need to allow for advancing the tail and cutting a message in half.
-  if (ply->record_length + sizeof(footer) > ply->read.offset){
+  if (ply->record_length + sizeof footer > ply->read.offset){
     if (config.debug.meshms)
       DEBUGF("EOF");
     return 1;
@@ -281,9 +300,9 @@ static int ply_read_next(struct ply_read *ply){
     ply->buffer = b;
   }
   
-  int read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
+  ssize_t read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
-  if (read!=ply->record_length)
+  if (read != ply->record_length)
-    return WHYF("Expected %d bytes read, got %d", ply->record_length, read);
+    return WHYF("Expected %u bytes read, got %zd", ply->record_length, read);
   
   ply->read.offset = record_start;
   return 0;
@@ -308,8 +327,6 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv,
   if (conv->found_my_ply){
     if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m))
       goto end;
-    // set the author of the manifest as we should already know that
-    m->author = *my_sid;
     if (rhizome_find_bundle_author(m))
       goto end;
   }else{
@@ -481,16 +498,19 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
     goto end;
   
   unsigned char version=0xFF;
-  ret=rhizome_read_buffered(&read, &buff, &version, 1);
+  ssize_t r = rhizome_read_buffered(&read, &buff, &version, 1);
-  if (version!=1){
+  ret = -1;
-    WARN("Expected version 1");
+  if (r == -1)
+    goto end;
+  if (version != 1) {
+    WARNF("Expected version 1 (got 0x%02x)", version);
     goto end;
   }
   
-  while (1){
+  while (1) {
     sid_t sid;
-    ret=rhizome_read_buffered(&read, &buff, sid.binary, sizeof(sid));
+    r = rhizome_read_buffered(&read, &buff, sid.binary, sizeof sid.binary);
-    if (ret<sizeof(sid))
+    if (r != sizeof sid.binary)
       break;
     if (config.debug.meshms)
       DEBUGF("Reading existing conversation for %s", alloca_tohex_sid_t(sid));
@@ -500,30 +520,29 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
     if (!ptr)
       goto end;
     unsigned char details[8*3];
-    ret = rhizome_read_buffered(&read, &buff, details, sizeof(details));
+    r = rhizome_read_buffered(&read, &buff, details, sizeof details);
-    if (ret<0)
+    if (r == -1)
       break;
-    int bytes=ret;
+    int bytes = r;
-    int ofs=0;
+    int ofs = 0;
-    ret=unpack_uint(details, bytes, &ptr->their_last_message);
+    int unpacked = unpack_uint(details, bytes, &ptr->their_last_message);
-    if (ret<0)
+    if (unpacked == -1)
       break;
-    ofs+=ret;
+    ofs += unpacked;
-    
+    unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->read_offset);
-    ret=unpack_uint(details+ofs,bytes-ofs, &ptr->read_offset);
+    if (unpacked == -1)
-    if (ret<0)
       break;
-    ofs+=ret;
+    ofs += unpacked;
-    
+    unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->their_size);
-    ret=unpack_uint(details+ofs,bytes-ofs, &ptr->their_size);
+    if (unpacked == -1)
-    if (ret<0)
       break;
-    ofs+=ret;
+    ofs += unpacked;
     read.offset += ofs - bytes;
   }
+  ret = 0;
 end:
   rhizome_read_close(&read);
-  return 0;
+  return ret;
 }
 
 static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv)
@@ -583,10 +602,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
     goto end;
   
   // then write it
-  m->version++;
+  rhizome_manifest_set_version(m, m->version + 1);
-  rhizome_manifest_set_ll(m,"version",m->version);
+  rhizome_manifest_set_filesize(m, (size_t)len + 1);
-  m->fileLength = (size_t) len + 1;
-  rhizome_manifest_set_ll(m,"filesize",m->fileLength);
   
   if (rhizome_write_open_manifest(&write, m) == -1)
     goto end;
@@ -597,8 +614,7 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
     goto end;
   if (rhizome_finish_write(&write))
     goto end;
-  m->filehash = write.id;
+  rhizome_manifest_set_filehash(m, &write.id);
-  rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
   if (rhizome_manifest_finalise(m, &mout, 1))
     goto end;
   
@@ -612,7 +628,8 @@ end:
 }
 
 // read information about existing conversations from a rhizome payload
-static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv){
+static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv)
+{
   int ret=-1;
   rhizome_manifest *m = rhizome_new_manifest();
   if (!m)
@@ -730,7 +747,8 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context
   return ret;
 }
 
-int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context){
+int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context)
+{
   const char *my_sidhex, *their_sidhex;
   if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1
     || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1)
@@ -744,8 +762,10 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
     return -1;
     
   sid_t my_sid, their_sid;
-  fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary));
+  if (str_to_sid_t(&my_sid, my_sidhex) == -1)
-  fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary));
+    return WHY("invalid sender SID");
+  if (str_to_sid_t(&their_sid, their_sidhex) == -1)
+    return WHY("invalid recipient SID");
   
   struct conversations *conv=find_or_create_conv(&my_sid, &their_sid);
   if (!conv)
@@ -915,7 +935,8 @@ static int mark_read(struct conversations *conv, const sid_t *their_sid, const c
   return ret;
 }
 
-int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context){
+int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context)
+{
   const char *my_sidhex, *their_sidhex, *offset_str;
   if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1
     || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, NULL) == -1

os.h

@@ -54,6 +54,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  *      }
  */
 typedef int64_t time_ms_t;
+#define PRItime_ms_t PRId64
 
 time_ms_t gettime_ms();
 time_ms_t sleep_ms(time_ms_t milliseconds);

overlay_mdp_services.c

@@ -137,11 +137,11 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp)
       unsigned char *bidprefix=&mdp->out.payload[1];
       uint64_t version=read_uint64(&mdp->out.payload[1+16]);
       uint64_t offset=read_uint64(&mdp->out.payload[1+16+8]);
-      int count=mdp->out.payload_length-(1+16+8+8);
+      size_t count = mdp->out.payload_length-(1+16+8+8);
       unsigned char *bytes=&mdp->out.payload[1+16+8+8];
 
       if (config.debug.rhizome_mdp_rx)
-	DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%d",
+	DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%zu",
 	       bidprefix[0],bidprefix[1],bidprefix[2],bidprefix[3],offset,count);
 
       /* Now see if there is a slot that matches.  If so, then
@@ -151,7 +151,7 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp)
 	 a slot to capture this files as it is being requested
 	 by someone else.
       */
-      rhizome_received_content(bidprefix,version,offset,count,bytes,type);
+      rhizome_received_content(bidprefix,version,offset, count, bytes, type);
 
       RETURN(0);
     }
@@ -359,8 +359,8 @@ static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, ov
     if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){
       rhizome_advertise_manifest(frame->source, m);
       // pre-emptively send the payload if it will fit in a single packet
-      if (m->fileLength > 0 && m->fileLength <= 1024)
+      if (m->filesize > 0 && m->filesize <= 1024)
-	rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->fileLength);
+	rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->filesize);
     }
     rhizome_manifest_free(m);
     offset+=RHIZOME_BAR_BYTES;

performance_timing.c

@@ -152,7 +152,7 @@ int fd_showstats()
   // Show periodic rhizome transfer information, but only
   // if there are some active rhizome transfers.
   if (rhizome_active_fetch_count()!=0)
-    INFOF("Rhizome transfer progress: %d,%d,%d,%d,%d,%d (remaining %d)",
+    INFOF("Rhizome transfer progress: %"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64" (remaining %"PRIu64")",
 	  rhizome_active_fetch_bytes_received(0),
 	  rhizome_active_fetch_bytes_received(1),
 	  rhizome_active_fetch_bytes_received(2),

rhizome.c

@@ -17,11 +17,12 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
+#include <stdlib.h>
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "str.h"
 #include "rhizome.h"
-#include <stdlib.h>
 
 int is_rhizome_enabled()
 {
@@ -152,43 +153,33 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
   return rhizome_add_manifest(m, 1);
 }
 
-int rhizome_manifest_check_sanity(rhizome_manifest *m_in)
+int rhizome_manifest_check_sanity(rhizome_manifest *m)
 {
   /* Ensure manifest meets basic sanity checks. */
-  const char *service = rhizome_manifest_get(m_in, "service", NULL, 0);
+  if (m->filesize == RHIZOME_SIZE_UNSET)
-  const char *sender = rhizome_manifest_get(m_in, "sender", NULL, 0);
+    return WHY("Manifest missing 'filesize' field");
-  const char *recipient = rhizome_manifest_get(m_in, "recipient", NULL, 0);
+  if (m->filesize && rhizome_filehash_t_is_zero(m->filehash))
-  
+    return WHY("Manifest 'filehash' field has not been set");
-  if (service == NULL || !service[0])
+  if (m->service == NULL || !m->service[0])
-      return WHY("Manifest missing 'service' field");
+    return WHY("Manifest missing 'service' field");
-  if (rhizome_manifest_get_ll(m_in, "date") == -1)
+  if (!m->has_date)
-      return WHY("Manifest missing 'date' field");
+    return WHY("Manifest missing 'date' field");
-  
+  if (m->version == 0)
-  /* Get manifest version number. */
-  m_in->version = rhizome_manifest_get_ll(m_in, "version");
-  if (m_in->version==-1)
     return WHY("Manifest must have a version number");
-  
+  if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
-  if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
+    if (m->name == NULL)
-    const char *name = rhizome_manifest_get(m_in, "name", NULL, 0);
-    if (name == NULL)
       return WHY("Manifest missing 'name' field");
-  } else if (strcasecmp(service, RHIZOME_SERVICE_MESHMS) == 0 
+  } else if (strcasecmp(m->service, RHIZOME_SERVICE_MESHMS) == 0 
-    || strcasecmp(service, RHIZOME_SERVICE_MESHMS2) == 0) {
+	  || strcasecmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) {
-    if (sender == NULL || !sender[0])
+    if (!m->has_sender)
       return WHY("MeshMS Manifest missing 'sender' field");
-    if (!str_is_subscriber_id(sender))
+    if (!m->has_recipient)
-      return WHYF("MeshMS Manifest contains invalid 'sender' field: %s", sender);
-    if (recipient == NULL || !recipient[0])
       return WHY("MeshMS Manifest missing 'recipient' field");
-    if (!str_is_subscriber_id(recipient))
-      return WHYF("MeshMS Manifest contains invalid 'recipient' field: %s", recipient);
   } else {
-    return WHY("Invalid service type");
+    return WHYF("Invalid service=%s", m->service);
   }
   if (config.debug.rhizome)
-    DEBUGF("sender='%s'", sender ? sender : "(null)");
+    DEBUGF("sender=%s", m->has_sender ? alloca_tohex_sid_t(m->sender) : "(null)");
-
   /* passes all sanity checks */
   return 0;
 }
@@ -204,15 +195,15 @@ int rhizome_manifest_check_sanity(rhizome_manifest *m_in)
     by joining the parent group.
 */
 
-int rhizome_manifest_bind_id(rhizome_manifest *m_in)
+int rhizome_manifest_bind_id(rhizome_manifest *m)
 {
-  if (rhizome_manifest_createid(m_in) == -1)
+  if (rhizome_manifest_createid(m) == -1)
     return -1;
   /* The ID is implicit in transit, but we need to store it in the file, so that reimporting
      manifests on receiver nodes works easily.  We might implement something that strips the id
      variable out of the manifest when sending it, or some other scheme to avoid sending all the
      extra bytes. */
-  if (!is_sid_t_any(m_in->author)) {
+  if (m->has_author) {
     /* Set the BK using the provided authorship information.
        Serval Security Framework defines BK as being:
        BK = privateKey XOR sha512(RS##BID), where BID = cryptoSignPublic, 
@@ -222,45 +213,42 @@ int rhizome_manifest_bind_id(rhizome_manifest *m_in)
        to encrypt and decrypt the BK field. */
     const unsigned char *rs;
     int rs_len=0;
-    unsigned char bkbytes[RHIZOME_BUNDLE_KEY_BYTES];
+    if (rhizome_find_secret(&m->author, &rs_len, &rs))
-
+      return WHYF("Failed to obtain RS for %s to calculate BK", alloca_tohex_sid_t(m->author));
-    if (rhizome_find_secret(&m_in->author, &rs_len, &rs))
+    rhizome_bk_t bkey;
-      return WHYF("Failed to obtain RS for %s to calculate BK", alloca_tohex_sid_t(m_in->author));
+    if (!rhizome_secret2bk(&m->cryptoSignPublic, rs, rs_len, bkey.binary, m->cryptoSignSecret))
-    if (!rhizome_secret2bk(&m_in->cryptoSignPublic, rs, rs_len, bkbytes, m_in->cryptoSignSecret)) {
+      rhizome_manifest_set_bundle_key(m, &bkey);
-      char bkhex[RHIZOME_BUNDLE_KEY_STRLEN + 1];
+    else
-      (void) tohex(bkhex, RHIZOME_BUNDLE_KEY_STRLEN, bkbytes);
-      if (config.debug.rhizome) DEBUGF("set BK=%s", bkhex);
-      rhizome_manifest_set(m_in, "BK", bkhex);
-    } else
       return WHY("Failed to set BK");
   }
   return 0;
 }
 
-int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
+int rhizome_add_manifest(rhizome_manifest *m, int ttl)
 {
   if (config.debug.rhizome)
-    DEBUGF("rhizome_add_manifest(m_in=%p, ttl=%d)",m_in, ttl);
+    DEBUGF("rhizome_add_manifest(m=%p, ttl=%d)",m, ttl);
 
-  if (m_in->finalised==0)
+  if (m->finalised==0)
     return WHY("Manifest must be finalised before being stored");
 
   /* Store time to live, clamped to within legal range */
-  m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
+  m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
 
-  if (rhizome_manifest_check_sanity(m_in))
+  if (rhizome_manifest_check_sanity(m))
     return -1;
 
-  if (m_in->fileLength && !rhizome_exists(&m_in->filehash))
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize > 0 && !rhizome_exists(&m->filehash))
     return WHY("File has not been imported");
 
   /* If the manifest already has an ID */
-  if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic))
+  if (rhizome_bid_t_is_zero(m->cryptoSignPublic))
     return WHY("Manifest does not have an ID");   
   
   /* Discard the new manifest unless it is newer than the most recent known version with the same ID */
   int64_t storedversion = -1;
-  switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m_in->cryptoSignPublic, END)) {
+  switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END)) {
     case -1:
       return WHY("Select failed");
     case 0:
@@ -268,18 +256,18 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
       break;
     case 1:
       if (config.debug.rhizome) 
-	DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m_in->version);
+	DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m->version);
-      if (m_in->version < storedversion)
+      if (m->version < storedversion)
 	return WHY("Newer version exists");
-      if (m_in->version == storedversion)
+      if (m->version == storedversion)
-	return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m_in->cryptoSignPublic), m_in->version);
+	return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
       break;
     default:
       return WHY("Select found too many rows!");
   }
 
   /* Okay, it is written, and can be put directly into the rhizome database now */
-  return rhizome_store_bundle(m_in);
+  return rhizome_store_bundle(m);
 }
 
 /* When voice traffic is being carried, we need to throttle Rhizome down

rhizome.h

@@ -21,12 +21,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #define __SERVALDNA__RHIZOME_H
 
 #include <sqlite3.h>
+#include <limits.h>
 #include "sha2.h"
 #include "str.h"
 #include "strbuf.h"
 #include "http_server.h"
 #include "nacl.h"
-#include <sys/stat.h>
 
 #ifndef __RHIZOME_INLINE
 # if __GNUC__ && !__GNUC_STDC_INLINE__
@@ -104,6 +104,8 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) {
 }
 
 #define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary)
+int cmp_rhizome_bk_t(const rhizome_bk_t *a, const rhizome_bk_t *b);
+int str_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex);
 
 
 extern time_ms_t rhizome_voice_timeout;
@@ -118,9 +120,6 @@ extern time_ms_t rhizome_voice_timeout;
 
 #define RHIZOME_IDLE_TIMEOUT 20000
 
-#define EXISTING_BUNDLE_ID 1
-#define NEW_BUNDLE_ID 2
-
 typedef struct rhizome_signature {
   unsigned char signature[crypto_sign_edwards25519sha512batch_BYTES
 			  +crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES+1];
@@ -138,85 +137,211 @@ typedef struct rhizome_signature {
 
 #define MAX_MANIFEST_VARS 256
 #define MAX_MANIFEST_BYTES 8192
-typedef struct rhizome_manifest {
-  int manifest_record_number;
-  int manifest_bytes;
-  int manifest_all_bytes;
-  unsigned char manifestdata[MAX_MANIFEST_BYTES];
-  unsigned char manifesthash[crypto_hash_sha512_BYTES];
 
-  /* CryptoSign key pair for this manifest.
+#define RHIZOME_SIZE_UNSET    UINT64_MAX
-     The filename as distributed on Rhizome will be the public key
+
-     of this pair, thus ensuring that noone can tamper with a bundle
+typedef struct rhizome_manifest
-     except the creator. */
+{
+  int manifest_record_number;
+
+  /* CryptoSign key pair for this manifest.  The public key is the Bundle ID
+   * (aka Manifest ID).
+   */
   rhizome_bid_t cryptoSignPublic;
   unsigned char cryptoSignSecret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES];
-  /* Whether we have the secret for this manifest on hand */
-  int haveSecret;
 
-  int var_count;
+  /* Whether cryptoSignSecret is correct (ie, bundle secret is known)
-  char *vars[MAX_MANIFEST_VARS];
+   */
-  char *values[MAX_MANIFEST_VARS];
+  enum { SECRET_UNKNOWN = 0, EXISTING_BUNDLE_ID, NEW_BUNDLE_ID } haveSecret;
 
-  int sig_count;
+  /* Version of the manifest.  Typically the number of milliseconds since 1970.
-  /* Parties who have signed this manifest (raw byte format) */
+   * A value of zero (0) means it has not been set yet.
+   * TODO: change this to uint64_t.
+   */
+  int64_t version;
+
+  /* Payload is described by the offset of its tail (number of missing bytes
+   * before the first byte in the payload), its size (number of bytes) and the
+   * hash of its content.  Bundle size = tail + filesize.
+   */
+  uint64_t tail;
+  uint64_t filesize;
+  rhizome_filehash_t filehash;
+
+  /* All the manifest fields in original order (the order affects the manifest
+   * hash which was used to sign the manifest, so the signature can only be
+   * checked if order is preserved).
+   *
+   * TODO: reduce to only unknown fields.
+   *
+   * TODO: store all vars and values as NUL terminated strings within
+   * manifestdata[], not malloc()/free() heap, to reduce memory fragmentation
+   * and allow manifest struct copying without string lifetime issues.
+   */
+  unsigned short var_count;
+  const char *vars[MAX_MANIFEST_VARS];
+  const char *values[MAX_MANIFEST_VARS];
+
+  /* Parties who have signed this manifest (binary format, malloc(3)).
+   * Recognised signature types:
+   *    0x17 = crypto_sign_edwards25519sha512batch()
+   */
+  unsigned short sig_count;
   unsigned char *signatories[MAX_MANIFEST_VARS];
-  /*
+  uint8_t signatureTypes[MAX_MANIFEST_VARS];
-    0x17 = crypto_sign_edwards25519sha512batch()
-  */
-  unsigned char signatureTypes[MAX_MANIFEST_VARS];
 
-  // errors only involve the correctness of fields that are mandatory for 
+  /* Imperfections.
-  // proper operation of the transport and storage layer
+   *  - Errors involve the correctness of fields that are mandatory for proper
-  int errors;
+   *    operation of the transport and storage layer.  A manifest with errors > 0
-  // a warning indicates that the manifest cannot be perfectly understood by this version of rhizome
+   *    must not be stored, transmitted or supplied via any API.
-  // during add, the manifest should not be finalised and imported
+   *  - Warnings indicate a manifest that cannot be fully understood by this
-  // during extract an error should be displayed.
+   *    version of Rhizome (probably from a future or a very old past version
-  int warnings;
+   *    of Rhizome).  During add or import (local injection), the manifest
-  time_ms_t inserttime;
+   *    should not be imported.  During extract or export (local) a warning or
+   *    error message should be logged.
+   */
+  unsigned short errors;
+  unsigned short warnings;
 
-  /* Set non-zero after variables have been packed and
+  /* Set non-zero after variables have been packed and signature blocks
-     signature blocks appended.
+   * appended.  All fields below may not be valid until the manifest has been
-     All fields below may not be valid until the manifest has been finalised */
+   * finalised.
-  int finalised;
+   */
+  bool_t finalised;
+
+  /* Whether the manifest contains a signature that corresponds to the manifest
+   * id (ie public key).  Caches the result of 
+   */
+  bool_t selfSigned;
+
+  /* If set, unlink(2) the associated file when freeing the manifest.
+   */
+  bool_t dataFileUnlinkOnFree;
+
+  /* Set if the tail field is valid, ie, the bundle is a journal.
+   */
+  bool_t is_journal;
+
+  /* Set if the date field is valid, ie, the manifest contains a valid "date"
+   * field.
+   */
+  bool_t has_date;
+
+  /* Set if the bundle_key field is valid, ie, the manifest contains a valid
+   * "BK" field.
+   */
+  bool_t has_bundle_key;
+
+  /* Set if the sender and recipient fields are valid, ie, the manifest
+   * contains a valid "sender"/"recipient" field.
+   */
+  bool_t has_sender;
+  bool_t has_recipient;
+
+  /* Set if the 'author' element is valid, ie, a SID has been assigned.
+   */
+  bool_t has_author;
 
   /* time-to-live in hops of this manifest. */
   int ttl;
 
-  /* When finalised, we keep the filehash and maximum priority due to any
-     group membership handy */
-  int64_t fileLength;
-  int64_t journalTail;
-  rhizome_filehash_t filehash;
-
   int fileHighestPriority;
+
   /* Absolute path of the file associated with the manifest */
-  char *dataFileName;
+  const char *dataFileName;
-  /* If set, unlink(2) the associated file when freeing the manifest */
-  int dataFileUnlinkOnFree;
 
   /* Whether the paylaod is encrypted or not */
-  int payloadEncryption;
+  enum rhizome_manifest_crypt {
+        PAYLOAD_CRYPT_UNKNOWN = 0,
+        PAYLOAD_CLEAR,
+        PAYLOAD_ENCRYPTED
+    } payloadEncryption;
   unsigned char payloadKey[RHIZOME_CRYPT_KEY_BYTES];
   unsigned char payloadNonce[crypto_stream_xsalsa20_NONCEBYTES];
 
-  /* Whether the manifest contains a signature that corresponds to the 
+  /* From the "date" field, if present.  The number of milliseconds since 1970
-     manifest id (ie public key) */
+   * when the bundle was last modified.
-  int selfSigned;
+   */
+  time_ms_t date;
 
-  /* Version of the manifest.  Typically the number of milliseconds since 1970. */
+  /* From the "service" field, which should always be present.
-  int64_t version;
+   */
+  const char *service;
 
-  int group_count;
+  /* From the optional "name" field.  NULL if there is no "name" field in the
-  char *groups[MAX_MANIFEST_VARS];
+   * manifest.
+   */
+  const char *name;
 
-  /* Author of the manifest.  A reference to a local keyring entry.  Manifests
+  /* Bundle Key "BK" field from the manifest.
-   * not authored locally will have the ANY author (all zeros).
+   */
+  rhizome_bk_t bundle_key;
+
+  /* Sender and recipient fields, if present in the manifest.
+   */
+  sid_t sender;
+  sid_t recipient;
+
+  /* Local data, not encapsulated in the bundle.  The system time of the most
+   * recent INSERT or UPDATE of the manifest into the store.
+   */
+  time_ms_t inserttime;
+
+  /* Local data, not encapsulated in the bundle.  The author of the manifest.
+   * A reference to a local keyring entry.  Manifests not authored locally will
+   * have an ANY author (all zeros).
    */
   sid_t author;
 
+  /* Unused.  SHOULD BE DELETED.
+   */
+  unsigned group_count;
+  char *groups[MAX_MANIFEST_VARS];
+
+  unsigned manifest_bytes;
+  unsigned manifest_all_bytes;
+  unsigned char manifestdata[MAX_MANIFEST_BYTES];
+  unsigned char manifesthash[crypto_hash_sha512_BYTES];
+
 } rhizome_manifest;
 
+/* These setter functions (methods) are needed because the relevant attributes
+ * are stored in two places: in the vars[] array and in a dedicated struct
+ * element.
+ *
+ * TODO: refactor to remove the redundancy, possibly removing these setter
+ * functions as well.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+#define rhizome_manifest_set_id(m,v)  _rhizome_manifest_set_id(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_version(m,v)  _rhizome_manifest_set_version(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_filesize(m,v)  _rhizome_manifest_set_filesize(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_filehash(m,v)  _rhizome_manifest_set_filehash(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_tail(m,v)  _rhizome_manifest_set_tail(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_bundle_key(m,v)  _rhizome_manifest_set_bundle_key(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_service(m,v)  _rhizome_manifest_set_service(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_name(m,v)  _rhizome_manifest_set_name(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_date(m,v)  _rhizome_manifest_set_date(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_sender(m,v)  _rhizome_manifest_set_sender(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_recipient(m,v)  _rhizome_manifest_set_recipient(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_crypt(m,v)  _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_author(m,v)  _rhizome_manifest_set_author(__WHENCE__,(m),(v))
+
+void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *);
+void _rhizome_manifest_set_version(struct __sourceloc, rhizome_manifest *, int64_t); // TODO change to uint64_t
+void _rhizome_manifest_set_filesize(struct __sourceloc, rhizome_manifest *, uint64_t);
+void _rhizome_manifest_set_filehash(struct __sourceloc, rhizome_manifest *, const rhizome_filehash_t *);
+void _rhizome_manifest_set_tail(struct __sourceloc, rhizome_manifest *, uint64_t);
+void _rhizome_manifest_set_bundle_key(struct __sourceloc, rhizome_manifest *, const rhizome_bk_t *);
+void _rhizome_manifest_set_service(struct __sourceloc, rhizome_manifest *, const char *);
+void _rhizome_manifest_set_name(struct __sourceloc, rhizome_manifest *, const char *);
+void _rhizome_manifest_set_date(struct __sourceloc, rhizome_manifest *, time_ms_t);
+void _rhizome_manifest_set_sender(struct __sourceloc, rhizome_manifest *, const sid_t *);
+void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *);
+void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
+void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
+
 /* Supported service identifiers.  These go in the 'service' field of every
  * manifest, and indicate which application must be used to process the bundle
  * after it is received by Rhizome.
@@ -261,6 +386,7 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report);
 
 int rhizome_manifest_createid(rhizome_manifest *m);
 int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
+
 int rhizome_strn_is_manifest_id(const char *text);
 int rhizome_str_is_manifest_id(const char *text);
 int rhizome_strn_is_bundle_key(const char *text);
@@ -269,6 +395,7 @@ int rhizome_strn_is_bundle_crypt_key(const char *text);
 int rhizome_str_is_bundle_crypt_key(const char *text);
 int rhizome_strn_is_file_hash(const char *text);
 int rhizome_str_is_file_hash(const char *text);
+int rhizome_str_is_manifest_service(const char *text);
 
 int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
 
@@ -291,16 +418,12 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
 int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
 int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize);
 int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
-char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen);
+
-int64_t  rhizome_manifest_get_ll(rhizome_manifest *m, const char *var);
-int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value);
-int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value);
-int rhizome_manifest_del(rhizome_manifest *m, const char *var);
-int64_t rhizome_file_size(char *filename);
 void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m);
 #define rhizome_manifest_free(m) _rhizome_manifest_free(__WHENCE__,m)
 rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence);
 #define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
+
 int rhizome_manifest_pack_variables(rhizome_manifest *m);
 int rhizome_store_bundle(rhizome_manifest *m);
 int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
@@ -402,7 +525,7 @@ int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, st
 #define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...)     _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__)
 
 double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
-int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
+int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
 int rhizome_update_file_priority(const char *fileid);
 int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
 int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
@@ -448,7 +571,7 @@ int rhizome_secret2bk(
   const unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]
 );
 unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m);
-int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk);
+int rhizome_extract_privatekey(rhizome_manifest *m, const rhizome_bk_t *bsk);
 int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk);
 int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
 			       const unsigned char *pk,rhizome_signature *out);
@@ -471,7 +594,7 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng
 struct rhizome_write_buffer
 {
   struct rhizome_write_buffer *_next;
-  int64_t offset;
+  uint64_t offset;
   size_t buffer_size;
   size_t data_size;
   unsigned char data[0];
@@ -483,10 +606,10 @@ struct rhizome_write
   uint64_t temp_id;
   char id_known;
   
-  int64_t tail;
+  uint64_t tail;
-  int64_t file_offset;
+  uint64_t file_offset;
-  int64_t written_offset;
+  uint64_t written_offset;
-  int64_t file_length;
+  uint64_t file_length;
   struct rhizome_write_buffer *buffer_list;
   size_t buffer_size;
   
@@ -521,9 +644,9 @@ struct rhizome_read
   int64_t blob_rowid;
   int blob_fd;
   
-  int64_t tail;
+  uint64_t tail;
-  int64_t offset;
+  uint64_t offset;
-  int64_t length;
+  uint64_t length;
 };
 
 /* Rhizome-specific HTTP request handling.
@@ -565,7 +688,7 @@ typedef struct rhizome_http_request
 } rhizome_http_request;
 
 int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, 
-			     uint64_t offset,int count,unsigned char *bytes,
+			     uint64_t offset, size_t count,unsigned char *bytes,
 			     int type);
 int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
 					 int64_t fileLength,int priority);
@@ -702,15 +825,15 @@ enum rhizome_start_fetch_result {
 enum rhizome_start_fetch_result rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip, const sid_t *sidp, const unsigned char *prefix, size_t prefix_length);
 int rhizome_any_fetch_active();
 int rhizome_any_fetch_queued();
-int rhizome_fetch_queue_bytes();
+uint64_t rhizome_fetch_queue_bytes();
 int rhizome_fetch_status_html(struct strbuf *b);
 int rhizome_fetch_has_queue_space(unsigned char log2_size);
 
 struct http_response_parts {
   uint16_t code;
   char *reason;
-  int64_t range_start;
+  uint64_t range_start;
-  int64_t content_length;
+  uint64_t content_length;
   char *content_start;
 };
 
@@ -719,9 +842,9 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
 /* rhizome storage methods */
 
 int rhizome_exists(const rhizome_filehash_t *hashp);
-int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
+int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
 int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
-int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size);
+int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size);
 int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
 int rhizome_write_file(struct rhizome_write *write, const char *filename);
 int rhizome_fail_write(struct rhizome_write *write);
@@ -737,10 +860,10 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
 int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
 int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
 
-int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, 
+int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset, 
 			    const unsigned char *key, const unsigned char *nonce);
 int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
-int rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
+ssize_t rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
 int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
 int rhizome_read_close(struct rhizome_read *read);
 int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);

rhizome_bundle.c

@@ -24,10 +24,253 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "conf.h"
 #include "rhizome.h"
 #include "str.h"
+#include "mem.h"
+
+static const char *rhizome_manifest_get(const rhizome_manifest *m, const char *var)
+{
+  unsigned i;
+  for (i = 0; i < m->var_count; ++i)
+    if (strcmp(m->vars[i], var) == 0)
+      return m->values[i];
+  return NULL;
+}
+
+#if 0
+static int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var)
+{
+  unsigned i;
+  for (i = 0; i < m->var_count; ++i)
+    if (strcmp(m->vars[i], var) == 0) {
+      int64_t val;
+      return str_to_int64(m->values[i], 10, &val, NULL) ? val : -1;
+    }
+  return -1;
+}
+#endif
+
+/* @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int _rhizome_manifest_del(struct __sourceloc __whence, rhizome_manifest *m, const char *var)
+{
+  if (config.debug.rhizome_manifest)
+    DEBUGF("DEL manifest[%d].%s", m->manifest_record_number, var);
+  int ret = 0;
+  unsigned i;
+  for (i = 0; i < m->var_count; ++i)
+    if (strcmp(m->vars[i], var) == 0) {
+      free((char *) m->vars[i]);
+      free((char *) m->values[i]);
+      --m->var_count;
+      m->finalised = 0;
+      ret = 1;
+      break;
+    }
+  for (; i < m->var_count; ++i) {
+    m->vars[i] = m->vars[i + 1];
+    m->values[i] = m->values[i + 1];
+  }
+  return ret;
+}
+
+#define rhizome_manifest_set(m,var,value) _rhizome_manifest_set(__WHENCE__, (m), (var), (value))
+#define rhizome_manifest_set_ll(m,var,value) _rhizome_manifest_set_ll(__WHENCE__, (m), (var), (value))
+#define rhizome_manifest_del(m,var) _rhizome_manifest_del(__WHENCE__, (m), (var))
+
+static const char *_rhizome_manifest_set(struct __sourceloc __whence, rhizome_manifest *m, const char *var, const char *value)
+{
+  if (config.debug.rhizome_manifest)
+    DEBUGF("SET manifest[%d].%s = %s", m->manifest_record_number, var, alloca_str_toprint(value));
+  unsigned i;
+  for(i=0;i<m->var_count;i++)
+    if (strcmp(m->vars[i],var) == 0) {
+      const char *ret = str_edup(value);
+      if (ret == NULL)
+	return NULL;
+      free((char *)m->values[i]);
+      m->values[i] = ret;
+      m->finalised = 0;
+      return ret;
+    }
+  if (m->var_count >= MAX_MANIFEST_VARS)
+    return WHYNULL("no more manifest vars");
+  if ((m->vars[m->var_count] = str_edup(var)) == NULL)
+    return NULL;
+  const char *ret = m->values[m->var_count] = str_edup(value);
+  if (ret == NULL) {
+    free((char *)m->vars[i]);
+    m->vars[i] = NULL;
+    return NULL;
+  }
+  m->var_count++;
+  m->finalised = 0;
+  return ret;
+}
+
+static const char *_rhizome_manifest_set_ll(struct __sourceloc __whence, rhizome_manifest *m, char *var, int64_t value)
+{
+  char str[50];
+  snprintf(str, sizeof str, "%" PRId64, value);
+  return rhizome_manifest_set(m, var, str);
+}
+
+void _rhizome_manifest_set_id(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_bid_t *bidp)
+{
+  const char *v = rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(*bidp));
+  assert(v); // TODO: remove known manifest fields from vars[]
+  if (bidp != &m->cryptoSignPublic)
+    m->cryptoSignPublic = *bidp;
+}
+
+void _rhizome_manifest_set_version(struct __sourceloc __whence, rhizome_manifest *m, int64_t version)
+{
+  const char *v = rhizome_manifest_set_ll(m, "version", version);
+  assert(v); // TODO: remove known manifest fields from vars[]
+  m->version = version;
+}
+
+void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifest *m, uint64_t size)
+{
+  const char *v = rhizome_manifest_set_ll(m, "filesize", size);
+  assert(v); // TODO: remove known manifest fields from vars[]
+  m->filesize = size;
+  if (m->filesize == 0)
+    rhizome_manifest_set_filehash(m, NULL);
+}
+
+/* Must always set file size before setting the file hash, to avoid assertion failures.
+ */
+void _rhizome_manifest_set_filehash(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_filehash_t *hash)
+{
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (hash) {
+    assert(m->filesize > 0);
+    const char *v = rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(*hash));
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->filehash = *hash;
+  } else {
+    assert(m->filesize == 0);
+    rhizome_manifest_del(m, "filehash");
+    m->filehash = RHIZOME_FILEHASH_NONE;
+  }
+}
+
+void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m, uint64_t tail)
+{
+  const char *v = rhizome_manifest_set_ll(m, "tail", tail);
+  assert(v); // TODO: remove known manifest fields from vars[]
+  m->tail = tail;
+  m->is_journal = (tail != RHIZOME_SIZE_UNSET);
+}
+
+void _rhizome_manifest_set_bundle_key(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_bk_t *bkp)
+{
+  if (bkp) {
+    const char *v = rhizome_manifest_set(m, "BK", alloca_tohex_rhizome_bk_t(*bkp));
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->bundle_key = *bkp;
+    m->has_bundle_key = 1;
+  } else {
+    rhizome_manifest_del(m, "BK");
+    m->bundle_key = RHIZOME_BK_NONE;
+    m->has_bundle_key = 0;
+  }
+}
+
+void _rhizome_manifest_set_service(struct __sourceloc __whence, rhizome_manifest *m, const char *service)
+{
+  if (service) {
+    const char *v = rhizome_manifest_set(m, "service", service);
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->service = v;
+  } else {
+    rhizome_manifest_del(m, "service");
+    m->service = NULL;
+  }
+}
+
+void _rhizome_manifest_set_name(struct __sourceloc __whence, rhizome_manifest *m, const char *name)
+{
+  if (name) {
+    const char *v = rhizome_manifest_set(m, "name", name);
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->name = v;
+  } else {
+    rhizome_manifest_del(m, "name");
+    m->name = NULL;
+  }
+}
+
+void _rhizome_manifest_set_date(struct __sourceloc __whence, rhizome_manifest *m, time_ms_t date)
+{
+  const char *v = rhizome_manifest_set_ll(m, "date", date);
+  assert(v); // TODO: remove known manifest fields from vars[]
+  m->date = date;
+  m->has_date = 1;
+}
+
+void _rhizome_manifest_set_sender(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp)
+{
+  if (sidp) {
+    const char *v = rhizome_manifest_set(m, "sender", alloca_tohex_sid_t(*sidp));
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->sender = *sidp;
+    m->has_sender = 1;
+  } else {
+    rhizome_manifest_del(m, "sender");
+    m->sender = SID_ANY;
+    m->has_sender = 0;
+  }
+}
+
+void _rhizome_manifest_set_recipient(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp)
+{
+  if (sidp) {
+    const char *v = rhizome_manifest_set(m, "recipient", alloca_tohex_sid_t(*sidp));
+    assert(v); // TODO: remove known manifest fields from vars[]
+    m->recipient = *sidp;
+    m->has_recipient = 1;
+  } else {
+    rhizome_manifest_del(m, "recipient");
+    m->recipient = SID_ANY;
+    m->has_recipient = 0;
+  }
+}
+
+void _rhizome_manifest_set_crypt(struct __sourceloc __whence, rhizome_manifest *m, enum rhizome_manifest_crypt flag)
+{
+  switch (flag) {
+    case PAYLOAD_CRYPT_UNKNOWN:
+      rhizome_manifest_del(m, "crypt");
+      break;
+    case PAYLOAD_CLEAR: {
+      const char *v = rhizome_manifest_set(m, "crypt", "0");
+      assert(v); // TODO: remove known manifest fields from vars[]
+      break;
+    }
+    case PAYLOAD_ENCRYPTED: {
+      const char *v = rhizome_manifest_set(m, "crypt", "1");
+      assert(v); // TODO: remove known manifest fields from vars[]
+      break;
+    }
+    default: abort();
+  }
+  m->payloadEncryption = flag;
+}
+
+void _rhizome_manifest_set_author(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp)
+{
+  if (sidp) {
+    m->author = *sidp;
+    m->has_author = 1;
+  } else {
+    m->author = SID_ANY;
+    m->has_author = 0;
+  }
+}
 
 int rhizome_manifest_verify(rhizome_manifest *m)
 {
-  int end_of_text=0;
+  unsigned end_of_text=0;
 
   /* find end of manifest body and start of signatures */
   while(m->manifestdata[end_of_text]&&end_of_text<m->manifest_all_bytes)
@@ -39,10 +282,12 @@ int rhizome_manifest_verify(rhizome_manifest *m)
   crypto_hash_sha512(m->manifesthash,m->manifestdata,end_of_text);
 
   /* Read signature blocks from file. */
-  int ofs=end_of_text;  
+  unsigned ofs = end_of_text;
   while(ofs<m->manifest_all_bytes) {
-    if (config.debug.rhizome) DEBUGF("ofs=0x%x, m->manifest_bytes=0x%x", ofs,m->manifest_all_bytes);
+    if (config.debug.rhizome)
-    if (rhizome_manifest_extract_signature(m,&ofs)) break;
+      DEBUGF("ofs=0x%x, m->manifest_bytes=0x%x", ofs,m->manifest_all_bytes);
+    if (rhizome_manifest_extract_signature(m, &ofs))
+      break;
   }
 
   if (m->sig_count==0) {
@@ -53,12 +298,18 @@ int rhizome_manifest_verify(rhizome_manifest *m)
   /* Make sure that id variable is correct */
   {
     rhizome_bid_t bid;
-    char *id = rhizome_manifest_get(m,"id",NULL,0);
+    const char *id = rhizome_manifest_get(m,"id");
     if (!id) {
-      WARN("Manifest lacks 'id' field");
+      WHY("Manifest lacks 'id' field");
       m->errors++;
     } else if (str_to_rhizome_bid_t(&bid, id) == -1) {
-      WARN("Invalid manifest 'id' field");
+      WHY("Invalid manifest 'id' field");
+      m->errors++;
+    } else if (cmp_rhizome_bid_t(&bid, &m->cryptoSignPublic) != 0) {
+      WHYF("Manifest id field does not match cryptoSignPublic: id=%s, cryptoSignPublic=%s",
+	  alloca_tohex_rhizome_bid_t(bid),
+	  alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)
+	);
       m->errors++;
     } else if (m->sig_count == 0 || memcmp(m->signatories[0], bid.binary, sizeof bid.binary) != 0) {
       if (config.debug.rhizome) {
@@ -71,10 +322,9 @@ int rhizome_manifest_verify(rhizome_manifest *m)
 	}
       }
       m->errors++;
-      m->selfSigned=0;
+      m->selfSigned = 0;
-    } else {
+    } else
-      m->selfSigned=1;
+      m->selfSigned = 1;
-    }
   }
 
   /* Mark as finalised, as it is all read and intact,
@@ -103,21 +353,19 @@ int rhizome_manifest_parse(rhizome_manifest *m)
 {
   IN();
   m->manifest_all_bytes=m->manifest_bytes;
-  m->var_count=0;
+  m->var_count = 0;
-  m->journalTail=-1;
+  m->filesize = RHIZOME_SIZE_UNSET;
+  m->tail = RHIZOME_SIZE_UNSET;
 
   /* Parse out variables, signature etc */
-  int have_service = 0;
   int have_id = 0;
   int have_version = 0;
-  int have_date = 0;
-  int have_filesize = 0;
   int have_filehash = 0;
 
-  int ofs = 0;
+  unsigned ofs = 0;
   while (ofs < m->manifest_bytes && m->manifestdata[ofs]) {
     char line[1024];
-    int limit = ofs + sizeof line - 1;
+    unsigned limit = ofs + sizeof line - 1;
     if (limit > m->manifest_bytes)
       limit = m->manifest_bytes;
     char *p = line;
@@ -144,9 +392,9 @@ int rhizome_manifest_parse(rhizome_manifest *m)
       *p++ = '\0';
       char *var = line;
       char *value = p;
-      if (rhizome_manifest_get(m, var, NULL, 0)) {
+      if (rhizome_manifest_get(m, var)) {
 	if (config.debug.rejecteddata)
-	  WARNF("Ill formed manifest file, duplicate variable \"%s\"", var);
+	  DEBUGF("Ill formed manifest file, duplicate variable \"%s\"", var);
 	m->errors++;
       } else if (m->var_count >= MAX_MANIFEST_VARS) {
 	if (config.debug.rejecteddata)
@@ -155,114 +403,140 @@ int rhizome_manifest_parse(rhizome_manifest *m)
       } else {
 	m->vars[m->var_count] = strdup(var);
 	m->values[m->var_count] = strdup(value);
-	
 	// if any of these fields are not well formed, the manifest is invalid and cannot be imported
-	
 	if (strcasecmp(var, "id") == 0) {
 	  have_id = 1;
 	  if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) == -1) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid manifest id: %s", value);
+	      DEBUGF("Invalid manifest id: %s", value);
 	    m->errors++;
 	  } else {
-	    /* Force to upper case to avoid case sensitive comparison problems later. */
+	    if (config.debug.rhizome_manifest)
-	    str_toupper_inplace(m->values[m->var_count]);
+	      DEBUGF("PARSE manifest[%d].id = %s", m->manifest_record_number, alloca_tohex_sid_t(m->cryptoSignPublic));
 	  }
 	} else if (strcasecmp(var, "filehash") == 0) {
 	  have_filehash = 1;
 	  if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid filehash: %s", value);
+	      DEBUGF("Invalid filehash: %s", value);
 	    m->errors++;
 	  } else {
-	    /* Force to upper case to avoid case sensitive comparison problems later. */
+	    if (config.debug.rhizome_manifest)
-	    str_toupper_inplace(m->values[m->var_count]);
+	      DEBUGF("PARSE manifest[%d].filehash = %s", m->manifest_record_number, alloca_tohex_rhizome_filehash_t(m->filehash));
 	  }
 	} else if (strcasecmp(var, "filesize") == 0) {
-	  have_filesize = 1;
 	  uint64_t filesize;
-	  if (!str_to_uint64(value, 10, &filesize, NULL)) {
+	  if (!str_to_uint64(value, 10, &filesize, NULL) || filesize == RHIZOME_SIZE_UNSET) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid filesize: %s", value);
+	      DEBUGF("Invalid filesize: %s", value);
 	    m->errors++;
 	  } else {
-	    m->fileLength = filesize;
+	    m->filesize = filesize;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].filesize = %"PRIu64, m->manifest_record_number, m->filesize);
 	  }
 	} else if (strcasecmp(var, "version") == 0) {
 	  have_version = 1;
 	  uint64_t version;
 	  if (!str_to_uint64(value, 10, &version, NULL)) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid version: %s", value);
+	      DEBUGF("Invalid version: %s", value);
 	    m->errors++;
 	  } else {
 	    m->version = version;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].version = %"PRIu64, m->manifest_record_number, m->version);
 	  }
 	// since rhizome *MUST* be able to carry future manifest versions
 	// if any of these fields are not well formed, the manifest can still be imported and exported
 	// but the bundle should not be added or exported
 	} else if (strcasecmp(var, "tail") == 0) {
 	  uint64_t tail;
-	  if (!str_to_uint64(value, 10, &tail, NULL)) {
+	  if (!str_to_uint64(value, 10, &tail, NULL) || tail == RHIZOME_SIZE_UNSET) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid tail: %s", value);
+	      DEBUGF("Invalid tail: %s", value);
 	    m->warnings++;
 	  } else {
-	    m->journalTail = tail;
+	    m->tail = tail;
+	    m->is_journal = 1;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].tail = %"PRIu64, m->manifest_record_number, m->tail);
 	  }
 	} else if (strcasecmp(var, "BK") == 0) {
-	  if (!rhizome_str_is_bundle_key(value)) {
+	  if (str_to_rhizome_bk_t(&m->bundle_key, value) == -1) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid BK: %s", value);
+	      DEBUGF("Invalid BK: %s", value);
 	    m->warnings++;
 	  } else {
-	    /* Force to upper case to avoid case sensitive comparison problems later. */
+	    m->has_bundle_key = 1;
-	    str_toupper_inplace(m->values[m->var_count]);
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].BK = %s", m->manifest_record_number, alloca_tohex_rhizome_bk_t(m->bundle_key));
 	  }
 	} else if (strcasecmp(var, "service") == 0) {
-	  have_service = 1;
+	  if (rhizome_str_is_manifest_service(value)) {
-	  if ( strcasecmp(value, RHIZOME_SERVICE_FILE) == 0
+	    m->service = m->values[m->var_count]; // will be free()d when vars[] and values[] are free()d
-	    || strcasecmp(value, RHIZOME_SERVICE_MESHMS) == 0
+	    if (config.debug.rhizome_manifest)
-	    || strcasecmp(value, RHIZOME_SERVICE_MESHMS2) == 0) {
+	      DEBUGF("PARSE manifest[%d].service = %s", m->manifest_record_number, alloca_str_toprint(m->service));
 	  } else {
 	    if (config.debug.rejecteddata)
-	      WARNF("Unsupported service: %s", value);
+	      DEBUGF("Invalid service: %s", value);
 	    m->warnings++;
 	  }
 	} else if (strcasecmp(var, "date") == 0) {
-	  have_date = 1;
 	  int64_t date;
 	  if (!str_to_int64(value, 10, &date, NULL)) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid date: %s", value);
+	      DEBUGF("Invalid date: %s", value);
-	    m->warnings++;
-	  }
-	  // TODO: store date in manifest struct
-	} else if (strcasecmp(var, "sender") == 0 || strcasecmp(var, "recipient") == 0) {
-	  if (!str_is_subscriber_id(value)) {
-	    if (config.debug.rejecteddata)
-	      WARNF("Invalid %s: %s", var, value);
 	    m->warnings++;
 	  } else {
-	    /* Force to upper case to avoid case sensitive comparison problems later. */
+	    m->date = date;
-	    str_toupper_inplace(m->values[m->var_count]);
+	    m->has_date = 1;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].date = %"PRItime_ms_t, m->manifest_record_number, m->date);
+	  }
+	} else if (strcasecmp(var, "sender") == 0) {
+	  if (str_to_sid_t(&m->sender, value) == -1) {
+	    if (config.debug.rejecteddata)
+	      DEBUGF("Invalid sender: %s", value);
+	    m->warnings++;
+	  } else {
+	    m->has_sender = 1;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].sender = %s", m->manifest_record_number, alloca_tohex_sid_t(m->sender));
+	  }
+	} else if (strcasecmp(var, "recipient") == 0) {
+	  if (str_to_sid_t(&m->recipient, value) == -1) {
+	    if (config.debug.rejecteddata)
+	      DEBUGF("Invalid recipient: %s", value);
+	    m->warnings++;
+	  } else {
+	    m->has_recipient = 1;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].recipient = %s", m->manifest_record_number, alloca_tohex_sid_t(m->recipient));
 	  }
 	} else if (strcasecmp(var, "name") == 0) {
 	  if (value[0] == '\0') {
 	    if (config.debug.rejecteddata)
 	      WARN("Empty name");
-	    m->warnings++;
+	    //m->warnings++;  TODO Meshms code should set a name for its "conversations" bundle
 	  }
+	  m->name = m->values[m->var_count]; // will be free()d when vars[] and values[] are free()d
+	  if (config.debug.rhizome_manifest)
+	    DEBUGF("PARSE manifest[%d].name = %s", m->manifest_record_number, alloca_str_toprint(m->name));
 	} else if (strcasecmp(var, "crypt") == 0) {
 	  if (!(strcmp(value, "0") == 0 || strcmp(value, "1") == 0)) {
 	    if (config.debug.rejecteddata)
-	      WARNF("Invalid crypt: %s", value);
+	      DEBUGF("Invalid crypt: %s", value);
 	    m->warnings++;
 	  } else {
-	    m->payloadEncryption = atoi(value);
+	    m->payloadEncryption = (value[0] == '1') ? PAYLOAD_ENCRYPTED : PAYLOAD_CLEAR;
+	    if (config.debug.rhizome_manifest)
+	      DEBUGF("PARSE manifest[%d].crypt = %u", m->manifest_record_number, m->payloadEncryption == PAYLOAD_ENCRYPTED ? 1 : 0);
 	  }
 	} else {
 	  // An unknown field is not an error... older rhizome nodes must carry newer manifests.
+	  if (config.debug.rhizome_manifest)
+	    DEBUGF("SKIP manifest[%d].%s = %s", m->manifest_record_number, var, alloca_str_toprint(value));
 	}
 	m->var_count++;
       }
@@ -273,53 +547,51 @@ int rhizome_manifest_parse(rhizome_manifest *m)
     ++ofs;
 
   /* Remember where the text ends */
-  int end_of_text=ofs;
+  unsigned end_of_text = ofs;
   m->manifest_bytes = end_of_text;
 
   // verify that all required fields are consistent.
   if (!have_id) {
     if (config.debug.rejecteddata)
-      WARNF("Missing manifest id field");
+      DEBUG("Missing manifest id field");
     m->errors++;
   }
   if (!have_version) {
     if (config.debug.rejecteddata)
-      WARNF("Missing version field");
+      DEBUG("Missing version field");
     m->errors++;
   }
-  if (!have_filesize) {
+  if (m->filesize == RHIZOME_SIZE_UNSET) {
     if (config.debug.rejecteddata)
-      WARNF("Missing filesize field");
+      DEBUG("Missing filesize field");
     m->errors++;
   }
-  if (!have_filehash && m->fileLength != 0) {
+  if (!have_filehash && m->filesize > 0) {
     if (config.debug.rejecteddata)
-      WARNF("Missing filehash field");
+      DEBUG("Missing filehash field");
     m->errors++;
   }
-  if (have_filehash && m->fileLength == 0) {
+  if (have_filehash && m->filesize == 0) {
     if (config.debug.rejecteddata)
-      WARNF("Spurious filehash field");
+      DEBUG("Spurious filehash field");
     m->errors++;
   }
 
   // warn if expected fields are missing
-  if (!have_service) {
+  if (m->service == NULL) {
     if (config.debug.rejecteddata)
-      WARNF("Missing service field");
+      DEBUG("Missing service field");
     m->warnings++;
   }
-  if (!have_date) {
+  if (!m->has_date) {
     if (config.debug.rejecteddata)
-      WARNF("Missing date field");
+      DEBUG("Missing date field");
     m->warnings++;
   }
 
-  // TODO Determine group membership here.
-
   if (m->errors || m->warnings) {
     if (config.debug.rejecteddata)
-      dump("manifest body",m->manifestdata,m->manifest_bytes);
+      dump("manifest body", m->manifestdata, (size_t) m->manifest_bytes);
   }
 
   RETURN(0);
@@ -353,7 +625,7 @@ int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t
      implementation.
   */
   // TODO encrypted payloads
-  if (m && m->payloadEncryption) 
+  if (m && m->payloadEncryption == PAYLOAD_ENCRYPTED)
     return WHY("Encryption of payloads not implemented");
 
   uint64_t filesize = 0;
@@ -389,100 +661,6 @@ int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t
   return 0;
 }
 
-char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen)
-{
-  int i,j;
-
-  if (!m) return NULL;
-
-  for(i=0;i<m->var_count;i++)
-    if (!strcmp(m->vars[i],var)) {
-      if (out) {
-	for(j=0;(j<maxlen);j++) {
-	  out[j]=m->values[i][j];
-	  if (!out[j]) break;
-	}
-      }
-      return m->values[i];
-    }
-  return NULL;
-}
-
-int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var)
-{
-  if (!m)
-    return -1;
-  int i;
-  for (i = 0; i < m->var_count; ++i)
-    if (!strcmp(m->vars[i], var)) {
-      int64_t val;
-      return str_to_int64(m->values[i], 10, &val, NULL) ? val : -1;
-    }
-  return -1;
-}
-
-double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value)
-{
-  int i;
-
-  if (!m) return default_value;
-
-  for(i=0;i<m->var_count;i++)
-    if (!strcmp(m->vars[i],var))
-      return strtod(m->values[i],NULL);
-  return default_value;
-}
-
-/* @author Andrew Bettison <andrew@servalproject.com>
- */
-int rhizome_manifest_del(rhizome_manifest *m, const char *var)
-{
-  int ret = 0;
-  int i;
-  for (i = 0; i < m->var_count; ++i)
-    if (strcmp(m->vars[i], var) == 0) {
-      free(m->vars[i]); 
-      free(m->values[i]); 
-      --m->var_count;
-      m->finalised = 0;
-      ret = 1;
-      break;
-    }
-  for (; i < m->var_count; ++i) {
-    m->vars[i] = m->vars[i + 1];
-    m->values[i] = m->values[i + 1];
-  }
-  return ret;
-}
-
-int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value)
-{
-  if (!m)
-    return WHY("m == NULL");
-  int i;
-  for(i=0;i<m->var_count;i++)
-    if (!strcmp(m->vars[i],var)) {
-      free(m->values[i]); 
-      m->values[i]=strdup(value);
-      m->finalised=0;
-      return 0;
-    }
-  if (m->var_count >= MAX_MANIFEST_VARS)
-    return WHY("no more manifest vars");
-  m->vars[m->var_count]=strdup(var);
-  m->values[m->var_count]=strdup(value);
-  m->var_count++;
-  m->finalised=0;
-  return 0;
-}
-
-int rhizome_manifest_set_ll(rhizome_manifest *m, char *var, int64_t value)
-{
-  char str[50];
-  snprintf(str, sizeof str, "%" PRId64, value);
-  return rhizome_manifest_set(m, var, str);
-}
-
 rhizome_manifest manifests[MAX_RHIZOME_MANIFESTS];
 char manifest_free[MAX_RHIZOME_MANIFESTS];
 int manifest_first_free=-1;
@@ -492,7 +670,7 @@ struct __sourceloc manifest_free_whence[MAX_RHIZOME_MANIFESTS];
 static void _log_manifest_trace(struct __sourceloc __whence, const char *operation)
 {
   int count_free = 0;
-  int i;
+  unsigned i;
   for (i = 0; i != MAX_RHIZOME_MANIFESTS; ++i)
     if (manifest_free[i])
       ++count_free;
@@ -503,7 +681,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
 {
   if (manifest_first_free<0) {
     /* Setup structures */
-    int i;
+    unsigned i;
     for(i=0;i<MAX_RHIZOME_MANIFESTS;i++) {
       manifest_alloc_whence[i]=__NOWHERE__;
       manifest_free_whence[i]=__NOWHERE__;
@@ -515,7 +693,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
   /* No free manifests */
   if (manifest_first_free>=MAX_RHIZOME_MANIFESTS)
     {
-      int i;
+      unsigned i;
       WHYF("%s(): no free manifest records, this probably indicates a memory leak", __FUNCTION__);
       WHYF("   Slot# | Last allocated by");
       for(i=0;i<MAX_RHIZOME_MANIFESTS;i++) {
@@ -545,8 +723,9 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
 
   if (config.debug.manifests) _log_manifest_trace(__whence, __FUNCTION__);
 
-  // Set global defaults for a manifest
+  // Set global defaults for a manifest (which are not zero)
-  m->journalTail = -1;
+  m->filesize = RHIZOME_SIZE_UNSET;
+  m->tail = RHIZOME_SIZE_UNSET;
 
   return m;
 }
@@ -554,7 +733,6 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
 void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
 {
   if (!m) return;
-  int i;
   int mid=m->manifest_record_number;
 
   if (m!=&manifests[mid]) {
@@ -574,20 +752,22 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
     exit(-1);
   }
 
-  /* Free variable and signature blocks.
+  /* Free variable and signature blocks. */
-     XXX These should be moved to malloc-free storage eventually */
+  unsigned i;
-  for(i=0;i<m->var_count;i++)
+  for(i=0;i<m->var_count;i++) {
-    { free(m->vars[i]); free(m->values[i]); 
+    free((char *) m->vars[i]);
-      m->vars[i]=NULL; m->values[i]=NULL; }
+    free((char *) m->values[i]);
-  for(i=0;i<m->sig_count;i++)
+    m->vars[i] = m->values[i] = NULL;
-    { free(m->signatories[i]);
+  }
-      m->signatories[i]=NULL;
+  for(i=0;i<m->sig_count;i++) {
-    }
+    free(m->signatories[i]);
+    m->signatories[i] = NULL;
+  }
 
   if (m->dataFileName) {
     if (m->dataFileUnlinkOnFree && unlink(m->dataFileName) == -1)
       WARNF_perror("unlink(%s)", alloca_str_toprint(m->dataFileName));
-    free(m->dataFileName);
+    free((char *) m->dataFileName);
     m->dataFileName = NULL;
   }
 
@@ -605,8 +785,8 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
    Signatures etc will be added later. */
 int rhizome_manifest_pack_variables(rhizome_manifest *m)
 {
-  int i,ofs=0;
+  unsigned i;
-
+  unsigned ofs = 0;
   for(i=0;i<m->var_count;i++)
     {
       if ((ofs+strlen(m->vars[i])+1+strlen(m->values[i])+1+1)>MAX_MANIFEST_BYTES)
@@ -690,7 +870,7 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid)
 
 int rhizome_manifest_dump(rhizome_manifest *m, const char *msg)
 {
-  int i;
+  unsigned i;
   WHYF("Dumping manifest %s:", msg);
   for(i=0;i<m->var_count;i++)
     WHYF("[%s]=[%s]\n", m->vars[i], m->values[i]);
@@ -702,6 +882,9 @@ int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int
   IN();
   int ret=0;
 
+  if (m->filesize == RHIZOME_SIZE_UNSET)
+    RETURN(WHY("Manifest filesize unknown"));
+
   // if a manifest was supplied with an ID, don't bother to check for a duplicate.
   // we only want to filter out added files with no existing manifest.
   if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID && rhizome_find_duplicate(m, mout) == 1)
@@ -719,101 +902,85 @@ int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int
 
   /* mark manifest as finalised */
   m->finalised=1;
-  ret=rhizome_add_manifest(m, 255 /* TTL */);
+  ret = rhizome_add_manifest(m, 255 /* TTL */);
 
   RETURN(ret);
   OUT();
 }
 
-int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp, rhizome_bk_t *bsk){
+int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp, rhizome_bk_t *bsk)
+{
   /* Fill in a few missing manifest fields, to make it easier to use when adding new files:
    - the default service is FILE
    - use the current time for "date"
    - if service is file, then use the payload file's basename for "name"
    */
-  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
-  if (service == NULL)
-    return WHYF("missing 'service'");
-  if (config.debug.rhizome)
-    DEBUGF("manifest service=%s", service);
-  if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) {
-    rhizome_manifest_set_ll(m, "date", (int64_t) gettime_ms());
-    if (config.debug.rhizome) DEBUGF("missing 'date', set default date=%s", rhizome_manifest_get(m, "date", NULL, 0));
-  }
 
-  if (strcasecmp(RHIZOME_SERVICE_FILE, service) == 0) {
+  /* Set version of manifest, either from version variable, or using current time */
-    const char *name = rhizome_manifest_get(m, "name", NULL, 0);
+  if (m->version == 0)
-    if (name == NULL) {
+    rhizome_manifest_set_version(m, gettime_ms());
-      if (filepath && *filepath){
-	name = strrchr(filepath, '/');
-	name = name ? name + 1 : filepath;
-      }else
-	name="";
-      rhizome_manifest_set(m, "name", name);
-      if (config.debug.rhizome) DEBUGF("missing 'name', set default name=\"%s\"", name);
-    } else {
-      if (config.debug.rhizome) DEBUGF("manifest contains name=\"%s\"", name);
-    }
-  }
 
-  /* If the author was not specified, then the manifest's "sender"
+  /* Set the manifest's author (not stored).  This must be done before binding to a new ID (below).
-   field is used, if present. */
+   * If no author was specified, then the manifest's "sender" field is used, if present.
+   */
   if (authorSidp)
-    m->author = *authorSidp;
+    rhizome_manifest_set_author(m, authorSidp);
-  else{
+  else if (m->has_sender)
-    const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
+    rhizome_manifest_set_author(m, &m->sender);
-    if (sender){
-      if (str_to_sid_t(&m->author, sender) == -1)
-	return WHYF("invalid sender: %s", sender);
-    }
-  }
 
-  /* set version of manifest, either from version variable, or using current time */
+  if (!m->haveSecret) {
-  if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
+    if (rhizome_bid_t_is_zero(m->cryptoSignPublic)) {
-  {
+      if (config.debug.rhizome)
-    /* No version set, default to the current time */
+	DEBUG("creating new bundle");
-    m->version = gettime_ms();
+      if (rhizome_manifest_bind_id(m) == -1)
-    rhizome_manifest_set_ll(m,"version",m->version);
-  }
-  
-  if (!m->haveSecret){
-    const char *id = rhizome_manifest_get(m, "id", NULL, 0);
-    if (id == NULL) {
-      if (config.debug.rhizome) DEBUG("creating new bundle");
-      if (rhizome_manifest_bind_id(m) == -1) {
 	return WHY("Could not bind manifest to an ID");
-      }
     } else {
-      if (config.debug.rhizome) DEBUGF("modifying existing bundle bid=%s", id);
+      if (config.debug.rhizome)
-      
+	DEBUGF("modifying existing bundle bid=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
       // Modifying an existing bundle.  Make sure we can find the bundle secret.
-      if (rhizome_extract_privatekey_required(m, bsk))
+      if (rhizome_extract_privatekey_required(m, bsk) == -1)
 	return -1;
-      
       // TODO assert that new version > old version?
     }
   }
   assert(m->haveSecret);
 
-  int crypt = rhizome_manifest_get_ll(m,"crypt"); 
+  if (m->service == NULL)
-  if (crypt==-1){
+    return WHYF("missing 'service'");
-    // no explicit crypt flag, should we encrypt this bundle?
+  if (config.debug.rhizome)
-    char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
+    DEBUGF("manifest service=%s", m->service);
-    char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
 
-    // anything sent from one person to another should be considered private and encrypted by default
+  if (!m->has_date) {
-    if (sender && recipient){
+    rhizome_manifest_set_date(m, (int64_t) gettime_ms());
-      sid_t s_sender, s_recipient;
+    if (config.debug.rhizome)
-      if (cf_opt_sid(&s_sender, sender)==CFOK 
+      DEBUGF("missing 'date', set default date=%"PRItime_ms_t, m->date);
-	&& cf_opt_sid(&s_recipient, recipient)==CFOK
+  }
-	&& !is_sid_t_broadcast(s_recipient)){
+
-	if (config.debug.rhizome)
+  if (strcasecmp(RHIZOME_SERVICE_FILE, m->service) == 0) {
-	  DEBUGF("Implicitly adding payload encryption due to presense of sender & recipient fields");
+    if (m->name == NULL) {
-	m->payloadEncryption=1;
+      if (filepath && *filepath) {
-	rhizome_manifest_set_ll(m,"crypt",1LL); 
+	const char *name = strrchr(filepath, '/');
-      }
+	rhizome_manifest_set_name(m, name ? name + 1 : filepath);
+      } else
+	rhizome_manifest_set_name(m, "");
+      if (config.debug.rhizome)
+	DEBUGF("missing 'name', set default name=%s", alloca_str_toprint(m->name));
+    } else {
+      if (config.debug.rhizome)
+	DEBUGF("manifest contains name=%s", alloca_str_toprint(m->name));
     }
   }
 
+  // Anything sent from one person to another should be considered private and encrypted by default.
+  if (   m->payloadEncryption == PAYLOAD_CRYPT_UNKNOWN
+      && m->has_sender
+      && m->has_recipient
+      && !is_sid_t_broadcast(m->recipient)
+  ) {
+    if (config.debug.rhizome)
+      DEBUGF("Implicitly adding payload encryption due to presense of sender & recipient fields");
+    rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED);
+  }
+
   return 0;
 }

rhizome_crypto.c

@@ -17,6 +17,10 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
+#include <stdlib.h>
+#include <ctype.h>
+#include <assert.h>
+
 #include "crypto_sign_edwards25519sha512batch.h"
 #include "nacl/src/crypto_sign_edwards25519sha512batch_ref/ge.h"
 
@@ -25,8 +29,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "str.h"
 #include "rhizome.h"
 #include "crypto.h"
-#include <stdlib.h>
-#include <ctype.h>
 
 /* Work out the encrypt/decrypt key for the supplied manifest.
    If the manifest is not encrypted, then return NULL.
@@ -40,7 +42,7 @@ int rhizome_manifest_createid(rhizome_manifest *m)
 {
   if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic.binary, m->cryptoSignSecret))
     return WHY("Failed to create keypair for manifest ID.");
-  rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
+  rhizome_manifest_set_id(m, &m->cryptoSignPublic);
   m->haveSecret = NEW_BUNDLE_ID;
   return 0;
 }
@@ -56,10 +58,14 @@ static int generate_keypair(const char *seed, struct signing_key *key)
   unsigned char hash[crypto_hash_sha512_BYTES];
   crypto_hash_sha512(hash, (unsigned char *)seed, strlen(seed));
   
-  // The first 256 bits of the hash will be used as the private key of the BID.
+  // The first 256 bits (32 bytes) of the hash will be used as the private key of the BID.
-  bcopy(hash, key->Private, sizeof(key->Private));
+  bcopy(hash, key->Private, sizeof key->Private);
   if (crypto_sign_compute_public_key(key->Private, key->Public.binary))
     return WHY("Could not generate public key");
+  // The last 32 bytes of the private key should be identical to the public key.  This is what
+  // crypto_sign_edwards25519sha512batch_keypair() returns, and there is code that depends on it.
+  // TODO: Refactor the Rhizome private/public keypair to eliminate this duplication.
+  bcopy(key->Public.binary, key->Private + RHIZOME_BUNDLE_KEY_BYTES, sizeof key->Public.binary);
   return 0;
 }
 
@@ -70,16 +76,20 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
   struct signing_key key;
   if (generate_keypair(seed, &key))
     return -1;
-  
+  int ret = rhizome_retrieve_manifest(&key.Public, m);
-  int ret=rhizome_retrieve_manifest(&key.Public, m);
   if (ret == -1)
     return -1;
-  
+  if (ret == 1) {
-  m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID;
+    // manifest not retrieved
-  m->cryptoSignPublic = key.Public;
+    rhizome_manifest_set_id(m, &key.Public);
+    m->haveSecret = NEW_BUNDLE_ID;
+  } else {
+    m->haveSecret = EXISTING_BUNDLE_ID;
+  }
   bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
-  if (ret == 1)
+  //Disabled for performance, but these asserts should always hold.
-    rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
+  //assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &key.Public) == 0);
+  //assert(memcmp(m->cryptoSignPublic.binary, m->cryptoSignSecret + RHIZOME_BUNDLE_KEY_BYTES, sizeof m->cryptoSignPublic.binary) == 0);
   return ret;
 }
 
@@ -230,7 +240,7 @@ int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned cha
  * which is used to look up the author's rhizome secret in the keyring.
  *
  * Returns 0 if a valid private key was extracted, with the private key in the manifest
- * 'cryptoSignSecret' field and the 'haveSecret' field set to 1.
+ * 'cryptoSignSecret' field and the 'haveSecret' field set to EXISTING_BUNDLE_ID.
  *
  * Returns 1 if the manifest does not have a BK field.
  *
@@ -248,37 +258,31 @@ int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned cha
  * @author Andrew Bettison <andrew@servalproject.com>
 
  */
-int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
+int rhizome_extract_privatekey(rhizome_manifest *m, const rhizome_bk_t *bsk)
 {
+  if (config.debug.rhizome)
+    DEBUGF("manifest[%d] bsk=%s", m->manifest_record_number, bsk ? alloca_tohex_rhizome_bk_t(*bsk) : "NULL");
   IN();
-  unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES];
-  char *bk = rhizome_manifest_get(m, "BK", NULL, 0);
   int result;
-  
+  if (m->has_bundle_key) {
-  if (bk){
+    if (!m->has_author) {
-    if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1)
+      result = rhizome_find_bundle_author(m);
-      RETURN(WHYF("invalid BK field: %s", bk));
+    } else {
-    
-    if (is_sid_t_any(m->author)) {
-      result=rhizome_find_bundle_author(m);
-    }else{
       int rs_len;
       const unsigned char *rs;
       result = rhizome_find_secret(&m->author, &rs_len, &rs);
-      if (result==0)
+      if (result == 0)
-	result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret);
+	result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, m->cryptoSignSecret);
     }
-    
     if (result == 0 && bsk && !rhizome_is_bk_none(bsk)){
       // If a bundle secret key was supplied that does not match the secret key derived from the
       // author, then warn but carry on using the author's.
-      if (memcmp(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES) != 0)
+      if (memcmp(bsk->binary, m->cryptoSignSecret, sizeof bsk->binary) != 0)
 	WARNF("Supplied bundle secret key is invalid -- ignoring");
     }
-    
+  }else if (bsk && !rhizome_is_bk_none(bsk)){
-  }else if(bsk && !rhizome_is_bk_none(bsk)){
+    bcopy(bsk->binary, m->cryptoSignSecret, sizeof bsk->binary);
-    bcopy(m->cryptoSignPublic.binary, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof m->cryptoSignPublic.binary);
+    bcopy(m->cryptoSignPublic.binary, m->cryptoSignSecret + sizeof bsk->binary, sizeof m->cryptoSignPublic.binary);
-    bcopy(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES);
     if (rhizome_verify_bundle_privatekey(m, m->cryptoSignSecret, m->cryptoSignPublic.binary))
       result=5;
     else
@@ -286,14 +290,12 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
   }else{
     result=1;
   }
-  
+  if (result == 0)
-  if (result == 0){
+    m->haveSecret = EXISTING_BUNDLE_ID;
-    m->haveSecret=EXISTING_BUNDLE_ID;
+  else {
-  }else{
     memset(m->cryptoSignSecret, 0, sizeof m->cryptoSignSecret);
-    m->haveSecret=0;
+    m->haveSecret = SECRET_UNKNOWN;
   }
-  
   RETURN(result);
   OUT();
 }
@@ -326,7 +328,7 @@ int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk)
  *
  * Returns 0 if an identity is found with permission to alter the bundle, after setting the manifest
  * 'author' field to the SID of the identity and the manifest 'cryptoSignSecret' field to the bundle
- * secret key and the 'haveSecret' field to 1.
+ * secret key and the 'haveSecret' field to EXISTING_BUNDLE_ID.
  *
  * Returns 1 if no identity in the keyring is the author of this bundle.
  *
@@ -339,15 +341,11 @@ int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk)
 int rhizome_find_bundle_author(rhizome_manifest *m)
 {
   IN();
-  char *bk = rhizome_manifest_get(m, "BK", NULL, 0);
+  if (!m->has_bundle_key) {
-  if (!bk) {
     if (config.debug.rhizome)
-      DEBUGF("missing BK field");
+      DEBUG("missing BK");
     RETURN(4);
   }
-  unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES];
-  if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1)
-    RETURN(WHYF("invalid BK field: %s", bk));
   int cn = 0, in = 0, kp = 0;
   for (; keyring_next_identity(keyring, &cn, &in, &kp); ++kp) {
     const sid_t *authorSidp = (const sid_t *) keyring->contexts[cn]->identities[in]->keypairs[kp]->public_key;
@@ -358,13 +356,12 @@ int rhizome_find_bundle_author(rhizome_manifest *m)
       if (rs_len < 16 || rs_len > 1024)
 	RETURN(WHYF("invalid Rhizome Secret: length=%d", rs_len));
       const unsigned char *rs = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key;
-
+      if (rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, m->cryptoSignSecret) == 0) {
-      if (!rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret)) {
+	m->haveSecret = EXISTING_BUNDLE_ID;
-	m->haveSecret=EXISTING_BUNDLE_ID;
+	if (!m->has_author || cmp_sid_t(&m->author, authorSidp) != 0){
-	if (cmp_sid_t(&m->author, authorSidp) != 0){
-	  m->author = *authorSidp;
 	  if (config.debug.rhizome)
-	    DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(m->author));
+	    DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(*authorSidp));
+	  rhizome_manifest_set_author(m, authorSidp);
 	  // if this bundle is already in the database, update the author.
 	  if (m->inserttime)
 	    sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
@@ -373,7 +370,6 @@ int rhizome_find_bundle_author(rhizome_manifest *m)
 		RHIZOME_BID_T, &m->cryptoSignPublic,
 		END);
 	}
-	
 	RETURN(0); // bingo
       }
     }
@@ -402,13 +398,13 @@ int rhizome_verify_bundle_privatekey(rhizome_manifest *m,
   for (i = 0;i < 32;++i) 
     if (pkin[i] != pk[i]) {
       if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary)
-	m->haveSecret=0;
+	m->haveSecret = SECRET_UNKNOWN;
       RETURN(-1);
     }
   if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) {
     if (config.debug.rhizome)
       DEBUGF("We have the private key for this bundle.");
-    m->haveSecret=EXISTING_BUNDLE_ID;
+    m->haveSecret = EXISTING_BUNDLE_ID;
   }
   RETURN(0);
   OUT();
@@ -504,18 +500,19 @@ int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char
   OUT();
 }
 
-int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs)
+int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs)
 {
   IN();
   if (!m)
     RETURN(WHY("NULL pointer passed in as manifest"));
   if (config.debug.rhizome)
-    DEBUGF("m->manifest_all_bytes=%d m->manifest_bytes=%d *ofs=%d", m->manifest_all_bytes, m->manifest_bytes, *ofs);
+    DEBUGF("m->manifest_all_bytes=%u m->manifest_bytes=%u *ofs=%u", m->manifest_all_bytes, m->manifest_bytes, *ofs);
 
-  if ((*ofs)>=m->manifest_all_bytes) { RETURN(0); }
+  if ((*ofs) >= m->manifest_all_bytes)
+    RETURN(0);
 
-  int sigType=m->manifestdata[*ofs];
+  uint8_t sigType = m->manifestdata[*ofs];
-  int len=(sigType&0x3f)*4+4+1;
+  uint8_t len = (sigType << 2) + 4 + 1;
 
   /* Each signature type is required to have a different length to detect it.
      At present only crypto_sign_edwards25519sha512batch() signatures are
@@ -577,7 +574,8 @@ int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs)
 
 // add value to nonce, with the same result regardless of CPU endian order
 // allowing for any carry value up to the size of the whole nonce
-static void add_nonce(unsigned char *nonce, int64_t value){
+static void add_nonce(unsigned char *nonce, uint64_t value)
+{
   int i=crypto_stream_xsalsa20_NONCEBYTES -1;
   while(i>=0 && value>0){
     int x = nonce[i]+(value & 0xFF);
@@ -590,13 +588,10 @@ static void add_nonce(unsigned char *nonce, int64_t value){
 /* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries
  * for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE)
  */
-int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, 
+int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset, 
-			    const unsigned char *key, const unsigned char *nonce){
+			    const unsigned char *key, const unsigned char *nonce)
-  
+{
-  if (stream_offset<0)
+  uint64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
-    return WHY("Invalid stream offset");
-  
-  int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
   size_t offset=0;
   
   unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES];
@@ -635,31 +630,21 @@ int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t s
 int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
 {
   // don't do anything if the manifest isn't flagged as being encrypted
-  if (!m->payloadEncryption)
+  if (m->payloadEncryption != PAYLOAD_ENCRYPTED)
     return 0;
-  if (m->payloadEncryption!=1)
+  if (m->has_sender && m->has_recipient){
-    return WHYF("Unsupported encryption scheme %d", m->payloadEncryption);
-  
-  char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
-  char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
-  
-  if (sender && recipient){
-    sid_t sender_sid, recipient_sid;
-    if (cf_opt_sid(&sender_sid, sender)!=CFOK)
-      return WHYF("Unable to parse sender sid");
-    if (cf_opt_sid(&recipient_sid, recipient)!=CFOK)
-      return WHYF("Unable to parse recipient sid");
-    
     unsigned char *nm_bytes=NULL;
     int cn=0,in=0,kp=0;
-    if (!keyring_find_sid(keyring, &cn, &in, &kp, &sender_sid)){
+    if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->sender)){
       cn=in=kp=0;
-      if (!keyring_find_sid(keyring, &cn, &in, &kp, &recipient_sid)){
+      if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->recipient)){
-	return WHYF("Neither the sender %s nor the recipient %s appears in our keyring", sender, recipient);
+	return WHYF("Neither the sender %s nor the recipient %s appears in our keyring",
+	    alloca_tohex_sid_t(m->sender),
+	    alloca_tohex_sid_t(m->recipient));
       }
-      nm_bytes=keyring_get_nm_bytes(&recipient_sid, &sender_sid);
+      nm_bytes=keyring_get_nm_bytes(&m->recipient, &m->sender);
     }else{
-      nm_bytes=keyring_get_nm_bytes(&sender_sid, &recipient_sid);
+      nm_bytes=keyring_get_nm_bytes(&m->sender, &m->recipient);
     }
     
     if (!nm_bytes)
@@ -670,10 +655,9 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
     bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
     
   }else{
-    if(!m->haveSecret){
+    if (!m->haveSecret && rhizome_extract_privatekey_required(m, bsk))
-      if (rhizome_extract_privatekey_required(m, bsk))
+      return -1;
-	return -1;
+    assert(m->haveSecret);
-    }
     
     unsigned char raw_key[9+crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]="sasquatch";
     bcopy(m->cryptoSignSecret, &raw_key[9], crypto_sign_edwards25519sha512batch_SECRETKEYBYTES);
@@ -687,9 +671,9 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
   // journal bundles must always have the same nonce, regardless of version.
   // otherwise, generate nonce from version#bundle id#version;
   unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary];
-  write_uint64(&raw_nonce[0], m->journalTail>=0?0:m->version);
+  write_uint64(&raw_nonce[0], m->is_journal ? 0 : m->version);
   bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary);
-  write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->journalTail>=0?0:m->version);
+  write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->is_journal ? 0 : m->version);
   
   unsigned char hash[crypto_hash_sha512_BYTES];
   

rhizome_database.c

@@ -176,20 +176,25 @@ void verify_bundles(){
 }
 
 /*
- * The MANIFESTS table 'author' column records the cryptographically verified SID of the author
+ * The MANIFESTS table 'author' column records the cryptographically verified SID of the author that
- * that has write permission on the bundle, ie, possesses the Rhizome secret key that generated the
+ * has write permission on the bundle, ie, possesses the Rhizome secret key that generated the BID,
- * BID, and hence can derive the Bundle Secret from the bundle's BK field:
+ * and hence can derive the Bundle Secret from the bundle's BK field:
- * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created
+ *
- *   locally by a non-secret identity, so no verification need be performed for one's own
+ * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created locally
- *   bundles while they remain in the Rhizome store.
+ *   by a non-secret identity, so no verification need be performed for one's own bundles while they
- * - When a bundle is imported, the 'author' column is set to NULL to indicate that no
+ *   remain in the local Rhizome store.
- *   verification has passed yet.  This includes one's own bundles that have been purged from
+ *
- *   the local Rhizome store then recovered from a remote Rhizome node.
+ * - When a bundle is imported, the 'author' column is set to NULL to indicate that no verification
- * - When a manifest with NULL 'author' is examined closely, ie extracted, not merely
+ *   has passed yet.  This includes one's own bundles that have been purged from the local Rhizome
- *   listed, the keyring is searched for an identity that is the author.  If an author is
+ *   store then recovered from a remote Rhizome node.
- *   found, the MANIFESTS table 'author' column is updated.  This allows one to regain the
+ *
- *   ability to overwrite one's own bundles that have been lost but recovered from an exterior
+ * - When a manifest with NULL 'author' is examined closely, ie extracted, not merely listed, the
- *   Rhizome node.
+ *   keyring is searched for an identity that is the author.  If the identity is found and its
+ *   Rhizome Secret unlocks the Bundle Key (ie, reveals a Bundle Secret that yields the Bundle's ID
+ *   as its public key), the MANIFESTS table 'author' column is updated.  This allows one to regain
+ *   the ability to overwrite one's own bundles that have been lost but
+ *   recovered from an exterior Rhizome node.
+ *
  * - The above check automates the "own bundle recovery" mechanism at the expense of a CPU-heavy
  *   cryptographic check every time a foreign bundle is examined, but at least listing is fast.
  *   This will not scale as many identities are added to the keyring.  It will eventually have to be
@@ -504,7 +509,7 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
       return -1;
     }
 #define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \
-	if (config.debug.rhizome_bind) \
+	if (config.debug.rhizome_sql_bind) \
 	  DEBUGF("%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement))
 #define BIND_RETRY(FUNC, ...) \
 	do { \
@@ -1270,7 +1275,8 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
  */
 int rhizome_store_bundle(rhizome_manifest *m)
 {
-  if (!m->finalised) return WHY("Manifest was not finalised");
+  if (!m->finalised)
+    return WHY("Manifest was not finalised");
 
   if (m->haveSecret) {
     /* We used to store the secret in the database, but we don't anymore, as we use 
@@ -1287,28 +1293,10 @@ int rhizome_store_bundle(rhizome_manifest *m)
   rhizome_manifest_to_bar(m,bar);
 
   /* Store the file (but not if it is already in the database) */
-  if (m->fileLength > 0 && !rhizome_exists(&m->filehash))
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize > 0 && !rhizome_exists(&m->filehash))
     return WHY("File should already be stored by now");
 
-  const char *name = rhizome_manifest_get(m, "name", NULL, 0);
-  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
-
-  sid_t *sender = NULL;
-  const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
-  if (sender_field) {
-    sender = (sid_t *) alloca(sizeof *sender);
-    if (str_to_sid_t(sender, sender_field) == -1)
-      return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
-  }
-
-  sid_t *recipient = NULL;
-  const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
-  if (recipient_field) {
-    recipient = (sid_t *) alloca(sizeof *recipient);
-    if (str_to_sid_t(recipient, recipient_field) == -1)
-      return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
-  }
-
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
     return WHY("Failed to begin transaction");
@@ -1337,14 +1325,14 @@ int rhizome_store_bundle(rhizome_manifest *m)
 	INT64, m->version,
 	INT64, (int64_t) gettime_ms(),
 	STATIC_BLOB, bar, RHIZOME_BAR_BYTES,
-	INT64, m->fileLength,
+	INT64, m->filesize,
-	RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL,
+	RHIZOME_FILEHASH_T|NUL, m->filesize > 0 ? &m->filehash : NULL,
-	SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author,
+	SID_T|NUL, m->has_author ? &m->author : NULL,
-	STATIC_TEXT, service,
+	STATIC_TEXT, m->service,
-	STATIC_TEXT|NUL, name,
+	STATIC_TEXT|NUL, m->name,
-	SID_T|NUL, sender,
+	SID_T|NUL, m->has_sender ? &m->sender : NULL,
-	SID_T|NUL, recipient,
+	SID_T|NUL, m->has_recipient ? &m->recipient : NULL,
-	INT64, m->journalTail,
+	INT64, m->tail,
 	END
       )
   ) == NULL)
@@ -1359,6 +1347,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
 
   // TODO remove old payload?
   
+#if 0
   if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
     int closed=rhizome_manifest_get_ll(m,"closedgroup");
     if (closed<1) closed=0;
@@ -1380,11 +1369,13 @@ int rhizome_store_bundle(rhizome_manifest *m)
     sqlite3_finalize(stmt);
     stmt = NULL;
   }
+#endif
 
+#if 0
   if (m->group_count > 0) {
     if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS (manifestid, groupid) VALUES (?, ?);")) == NULL)
       goto rollback;
-    int i;
+    unsigned i;
     for (i=0;i<m->group_count;i++){
       if (sqlite_bind(&retry, stmt, RHIZOME_BID_T, &m->cryptoSignPublic, TEXT, m->groups[i]) == -1)
 	goto rollback;
@@ -1395,14 +1386,15 @@ int rhizome_store_bundle(rhizome_manifest *m)
     sqlite3_finalize(stmt);
     stmt = NULL;
   }
+#endif
+
   if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
     // This message used in tests; do not modify or remove.
-    const char *service = rhizome_manifest_get(m, "service", NULL, 0);
     INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRId64,
-	  service ? service : "NULL",
+	  m->service ? m->service : "NULL",
 	  alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
 	  m->version
-	  );
+	);
     monitor_announce_bundle(m);
     if (serverMode)
       rhizome_sync_announce();
@@ -1417,10 +1409,12 @@ rollback:
 }
 
 int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, 
-			   const char *sender_sid, const char *recipient_sid, 
+			   const char *sender_hex, const char *recipient_hex, 
 			   int limit, int offset, char count_rows)
 {
   IN();
+  sid_t sender;
+  sid_t recipient;
   strbuf b = strbuf_alloca(1024);
   strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
   
@@ -1428,10 +1422,16 @@ int rhizome_list_manifests(struct cli_context *context, const char *service, con
     strbuf_sprintf(b, " AND service = ?1");
   if (name && *name)
     strbuf_sprintf(b, " AND name like ?2");
-  if (sender_sid && *sender_sid)
+  if (sender_hex && *sender_hex) {
+    if (str_to_sid_t(&sender, sender_hex) == -1)
+      RETURN(WHYF("Invalid sender SID: %s", sender_hex));
     strbuf_sprintf(b, " AND sender = ?3");
-  if (recipient_sid && *recipient_sid)
+  }
+  if (recipient_hex && *recipient_hex) {
+    if (str_to_sid_t(&recipient, recipient_hex) == -1)
+      RETURN(WHYF("Invalid recipient SID: %s", recipient_hex));
     strbuf_sprintf(b, " AND recipient = ?4");
+  }
   
   strbuf_sprintf(b, " ORDER BY inserttime DESC");
   
@@ -1452,10 +1452,10 @@ int rhizome_list_manifests(struct cli_context *context, const char *service, con
     ret = sqlite3_bind_text(statement, 1, service, -1, SQLITE_STATIC);
   if (ret==SQLITE_OK && name && *name)
     ret = sqlite3_bind_text(statement, 2, name, -1, SQLITE_STATIC);
-  if (ret==SQLITE_OK && sender_sid && *sender_sid)
+  if (ret==SQLITE_OK && sender_hex && *sender_hex)
-    ret = sqlite3_bind_text(statement, 3, sender_sid, -1, SQLITE_STATIC);
+    ret = sqlite3_bind_text(statement, 3, sender_hex, -1, SQLITE_STATIC);
-  if (ret==SQLITE_OK && recipient_sid && *recipient_sid)
+  if (ret==SQLITE_OK && recipient_hex && *recipient_hex)
-    ret = sqlite3_bind_text(statement, 4, recipient_sid, -1, SQLITE_STATIC);
+    ret = sqlite3_bind_text(statement, 4, recipient_hex, -1, SQLITE_STATIC);
   
   if (ret!=SQLITE_OK){
     ret = WHYF("Failed to bind parameters: %s", sqlite3_errmsg(rhizome_db));
@@ -1514,70 +1514,57 @@ int rhizome_list_manifests(struct cli_context *context, const char *service, con
     if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
       WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
     } else {
-      int64_t blob_version = rhizome_manifest_get_ll(m, "version");
+
-      if (blob_version != q_version)
+      if (m->version != q_version)
 	WARNF("MANIFESTS row id=%s version=%"PRId64" does not match manifest blob.version=%"PRId64, 
-	  q_manifestid, q_version, blob_version);
+	  q_manifestid, q_version, m->version);
       int match = 1;
       
-      const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
+      if (service && service[0] && !(m->service && strcasecmp(m->service, service) == 0))
-      if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0))
 	match = 0;
-      const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0);
+      if (match && sender_hex && sender_hex[0]) {
-      const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
+	if (!(m->has_sender && cmp_sid_t(&sender, &m->sender) == 0))
-      if (match && sender_sid[0]) {
-	if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0))
 	  match = 0;
       }
-      if (match && recipient_sid[0]) {
+      if (match && recipient_hex && recipient_hex[0]) {
-	if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0))
+	if (!(m->has_recipient && cmp_sid_t(&recipient, &m->recipient) == 0))
 	  match = 0;
       }
       
       if (match) {
-	const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
-	int64_t blob_date = rhizome_manifest_get_ll(m, "date");
-	const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
 	int from_here = 0;
-	sid_t senderSid;
-	sid_t recipientSid;
-	
-	if (blob_sender)
-	  str_to_sid_t(&senderSid, blob_sender);
-	if (blob_recipient)
-	  str_to_sid_t(&recipientSid, blob_recipient);
 	
 	if (q_author) {
-	  if (config.debug.rhizome) DEBUGF("q_author=%s", alloca_str_toprint(q_author));
+	  sid_t author;
-	  str_to_sid_t(&m->author, q_author);
+	  if (str_to_sid_t(&author, q_author) == -1) {
-	  int cn = 0, in = 0, kp = 0;
+	    WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_manifestid, alloca_str_toprint(q_author));
-	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, &m->author);
+	  } else {
+	    rhizome_manifest_set_author(m, &author);
+	    int cn = 0, in = 0, kp = 0;
+	    from_here = keyring_find_sid(keyring, &cn, &in, &kp, &m->author);
+	  }
 	}
-	if (!from_here && blob_sender) {
+	if (!from_here && m->has_sender) {
-	  if (config.debug.rhizome) DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender));
+	  if (config.debug.rhizome)
+	    DEBUGF("blob.sender=%s", alloca_tohex_sid_t(m->sender));
 	  int cn = 0, in = 0, kp = 0;
-	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, &senderSid);
+	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, &m->sender);
 	}
 	
 	cli_put_long(context, rowid, ":");
-	cli_put_string(context, blob_service, ":");
+	cli_put_string(context, m->service, ":");
 	cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
-	cli_put_long(context, blob_version, ":");
+	cli_put_long(context, m->version, ":");
-	cli_put_long(context, blob_date, ":");
+	cli_put_long(context, m->has_date ? m->date : 0, ":");
 	cli_put_long(context, q_inserttime, ":");
-	cli_put_hexvalue(context, q_author ? m->author.binary : NULL, sizeof m->author.binary, ":");
+	cli_put_hexvalue(context, m->has_author ? m->author.binary : NULL, sizeof m->author.binary, ":");
 	cli_put_long(context, from_here, ":");
-	cli_put_long(context, m->fileLength, ":");
+	assert(m->filesize != RHIZOME_SIZE_UNSET);
-	
+	cli_put_long(context, m->filesize, ":");
-	unsigned char filehash[SHA512_DIGEST_LENGTH];
+	cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":");
-	if (m->fileLength)
+	cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":");
-	  fromhex(filehash, blob_filehash, SHA512_DIGEST_LENGTH);
+	cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":");
-	
+	cli_put_string(context, m->name, "\n");
-	cli_put_hexvalue(context, m->fileLength?filehash:NULL, SHA512_DIGEST_LENGTH, ":");
-	
-	cli_put_hexvalue(context, blob_sender ? senderSid.binary : NULL, sizeof senderSid.binary, ":");
-	cli_put_hexvalue(context, blob_recipient ? recipientSid.binary : NULL, sizeof recipientSid.binary, ":");
-	cli_put_string(context, blob_name, "\n");
       }
     }
     if (m) rhizome_manifest_free(m);
@@ -1631,56 +1618,37 @@ int rhizome_update_file_priority(const char *fileid)
  */
 int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
 {
-  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
+  if (m->service == NULL)
-  if (service == NULL)
     return WHY("Manifest has no service");
-
-  const char *name = rhizome_manifest_get(m, "name", NULL, 0);
-
-  sid_t *sender = NULL;
-  const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
-  if (sender_field) {
-    sender = (sid_t *) alloca(sizeof *sender);
-    if (str_to_sid_t(sender, sender_field) == -1)
-      return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
-  }
-
-  sid_t *recipient = NULL;
-  const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
-  if (recipient_field) {
-    recipient = (sid_t *) alloca(sizeof *recipient);
-    if (str_to_sid_t(recipient, recipient_field) == -1)
-      return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
-  }
-
   char sqlcmd[1024];
   strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
   strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
-  if (m->fileLength != 0)
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize > 0)
     strbuf_puts(b, " AND filehash = ?");
-  if (name)
+  if (m->name)
     strbuf_puts(b, " AND name = ?");
-  if (sender)
+  if (m->has_sender)
     strbuf_puts(b, " AND sender = ?");
-  if (recipient)
+  if (m->has_recipient)
     strbuf_puts(b, " AND recipient = ?");
   if (strbuf_overrun(b))
     return WHYF("SQL command too long: %s", strbuf_str(b));
 
   int ret = 0;
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END);
+  sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END);
   if (!statement)
     return -1;
   int field = 2;
-  if (m->fileLength != 0)
+  if (m->filesize > 0)
     sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END);
-  if (name)
+  if (m->name)
-    sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END);
+    sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, m->name, END);
-  if (sender)
+  if (m->has_sender)
-    sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END);
+    sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->sender, END);
-  if (recipient)
+  if (m->has_recipient)
-    sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END);
+    sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->recipient, END);
 
   int rows = 0;
   while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
@@ -1705,9 +1673,11 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
     }
     const char *q_author = (const char *) sqlite3_column_text(statement, 2);
     if (q_author) {
-      if (config.debug.rhizome)
+      sid_t author;
-	strbuf_sprintf(b, " .author=%s", q_author);
+      if (str_to_sid_t(&author, q_author) == -1)
-      str_to_sid_t(&blob_m->author, q_author);
+	WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_manifestid, alloca_str_toprint(q_author));
+      else
+	rhizome_manifest_set_author(blob_m, &author);
     }
     // check that we can re-author this manifest
     if (rhizome_extract_privatekey(blob_m, NULL)){
@@ -1737,8 +1707,11 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
   if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1)
     return WHYF("Manifest %s exists but is invalid", q_id);
   if (q_author) {
-    if (str_to_sid_t(&m->author, q_author) == -1)
+    sid_t author;
-      WARNF("manifest id=%s contains invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author));
+    if (str_to_sid_t(&author, q_author) == -1)
+      WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author));
+    else
+      rhizome_manifest_set_author(m, &author);
   }
   if (m->version != q_version)
     WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version);
@@ -1945,10 +1918,5 @@ int rhizome_is_bar_interesting(unsigned char *bar)
 
 int rhizome_is_manifest_interesting(rhizome_manifest *m)
 {
-  char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
+  return is_interesting(alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
-  if (!rhizome_manifest_get(m, "id", id, sizeof id))
-    // dodgy manifest, we don't want to receive it
-    return WHY("Ignoring bad manifest (no ID field)");
-  str_toupper_inplace(id);
-  return is_interesting(id, m->version);
 }

rhizome_direct_http.c

@@ -222,8 +222,8 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
       return 0;
     }
     // If manifest template did not specify a service field, then by default it is "file".
-    if (rhizome_manifest_get(m, "service", NULL, 0) == NULL)
+    if (m->service == NULL)
-      rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
+      rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
     sid_t *author = NULL;
     if (!is_sid_t_any(config.rhizome.api.addfile.default_author))
       author = &config.rhizome.api.addfile.default_author;
@@ -234,11 +234,11 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
       http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest");
       return 0;
     }
-    m->payloadEncryption=0;
+    rhizome_manifest_set_crypt(m, PAYLOAD_CLEAR);
-    rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
     // import file contents
     // TODO, stream file into database
-    if (m->fileLength) {
+    assert(m->filesize != RHIZOME_SIZE_UNSET);
+    if (m->filesize > 0) {
       if (rhizome_add_file(m, payload_path)) {
 	rhizome_manifest_free(m);
 	rhizome_direct_clear_temporary_files(r);
@@ -637,12 +637,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	/* Get filehash and size from manifest if present */
 	if (config.debug.rhizome_tx) {
 	  DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
-	  DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
+	  DEBUGF("bundle filehash = %s", alloca_tohex_rhizome_filehash_t(m->filehash));
-	  DEBUGF("file size = %"PRId64, m->fileLength);
+	  DEBUGF("file size = %"PRId64, m->filesize);
+	  DEBUGF("version = %"PRId64, m->version);
 	}
-	int64_t version = rhizome_manifest_get_ll(m, "version");
-	if (config.debug.rhizome_tx)
-	  DEBUGF("version = %"PRId64,version);
 
 	/* We now have everything we need to compose the POST request and send it.
 	 */
@@ -661,15 +659,16 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  "\r\n";
 	/* Work out what the content length should be */
 	if (config.debug.rhizome_tx)
-	  DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes);
+	  DEBUGF("manifest_all_bytes=%u, manifest_bytes=%u", m->manifest_all_bytes, m->manifest_bytes);
-	int content_length
+	assert(m->filesize != RHIZOME_SIZE_UNSET);
-	  =strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */
+	size_t content_length =
-	  +strlen(boundary)
+	    strlen(template2) - 2 /* minus 2 for the "%s" that gets replaced */
-	  +m->manifest_all_bytes
+	  + strlen(boundary)
-	  +strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */
+	  + m->manifest_all_bytes
-	  +strlen(boundary)
+	  + strlen(template3) - 2 /* minus 2 for the "%s" that gets replaced */
-	  +m->fileLength
+	  + strlen(boundary)
-	  +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n");
+	  + m->filesize
+	  + strlen("\r\n--") + strlen(boundary) + strlen("--\r\n");
 
 	int len=snprintf(buffer,8192,template,content_length,boundary);
 	len+=snprintf(&buffer[len],8192-len,template2,boundary);
@@ -705,7 +704,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	/* send file contents */
 	{
 	  rhizome_filehash_t filehash;
-	  if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1)
+	  if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, m->version, &filehash) == -1)
 	    goto closeit;
 
 	  struct rhizome_read read;
@@ -713,28 +712,26 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  if (rhizome_open_read(&read, &filehash))
 	    goto closeit;
 
-	  int64_t read_ofs;
+	  uint64_t read_ofs;
-	  for(read_ofs=0;read_ofs<m->fileLength;){
+	  for(read_ofs=0;read_ofs<m->filesize;){
 	    unsigned char buffer[4096];
 	    read.offset=read_ofs;
-	    int bytes_read = rhizome_read(&read, buffer, sizeof buffer);
+	    ssize_t bytes_read = rhizome_read(&read, buffer, sizeof buffer);
-	    if (bytes_read<0){
+	    if (bytes_read == -1) {
 	      rhizome_read_close(&read);
 	      goto closeit;
 	    }
-
+	    size_t write_ofs = 0;
-	    int write_ofs=0;
+	    while (write_ofs < (size_t) bytes_read){
-	    while(write_ofs < bytes_read){
+	      ssize_t written = write(sock, buffer + write_ofs, (size_t) bytes_read - write_ofs);
-	      int written = write(sock, buffer + write_ofs, bytes_read - write_ofs);
+	      if (written == -1){
-	      if (written<0){
 		WHY_perror("write");
 		rhizome_read_close(&read);
 		goto closeit;
 	      }
-	      write_ofs+=written;
+	      write_ofs += (size_t) written;
 	    }
-
+	    read_ofs += (size_t) bytes_read;
-	    read_ofs+=bytes_read;
 	  }
 	  rhizome_read_close(&read);
 	}

rhizome_fetch.c

@@ -73,7 +73,7 @@ struct rhizome_fetch_slot {
 
   /* HTTP streaming reception of manifests */
   char manifest_buffer[1024];
-  int manifest_bytes;
+  unsigned manifest_bytes;
 
   /* MDP transport specific elements */
   rhizome_bid_t bid;
@@ -100,7 +100,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot);
  */
 struct rhizome_fetch_queue {
   struct rhizome_fetch_slot active; // must be first element in struct
-  int candidate_queue_size;
+  unsigned candidate_queue_size;
   struct rhizome_fetch_candidate *candidate_queue;
   unsigned char log_size_threshold; // will only queue payloads smaller than this.
 };
@@ -159,23 +159,28 @@ int rhizome_active_fetch_count()
   return active;
 }
 
-int rhizome_active_fetch_bytes_received(int q)
+uint64_t rhizome_active_fetch_bytes_received(int q)
 {
   if (q<0 || q>=NQUEUES) return -1;
   if (rhizome_fetch_queues[q].active.state==RHIZOME_FETCH_FREE) return -1;
-  return (int)rhizome_fetch_queues[q].active.write_state.file_offset;
+  return rhizome_fetch_queues[q].active.write_state.file_offset;
 }
 
-int rhizome_fetch_queue_bytes(){
+uint64_t rhizome_fetch_queue_bytes()
-  int i,j,bytes=0;
+{
+  uint64_t bytes = 0;
+  unsigned i;
   for(i=0;i<NQUEUES;i++){
     if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE){
-      int received=rhizome_fetch_queues[i].active.write_state.file_offset;
+      assert(rhizome_fetch_queues[i].active.manifest->filesize != RHIZOME_SIZE_UNSET);
-      bytes+=rhizome_fetch_queues[i].active.manifest->fileLength - received;
+      bytes += rhizome_fetch_queues[i].active.manifest->filesize - rhizome_fetch_queues[i].active.write_state.file_offset;
     }
+    unsigned j;
     for (j=0;j<rhizome_fetch_queues[i].candidate_queue_size;j++){
-      if (rhizome_fetch_queues[i].candidate_queue[j].manifest)
+      if (rhizome_fetch_queues[i].candidate_queue[j].manifest) {
-        bytes+=rhizome_fetch_queues[i].candidate_queue[j].manifest->fileLength;
+        assert(rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET);
+        bytes += rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize;
+      }
     }
   }
   return bytes;
@@ -183,20 +188,21 @@ int rhizome_fetch_queue_bytes(){
 
 int rhizome_fetch_status_html(strbuf b)
 {
-  int i,j;
+  unsigned i;
   for(i=0;i<NQUEUES;i++){
     struct rhizome_fetch_queue *q=&rhizome_fetch_queues[i];
-    int used=0;
+    unsigned used=0;
+    unsigned j;
     for (j=0;j<q->candidate_queue_size;j++){
       if (q->candidate_queue[j].manifest)
 	used++;
     }
-    strbuf_sprintf(b, "<p>Slot %d, (%d of %d): ", i, used, q->candidate_queue_size);
+    strbuf_sprintf(b, "<p>Slot %u, (%u of %u): ", i, used, q->candidate_queue_size);
     if (q->active.state!=RHIZOME_FETCH_FREE){
-      strbuf_sprintf(b, "%s %"PRId64" of %"PRId64" from %s*",
+      strbuf_sprintf(b, "%s %"PRIu64" of %"PRIu64" from %s*",
 	fetch_state(q->active.state),
 	q->active.write_state.file_offset,
-	q->active.manifest->fileLength,
+	q->active.manifest->filesize,
 	alloca_tohex_sid_t_trunc(q->active.peer_sid, 16));
     }else{
       strbuf_puts(b, "inactive");
@@ -206,7 +212,8 @@ int rhizome_fetch_status_html(strbuf b)
     for (j=0; j< q->candidate_queue_size;j++){
       if (q->candidate_queue[j].manifest){
 	candidates++;
-	candidate_size += q->candidate_queue[j].manifest->fileLength;
+	assert(q->candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET);
+	candidate_size += q->candidate_queue[j].manifest->filesize;
       }
     }
     if (candidates)
@@ -272,9 +279,10 @@ static struct rhizome_fetch_slot *fetch_search_slot(const unsigned char *id, int
 // find the first matching candidate for this bundle
 static struct rhizome_fetch_candidate *fetch_search_candidate(const unsigned char *id, int prefix_length)
 {
-  int i, j;
+  unsigned i;
   for (i = 0; i < NQUEUES; ++i) {
     struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
+    unsigned j;
     for (j = 0; j < q->candidate_queue_size; j++) {
       struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
       if (!c->manifest)
@@ -304,14 +312,14 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, int i)
+static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, unsigned i)
 {
   struct rhizome_fetch_candidate * const c = &q->candidate_queue[i];
   struct rhizome_fetch_candidate * e = &q->candidate_queue[q->candidate_queue_size - 1];
   if (config.debug.rhizome_rx)
-    DEBUGF("insert queue[%d] candidate[%d]", (int)(q - rhizome_fetch_queues), i);
+    DEBUGF("insert queue[%d] candidate[%u]", (int)(q - rhizome_fetch_queues), i);
   assert(i >= 0 && i < q->candidate_queue_size);
-  assert(i == 0 || c[-1].manifest);
+  assert(i == 0 || c[i-1].manifest);
   if (e->manifest) // queue is full
     rhizome_manifest_free(e->manifest);
   else
@@ -330,7 +338,7 @@ static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i)
+static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, unsigned i)
 {
   assert(i >= 0 && i < q->candidate_queue_size);
   struct rhizome_fetch_candidate *c = &q->candidate_queue[i];
@@ -348,10 +356,10 @@ static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i)
 
 static void candidate_unqueue(struct rhizome_fetch_candidate *c)
 {
-  int i, index;
+  unsigned i;
   for (i = 0; i < NQUEUES; ++i) {
     struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
-    index = c - q->candidate_queue;
+    unsigned index = c - q->candidate_queue;
     if (index>=0 && index < q->candidate_queue_size){
       rhizome_fetch_unqueue(q, index);
       return;
@@ -459,8 +467,8 @@ static int rhizome_import_received_bundle(struct rhizome_manifest *m)
   m->finalised = 1;
   m->manifest_bytes = m->manifest_all_bytes; // store the signatures too
   if (config.debug.rhizome_rx) {
-    DEBUGF("manifest len=%d has %d signatories. Associated file = %"PRId64" bytes", 
+    DEBUGF("manifest len=%u has %u signatories. Associated filesize=%"PRIu64" bytes", 
-	   m->manifest_bytes, m->sig_count, m->fileLength);
+	   m->manifest_bytes, m->sig_count, m->filesize);
     dump("manifest", m->manifestdata, m->manifest_all_bytes);
   }
   return rhizome_add_manifest(m, m->ttl - 1 /* TTL */);
@@ -489,24 +497,27 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
     strbuf r = strbuf_local(slot->request, sizeof slot->request);
     strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
     
-    if (slot->manifest->journalTail>=0){
+    if (slot->manifest->is_journal){
       // if we're fetching a journal bundle, work out how many bytes we have of a previous version
       // and therefore what range of bytes we should ask for
       slot->previous = rhizome_new_manifest();
       if (rhizome_retrieve_manifest(&slot->manifest->cryptoSignPublic, slot->previous)){
 	rhizome_manifest_free(slot->previous);
 	slot->previous=NULL;
-	
+      // check that the new journal is valid and has some overlapping bytes
-	// check that the new journal is valid and has some overlapping bytes
+      }else if (   !slot->previous->is_journal
-      }else if (slot->previous->journalTail > slot->manifest->journalTail
+		|| slot->previous->tail > slot->manifest->tail
-	  || slot->previous->fileLength + slot->previous->journalTail < slot->manifest->journalTail){
+		|| slot->previous->filesize + slot->previous->tail < slot->manifest->tail
+      ){
 	rhizome_manifest_free(slot->previous);
 	slot->previous=NULL;
       }else{
-	assert(slot->previous->fileLength >= slot->manifest->journalTail);
+	assert(slot->previous->filesize >= slot->manifest->tail);
-	assert(slot->manifest->fileLength > 0);
+	assert(slot->manifest->filesize > 0);
-	strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n",
+	strbuf_sprintf(r, "Range: bytes=%"PRIu64"-%"PRIu64"\r\n",
-	  slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1);
+	    slot->previous->filesize - slot->manifest->tail,
+	    slot->manifest->filesize - 1
+	  );
       }
     }
 
@@ -516,7 +527,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
       RETURN(WHY("request overrun"));
     slot->request_len = strbuf_len(r);
 
-    if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT))
+    if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->filesize, RHIZOME_PRIORITY_DEFAULT))
       RETURN(-1);
   } else {
     strbuf r = strbuf_local(slot->request, sizeof slot->request);
@@ -526,8 +537,8 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
     slot->request_len = strbuf_len(r);
 
     slot->manifest_bytes=0;
-    slot->write_state.file_offset=0;
+    slot->write_state.file_offset = 0;
-    slot->write_state.file_length=-1;
+    slot->write_state.file_length = RHIZOME_SIZE_UNSET;
   }
 
   slot->request_ofs = 0;
@@ -647,16 +658,16 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
   */
 
   if (config.debug.rhizome_rx)
-    DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRId64" peerip=%s",
+    DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRIu64" peerip=%s",
 	   slotno(slot),
 	   alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
 	   m->version,
-	   m->fileLength,
+	   m->filesize,
 	   alloca_sockaddr(peerip, sizeof(struct sockaddr_in))
 	   );
 
   // If the payload is empty, no need to fetch, so import now.
-  if (m->fileLength == 0) {
+  if (m->filesize == 0) {
     if (config.debug.rhizome_rx)
       DEBUGF("   manifest fetch not started -- nil payload, so importing instead");
     if (rhizome_import_received_bundle(m) == -1)
@@ -769,7 +780,7 @@ static void rhizome_start_next_queued_fetch(struct rhizome_fetch_slot *slot)
   IN();
   struct rhizome_fetch_queue *q;
   for (q = (struct rhizome_fetch_queue *) slot; q >= rhizome_fetch_queues; --q) {
-    int i = 0;
+    unsigned i = 0;
     struct rhizome_fetch_candidate *c;
     while (i < q->candidate_queue_size && (c = &q->candidate_queue[i])->manifest) {
       int result = rhizome_fetch(slot, c->manifest, &c->peer_ipandport, &c->peer_sid);
@@ -820,7 +831,7 @@ int rhizome_fetch_has_queue_space(unsigned char log2_size){
     struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
     if (log2_size < q->log_size_threshold){
       // is there an empty candidate?
-      int j=0;
+      unsigned j;
       for (j=0;j < q->candidate_queue_size;j++)
 	if (!q->candidate_queue[j].manifest)
 	  return 1;
@@ -859,8 +870,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
   int priority=100; /* normal priority */
 
   if (config.debug.rhizome_rx)
-    DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:",
+    DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRIu64" priority=%d:",
-	alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, priority);
+	alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->filesize, priority);
 
   if (!rhizome_is_manifest_interesting(m)) {
     if (config.debug.rhizome_rx)
@@ -875,7 +886,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
       DEBUGF("   is new (have version %"PRId64")", stored_version);
   }
 
-  if (m->fileLength == 0) {
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize == 0) {
     if (rhizome_manifest_verify(m) != 0) {
       WHY("Error verifying manifest when considering for import");
       /* Don't waste time looking at this manifest again for a while */
@@ -889,9 +901,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
   }
 
   // Find the proper queue for the payload.  If there is none suitable, it is an error.
-  struct rhizome_fetch_queue *qi = rhizome_find_queue(m->fileLength);
+  struct rhizome_fetch_queue *qi = rhizome_find_queue(m->filesize);
   if (!qi) {
-    WHYF("No suitable fetch queue for bundle size=%"PRId64, m->fileLength);
+    WHYF("No suitable fetch queue for bundle size=%"PRIu64, m->filesize);
     rhizome_manifest_free(m);
     RETURN(-1);
   }
@@ -900,9 +912,10 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
   // may have changed between versions.) If a newer or the same version is already queued, then
   // ignore this one.  Otherwise, unqueue all older candidates.
   int ci = -1;
-  int i, j;
+  unsigned i;
   for (i = 0; i < NQUEUES; ++i) {
     struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
+    unsigned j;
     for (j = 0; j < q->candidate_queue_size; ) {
       struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
       if (c->manifest) {
@@ -953,18 +966,18 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
 
   if (config.debug.rhizome_rx) {
     DEBUG("Rhizome fetch queues:");
-    int i, j;
+    unsigned i, j;
     for (i = 0; i < NQUEUES; ++i) {
       struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
       for (j = 0; j < q->candidate_queue_size; ++j) {
 	struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
 	if (!c->manifest)
 	  break;
-	DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRId64, i, j,
+	DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRIu64, i, j,
 	    c->manifest,
 	    alloca_tohex_rhizome_bid_t(c->manifest->cryptoSignPublic),
 	    c->priority,
-	    c->manifest->fileLength
+	    c->manifest->filesize
 	  );
       }
     }
@@ -1026,16 +1039,18 @@ static void rhizome_fetch_mdp_slot_callback(struct sched_ent *alarm)
 
   time_ms_t now = gettime_ms();
   if (now-slot->last_write_time>slot->mdpIdleTimeout) {
-    DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRId64" of %"PRId64" bytes)",
+    DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRIu64" of %"PRIu64" bytes)",
 	   now-slot->last_write_time,
-	   slot->write_state.file_offset, slot->write_state.file_length);
+	   slot->write_state.file_offset,
+	   slot->write_state.file_length);
     rhizome_fetch_close(slot);
     OUT();
     return;
   }
   if (config.debug.rhizome_rx)
-    DEBUGF("Timeout: Resending request for slot=0x%p (%"PRId64" of %"PRId64" received)",
+    DEBUGF("Timeout: Resending request for slot=0x%p (%"PRIu64" of %"PRIu64" received)",
-	   slot,slot->write_state.file_offset, slot->write_state.file_length);
+	   slot, slot->write_state.file_offset,
+	   slot->write_state.file_length);
   rhizome_fetch_mdp_requestblocks(slot);
   OUT();
 }
@@ -1133,11 +1148,14 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){
    *        [               | written | overlap |  new content]
    */
   
-  uint64_t start = slot->manifest->journalTail - slot->previous->journalTail + slot->write_state.file_offset;
+  assert(slot->manifest->tail != RHIZOME_SIZE_UNSET);
-  uint64_t length = slot->previous->fileLength - slot->manifest->journalTail - slot->write_state.file_offset;
+  assert(slot->previous->tail != RHIZOME_SIZE_UNSET);
+  assert(slot->previous->filesize != RHIZOME_SIZE_UNSET);
+  uint64_t start = slot->manifest->tail - slot->previous->tail + slot->write_state.file_offset;
+  uint64_t length = slot->previous->filesize - slot->manifest->tail - slot->write_state.file_offset;
   
   // of course there might not be any overlap
-  if (start>=0 && start < slot->previous->fileLength && length>0){
+  if (start>=0 && start < slot->previous->filesize && length>0){
     if (config.debug.rhizome)
       DEBUGF("Copying %"PRId64" bytes from previous journal", length);
     rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length);
@@ -1170,8 +1188,8 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
   }
 
   if (config.debug.rhizome_rx)
-    DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRId64" bytes)",
+    DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRIu64" bytes)",
-	   slot,slot->write_state.file_length);
+	   slot, slot->write_state.file_length);
   
   /* close socket and stop watching it */
   if (slot->alarm.poll.fd>=0) {
@@ -1308,17 +1326,18 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
     time_ms_t interval = now - slot->start_time;
     if (interval <= 0)
       interval = 1;
-    DEBUGF("Closing rhizome fetch slot = 0x%p.  Received %"PRId64" bytes in %"PRId64"ms (%"PRId64"KB/sec).",
+    DEBUGF("Closing rhizome fetch slot = 0x%p.  Received %"PRIu64" bytes in %"PRIu64"ms (%"PRIu64"KB/sec).",
            slot, slot->write_state.file_offset,
-           (int64_t)interval,
+           (uint64_t)interval,
-           (int64_t)((slot->write_state.file_offset) / interval));
+           slot->write_state.file_offset / (uint64_t)interval
+	  );
   }
 
   rhizome_fetch_close(slot);
   RETURN(-1);
 }
 
-int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, int bytes)
+int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, size_t bytes)
 {
   IN();
   
@@ -1327,17 +1346,18 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer
   
   // Truncate to known length of file (handy for reading from journal bundles that
   // might grow while we are reading from them).
-  if (bytes>(slot->write_state.file_length-(slot->write_state.file_offset))){
+  if (bytes > slot->write_state.file_length - slot->write_state.file_offset) {
-    bytes=slot->write_state.file_length-(slot->write_state.file_offset);
+    bytes = slot->write_state.file_length - slot->write_state.file_offset;
   }
 
   if (!slot->manifest){
     /* We are reading a manifest.  Read it into a buffer. */
-    int count=bytes;
+    unsigned count = bytes;
-    if (count+slot->manifest_bytes>1024) count=1024-slot->manifest_bytes;
+    if (count + slot->manifest_bytes > 1024)
+      count = 1024 - slot->manifest_bytes;
     bcopy(buffer,&slot->manifest_buffer[slot->manifest_bytes],count);
     slot->manifest_bytes+=count;
-    slot->write_state.file_offset+=count;
+    slot->write_state.file_offset += count;
   } else {
     
     /* We are reading a file. Stream it into the database. */
@@ -1358,7 +1378,7 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer
 
 int rhizome_received_content(const unsigned char *bidprefix,
 			     uint64_t version, uint64_t offset,
-			     int count,unsigned char *bytes,int type)
+			     size_t count, unsigned char *bytes, int type)
 {
   IN();
   if (!is_rhizome_mdp_enabled()) {
@@ -1368,7 +1388,7 @@ int rhizome_received_content(const unsigned char *bidprefix,
   
   if (slot && slot->bidVersion == version && slot->state == RHIZOME_FETCH_RXFILEMDP){
     if (config.debug.rhizome)
-      DEBUGF("Rhizome over MDP receiving %d bytes.",count);
+      DEBUGF("Rhizome over MDP receiving %zu bytes.", count);
     if (rhizome_random_write(&slot->write_state, offset, bytes, count)){
       if (config.debug.rhizome)
 	DEBUGF("Write failed!");
@@ -1398,18 +1418,18 @@ int rhizome_received_content(const unsigned char *bidprefix,
     rhizome_manifest *m = NULL;
     struct rhizome_fetch_candidate *c = NULL;
     
-    if (slot && slot->bidVersion == version && slot->manifest->fileLength==count 
+    if (slot && slot->bidVersion == version && slot->manifest->filesize == count 
-	  && slot->state!=RHIZOME_FETCH_RXFILEMDP){
+	  && slot->state != RHIZOME_FETCH_RXFILEMDP) {
       m=slot->manifest;
     }else{
       slot = NULL;
       c = fetch_search_candidate(bidprefix, 16);
-      if (c && c->manifest->version==version && c->manifest->fileLength==count)
+      if (c && c->manifest->version == version && c->manifest->filesize == count)
 	m=c->manifest;
     }
     
     if (m){
-      if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){
+      if (rhizome_import_buffer(m, bytes, count) >= 0 && !rhizome_import_received_bundle(m)){
 	INFOF("Completed MDP transfer in one hit for file %s",
 	    alloca_tohex_rhizome_filehash_t(m->filehash));
 	if (c)
@@ -1459,8 +1479,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
       } else {
 	if (errno!=EAGAIN) {
 	  if (config.debug.rhizome_rx)
-	    DEBUGF("Empty read, closing connection: received %"PRId64" of %"PRId64" bytes",
+	    DEBUGF("Empty read, closing connection: received %"PRIu64" of %"PRIu64" bytes",
-		   slot->write_state.file_offset,slot->write_state.file_length);
+		   slot->write_state.file_offset,
+		   slot->write_state.file_length);
 	  rhizome_fetch_switch_to_mdp(slot);
 	}
 	return;
@@ -1509,18 +1530,18 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
 	    rhizome_fetch_switch_to_mdp(slot);
 	    return;
 	  }
-	  if (slot->write_state.file_length==-1)
+	  if (slot->write_state.file_length == RHIZOME_SIZE_UNSET)
-	    slot->write_state.file_length=parts.content_length;
+	    slot->write_state.file_length = parts.content_length;
 	  else if (parts.content_length + parts.range_start != slot->write_state.file_length)
-	    WARNF("Expected content length %"PRId64", got %"PRId64" + %"PRId64, 
+	    WARNF("Expected content length %"PRIu64", got %"PRIu64" + %"PRIu64, 
 	      slot->write_state.file_length, parts.content_length, parts.range_start);
 	  /* We have all we need.  The file is already open, so just write out any initial bytes of
 	     the body we read.
 	  */
 	  slot->state = RHIZOME_FETCH_RXFILE;
 	  if (slot->previous && parts.range_start){
-	    if (parts.range_start != slot->previous->fileLength - slot->manifest->journalTail)
+	    if (parts.range_start != slot->previous->filesize - slot->manifest->tail)
-	      WARNF("Expected Content-Range header to start @%"PRId64, slot->previous->fileLength - slot->manifest->journalTail);
+	      WARNF("Expected Content-Range header to start @%"PRIu64, slot->previous->filesize - slot->manifest->tail);
 	    pipe_journal(slot);
 	  }
 	  

rhizome_packetformats.c

@@ -80,17 +80,18 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
 
   if (!m) { RETURN(WHY("null manifest passed in")); }
 
-  int i;
-
   /* Manifest prefix */
+  unsigned i;
   for(i=0;i<RHIZOME_BAR_PREFIX_BYTES;i++) 
     bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic.binary[i];
   /* file length */
-  bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->fileLength);
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->filesize);
   /* Version */
   for(i=0;i<7;i++) bar[RHIZOME_BAR_VERSION_OFFSET+6-i]=(m->version>>(8*i))&0xff;
 
-  /* geo bounding box */
+#if 0
+  /* geo bounding box TODO: replace with bounding circle!!! */
   double minLat=rhizome_manifest_get_double(m,"min_lat",-90);
   if (minLat<-90) minLat=-90; if (minLat>90) minLat=90;
   double minLong=rhizome_manifest_get_double(m,"min_long",-180);
@@ -99,6 +100,12 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
   if (maxLat<-90) maxLat=-90; if (maxLat>90) maxLat=90;
   double maxLong=rhizome_manifest_get_double(m,"max_long",+180);
   if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180;  
+#else
+  double minLat = -90;
+  double minLong = -180;
+  double maxLat = +90;
+  double maxLong = +180;
+#endif
   unsigned short v;
   int o=RHIZOME_BAR_GEOBOX_OFFSET;
   v=(minLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
@@ -338,8 +345,7 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
       /* trim manifest ID to a prefix for ease of debugging
 	 (that is the only use of this */
       if (config.debug.rhizome_ads){
-	int64_t version = rhizome_manifest_get_ll(m, "version");
+	DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
-	DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), version);
       }
 
       /* Crude signature presence test */

rhizome_store.c

@@ -14,7 +14,7 @@ int rhizome_exists(const rhizome_filehash_t *hashp)
   return gotfile;
 }
 
-int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority)
+int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority)
 {
   write->blob_fd=-1;
   
@@ -136,13 +136,14 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
  * */
 
 // encrypt and hash data, data buffers must be passed in file order.
-static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, int data_size){
+static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size)
-  if (data_size<=0)
+{
+  if (data_size <= 0)
     return WHY("No content supplied");
     
   /* Make sure we aren't being asked to write more data than we expected */
   if (write_state->file_offset + data_size > write_state->file_length)
-    return WHYF("Too much content supplied, %"PRId64" + %d > %"PRId64,
+    return WHYF("Too much content supplied, %"PRIu64" + %zu > %"PRIu64,
 		write_state->file_offset, data_size, write_state->file_length);
 
   if (write_state->crypt){
@@ -157,7 +158,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer
   write_state->file_offset+=data_size;
   
   if (config.debug.rhizome)
-    DEBUGF("Processed %"PRId64" of %"PRId64, write_state->file_offset, write_state->file_length);
+    DEBUGF("Processed %"PRIu64" of %"PRIu64, write_state->file_offset, write_state->file_length);
   return 0;
 }
 
@@ -189,6 +190,11 @@ static int write_get_lock(struct rhizome_write *write_state){
 // write data to disk
 static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size)
 {
+  if (config.debug.rhizome) {
+    DEBUGF("write_state->file_length=%"PRIu64" file_offset=%"PRIu64, write_state->file_length, file_offset);
+    //dump("buffer", buffer, data_size);
+  }
+
   if (data_size<=0)
     return 0;
   
@@ -198,7 +204,8 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
   if (write_state->blob_fd != -1) {
     int ofs=0;
     // keep trying until all of the data is written.
-    lseek(write_state->blob_fd, file_offset, SEEK_SET);
+    if (lseek64(write_state->blob_fd, (off64_t) file_offset, SEEK_SET) == -1)
+      return WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", write_state->blob_fd, file_offset);
     while(ofs < data_size){
       int r=write(write_state->blob_fd, buffer + ofs, data_size - ofs);
       if (r<0)
@@ -225,10 +232,10 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
     }
   }
   
-  write_state->written_offset=file_offset + data_size;
+  write_state->written_offset = file_offset + data_size;
   
   if (config.debug.rhizome)
-    DEBUGF("Wrote %"PRId64" of %"PRId64, file_offset + data_size, write_state->file_length);
+    DEBUGF("Wrote %"PRIu64" of %"PRIu64, file_offset + data_size, write_state->file_length);
   return 0;
 }
 
@@ -254,8 +261,12 @@ static int write_release_lock(struct rhizome_write *write_state){
 
 // Write data buffers in any order, the data will be cached and streamed into the database in file order. 
 // Though there is an upper bound on the amount of cached data
-int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size)
+int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size)
 {
+  if (config.debug.rhizome) {
+    DEBUGF("write_state->file_length=%"PRIu64" offset=%"PRIu64, write_state->file_length, offset);
+    //dump("buffer", buffer, data_size);
+  }
   if (offset + data_size > write_state->file_length)
     data_size = write_state->file_length - offset;
   
@@ -269,10 +280,10 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
     // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go.
     // (Not perfect if the range overlaps)
     uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
-    if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE)
+    if (new_size >= write_state->file_length || new_size >= RHIZOME_BUFFER_MAXIMUM_SIZE)
       should_write = 1;
   }
-  int64_t last_offset = write_state->written_offset;
+  uint64_t last_offset = write_state->written_offset;
   
   while(1){
     
@@ -396,12 +407,10 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){
   ret = write_get_lock(write);
   if (ret)
     goto end;
-  while(write->file_offset < write->file_length){
+  while(write->file_offset < write->file_length) {
-    
+    size_t size = sizeof buffer;
-    int size=sizeof(buffer);
     if (write->file_offset + size > write->file_length)
-      size=write->file_length - write->file_offset;
+      size = write->file_length - write->file_offset;
-    
     size_t r = fread(buffer, 1, size, f);
     if (ferror(f)){
       ret = WHY_perror("fread");
@@ -451,7 +460,7 @@ int rhizome_finish_write(struct rhizome_write *write)
   }
   
   if (write->file_offset < write->file_length){
-    WHYF("Only processed %"PRId64" bytes, expected %"PRId64, write->file_offset, write->file_length);
+    WHYF("Only processed %"PRIu64" bytes, expected %"PRIu64, write->file_offset, write->file_length);
   }
     
   int fd = write->blob_fd;
@@ -550,14 +559,15 @@ failure:
 // import a file for an existing bundle with a known file hash
 int rhizome_import_file(rhizome_manifest *m, const char *filepath)
 {
-  if (m->fileLength<=0)
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize == 0)
     return 0;
   
   /* Import the file first, checking the hash as we go */
   struct rhizome_write write;
   bzero(&write, sizeof(write));
   
-  int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
+  int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
   if (ret!=0)
     return ret;
   
@@ -578,16 +588,18 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
 // store a whole payload from a single buffer
 int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length)
 {
-  if (m->fileLength<=0)
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (m->filesize == 0)
     return 0;
-  if (length!=m->fileLength)
+
-    return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length);
+  if (length != m->filesize)
+    return WHYF("Expected %"PRIu64" bytes, got %zu", m->filesize, length);
   
   /* Import the file first, checking the hash as we go */
   struct rhizome_write write;
   bzero(&write, sizeof(write));
   
-  int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
+  int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
   if (ret!=0)
     return ret;
   
@@ -607,36 +619,27 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t len
 
 int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
 {
-  int64_t existing = rhizome_manifest_get_ll(m, "filesize");
+  uint64_t size = 0;
-  
-  m->fileLength = 0;
   if (filepath[0]) {
     struct stat stat;
-    if (lstat(filepath,&stat))
+    if (lstat(filepath, &stat))
-      return WHYF("Could not stat() payload file '%s'",filepath);
+      return WHYF_perror("lstat(%s)", alloca_str_toprint(filepath));
-    m->fileLength = stat.st_size;
+    size = stat.st_size;
   }
   
-  // fail if the file is shorter than specified by the manifest
+  // Fail if the file is shorter than already specified by the manifest.
-  if (existing > m->fileLength)
+  if (m->filesize != RHIZOME_SIZE_UNSET && size < m->filesize)
     return WHY("Manifest length is longer than the file");
   
-  // if the file is longer than specified by the manifest, ignore the end.
+  // If the file is longer than already specified by the manifest, ignore the end of the file.
-  if (existing!=-1 && existing < m->fileLength)
+  if (m->filesize == RHIZOME_SIZE_UNSET || size > m->filesize)
-    m->fileLength = existing;
+    rhizome_manifest_set_filesize(m, size);
-  
-  rhizome_manifest_set_ll(m, "filesize", m->fileLength);
-  
-  if (m->fileLength == 0){
-    m->filehash = RHIZOME_FILEHASH_NONE;
-    rhizome_manifest_del(m, "filehash");
-  }
   return 0;
 }
 
 static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_write *write)
 {
-  if (!m->payloadEncryption)
+  if (m->payloadEncryption != PAYLOAD_ENCRYPTED)
     return 0;
   
   // if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
@@ -647,8 +650,8 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru
     DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
 
   write->crypt=1;
-  if (m->journalTail>0)
+  if (m->is_journal && m->tail > 0)
-    write->tail = m->journalTail;
+    write->tail = m->tail;
 
   bcopy(m->payloadKey, write->key, sizeof(write->key));
   bcopy(m->payloadNonce, write->nonce, sizeof(write->nonce));
@@ -657,7 +660,8 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru
 
 int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m)
 {
-  if (rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT))
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  if (rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT))
     return -1;
 
   if (rhizome_write_derive_key(m, NULL, write))
@@ -672,18 +676,14 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
   // Stream the file directly into the database, encrypting & hashing as we go.
   struct rhizome_write write;
   bzero(&write, sizeof(write));
-
   if (rhizome_write_open_manifest(&write, m))
     goto failure;
   if (rhizome_write_file(&write, filepath))
     goto failure;
   if (rhizome_finish_write(&write))
     goto failure;
-
+  rhizome_manifest_set_filehash(m, &write.id);
-  m->filehash = write.id;
-  rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
   return 0;
-
 failure:
   rhizome_fail_write(&write);
   return -1;
@@ -704,7 +704,7 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
       " AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1)
     return -1;
   if (read->blob_rowid != -1) {
-    read->length = -1; // discover the length on opening the db BLOB
+    read->length = RHIZOME_SIZE_UNSET; // discover the length on opening the db BLOB
   } else {
     // No row in FILEBLOBS, look for an external blob file.
     char blob_path[1024];
@@ -716,8 +716,10 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
 	return 1; // file not available
       return WHYF_perror("open(%s)", alloca_str_toprint(blob_path));
     }
-    if ((read->length = lseek(read->blob_fd, 0, SEEK_END)) == -1)
+    off64_t pos = lseek64(read->blob_fd, 0, SEEK_END);
-      return WHYF_perror("lseek(%s,0,SEEK_END)", alloca_str_toprint(blob_path));
+    if (pos == -1)
+      return WHYF_perror("lseek64(%s,0,SEEK_END)", alloca_str_toprint(blob_path));
+    read->length = pos;
     if (config.debug.externalblobs)
       DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length);
   }
@@ -731,8 +733,8 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
 {
   IN();
   if (read_state->blob_fd != -1) {
-    if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1)
+    if (lseek64(read_state->blob_fd, (off64_t) read_state->offset, SEEK_SET) == -1)
-      RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset));
+      RETURN(WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", read_state->blob_fd, read_state->offset));
     if (bufsz == 0)
       RETURN(0);
     ssize_t rd = read(read_state->blob_fd, buffer, bufsz);
@@ -755,7 +757,7 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
     RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db)));
   }
   assert(blob != NULL);
-  if (read_state->length == -1)
+  if (read_state->length == RHIZOME_SIZE_UNSET)
     read_state->length = sqlite3_blob_bytes(blob);
   // A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out
   // the length.
@@ -813,6 +815,7 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz
   }
   
   if (read_state->crypt && buffer && bytes_read>0){
+    dump("before decrypt", buffer, bytes_read);
     if(rhizome_crypt_xor_block(
 	buffer, bytes_read, 
 	read_state->offset + read_state->tail, 
@@ -820,62 +823,76 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz
       RETURN(-1);
     }
   }
-  read_state->offset+=bytes_read;
+  read_state->offset += bytes_read;
+  if (config.debug.rhizome) {
+    DEBUGF("read %zu bytes, read_state->offset=%"PRIu64, bytes_read, read_state->offset);
+    //dump("buffer", buffer, bytes_read);
+  }
   RETURN(bytes_read);
   OUT();
 }
 
 /* Read len bytes from read->offset into data, using *buffer to cache any reads */
-int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
+ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
 {
   size_t bytes_copied=0;
   
   while (len>0){
+    DEBUGF("len=%zu read->length=%"PRIu64" read->offset=%"PRIu64" buffer->offset=%"PRIu64"", len, read->length, read->offset, buffer->offset);
     // make sure we only attempt to read data that actually exists
-    if (read->length !=-1 && read->offset + len > read->length)
+    if (read->length != RHIZOME_SIZE_UNSET && read->offset + len > read->length)
       len = read->length - read->offset;
 
     // if we can supply either the beginning or end of the data from cache, do that first.
-    uint64_t ofs=read->offset - buffer->offset;
+    if (read->offset >= buffer->offset) {
-    if (ofs>=0 && ofs<=buffer->len){
+      assert(read->offset - buffer->offset <= SIZE_MAX);
-      size_t size=len;
+      size_t ofs = read->offset - buffer->offset;
-      if (size > buffer->len - ofs)
+      if (ofs <= buffer->len){
-	size = buffer->len - ofs;
+	size_t size = len;
-      if (size>0){
+	if (size > buffer->len - ofs)
-	// copy into the start of the data buffer
+	  size = buffer->len - ofs;
-	bcopy(buffer->data + ofs, data, size);
+	if (size > 0){
-	data+=size;
+	  // copy into the start of the data buffer
-	len-=size;
+	  bcopy(buffer->data + ofs, data, size);
-	read->offset+=size;
+	  data+=size;
-	bytes_copied+=size;
+	  len-=size;
-	continue;
+	  read->offset+=size;
+	  bytes_copied+=size;
+	  DEBUGF("read->offset=%"PRIu64, read->offset);
+	  continue;
+	}
       }
     }
     
-    ofs = (read->offset+len) - buffer->offset;
+    if (read->offset + len > buffer->offset) {
-    if (ofs>0 && ofs<=buffer->len){
+      assert(read->offset + len - buffer->offset <= SIZE_MAX);
-      size_t size=len;
+      size_t ofs = read->offset + len - buffer->offset;
-      if (size > ofs)
+      if (ofs <= buffer->len){
-	size = ofs;
+	size_t size = len;
-      if (size>0){
+	if (size > ofs)
-	// copy into the end of the data buffer
+	  size = ofs;
-	bcopy(buffer->data + ofs - size, data + len - size, size);
+	if (size>0){
-	len-=size;
+	  // copy into the end of the data buffer
-	bytes_copied+=size;
+	  bcopy(buffer->data + ofs - size, data + len - size, size);
-	continue;
+	  len-=size;
+	  bytes_copied+=size;
+	  DEBUGF("read->offset=%"PRIu64, read->offset);
+	  continue;
+	}
       }
     }
     
     // ok, so we need to read a new buffer to fulfill the request.
     // remember the requested read offset so we can put it back
-    ofs = read->offset;
+    uint64_t ofs = read->offset;
     buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
-    ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data));
+    ssize_t r = rhizome_read(read, buffer->data, sizeof buffer->data);
     read->offset = ofs;
     buffer->len = 0;
-    if (len == -1)
+    if (r == -1)
       return -1;
-    buffer->len = (size_t) len;
+    buffer->len = (size_t) r;
+    DEBUGF("read->offset=%"PRIu64, read->offset);
   }
   return bytes_copied;
 }
@@ -1066,8 +1083,9 @@ static int write_file(struct rhizome_read *read, const char *filepath){
   return ret;
 }
 
-static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){
+static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state)
-  read_state->crypt=m->payloadEncryption;
+{
+  read_state->crypt = m->payloadEncryption == PAYLOAD_ENCRYPTED;
   if (read_state->crypt){
     // if the manifest specifies encryption, make sure we can generate the payload key and encrypt
     // the contents as we go
@@ -1076,10 +1094,9 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
       return WHY("Unable to decrypt bundle, valid key not found");
     }
     if (config.debug.rhizome)
-      DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
+      DEBUGF("Decrypting payload contents for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
-    
+    if (m->is_journal && m->tail > 0)
-    if (m->journalTail>0)
+      read_state->tail = m->tail;
-      read_state->tail = m->journalTail;
     bcopy(m->payloadKey, read_state->key, sizeof(read_state->key));
     bcopy(m->payloadNonce, read_state->nonce, sizeof(read_state->nonce));
   }
@@ -1172,24 +1189,20 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m,
 {
   int ret = 0;
 
-  if (advance_by > m->fileLength)
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
+  assert(m->is_journal);
+  if (advance_by > m->filesize)
     return WHY("Cannot advance past the existing content");
 
-  uint64_t old_length = m->fileLength;
+  uint64_t copy_length = m->filesize - advance_by;
-  uint64_t copy_length = old_length - advance_by;
+  rhizome_manifest_set_filesize(m, m->filesize + new_size - advance_by);
 
-  m->fileLength = m->fileLength + new_size - advance_by;
+  if (advance_by > 0)
-  rhizome_manifest_set_ll(m, "filesize", m->fileLength);
+    rhizome_manifest_set_tail(m, m->tail + advance_by);
 
-  if (advance_by>0){
+  rhizome_manifest_set_version(m, m->filesize);
-    m->journalTail += advance_by;
-    rhizome_manifest_set_ll(m,"tail",m->journalTail);
-  }
 
-  m->version = m->fileLength;
+  ret = rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT);
-  rhizome_manifest_set_ll(m,"version",m->version);
-
-  ret = rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
   if (ret)
     goto failure;
 
@@ -1231,8 +1244,7 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
   if (ret)
     goto failure;
 
-  m->filehash = write.id;
+  rhizome_manifest_set_filehash(m, &write.id);
-  rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
   return 0;
 
 failure:
@@ -1245,7 +1257,7 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t
 {
   struct stat stat;
   if (lstat(filename,&stat))
-    return WHYF("Could not stat() payload file '%s'",filename);
+    return WHYF_perror("stat(%s)", alloca_str_toprint(filename));
 
   struct rhizome_write write;
   bzero(&write, sizeof write);
@@ -1263,8 +1275,7 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t
   if (ret)
     goto failure;
 
-  m->filehash = write.id;
+  rhizome_manifest_set_filehash(m, &write.id);
-  rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
 
   return 0;
 

serval.h

@@ -829,7 +829,7 @@ int upper7_decode(struct slip_decode_state *state,unsigned char byte);
 uint32_t Crc32_ComputeBuf( uint32_t inCrc32, const void *buf,
 			  size_t bufLen );
 int rhizome_active_fetch_count();
-int rhizome_active_fetch_bytes_received(int q);
+uint64_t rhizome_active_fetch_bytes_received(int q);
 extern int64_t bundles_available;
 extern char crash_handler_clue[1024];
 

testdefs_rhizome.sh

@@ -78,6 +78,63 @@ assert_rhizome_list() {
    rhizome_list_file_count=$(( $(replayStdout | wc -l) - 2 ))
 }
 
+# Parse the standard output produced by the immediately preceding "rhizome list"
+# command into the following shell variables:
+#  NCOLS the number of columns
+#  NROWS the number of data rows (not counting headers)
+#  HEADER[c] the C-th header label, 0 <= C <= NCOLS-1
+#  <label>[R] where <label> is a header label with all non-alphanumerics
+#  replaced by underscore '_' and all alphas converted to upper case, eg,
+#  .author -> _AUTHOR, is the value of that column in the R-th row, 0 <=
+#  R < NROWS
+#
+# Warning: overwrites existing shell variables.  Names of overwritten shell
+# variables are derived directly from the output of rhizome list, so cannot be
+# controlled.  If a prefix is supplied, all variables are prefixed with that.
+rhizome_list_unpack() {
+   local prefix="$1"
+   {
+      local n
+      read n
+      eval ${prefix}NCOLS=\"\$n\"
+      declare -a ${prefix}HEADER
+      local -a header
+      local oIFS="$IFS"
+      IFS=:
+      read -r -a header
+      IFS="$oIFS"
+      eval ${prefix}HEADER="(\"\${header[@]}\")"
+      local hdr
+      local -a colvars=()
+      for hdr in "${header[@]}"; do
+         case "$hdr" in
+         id)
+            hdr=BID;;
+         *)
+            hdr="${hdr//[^A-Za-z0-9_]/_}"
+            # hdr="${hdr^^*}" would do in Bash-4.0 and later
+            hdr="$(echo "$hdr" | sed -e 's/.*/\U&/')"
+            ;;
+         esac
+         colvars+=("$hdr")
+      done
+      local -a row
+      IFS=:
+      local i=0
+      while eval read -r -a row; do
+         local j=0
+         local val
+         for val in "${row[@]}"; do
+            eval ${prefix}${colvars[$j]}[$i]=\"\$val\"
+            let ++j
+         done
+         let ++i
+      done
+      IFS="$oIFS"
+      eval ${prefix}NROWS=$i
+   } < "$TFWSTDOUT"
+}
+
 rhizome_list_dump() {
    local ncols
    local -a headers

tests/meshms

@@ -28,15 +28,22 @@ teardown() {
    assert_no_servald_processes
 }
 
+setup_logging() {
+   executeOk_servald config \
+      set debug.meshms on \
+      set debug.rhizome on \
+      set debug.rhizome_manifest on \
+      set debug.rejecteddata on \
+      set log.console.level debug \
+      set log.console.show_time on
+}
+
 doc_MessageDelivery="Send messages, ack and read them in a 2 party conversation"
 setup_MessageDelivery() {
    setup_servald
    set_instance +A
    create_identities 2
-   executeOk_servald config \
+   setup_logging
-     set debug.meshms on \
-     set debug.rhizome on \
-     set log.console.level debug
 }
 test_MessageDelivery() {
    # 1. empty list
@@ -60,8 +67,12 @@ test_MessageDelivery() {
    assertStdoutGrep --stdout --matches=1 "^0:19:<:How are you\$"
    assertStdoutGrep --stdout --matches=1 "^1:5:<:Hi\$"
    assertStdoutLineCount '==' 4
+   CONV_BID=$(replayStderr | sed -n -e '/MESHMS CONVERSATION BUNDLE/s/.*bid=\([0-9A-F]*\).*/\1/p')
+   CONV_SECRET=$(replayStderr | sed -n -e '/MESHMS CONVERSATION BUNDLE/s/.*secret=\([0-9A-F]*\).*/\1/p')
+   tfw_log "CONV_BID=$CONV_BID CONV_SECRET=$CONV_SECRET"
    # 5. mark the first message as read
    executeOk_servald meshms read messages $SIDA2 $SIDA1 5
+   check_meshms_bundles
    executeOk_servald meshms list messages $SIDA2 $SIDA1
    assertStdoutGrep --stdout --matches=1 "^0:19:<:How are you\$"
    assertStdoutGrep --stdout --matches=1 "^1:5:MARK:read\$"
@@ -69,6 +80,7 @@ test_MessageDelivery() {
    assertStdoutLineCount '==' 5
    # 6. mark all messages as read
    executeOk_servald meshms read messages $SIDA2
+   check_meshms_bundles
    executeOk_servald meshms list messages $SIDA2 $SIDA1
    assertStdoutGrep --stdout --matches=1 "^0:19:MARK:read\$"
    assertStdoutGrep --stdout --matches=1 "^1:19:<:How are you\$"
@@ -82,17 +94,24 @@ test_MessageDelivery() {
    assertStdoutLineCount '==' 5
 }
 
-doc_MessageThreading="Messages sent at the same time, thread differently"
+check_meshms_bundles() {
-setup_MessageThreading() {
+   # Dump the MeshMS bundles to the log and check consistency
-   setup_servald
+   # The only "file" bundle should be the conversation list
-   foreach_instance +A +B create_single_identity
+   executeOk_servald rhizome list file
-   set_instance +A
+   rhizome_list_unpack X
-   executeOk_servald meshms send message $SIDA $SIDB "Hello can you hear me"
+   assert [ $XNROWS -eq 1 ]
-   executeOk_servald meshms send message $SIDA $SIDB "Still waiting"
+   assert [ ${XBID[0]} = $CONV_BID ]
-   set_instance +B
+   executeOk_servald rhizome extract bundle $CONV_BID manifest.conv payload.conv $CONV_SECRET
-   executeOk_servald meshms send message $SIDB $SIDA "Help Im trapped in a test case factory"
+   tfw_cat -v manifest.conv --hexdump payload.conv
-   executeOk_servald meshms send message $SIDB $SIDA "Never mind"
+   # The only "MeshMS2" bundles should be the two ply bundles
-   start_servald_instances +A +B
+   executeOk_servald rhizome list MeshMS2
+   rhizome_list_unpack X
+   assert [ $XNROWS -eq 2 ]
+   local bid
+   for bid in ${XBID[*]}; do
+      executeOk_servald rhizome extract bundle $bid manifest.$bid payload.$bid
+      tfw_cat -v manifest.$bid --hexdump payload.$bid
+   done
 }
 
 has_unread_messages() {
@@ -109,6 +128,19 @@ messages_delivered() {
    fi
 }
 
+doc_MessageThreading="Messages sent at the same time, thread differently"
+setup_MessageThreading() {
+   setup_servald
+   foreach_instance +A +B create_single_identity
+   foreach_instance +A +B setup_logging
+   set_instance +A
+   executeOk_servald meshms send message $SIDA $SIDB "Hello can you hear me"
+   executeOk_servald meshms send message $SIDA $SIDB "Still waiting"
+   set_instance +B
+   executeOk_servald meshms send message $SIDB $SIDA "Help Im trapped in a test case factory"
+   executeOk_servald meshms send message $SIDB $SIDA "Never mind"
+   start_servald_instances +A +B
+}
 test_MessageThreading() {
    set_instance +B
    wait_until has_unread_messages $SIDB
@@ -118,7 +150,6 @@ test_MessageThreading() {
    assertStdoutGrep --stdout --matches=1 "^2:54:>:Never mind\$"
    assertStdoutGrep --stdout --matches=1 "^3:41:>:Help Im trapped in a test case factory\$"
    assertStdoutLineCount '==' 6
-
    set_instance +A
    wait_until has_unread_messages $SIDA
    wait_until messages_delivered $SIDA $SIDB
@@ -136,10 +167,7 @@ setup_listConversations() {
    setup_servald
    set_instance +A
    create_identities 5
-   executeOk_servald config \
+   setup_logging
-     set debug.rhizome on \
-     set debug.meshms on \
-     set log.console.level debug
    # create 3 threads, with all permutations of incoming and outgoing messages
    executeOk_servald meshms send message $SIDA1 $SIDA2 "Message1"
    executeOk_servald meshms send message $SIDA3 $SIDA1 "Message2"

tests/rhizomeops

@@ -42,6 +42,8 @@ setup_rhizome() {
 set_rhizome_config() {
    executeOk_servald config \
       set debug.rhizome on \
+      set debug.rhizome_manifest on \
+      set debug.rejecteddata on \
       set debug.verbose on \
       set log.console.level debug
 }
@@ -215,7 +217,7 @@ setup_ExtractManifestAfterAdd() {
 test_ExtractManifestAfterAdd() {
    executeOk_servald rhizome export manifest $manifestid file1x.manifest
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
@@ -224,6 +226,7 @@ test_ExtractManifestAfterAdd() {
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    assert [ -e file1x.manifest ]
    assert diff file1.manifest file1x.manifest
@@ -244,7 +247,7 @@ setup_ExtractManifestFileAfterAdd() {
 test_ExtractManifestFileAfterAdd() {
    executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
@@ -253,6 +256,7 @@ test_ExtractManifestFileAfterAdd() {
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    assert [ -e file1x.manifest ]
    assert diff file1.manifest file1x.manifest
@@ -282,7 +286,7 @@ setup_ExtractManifestFileFromExtBlob() {
 test_ExtractManifestFileFromExtBlob() {
    executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
@@ -291,6 +295,7 @@ test_ExtractManifestFileFromExtBlob() {
    assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    assert [ -e file1x.manifest ]
    assert diff file1.manifest file1x.manifest
@@ -298,7 +303,7 @@ test_ExtractManifestFileFromExtBlob() {
    assert diff file1 file1x
    executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
    tfw_cat --stdout --stderr
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat file2 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
@@ -307,6 +312,7 @@ test_ExtractManifestFileFromExtBlob() {
    assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    assert [ -e file2x.manifest ]
    assert diff file2.manifest file2x.manifest
@@ -344,17 +350,18 @@ test_ExtractManifestToStdout() {
    executeOk_servald rhizome export manifest $manifestid -
    assertStdoutLineCount '>=' 9
    local size=$(( $(cat file1 | wc -c) + 0 ))
-   assertStdoutGrep --line=..8 --matches=1 "^service:file$"
+   assertStdoutGrep --line=..9 --matches=1 "^service:file$"
-   assertStdoutGrep --line=..8 --matches=1 "^manifestid:$manifestid\$"
+   assertStdoutGrep --line=..9 --matches=1 "^manifestid:$manifestid\$"
-   assertStdoutGrep --line=..8 --matches=1 "^version:$version\$"
+   assertStdoutGrep --line=..9 --matches=1 "^version:$version\$"
-   assertStdoutGrep --line=..8 --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --line=..9 --matches=1 "^inserttime:$rexp_date\$"
-   assertStdoutGrep --line=..8 --matches=1 "^filehash:$filehash\$"
+   assertStdoutGrep --line=..9 --matches=1 "^filehash:$filehash\$"
-   assertStdoutGrep --line=..8 --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --line=..9 --matches=1 "^filesize:$size\$"
-   assertStdoutGrep --line=..8 --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --line=..9 --matches=1 "^\.author:$SIDB1\$"
-   assertStdoutGrep --line=..8 --matches=1 "^\.readonly:0\$"
+   assertStdoutGrep --line=..9 --matches=1 "^\.secret:$rexp_bundlesecret\$"
-   assertStdoutGrep --line=9 --matches=1 "^manifest:"
+   assertStdoutGrep --line=..9 --matches=1 "^\.readonly:0\$"
-   replayStdout | $SED -n '9s/^manifest://p' >file1x.manifest
+   assertStdoutGrep --line=10 --matches=1 "^manifest:"
-   replayStdout | $SED -n '10,$p' >>file1x.manifest
+   replayStdout | $SED -n '10s/^manifest://p' >file1x.manifest
+   replayStdout | $SED -n '11,$p' >>file1x.manifest
    cat file1.manifest >file1n.manifest
    echo >>file1n.manifest
    tfw_cat file1n.manifest file1x.manifest
@@ -434,7 +441,7 @@ test_ExtractFileAfterAdd() {
    tfw_cat --stderr
    assert diff file1 file1x
    local size=$(( $(cat file1 | wc -c) + 0 ))
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
    assertStdoutGrep --matches=1 "^version:$version\$"
@@ -442,6 +449,7 @@ test_ExtractFileAfterAdd() {
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
 }
 
@@ -682,10 +690,11 @@ setup_AddUnsupportedService() {
    setup_servald
    setup_rhizome
    echo "Message1" >file1
-   echo -e 'service=Fubar' >file1.manifest
+   echo 'service=Fubar' >file1.manifest
 }
 test_AddUnsupportedService() {
    execute $servald rhizome add file $SIDB1 file1 file1.manifest
+   tfw_cat --stdout --stderr
    assertExitStatus '!=' 0
 }
 
@@ -732,6 +741,7 @@ test_RecipientIsEncrypted() {
    assert diff file1 file1x
    extract_manifest_filehash filehash file1.manifest
    executeOk_servald rhizome export file $filehash file1y
+   tfw_cat file1 -v file1y
    assert ! diff file1 file1y
 }
 
@@ -1020,7 +1030,7 @@ test_ImportOwnBundle() {
    tfw_cat --stderr
    assert cmp fileB.manifest fileBx.manifest
    assert cmp fileB fileBx
-   assertStdoutLineCount '==' 8
+   assertStdoutLineCount '==' 9
    local size=$(( $(cat fileB | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
    assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
@@ -1029,6 +1039,7 @@ test_ImportOwnBundle() {
    assertStdoutGrep --matches=1 "^filehash:$filehash\$"
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
+   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    # Now bundle author should be known, so appears to be from here
    executeOk_servald rhizome list

tests/routing

@@ -61,11 +61,11 @@ has_link() {
 
 path_exists() {
    local dest
-   for dest; do tru; done;
+   eval dest=\$$#
    local dest_sidvar=SID${dest#+}
    local next_inst=$1
    shift
-   local I N
+   local I
    for I; do
       local sidvar=SID${I#+}
       [ -n "${!sidvar}" ] || break