Encrypt payloads without needing a sender

- reworked keyring identity handling to reduce memory searching

conf_schema.h

@@ -405,6 +405,8 @@ END_STRUCT
 
 STRUCT(rhizome_mdp)
 ATOM(bool_t,                enable,     1, boolean,, "If true, Rhizome MDP server is started")
+ATOM(uint64_t,              stall_timeout,      1000, uint64_scaled,, "Timeout to request more data.")
+ATOM(uint64_t,              block_size, 512, uint64_scaled,, "Transfer block size.")
 END_STRUCT
 
 STRUCT(rhizome_advertise)
@@ -421,9 +423,7 @@ ATOM(uint64_t,              database_size,  UINT64_MAX, uint64_scaled,, "Maximum
 ATOM(uint64_t,              min_free_space, 100*1024*1024, uint64_scaled,, "Minimum free space to preserve on the disk")
 ATOM(uint32_t,              max_blob_size,  128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
 
-ATOM(uint64_t,              rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")
 ATOM(uint64_t,              idle_timeout,           RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")
-ATOM(uint64_t,              mdp_stall_timeout,      1000, uint64_scaled,, "Timeout to request more data via mdp.")
 ATOM(uint32_t,              fetch_delay_ms,         50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
 SUB_STRUCT(rhizome_direct,  direct,)
 SUB_STRUCT(rhizome_api,     api,)

keyring.c

@@ -209,7 +209,7 @@ keypair *keyring_next_keytype(keyring_iterator *it, unsigned keytype)
   return kp;
 }
 
-keypair *keyring_identity_keytype(keyring_identity *id, unsigned keytype)
+keypair *keyring_identity_keytype(const keyring_identity *id, unsigned keytype)
 {
   keypair *kp=id->keypairs;
   while(kp && kp->type!=keytype)
@@ -231,7 +231,22 @@ keypair *keyring_find_did(keyring_iterator *it, const char *did)
   return NULL;
 }
 
-static int keyring_find_box(keyring_iterator *it, const sid_t *sidp, const uint8_t **sk)
+const uint8_t * keyring_get_box(const keyring_identity *id)
+{
+  keypair *kp = id->keypairs;
+  while(kp){
+    if (kp->type == KEYTYPE_CRYPTOBOX)
+      return kp->private_key;
+    if (kp->type == KEYTYPE_CRYPTOCOMBINED){
+      struct combined_sk *secret = (struct combined_sk *)kp->private_key;
+      return secret->box_key;
+    }
+    kp = kp->next;
+  }
+  return NULL;
+}
+
+int keyring_find_box(keyring_iterator *it, const sid_t *sidp, const uint8_t **sk)
 {
   keypair *kp;
   while((kp=keyring_next_key(it))){
@@ -255,29 +270,26 @@ static int keyring_find_box(keyring_iterator *it, const sid_t *sidp, const uint8
   return 0;
 }
 
-int keyring_find_sid(keyring_iterator *it, const sid_t *sidp)
+keyring_identity *keyring_find_identity(keyring_file *k, const sid_t *sidp){
-{
+  keypair *kp;
-  return keyring_find_box(it, sidp, NULL);
+  keyring_iterator it;
-}
+  keyring_iterator_start(k, &it);
-
+  while((kp=keyring_next_key(&it))){
-const sid_t *keyring_identity_sid(const keyring_identity *id)
+    if (kp->type == KEYTYPE_CRYPTOBOX){
-{
+      if (memcmp(sidp->binary, kp->public_key, SID_SIZE) == 0)
-  keypair *kp=id->keypairs;
+	return it.identity;
-  while(kp){
+    }else if(kp->type == KEYTYPE_CRYPTOCOMBINED){
-    if (kp->type == KEYTYPE_CRYPTOBOX)
-      return (const sid_t *)kp->public_key;
-    if (kp->type == KEYTYPE_CRYPTOCOMBINED){
       struct combined_pk *pk = (struct combined_pk *)kp->public_key;
-      return &pk->box_key;
+      if (memcmp(sidp->binary, pk->box_key.binary, SID_SIZE) == 0)
+	return it.identity;
     }
-    kp=kp->next;
   }
   return NULL;
 }
 
-static void add_subscriber(keyring_identity *id, const sid_t *sid)
+static void add_subscriber(keyring_identity *id)
 {
-  id->subscriber = find_subscriber(sid->binary, SID_SIZE, 1);
+  id->subscriber = find_subscriber(id->box_pk->binary, SID_SIZE, 1);
   if (id->subscriber) {
     if (id->subscriber->reachable == REACHABLE_NONE){
       id->subscriber->reachable = REACHABLE_SELF;
@@ -1154,11 +1166,6 @@ static keyring_identity *keyring_unpack_identity(unsigned char *slot, const char
     keyring_free_identity(id);
     return NULL;
   }
-  if (!keyring_identity_sid(id)) {
-    DEBUGF(keyring, "identity does not have a SID");
-    keyring_free_identity(id);
-    return NULL;
-  }
   DEBUGF(keyring, "unpacked key pairs");
   return id;
 }
@@ -1196,6 +1203,41 @@ static int keyring_identity_mac(const keyring_identity *id, unsigned char *pkrsa
   return 0;
 }
 
+static int keyring_finalise_identity(keyring_file *k, keyring_identity *id)
+{
+  keypair *kp = id->keypairs;
+  while(kp){
+    switch(kp->type){
+      case KEYTYPE_CRYPTOBOX:
+	id->box_pk = (const sid_t *)kp->public_key;
+	id->box_sk = kp->private_key;
+	break;
+      case KEYTYPE_CRYPTOSIGN:
+	if (!rhizome_verify_bundle_privatekey(kp->private_key,kp->public_key)){
+	  /* SAS key is invalid (perhaps because it was a pre 0.90 format one),
+	     so replace it */
+	  WARN("SAS key is invalid -- regenerating.");
+	  crypto_sign_keypair(kp->public_key, kp->private_key);
+	  k->dirty = 1;
+	}
+	id->sign_pk = kp->public_key;
+	id->sign_sk = kp->private_key;
+	break;
+      case KEYTYPE_CRYPTOCOMBINED:{
+	struct combined_pk *pk = (struct combined_pk *)kp->public_key;
+	struct combined_sk *sk = (struct combined_sk *)kp->private_key;
+	id->box_pk = &pk->box_key;
+	id->box_sk = sk->box_key;
+	id->sign_pk = pk->sign_key;
+	id->sign_sk = sk->sign_key;
+	break;
+      }
+    }
+    kp = kp->next;
+  }
+  return 0;
+}
+
 
 /* Read the slot, and try to decrypt it.  Decryption is symmetric with encryption, so the same
  * function is used for munging the slot before making use of it, whichever way we are going.  Once
@@ -1220,7 +1262,7 @@ static int keyring_decrypt_pkr(keyring_file *k, const char *pin, int slot_number
   }
   /* 3. Unpack contents of slot into a new identity in the provided context. */
   DEBUGF(keyring, "unpack slot %u", slot_number);
-  if (((id = keyring_unpack_identity(slot, pin)) == NULL) || !id->keypairs)
+  if (((id = keyring_unpack_identity(slot, pin)) == NULL))
     goto kdp_safeexit; // Not a valid slot
   id->slot = slot_number;
   /* 4. Verify that slot is self-consistent (check MAC) */
@@ -1237,12 +1279,10 @@ static int keyring_decrypt_pkr(keyring_file *k, const char *pin, int slot_number
     goto kdp_safeexit;
   }
 
-  // Add any unlocked subscribers to our memory table, flagged as local SIDs.
+  if (keyring_finalise_identity(k, id)!=0)
-  {
+    goto kdp_safeexit;
-    const sid_t *sid = keyring_identity_sid(id);
+
-    if (sid)
+  add_subscriber(id);
-      add_subscriber(id, sid);
-  }
 
   /* All fine, so add the id into the context and return. */
   keyring_identity **i=&k->identities;
@@ -1306,6 +1346,9 @@ int keyring_enter_pin(keyring_file *k, const char *pin)
     }
   }
 
+  if (k->dirty)
+    keyring_commit(k);
+  
   RETURN(identitiesFound);
   OUT();
 }
@@ -1346,11 +1389,11 @@ static unsigned find_free_slot(const keyring_file *k)
 
 static int keyring_commit_identity(keyring_file *k, keyring_identity *id)
 {
-  const sid_t *sid=keyring_identity_sid(id);
+  keyring_finalise_identity(k, id);
   // Do nothing if an identity with this sid already exists
   keyring_iterator it;
   keyring_iterator_start(k, &it);
-  if (keyring_find_sid(&it, sid))
+  if (keyring_find_sid(&it, id->box_pk))
     return 0;
   set_slot(k, id->slot, 1);
 
@@ -1359,7 +1402,7 @@ static int keyring_commit_identity(keyring_file *k, keyring_identity *id)
     i=&(*i)->next;
 
   *i=id;
-  add_subscriber(id, sid);
+  add_subscriber(id);
   return 1;
 }
 
@@ -1405,9 +1448,11 @@ keyring_identity *keyring_create_identity(keyring_file *k, const char *pin)
   assert(id->keypairs);
 
   /* Mark slot as occupied and internalise new identity. */
-  keyring_commit_identity(k, id);
+  if (keyring_commit_identity(k, id)!=1)
+    goto kci_safeexit;
 
   /* Everything went fine */
+  k->dirty = 1;
   return id;
 
  kci_safeexit:
@@ -1472,6 +1517,8 @@ int keyring_commit(keyring_file *k)
     WHYF_perror("fflush(%d)", fileno(k->file));
     errorCount++;
   }
+  if (!errorCount)
+    k->dirty=0;
   return errorCount ? WHYF("%u errors commiting keyring to disk", errorCount) : 0;
 }
 
@@ -1611,62 +1658,16 @@ keypair * keyring_find_public_tag_value(keyring_iterator *it, const char *name,
   return NULL;
 }
 
-static int keyring_find_sign_key(keyring_file *k, keyring_identity *identity, const uint8_t **sk, const uint8_t **pk)
-{
-  IN();
-  assert(identity);
-
-  keypair *kp=identity->keypairs;
-  while(kp){
-    if (kp->type == KEYTYPE_CRYPTOSIGN){
-      if (!kp->verified){
-	if (!rhizome_verify_bundle_privatekey(kp->private_key,kp->public_key)){
-	  /* SAS key is invalid (perhaps because it was a pre 0.90 format one),
-	     so replace it */
-	  WARN("SAS key is invalid -- regenerating.");
-	  crypto_sign_keypair(kp->public_key, kp->private_key);
-	  keyring_commit(k);
-	}
-	kp->verified=1;
-      }
-      DEBUGF(keyring, "Found SAS entry");
-      if (sk)
-	*sk = kp->private_key;
-      if (pk)
-	*pk = kp->public_key;
-      RETURN(0);
-    }else if (kp->type == KEYTYPE_CRYPTOCOMBINED){
-      DEBUGF(keyring, "Found combined key");
-      if (sk){
-	struct combined_sk *secret = (struct combined_sk *)kp->private_key;
-	*sk = secret->sign_key;
-      }
-      if (pk){
-	struct combined_pk *secret = (struct combined_pk *)kp->public_key;
-	*pk = secret->sign_key;
-      }
-      RETURN(0);
-    }
-    kp=kp->next;
-  }
-  RETURN(WHY("Identity lacks SAS"));
-  OUT();
-}
-
 // sign the hash of a message, adding the signature to the end of the message buffer.
 int keyring_sign_message(struct keyring_identity *identity, unsigned char *content, size_t buffer_len, size_t *content_len)
 {
   if (*content_len + SIGNATURE_BYTES > buffer_len)
     return WHYF("Insufficient space in message buffer to add signature. %zu, need %zu",buffer_len, *content_len + SIGNATURE_BYTES);
 
-  const uint8_t *sk = NULL;
-  if (keyring_find_sign_key(keyring, identity, &sk, NULL)==-1)
-    return -1;
-
   unsigned char hash[crypto_hash_sha512_BYTES];
   crypto_hash_sha512(hash, content, *content_len);
 
-  if (crypto_sign_detached(&content[*content_len], NULL, hash, crypto_hash_sha512_BYTES, sk))
+  if (crypto_sign_detached(&content[*content_len], NULL, hash, crypto_hash_sha512_BYTES, identity->sign_sk))
     return WHY("Signing failed");
 
   *content_len += SIGNATURE_BYTES;
@@ -1704,12 +1705,9 @@ static int keyring_store_sas(struct internal_mdp_header *header, struct overlay_
   return 0;
 }
 
-static int keyring_respond_sas(keyring_file *k, struct internal_mdp_header *header)
+static int keyring_respond_sas(struct internal_mdp_header *header)
 {
-  const uint8_t *sk = NULL;
+  keyring_identity *id = header->destination->identity;
-  const uint8_t *pk = NULL;
-  if (keyring_find_sign_key(k, header->destination->identity, &sk, &pk)==-1)
-    return -1;
 
   /* It's a request, so find the SAS for the SID the request was addressed to,
      use that to sign that SID, and then return it in an authcrypted frame. */
@@ -1722,10 +1720,10 @@ static int keyring_respond_sas(keyring_file *k, struct internal_mdp_header *head
   ob_limitsize(response_payload, sizeof buff);
   
   ob_append_byte(response_payload, KEYTYPE_CRYPTOSIGN);
-  ob_append_bytes(response_payload, pk, crypto_sign_PUBLICKEYBYTES);
+  ob_append_bytes(response_payload, id->sign_pk, crypto_sign_PUBLICKEYBYTES);
   uint8_t *sig = ob_append_space(response_payload, crypto_sign_BYTES);
 
-  if (crypto_sign_detached(sig, NULL, header->destination->sid.binary, SID_SIZE, sk))
+  if (crypto_sign_detached(sig, NULL, header->destination->sid.binary, SID_SIZE, id->sign_sk))
     return WHY("crypto_sign() failed");
     
   DEBUGF(keyring, "Sending SID:SAS mapping, %zd bytes, %s:%"PRImdp_port_t" -> %s:%"PRImdp_port_t,
@@ -1863,7 +1861,7 @@ static int keyring_mapping_request(struct internal_mdp_header *header, struct ov
   switch(ob_get(payload)){
     case KEYTYPE_CRYPTOSIGN:
       if (ob_remaining(payload)==0)
-	return keyring_respond_sas(keyring, header);
+	return keyring_respond_sas(header);
       return keyring_store_sas(header, payload);
       break;
     case UNLOCK_REQUEST:
@@ -2049,7 +2047,7 @@ unsigned nm_slots_used=0;
 #define NM_CACHE_SLOTS 512
 struct nm_record nm_cache[NM_CACHE_SLOTS];
 
-unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp)
+unsigned char *keyring_get_nm_bytes(const uint8_t *box_sk, const sid_t *box_pk, const sid_t *unknown_sidp)
 {
   IN();
   assert(keyring != NULL);
@@ -2057,21 +2055,13 @@ unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknow
   /* See if we have it cached already */
   unsigned i;
   for(i=0;i<nm_slots_used;i++){
-    if (cmp_sid_t(&nm_cache[i].known_key, known_sidp) != 0) continue;
+    if (cmp_sid_t(&nm_cache[i].known_key, box_pk) != 0) continue;
     if (cmp_sid_t(&nm_cache[i].unknown_key, unknown_sidp) != 0) continue;
     RETURN(nm_cache[i].nm_bytes);
   }
 
   /* Not in the cache, so prepare to cache it (or return failure if known is not
      in fact a known key */
-  keyring_iterator it;
-  keyring_iterator_start(keyring, &it);
-  const uint8_t *sk;
-  if (!keyring_find_box(&it, known_sidp, &sk)) {
-    WHY("known key is not in fact known.");
-    RETURN(NULL);
-  }
-
   /* work out where to store it */
   if (nm_slots_used<NM_CACHE_SLOTS) {
     i=nm_slots_used; nm_slots_used++; 
@@ -2080,9 +2070,9 @@ unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknow
   }
 
   /* calculate and store */
-  nm_cache[i].known_key = *known_sidp;
+  nm_cache[i].known_key = *box_pk;
   nm_cache[i].unknown_key = *unknown_sidp;
-  if (crypto_box_beforenm(nm_cache[i].nm_bytes, unknown_sidp->binary, sk)){
+  if (crypto_box_beforenm(nm_cache[i].nm_bytes, unknown_sidp->binary, box_sk)){
     WHY("crypto_box_beforenm failed");
     RETURN(NULL);
   }
@@ -2196,8 +2186,12 @@ int keyring_load_from_dump(keyring_file *k, unsigned entry_pinc, const char **en
     }
     if (id == NULL || idn != last_idn) {
       last_idn = idn;
-      if (id)
+      if (id){
-	keyring_commit_identity(k, id);
+	if (keyring_commit_identity(k, id)!=1)
+	  keyring_free_identity(id);
+	else
+	  k->dirty=1;
+      }
       if ((id = emalloc_zero(sizeof(keyring_identity))) == NULL) {
 	keyring_free_keypair(kp);
 	return -1;
@@ -2216,8 +2210,12 @@ int keyring_load_from_dump(keyring_file *k, unsigned entry_pinc, const char **en
     if (!keyring_identity_add_keypair(id, kp))
       keyring_free_keypair(kp);
   }
-  if (id)
+  if (id){
-    keyring_commit_identity(k, id);
+    if (keyring_commit_identity(k, id)!=1)
+      keyring_free_identity(id);
+    else
+      k->dirty=1;
+  }
   if (ferror(input))
     return WHYF_perror("fscanf");
   return 0;

keyring.h

@@ -30,7 +30,6 @@ typedef struct keypair {
   size_t private_key_len;
   unsigned char *public_key;
   size_t public_key_len;
-  uint8_t verified;
   struct keypair *next;
 } keypair;
 
@@ -45,6 +44,10 @@ typedef struct keyring_identity {
   time_ms_t challenge_expires;
   unsigned char challenge[24];
   unsigned int slot;
+  const uint8_t *box_sk;
+  const sid_t *box_pk;
+  const uint8_t *sign_sk;
+  const uint8_t *sign_pk;
   struct keyring_identity *next;
   keypair *keypairs;
 } keyring_identity;
@@ -67,6 +70,7 @@ typedef struct keyring_file {
   keyring_identity *identities;
   FILE *file;
   size_t file_size;
+  uint8_t dirty;
 } keyring_file;
 
 typedef struct keyring_iterator{
@@ -79,10 +83,11 @@ void keyring_iterator_start(keyring_file *k, keyring_iterator *it);
 keyring_identity * keyring_next_identity(keyring_iterator *it);
 keypair * keyring_next_key(keyring_iterator *it);
 keypair * keyring_next_keytype(keyring_iterator *it, unsigned keytype);
-keypair *keyring_identity_keytype(keyring_identity *id, unsigned keytype);
+keypair *keyring_identity_keytype(const keyring_identity *id, unsigned keytype);
 keypair *keyring_find_did(keyring_iterator *it, const char *did);
-int keyring_find_sid(keyring_iterator *it, const sid_t *sidp);
+keyring_identity *keyring_find_identity(keyring_file *k, const sid_t *sidp);
-const sid_t *keyring_identity_sid(const keyring_identity *id);
+int keyring_find_box(keyring_iterator *it, const sid_t *sidp, const uint8_t **sk);
+#define keyring_find_sid(I, S) keyring_find_box(I, S, NULL)
 
 void keyring_free(keyring_file *k);
 int keyring_release_identity(keyring_iterator *it);
@@ -119,7 +124,7 @@ void keyring_identity_extract(const keyring_identity *id, const sid_t **sidp, co
 int keyring_load_from_dump(keyring_file *k, unsigned entry_pinc, const char **entry_pinv, FILE *input);
 int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret);
 
-unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp);
+unsigned char *keyring_get_nm_bytes(const uint8_t *box_sk, const sid_t *box_pk, const sid_t *unknown_sidp);
 
 struct internal_mdp_header;
 struct overlay_buffer;

keyring_cli.c

@@ -151,10 +151,9 @@ static int app_keyring_list(const struct cli_parsed *parsed, struct cli_context
 
 static void cli_output_identity(struct cli_context *context, const keyring_identity *id)
 {
-  const sid_t *sid = keyring_identity_sid(id);
+  if (id->box_pk){
-  if (sid){
       cli_field_name(context, "sid", ":");
-      cli_put_string(context, alloca_tohex_sid_t(*sid), "\n");
+      cli_put_string(context, alloca_tohex_sid_t(*id->box_pk), "\n");
   }
   keypair *kp=id->keypairs;
   while(kp){

meshms.c

@@ -46,18 +46,12 @@ void meshms_free_conversations(struct meshms_conversations *conv)
   }
 }
 
-static enum meshms_status get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
+static enum meshms_status get_my_conversation_bundle(const keyring_identity *id, rhizome_manifest *m)
 {
   /* Find our private key */
-  keyring_iterator it;
-  keyring_iterator_start(keyring, &it);
-  
-  if (!keyring_find_sid(&it, my_sidp))
-    return MESHMS_STATUS_SID_LOCKED;
-
   strbuf sb = strbuf_alloca(1024);
   strbuf_puts(sb, "incorrection");
-  strbuf_tohex(sb, crypto_box_SECRETKEYBYTES * 2, it.keypair->private_key);
+  strbuf_tohex(sb, crypto_box_SECRETKEYBYTES * 2, id->box_sk);
   strbuf_puts(sb, "concentrativeness");
   assert(!strbuf_overrun(sb));
   if (rhizome_get_bundle_from_seed(m, strbuf_str(sb)) == -1)
@@ -69,7 +63,8 @@ static enum meshms_status get_my_conversation_bundle(const sid_t *my_sidp, rhizo
   if (m->haveSecret == NEW_BUNDLE_ID) {
     rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
     rhizome_manifest_set_name(m, "");
-    struct rhizome_bundle_result result = rhizome_fill_manifest(m, NULL, my_sidp);
+    rhizome_manifest_set_author_identity(m, id);
+    struct rhizome_bundle_result result = rhizome_fill_manifest(m, NULL);
     switch (result.status) {
     case RHIZOME_BUNDLE_STATUS_NEW:
     case RHIZOME_BUNDLE_STATUS_SAME:
@@ -130,7 +125,7 @@ static struct meshms_conversations *add_conv(struct meshms_conversations **conv,
 
 // find matching conversations
 // if their_sid == my_sid, return all conversations with any recipient
-static enum meshms_status get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv)
+static enum meshms_status get_database_conversations(const keyring_identity *id, const sid_t *their_sid, struct meshms_conversations **conv)
 {
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
@@ -139,16 +134,16 @@ static enum meshms_status get_database_conversations(const sid_t *my_sid, const
       " WHERE service = ?3"
       " AND (sender=?1 or recipient=?1)"
       " AND (sender=?2 or recipient=?2)",
-      SID_T, my_sid,
+      SID_T, id->box_pk,
-      SID_T, their_sid ? their_sid : my_sid,
+      SID_T, their_sid ? their_sid : id->box_pk,
       STATIC_TEXT, RHIZOME_SERVICE_MESHMS2,
       END
     );
   if (!statement)
     return MESHMS_STATUS_ERROR;
   DEBUGF(meshms, "Looking for conversations for %s, %s",
-	 alloca_tohex_sid_t(*my_sid),
+	 alloca_tohex_sid_t(*id->box_pk),
-	 alloca_tohex_sid_t(*(their_sid ? their_sid : my_sid))
+	 alloca_tohex_sid_t(*(their_sid ? their_sid : id->box_pk))
 	);
   int r;
   while ((r=sqlite_step_retry(&retry, statement)) == SQLITE_ROW) {
@@ -170,7 +165,7 @@ static enum meshms_status get_database_conversations(const sid_t *my_sid, const
       WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them));
       continue;
     }
-    if (cmp_sid_t(&their_sid, my_sid) == 0) {
+    if (cmp_sid_t(&their_sid, id->box_pk) == 0) {
       them = sender;
       if (str_to_sid_t(&their_sid, them) == -1) {
 	WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them));
@@ -182,12 +177,11 @@ static enum meshms_status get_database_conversations(const sid_t *my_sid, const
       break;
     struct meshms_ply *p;
     if (them==sender){
-      ptr->found_their_ply=1;
       p=&ptr->their_ply;
     }else{
-      ptr->found_my_ply=1;
       p=&ptr->my_ply;
     }
+    p->found = 1;
     p->bundle_id = bid;
     p->version = version;
     p->tail = tail;
@@ -199,10 +193,10 @@ static enum meshms_status get_database_conversations(const sid_t *my_sid, const
   return MESHMS_STATUS_OK;
 }
 
-static enum meshms_status find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv)
+static enum meshms_status find_or_create_conv(keyring_identity *id, const sid_t *their_sid, struct meshms_conversations **conv)
 {
   enum meshms_status status;
-  if (meshms_failed(status = meshms_conversations_list(my_sid, their_sid, conv)))
+  if (meshms_failed(status = meshms_conversations_list(id, NULL, their_sid, conv)))
     return status;
   if (*conv == NULL) {
     if ((*conv = (struct meshms_conversations *) emalloc_zero(sizeof(struct meshms_conversations))) == NULL)
@@ -213,51 +207,6 @@ static enum meshms_status find_or_create_conv(const sid_t *my_sid, const sid_t *
   return status;
 }
 
-static enum meshms_status create_ply(const sid_t *my_sid, struct meshms_conversations *conv, rhizome_manifest *m)
-{
-  DEBUGF(meshms, "Creating ply for my_sid=%s them=%s",
-	 alloca_tohex_sid_t(conv->them),
-	 alloca_tohex_sid_t(*my_sid)
-	);
-  rhizome_manifest_set_service(m, RHIZOME_SERVICE_MESHMS2);
-  rhizome_manifest_set_sender(m, my_sid);
-  rhizome_manifest_set_recipient(m, &conv->them);
-  rhizome_manifest_set_filesize(m, 0);
-  rhizome_manifest_set_tail(m, 0);
-  struct rhizome_bundle_result result = rhizome_fill_manifest(m, NULL, my_sid);
-  switch (result.status) {
-  case RHIZOME_BUNDLE_STATUS_NEW:
-  case RHIZOME_BUNDLE_STATUS_SAME:
-  case RHIZOME_BUNDLE_STATUS_DUPLICATE:
-    break;
-  case RHIZOME_BUNDLE_STATUS_ERROR:
-  case RHIZOME_BUNDLE_STATUS_INVALID:
-  case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
-    WHYF("Error creating ply manifest: %s", alloca_rhizome_bundle_result(result));
-    rhizome_bundle_result_free(&result);
-    return MESHMS_STATUS_ERROR;
-  case RHIZOME_BUNDLE_STATUS_BUSY:
-    // TODO
-  case RHIZOME_BUNDLE_STATUS_OLD:
-  case RHIZOME_BUNDLE_STATUS_FAKE:
-  case RHIZOME_BUNDLE_STATUS_NO_ROOM:
-  case RHIZOME_BUNDLE_STATUS_MANIFEST_TOO_BIG:
-    WARNF("Cannot create ply manifest: %s", alloca_rhizome_bundle_result(result));
-    rhizome_bundle_result_free(&result);
-    return MESHMS_STATUS_PROTOCOL_FAULT;
-  case RHIZOME_BUNDLE_STATUS_READONLY:
-    INFOF("Cannot create ply manifest: %s", alloca_rhizome_bundle_result(result));
-    rhizome_bundle_result_free(&result);
-    return MESHMS_STATUS_SID_LOCKED;
-  }
-  rhizome_bundle_result_free(&result);
-  assert(m->haveSecret);
-  assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
-  conv->my_ply.bundle_id = m->cryptoSignPublic;
-  conv->found_my_ply = 1;
-  return 0;
-}
-
 static size_t append_footer(unsigned char *buffer, char type, size_t message_len)
 {
   assert(message_len <= MESHMS_MESSAGE_MAX_LEN);
@@ -375,15 +324,15 @@ static enum meshms_status ply_find_prev(struct meshms_ply_read *ply, char type)
   return status;
 }
 
-static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshms_conversations *conv, unsigned char *buffer, int len)
+static enum meshms_status append_meshms_buffer(const keyring_identity *id, const sid_t *recipient, struct meshms_ply *ply, unsigned char *buffer, int len)
 {
   enum meshms_status status = MESHMS_STATUS_ERROR;
   rhizome_manifest *mout = NULL;
   rhizome_manifest *m = rhizome_new_manifest();
   if (!m)
     goto end;
-  if (conv->found_my_ply){
+  if (ply->found){
-    switch (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m)) {
+    switch (rhizome_retrieve_manifest(&ply->bundle_id, m)) {
       case RHIZOME_BUNDLE_STATUS_SAME:
 	break;
       case RHIZOME_BUNDLE_STATUS_NEW: // bundle not found
@@ -400,17 +349,52 @@ static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshm
       status = MESHMS_STATUS_PROTOCOL_FAULT;
       goto end;
     }
+    assert(m->author_identity == id);
   } else {
-    status = create_ply(my_sid, conv, m);
+    DEBUGF(meshms, "Creating ply for sender=%s recipient=%s",
-    switch (status) {
+	   alloca_tohex_sid_t(*id->box_pk),
-    case MESHMS_STATUS_OK:
+	   alloca_tohex_sid_t(*recipient)
+	  );
+    rhizome_manifest_set_service(m, RHIZOME_SERVICE_MESHMS2);
+    rhizome_manifest_set_sender(m, id->box_pk);
+    rhizome_manifest_set_recipient(m, recipient);
+    rhizome_manifest_set_filesize(m, 0);
+    rhizome_manifest_set_tail(m, 0);
+    rhizome_manifest_set_author_identity(m, id);
+    struct rhizome_bundle_result result = rhizome_fill_manifest(m, NULL);
+    switch (result.status) {
+    case RHIZOME_BUNDLE_STATUS_NEW:
+    case RHIZOME_BUNDLE_STATUS_SAME:
+    case RHIZOME_BUNDLE_STATUS_DUPLICATE:
+      status = MESHMS_STATUS_OK;
+      break;
+    case RHIZOME_BUNDLE_STATUS_ERROR:
+    case RHIZOME_BUNDLE_STATUS_INVALID:
+    case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
+      WHYF("Error creating ply manifest: %s", alloca_rhizome_bundle_result(result));
+      status = MESHMS_STATUS_ERROR;
+      break;
+    case RHIZOME_BUNDLE_STATUS_BUSY:
+      // TODO
+    case RHIZOME_BUNDLE_STATUS_OLD:
+    case RHIZOME_BUNDLE_STATUS_FAKE:
+    case RHIZOME_BUNDLE_STATUS_NO_ROOM:
+    case RHIZOME_BUNDLE_STATUS_MANIFEST_TOO_BIG:
+      WARNF("Cannot create ply manifest: %s", alloca_rhizome_bundle_result(result));
+      status = MESHMS_STATUS_PROTOCOL_FAULT;
+      break;
+    case RHIZOME_BUNDLE_STATUS_READONLY:
+      INFOF("Cannot create ply manifest: %s", alloca_rhizome_bundle_result(result));
+      status = MESHMS_STATUS_SID_LOCKED;
       break;
-    case MESHMS_STATUS_ERROR:
-    case MESHMS_STATUS_UPDATED:
-    case MESHMS_STATUS_SID_LOCKED:
-    case MESHMS_STATUS_PROTOCOL_FAULT:
-      goto end;
     }
+    rhizome_bundle_result_free(&result);
+    if (status!=MESHMS_STATUS_OK)
+      goto end;
+    assert(m->haveSecret);
+    assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
+    ply->bundle_id = m->cryptoSignPublic;
+    ply->found = 1;
   }
   assert(m->haveSecret);
   assert(m->authorship == AUTHOR_AUTHENTIC);
@@ -469,12 +453,12 @@ end:
 
 // update if any conversations are unread or need to be acked.
 // return MESHMS_STATUS_UPDATED if the conversation index needs to be saved.
-static enum meshms_status update_conversation(const sid_t *my_sid, struct meshms_conversations *conv)
+static enum meshms_status update_conversation(const keyring_identity *id, struct meshms_conversations *conv)
 {
   DEBUG(meshms, "Checking if conversation needs to be acked");
     
   // Nothing to be done if they have never sent us anything
-  if (!conv->found_their_ply)
+  if (!conv->their_ply.found)
     return MESHMS_STATUS_OK;
   
   rhizome_manifest *m_ours = NULL;
@@ -503,7 +487,7 @@ static enum meshms_status update_conversation(const sid_t *my_sid, struct meshms
   // find our previous ack
   uint64_t previous_ack = 0;
   
-  if (conv->found_my_ply){
+  if (conv->my_ply.found){
     DEBUG(meshms, "Locating our previous ack");
       
     m_ours = rhizome_new_manifest();
@@ -541,7 +525,7 @@ static enum meshms_status update_conversation(const sid_t *my_sid, struct meshms
     ofs+=pack_uint(&buffer[ofs], conv->their_last_message - previous_ack);
   ofs+=append_footer(buffer+ofs, MESHMS_BLOCK_TYPE_ACK, ofs);
   ofs+=append_timestamp(buffer+ofs);
-  status = append_meshms_buffer(my_sid, conv, buffer, ofs);
+  status = append_meshms_buffer(id, &conv->them, &conv->my_ply, buffer, ofs);
   DEBUGF(meshms, "status=%d", status);
 end:
   ply_read_close(&ply);
@@ -556,7 +540,7 @@ end:
 }
 
 // update conversations, and return MESHMS_STATUS_UPDATED if the conversation index should be saved
-static enum meshms_status update_conversations(const sid_t *my_sid, struct meshms_conversations **conv)
+static enum meshms_status update_conversations(const keyring_identity *id, struct meshms_conversations **conv)
 {
   enum meshms_status rstatus = MESHMS_STATUS_OK;
   struct meshms_conversations **ptr = conv;
@@ -564,7 +548,7 @@ static enum meshms_status update_conversations(const sid_t *my_sid, struct meshm
     struct meshms_conversations *n = *ptr;
     if (n->their_size != n->their_ply.size) {
       enum meshms_status status;
-      if (meshms_failed(status = update_conversation(my_sid, n)))
+      if (meshms_failed(status = update_conversation(id, n)))
 	return status;
       if (status == MESHMS_STATUS_UPDATED){
 	rstatus = MESHMS_STATUS_UPDATED;
@@ -797,20 +781,32 @@ end:
 }
 
 // read information about existing conversations from a rhizome payload
-enum meshms_status meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv)
+enum meshms_status meshms_conversations_list(const keyring_identity *id, const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv)
 {
   enum meshms_status status = MESHMS_STATUS_ERROR;
   rhizome_manifest *m = rhizome_new_manifest();
   if (!m)
     goto end;
-  if (meshms_failed(status = get_my_conversation_bundle(my_sid, m)))
+
+  assert(id || my_sid);
+  if (!my_sid){
+    my_sid = id->box_pk;
+  }else if(!id){
+    id = keyring_find_identity(keyring, my_sid);
+    if (!id){
+      status = MESHMS_STATUS_SID_LOCKED;
+      goto end;
+    }
+  }
+
+  if (meshms_failed(status = get_my_conversation_bundle(id, m)))
     goto end;
   // read conversations payload
   if (meshms_failed(status = read_known_conversations(m, their_sid, conv)))
     goto end;
-  if (meshms_failed(status = get_database_conversations(my_sid, their_sid, conv)))
+  if (meshms_failed(status = get_database_conversations(id, their_sid, conv)))
     goto end;
-  if ((status = update_conversations(my_sid, conv)) == MESHMS_STATUS_UPDATED && their_sid == NULL)
+  if ((status = update_conversations(id, conv)) == MESHMS_STATUS_UPDATED && their_sid == NULL)
     status = write_known_conversations(m, *conv);
 end:
   rhizome_manifest_free(m);
@@ -843,11 +839,20 @@ enum meshms_status meshms_message_iterator_open(struct meshms_message_iterator *
 	 me ? alloca_tohex_sid_t(*me) : "NULL",
 	 them ? alloca_tohex_sid_t(*them) : "NULL"
 	);
-  enum meshms_status status;
+  enum meshms_status status = MESHMS_STATUS_ERROR;
   bzero(iter, sizeof *iter);
-  if (meshms_failed(status = find_or_create_conv(me, them, &iter->_conv)))
+
+  keyring_identity *id = keyring_find_identity(keyring, me);
+  if (!id){
+    status = MESHMS_STATUS_SID_LOCKED;
+    WHY("Identity not found");
+    goto fail;
+  }
+
+  if (meshms_failed(status = find_or_create_conv(id, them, &iter->_conv)))
     goto fail;
   assert(iter->_conv != NULL);
+  iter->identity = id;
   iter->_my_sid = *me;
   iter->my_sid = &iter->_my_sid;
   iter->their_sid = &iter->_conv->them;
@@ -856,12 +861,12 @@ enum meshms_status meshms_message_iterator_open(struct meshms_message_iterator *
   iter->read_offset = iter->_conv->read_offset;
   iter->timestamp = 0;
   // If I have never sent a message (or acked any of theirs), there are no messages in the thread.
-  if (iter->_conv->found_my_ply) {
+  if (iter->_conv->my_ply.found) {
     if ((iter->_my_manifest = rhizome_new_manifest()) == NULL)
       goto error;
     if (meshms_failed(status = ply_read_open(&iter->_my_reader, &iter->_conv->my_ply.bundle_id, iter->_my_manifest)))
       goto fail;
-    if (iter->_conv->found_their_ply) {
+    if (iter->_conv->their_ply.found) {
       if ((iter->_their_manifest = rhizome_new_manifest()) == NULL)
 	goto error;
       if (meshms_failed(status = ply_read_open(&iter->_their_reader, &iter->_conv->their_ply.bundle_id, iter->_their_manifest)))
@@ -916,9 +921,9 @@ void meshms_message_iterator_close(struct meshms_message_iterator *iter)
 enum meshms_status meshms_message_iterator_prev(struct meshms_message_iterator *iter)
 {
   assert(iter->_conv != NULL);
-  if (iter->_conv->found_my_ply) {
+  if (iter->_conv->my_ply.found) {
     assert(iter->_my_manifest != NULL);
-    if (iter->_conv->found_their_ply)
+    if (iter->_conv->their_ply.found)
       assert(iter->_their_manifest != NULL);
   }
   enum meshms_status status = MESHMS_STATUS_UPDATED;
@@ -972,7 +977,7 @@ enum meshms_status meshms_message_iterator_prev(struct meshms_message_iterator *
 	  break;
 	case MESHMS_BLOCK_TYPE_ACK:
 	  // Read the received messages up to the ack'ed offset
-	  if (iter->_conv->found_their_ply) {
+	  if (iter->_conv->their_ply.found) {
 	    int ofs = unpack_uint(iter->_my_reader.record, iter->_my_reader.record_length, (uint64_t*)&iter->_their_reader.read.offset);
 	    if (ofs == -1) {
 	      WHYF("Malformed ACK");
@@ -1012,47 +1017,63 @@ enum meshms_status meshms_send_message(const sid_t *sender, const sid_t *recipie
     return MESHMS_STATUS_ERROR;
   }
   struct meshms_conversations *conv = NULL;
-  enum meshms_status status;
+  enum meshms_status status = MESHMS_STATUS_ERROR;
-  if (!meshms_failed(status = find_or_create_conv(sender, recipient, &conv))) {
+  unsigned char buffer[message_len + 4 + 6];
+
+  keyring_identity *id = keyring_find_identity(keyring, sender);
+  if (!id)
+    return MESHMS_STATUS_SID_LOCKED;
+
+  if (meshms_failed(status = find_or_create_conv(id, recipient, &conv)))
+    goto end;
+
   assert(conv != NULL);
   // construct a message payload
-    // TODO, new format here.
+  // TODO, new format here for compressed text?
-    unsigned char buffer[message_len + 4 + 6];
   strncpy((char*)buffer, message, message_len);
   // ensure message is NUL terminated
   if (message[message_len - 1] != '\0')
     buffer[message_len++] = '\0';
   message_len += append_footer(buffer + message_len, MESHMS_BLOCK_TYPE_MESSAGE, message_len);
-    message_len+=append_timestamp(buffer + message_len);
+  message_len += append_timestamp(buffer + message_len);
-    status = append_meshms_buffer(sender, conv, buffer, message_len);
+  status = append_meshms_buffer(id, recipient, &conv->my_ply, buffer, message_len);
-  }
+
+end:
   meshms_free_conversations(conv);
   return status;
 }
 
 enum meshms_status meshms_mark_read(const sid_t *sender, const sid_t *recipient, uint64_t offset)
 {
+  rhizome_manifest *m=NULL;
+  enum meshms_status status = MESHMS_STATUS_ERROR;
+  struct meshms_conversations *conv = NULL;
+
+  keyring_identity *id = keyring_find_identity(keyring, sender);
+  if (!id){
+    status = MESHMS_STATUS_SID_LOCKED;
+    goto end;
+  }
+
   DEBUGF(meshms, "sender=%s recipient=%s offset=%"PRIu64,
 	 alloca_tohex_sid_t(*sender),
 	 recipient ? alloca_tohex_sid_t(*recipient) : "NULL",
 	 offset
 	);
-  enum meshms_status status = MESHMS_STATUS_ERROR;
+  m = rhizome_new_manifest();
-  struct meshms_conversations *conv = NULL;
-  rhizome_manifest *m = rhizome_new_manifest();
   if (!m)
     goto end;
-  if (meshms_failed(status = get_my_conversation_bundle(sender, m)))
+  if (meshms_failed(status = get_my_conversation_bundle(id, m)))
     goto end;
   // read all conversations, so we can write them again
   if (meshms_failed(status = read_known_conversations(m, NULL, &conv)))
     goto end;
   // read the full list of conversations from the database too
-  if (meshms_failed(status = get_database_conversations(sender, NULL, &conv)))
+  if (meshms_failed(status = get_database_conversations(id, NULL, &conv)))
     goto end;
   // check if any incoming conversations need to be acked or have new messages and update the read offset
   unsigned changed = 0;
-  if (meshms_failed(status = update_conversations(sender, &conv)))
+  if (meshms_failed(status = update_conversations(id, &conv)))
     goto end;
   if (status == MESHMS_STATUS_UPDATED)
     changed = 1;
@@ -1086,28 +1107,24 @@ static int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_
     return -1;
     
   sid_t sid;
-  fromhex(sid.binary, sidhex, sizeof(sid.binary));
+  struct meshms_conversations *conv = NULL;
+  enum meshms_status status = MESHMS_STATUS_ERROR;
 
+  fromhex(sid.binary, sidhex, sizeof(sid.binary));
   int offset=atoi(offset_str);
   int count=atoi(count_str);
 
   if (create_serval_instance_dir() == -1)
-    return -1;
+    goto end;
   if (!(keyring = keyring_open_instance_cli(parsed)))
-    return -1;
+    goto end;
-  if (rhizome_opendb() == -1){
+
-    keyring_free(keyring);
+  if (rhizome_opendb() == -1)
-    keyring = NULL;
+    goto end;
-    return -1;
+
-  }
+  if (meshms_failed(status = meshms_conversations_list(NULL, &sid, NULL, &conv)))
+    goto end;
 
-  struct meshms_conversations *conv=NULL;
-  enum meshms_status status;
-  if (meshms_failed(status = meshms_conversations_list(&sid, NULL, &conv))) {
-    keyring_free(keyring);
-    keyring = NULL;
-    return status;
-  }
   const char *names[]={
     "_id","recipient","read", "last_message", "read_offset"
   };
@@ -1130,11 +1147,15 @@ static int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_
     }
   }
   cli_row_count(context, rows);
+  status=MESHMS_STATUS_OK;
 
+end:
+  if (conv)
     meshms_free_conversations(conv);
+  if (keyring)
     keyring_free(keyring);
   keyring = NULL;
-  return 0;
+  return status;
 }
 
 DEFINE_CMD(app_meshms_send_message, 0,

meshms.h

@@ -55,6 +55,7 @@ struct meshms_ply {
   uint64_t version;
   uint64_t tail;
   uint64_t size;
+  uint8_t found;
 };
 
 struct meshms_conversations {
@@ -63,10 +64,7 @@ struct meshms_conversations {
   // who are we talking to?
   sid_t them;
   
-  char found_my_ply;
   struct meshms_ply my_ply;
-  
-  char found_their_ply;
   struct meshms_ply their_ply;
   
   // what is the offset of their last message
@@ -95,7 +93,7 @@ struct meshms_ply_read {
 /* Fetch the list of all MeshMS conversations into a binary tree whose nodes
  * are all allocated by malloc(3).
  */
-enum meshms_status meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv);
+enum meshms_status meshms_conversations_list(const struct keyring_identity *id, const sid_t *my_sid, const sid_t *their_sid, struct meshms_conversations **conv);
 void meshms_free_conversations(struct meshms_conversations *conv);
 
 /* For iterating over a binary tree of all MeshMS conversations, as created by
@@ -134,6 +132,7 @@ void meshms_conversation_iterator_advance(struct meshms_conversation_iterator *)
  */
 struct meshms_message_iterator {
   // Public fields that remain fixed for the life of the iterator:
+  struct keyring_identity *identity;
   const sid_t *my_sid;
   const sid_t *their_sid;
   const rhizome_bid_t *my_ply_bid;

meshms_restful.c

@@ -209,7 +209,7 @@ static int restful_meshms_conversationlist_json(httpd_request *r, const char *re
   r->u.mclist.rowcount = 0;
   r->u.mclist.conv = NULL;
   enum meshms_status status;
-  if (meshms_failed(status = meshms_conversations_list(&r->sid1, NULL, &r->u.mclist.conv)))
+  if (meshms_failed(status = meshms_conversations_list(NULL, &r->sid1, NULL, &r->u.mclist.conv)))
     return http_request_meshms_response(r, 0, NULL, status);
   if (r->u.mclist.conv != NULL)
     meshms_conversation_iterator_start(&r->u.mclist.iter, r->u.mclist.conv);

overlay_mdp.c

@@ -440,7 +440,9 @@ static struct overlay_buffer *overlay_mdp_decrypt(struct internal_mdp_header *he
       
   case 0:
     {
-      unsigned char *k=keyring_get_nm_bytes(&header->destination->sid, &header->source->sid);
+      unsigned char *k=keyring_get_nm_bytes(header->destination->identity->box_sk,
+	header->destination->identity->box_pk,
+	&header->source->sid);
       if (!k){
 	WHY("I don't have the private key required to decrypt that");
 	break;
@@ -764,7 +766,7 @@ static struct overlay_buffer * encrypt_payload(
   
   /* get pre-computed PKxSK bytes (the slow part of auth-cryption that can be
      retained and reused, and use that to do the encryption quickly. */
-  unsigned char *k=keyring_get_nm_bytes(&source->sid, &dest->sid);
+  unsigned char *k=keyring_get_nm_bytes(source->identity->box_sk, source->identity->box_pk, &dest->sid);
   if (!k) {
     ob_free(ret);
     WHY("could not compute Curve25519(NxM)");

rhizome.c

@@ -320,7 +320,11 @@ struct rhizome_bundle_result rhizome_manifest_add_file(int appending,
     WARNF("Manifest 'service' field not supplied - setting to '%s'", RHIZOME_SERVICE_FILE);
     rhizome_manifest_set_service(new_manifest, RHIZOME_SERVICE_FILE);
   }
-  result = rhizome_fill_manifest(new_manifest, file_path, author);
+
+  if (author)
+    rhizome_manifest_set_author(m, author);
+
+  result = rhizome_fill_manifest(new_manifest, file_path);
 error:
   switch (result.status) {
   case RHIZOME_BUNDLE_STATUS_NEW:
@@ -520,76 +524,6 @@ end:
   return ret;
 }
 
-/* Sets the bundle key "BK" field of a manifest.  Returns 1 if the field was set, 0 if not.
- *
- * This function must not be called unless the bundle secret is known.
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-int rhizome_manifest_add_bundle_key(rhizome_manifest *m)
-{
-  IN();
-  assert(m->haveSecret);
-  switch (m->authorship) {
-    case ANONYMOUS: // there can be no BK field without an author
-    case AUTHOR_UNKNOWN: // we already know the author is not in the keyring
-    case AUTHENTICATION_ERROR: // already tried and failed to get Rhizome Secret
-      break;
-    case AUTHOR_NOT_CHECKED:
-    case AUTHOR_LOCAL:
-    case AUTHOR_AUTHENTIC:
-    case AUTHOR_IMPOSTOR: {
-	/* Set the BK using the provided author.  Serval Security Framework defines BK as being:
-	*    BK = privateKey XOR sha512(RS##BID)
-	* where BID = cryptoSignPublic, 
-	*       RS is the rhizome secret for the specified author. 
-	* The nice thing about this specification is that:
-	*    privateKey = BK XOR sha512(RS##BID)
-	* so the same function can be used to encrypt and decrypt the BK field.
-	*/
-	const unsigned char *rs;
-	size_t rs_len = 0;
-	enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs);
-	switch (d) {
-	  case FOUND_RHIZOME_SECRET: {
-	      rhizome_bk_t bkey;
-	      if (rhizome_secret2bk(&m->cryptoSignPublic, rs, rs_len, bkey.binary, m->cryptoSignSecret) == 0) {
-		rhizome_manifest_set_bundle_key(m, &bkey);
-		m->authorship = AUTHOR_AUTHENTIC;
-		RETURN(1);
-	      } else
-		m->authorship = AUTHENTICATION_ERROR;
-	    }
-	    break;
-	  case IDENTITY_NOT_FOUND:
-	    m->authorship = AUTHOR_UNKNOWN;
-	    break;
-	  case IDENTITY_HAS_NO_RHIZOME_SECRET:
-	    m->authorship = AUTHENTICATION_ERROR;
-	    break;
-	  default:
-	    FATALF("find_rhizome_secret() returned unknown code %d", (int)d);
-	    break;
-	}
-      }
-      break;
-    default:
-      FATALF("m->authorship = %d", (int)m->authorship);
-  }
-  rhizome_manifest_del_bundle_key(m);
-  switch (m->authorship) {
-    case AUTHOR_UNKNOWN:
-      INFOF("Cannot set BK because author=%s is not in keyring", alloca_tohex_sid_t(m->author));
-      break;
-    case AUTHENTICATION_ERROR:
-      WHY("Cannot set BK due to error");
-      break;
-    default:
-      break;
-  }
-  RETURN(0);
-}
-
 /* Test the status of a given manifest 'm' (id, version) with respect to the Rhizome store, and
  * return a code which indicates whether 'm' should be stored or not, setting *mout to 'm' or
  * to point to a newly allocated manifest.  The caller is responsible for freeing *mout if *mout !=

rhizome.h

@@ -214,6 +214,7 @@ typedef struct rhizome_manifest
    * have an ANY author (all zeros).
    */
   sid_t author;
+  const struct keyring_identity *author_identity;
 
   size_t manifest_body_bytes;
   size_t manifest_all_bytes;
@@ -254,7 +255,8 @@ typedef struct rhizome_manifest
 #define rhizome_manifest_set_crypt(m,v)         _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
 #define rhizome_manifest_set_rowid(m,v)         _rhizome_manifest_set_rowid(__WHENCE__,(m),(v))
 #define rhizome_manifest_set_inserttime(m,v)    _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v))
-#define rhizome_manifest_set_author(m,v)        _rhizome_manifest_set_author(__WHENCE__,(m),(v))
+#define rhizome_manifest_set_author(m,v)        _rhizome_manifest_set_author(__WHENCE__,(m),NULL,(v))
+#define rhizome_manifest_set_author_identity(m,v) _rhizome_manifest_set_author(__WHENCE__,(m),(v),NULL)
 #define rhizome_manifest_del_author(m)          _rhizome_manifest_del_author(__WHENCE__,(m))
 
 void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *);
@@ -280,7 +282,7 @@ void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *);
 void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
 void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t);
 void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t);
-void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
+void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const struct keyring_identity *, const sid_t *);
 void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
 
 #define rhizome_manifest_overwrite(dstm,srcm)   _rhizome_manifest_overwrite(__WHENCE__,(dstm),(srcm))
@@ -486,12 +488,11 @@ struct rhizome_bundle_result rhizome_manifest_add_file(int appending,
 int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath, int zip_files);
 
 int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath);
-struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp);
+struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const char *filepath);
 
 int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *);
 int rhizome_manifest_add_bundle_key(rhizome_manifest *);
 
-void rhizome_find_bundle_author_and_secret(rhizome_manifest *m);
 int rhizome_lookup_author(rhizome_manifest *m);
 void rhizome_authenticate_author(rhizome_manifest *m);
 
@@ -635,12 +636,6 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp);
 #define RHIZOME_VERIFY 1
 
 int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
-enum rhizome_secret_disposition {
-    FOUND_RHIZOME_SECRET = 0,
-    IDENTITY_NOT_FOUND,
-    IDENTITY_HAS_NO_RHIZOME_SECRET,
-};
-enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs);
 int rhizome_bk2secret(
   const rhizome_bid_t *bidp,
   const unsigned char *rs, const size_t rs_len,

rhizome_bundle.c

@@ -385,16 +385,24 @@ void _rhizome_manifest_set_inserttime(struct __sourceloc __whence, rhizome_manif
   m->inserttime = time;
 }
 
-void _rhizome_manifest_set_author(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp)
+void _rhizome_manifest_set_author(struct __sourceloc __whence, rhizome_manifest *m, const keyring_identity *id, const sid_t *sidp)
 {
-  if (sidp) {
+  if (id) {
-    if (m->authorship == ANONYMOUS || cmp_sid_t(&m->author, sidp) != 0) {
+    if (m->author_identity == id)
+      return;
+    sidp = id->box_pk;
+  } else if (sidp) {
+    if (m->authorship != ANONYMOUS && cmp_sid_t(&m->author, sidp) == 0)
+      return;
+  } else {
+    _rhizome_manifest_del_author(__whence, m);
+    return;
+  }
+
   DEBUGF(rhizome_manifest, "SET manifest %p author = %s", m, alloca_tohex_sid_t(*sidp));
   m->author = *sidp;
+  m->author_identity = id;
   m->authorship = AUTHOR_NOT_CHECKED;
-    }
-  } else
-    _rhizome_manifest_del_author(__whence, m);
 }
 
 void _rhizome_manifest_del_author(struct __sourceloc __whence, rhizome_manifest *m)
@@ -402,6 +410,7 @@ void _rhizome_manifest_del_author(struct __sourceloc __whence, rhizome_manifest
   if (m->authorship != ANONYMOUS) {
     DEBUGF(rhizome_manifest, "DEL manifest %p author", m);
     m->author = SID_ANY;
+    m->author_identity = NULL;
     m->authorship = ANONYMOUS;
   }
 }
@@ -1420,25 +1429,16 @@ int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepat
  * pointer to a "free" method (eg, free(3)) that must be called to release the string before the
  * pointer is discarded.
  */
-struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp)
+struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const char *filepath)
 {
   /* Set version of manifest from current time if not already set. */
   if (m->version == 0)
     rhizome_manifest_set_version(m, gettime_ms());
 
-  /* Set the manifest's author.  This must be done before binding to a new ID (below).  If the
-   * 'authorSidp' parameter was not set, then don't use the 'sender' field here, as we only want to
-   * authenticate an explicitly supplied author, not the sender.
-   */
-  if (authorSidp)
-    rhizome_manifest_set_author(m, authorSidp);
-
   /* Fill in the bundle secret and bundle ID.
    */
-  int valid_haveSecret = 0;
   switch (m->haveSecret) {
   case SECRET_UNKNOWN:
-    valid_haveSecret = 1;
     // If the Bundle Id is already known, then derive the bundle secret from BK if known.
     if (m->has_id) {
       DEBUGF(rhizome, "discover secret for bundle bid=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
@@ -1452,7 +1452,6 @@ struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const ch
     }
     // fall through to set the BK field...
   case NEW_BUNDLE_ID:
-    valid_haveSecret = 1;
     assert(m->has_id);
     // If no 'authorSidp' parameter was supplied but the manifest has a 'sender' field, then use the
     // sender as the author.
@@ -1465,42 +1464,35 @@ struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const ch
     }
     break;
   case EXISTING_BUNDLE_ID:
-    valid_haveSecret = 1;
     // If modifying an existing bundle, try to discover the bundle secret key and the author.
     assert(m->has_id);
     DEBUGF(rhizome, "modifying existing bundle bid=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
     rhizome_authenticate_author(m);
     // TODO assert that new version > old version?
-  }
+    break;
-  if (!valid_haveSecret)
+  default:
     FATALF("haveSecret = %d", m->haveSecret);
-  int valid_authorship = 0;
+  }
+
   switch (m->authorship) {
   case ANONYMOUS:
-    assert(!authorSidp);
-    valid_authorship = 1;
-    break;
   case AUTHOR_AUTHENTIC:
-    valid_authorship = 1;
+    break; // all good
-    break;
   case AUTHOR_UNKNOWN:
     return rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Author is not in keyring");
   case AUTHOR_IMPOSTOR:
     return rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Incorrect author");
-  case AUTHOR_NOT_CHECKED:
-  case AUTHOR_LOCAL:
-    FATALF("logic error (bug): m->authorship = %d", m->authorship);
   case AUTHENTICATION_ERROR:
     return rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_ERROR, "Error authenticating author");
-  }
+  default:
-  if (!valid_authorship)
     FATALF("m->authorship = %d", (int)m->authorship);
+  }
 
   /* Service field must already be set.
    */
-  if (m->service == NULL) {
+  if (m->service == NULL)
     return rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, "Missing 'service' field");
-  }
+
   DEBUGF(rhizome, "manifest contains service=%s", m->service);
 
   /* Fill in 'date' field to current time unless already set.
@@ -1525,11 +1517,10 @@ struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const ch
    * and encrypted by default.
    */
   if (   m->payloadEncryption == PAYLOAD_CRYPT_UNKNOWN
-      && m->has_sender
       && m->has_recipient
       && !is_sid_t_broadcast(m->recipient)
   ) {
-    DEBUGF(rhizome, "Implicitly adding payload encryption due to presense of sender & recipient fields");
+    DEBUGF(rhizome, "Implicitly adding payload encryption due to presense of recipient field");
     rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED);
   }
 

rhizome_crypto.c

@@ -172,95 +172,124 @@ int rhizome_secret2bk(
   OUT();
 }
 
-
+static keypair *get_secret(const keyring_identity *id)
-/* Given a SID, search the keyring for an identity with the same SID and return its Rhizome secret
- * if found.
- *
- * Returns FOUND_RHIZOME_SECRET if the author's rhizome secret is found; '*rs' is set to point to
- * the secret key in the keyring, and '*rs_len' is set to the key length.
- *
- * Returns IDENTITY_NOT_FOUND if the SID is not in the keyring.
- *
- * Returns IDENTITY_HAS_NO_RHIZOME_SECRET if the SID is in the keyring but has no Rhizome Secret.
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs)
 {
-  IN();
+  keypair *kp=keyring_identity_keytype(id, KEYTYPE_RHIZOME);
-  keyring_iterator it;
-  keyring_iterator_start(keyring, &it);
-  if (!keyring_find_sid(&it, authorSidp)) {
-    DEBUGF(rhizome, "identity sid=%s is not in keyring", alloca_tohex_sid_t(*authorSidp));
-    RETURN(IDENTITY_NOT_FOUND);
-  }
-  keypair *kp=keyring_identity_keytype(it.identity, KEYTYPE_RHIZOME);
   if (!kp) {
-    WARNF("Identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*authorSidp));
+    WARNF("Identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*id->box_pk));
-    RETURN(IDENTITY_HAS_NO_RHIZOME_SECRET);
+    return NULL;
   }
-  int rslen = kp->private_key_len;
+  assert(kp->private_key_len >= 16);
-  assert(rslen >= 16);
+  assert(kp->private_key_len <= 1024);
-  assert(rslen <= 1024);
+  return kp;
-  if (rs_len)
+}
-    *rs_len = rslen;
+
-  if (rs)
+/*
-    *rs = kp->private_key;
+ * If this identity has permission to alter the bundle, then set;
-  RETURN(FOUND_RHIZOME_SECRET);
+ *  - the manifest 'authorship' field to AUTHOR_AUTHENTIC
+ *  - the 'author' field to the SID of the identity
+ *  - the manifest 'cryptoSignSecret' field to the bundle secret key
+ *  - the 'haveSecret' field to EXISTING_BUNDLE_ID.
+ * and finally update the database with the result.
+*/
+static enum rhizome_bundle_authorship try_author(rhizome_manifest *m, const keyring_identity *id, const sid_t *sid){
+  if (!sid)
+    return AUTHOR_UNKNOWN;
+
+  if (!id){
+    id = keyring_find_identity(keyring, sid);
+    if (!id)
+      return AUTHOR_UNKNOWN;
+  }
+
+  if (m->has_bundle_key){
+    keypair *kp = get_secret(id);
+    if (!kp)
+      return AUTHENTICATION_ERROR;
+    uint8_t secret[crypto_sign_SECRETKEYBYTES];
+    uint8_t *s = m->haveSecret ? secret : m->cryptoSignSecret;
+    switch (rhizome_bk2secret(&m->cryptoSignPublic, kp->private_key, kp->private_key_len, m->bundle_key.binary, s)) {
+      case 0:
+	if (m->haveSecret && memcmp(secret, m->cryptoSignSecret, sizeof m->cryptoSignSecret) != 0)
+	  FATALF("Bundle secret does not match derived secret");
+	break;
+      case -1:
+	return AUTHENTICATION_ERROR;
+      default:
+	return AUTHOR_IMPOSTOR;
+    }
+  }else{
+    if (memcmp(&m->cryptoSignPublic, id->sign_pk, crypto_sign_PUBLICKEYBYTES)==0){
+      bcopy(id->sign_sk, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
+    }else{
+      DEBUGF(rhizome, "   bundle has no BK field");
+      // TODO if cryptoSignPublic == id signing key...
+      return ANONYMOUS;
+    }
+  }
+
+  if (m->rowid && m->authorship == ANONYMOUS){
+    // if this bundle is already in the database, update the author.
+    sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
+	"UPDATE MANIFESTS SET author = ? WHERE rowid = ?;",
+	SID_T, sid,
+	INT64, m->rowid,
+	END);
+  }
+
+  m->authorship = AUTHOR_AUTHENTIC;
+  m->author = *sid;
+  m->author_identity = id;
+  if (!m->haveSecret)
+    m->haveSecret = EXISTING_BUNDLE_ID;
+  return m->authorship;
 }
 
 /* Attempt to authenticate the authorship of the given bundle, and set the 'authorship' element
- * accordingly.  If the manifest has no BK field, then no authentication can be performed.
+ * accordingly.
  *
- * @author Andrew Bettison <andrew@servalproject.com>
+ * If an author has already been set, confirm it is valid.
+ *
+ * If the bundle has a sender, try that identity first.
+ *
+ * Otherwise test each identity in the keyring to discover the author of the bundle.
+ *
+ * If the manifest has no BK field, then we can only test if the bundle ID is equal to the identities signing key.
+ *
+ * If no identity is found in the keyring that combines with the bundle key (BK) field to yield
+ * the bundle's secret key, then leaves the manifest 'authorship' field as ANONYMOUS.
  */
+
 void rhizome_authenticate_author(rhizome_manifest *m)
 {
   IN();
   DEBUGF(rhizome, "authenticate author for bid=%s", m->has_id ? alloca_tohex_rhizome_bid_t(m->cryptoSignPublic) : "(none)");
   switch (m->authorship) {
     case ANONYMOUS:
-      DEBUGF(rhizome, "   manifest %p author unknown", m);
+
-      rhizome_find_bundle_author_and_secret(m);
+      assert(is_sid_t_any(m->author));
+
+      // Optimisation: try 'sender' SID first, if present.
+      if (m->has_sender && try_author(m, NULL, &m->sender) == AUTHOR_AUTHENTIC)
 	RETURNVOID;
+
+      keyring_iterator it;
+      keyring_iterator_start(keyring, &it);
+      keyring_identity *id;
+      while((id = keyring_next_identity(&it))){
+	// skip the sender if we've already tried it.
+	if (m->has_sender && cmp_sid_t(&m->sender, id->box_pk)==0)
+	  continue;
+	if (try_author(m, id, id->box_pk) == AUTHOR_AUTHENTIC)
+	  RETURNVOID;
+      }
+
+      RETURNVOID;
+
     case AUTHOR_NOT_CHECKED:
-    case AUTHOR_LOCAL: {
+    case AUTHOR_LOCAL:
-	DEBUGF(rhizome, "   manifest %p authenticate author=%s", m, alloca_tohex_sid_t(m->author));
+      m->authorship = try_author(m, m->author_identity, &m->author);
-	size_t rs_len;
-	const unsigned char *rs;
-	enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs);
-	switch (d) {
-	  case FOUND_RHIZOME_SECRET:
-	    DEBUGF(rhizome, "   author has Rhizome secret");
-	    switch (rhizome_bk2secret(&m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, m->cryptoSignSecret)) {
-	      case 0:
-		DEBUGF(rhizome, "   is authentic");
-		m->authorship = AUTHOR_AUTHENTIC;
-		if (!m->haveSecret)
-		  m->haveSecret = EXISTING_BUNDLE_ID;
-		break;
-	      case -1:
-		DEBUGF(rhizome, "   error");
-		m->authorship = AUTHENTICATION_ERROR;
-		break;
-	      default:
-		DEBUGF(rhizome, "   author is impostor");
-		m->authorship = AUTHOR_IMPOSTOR;
-		break;
-	    }
       RETURNVOID;
-	  case IDENTITY_NOT_FOUND:
-	    DEBUGF(rhizome, "   author not found");
-	    m->authorship = AUTHOR_UNKNOWN;
-	    RETURNVOID;
-	  case IDENTITY_HAS_NO_RHIZOME_SECRET:
-	    DEBUGF(rhizome, "   author has no Rhizome secret");
-	    m->authorship = AUTHENTICATION_ERROR;
-	    RETURNVOID;
-	}
-	FATALF("find_rhizome_secret() returned unknown code %d", (int)d);
-      }
-      break;
     case AUTHENTICATION_ERROR:
     case AUTHOR_UNKNOWN:
     case AUTHOR_IMPOSTOR:
@@ -271,6 +300,76 @@ void rhizome_authenticate_author(rhizome_manifest *m)
   FATALF("m->authorship = %d", (int)m->authorship);
 }
 
+/* Sets the bundle key "BK" field of a manifest.  Returns 1 if the field was set, 0 if not.
+ *
+ * This function must not be called unless the bundle secret is known.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+int rhizome_manifest_add_bundle_key(rhizome_manifest *m)
+{
+  IN();
+  assert(m->haveSecret);
+  switch (m->authorship) {
+    case ANONYMOUS: // there can be no BK field without an author
+    case AUTHOR_UNKNOWN: // we already know the author is not in the keyring
+    case AUTHENTICATION_ERROR: // already tried and failed to get Rhizome Secret
+      break;
+    case AUTHOR_NOT_CHECKED:
+    case AUTHOR_LOCAL:
+    case AUTHOR_AUTHENTIC:
+    case AUTHOR_IMPOSTOR: {
+	/* Set the BK using the provided author.  Serval Security Framework defines BK as being:
+	*    BK = privateKey XOR sha512(RS##BID)
+	* where BID = cryptoSignPublic,
+	*       RS is the rhizome secret for the specified author.
+	* The nice thing about this specification is that:
+	*    privateKey = BK XOR sha512(RS##BID)
+	* so the same function can be used to encrypt and decrypt the BK field.
+	*/
+
+	if (!m->author_identity){
+	  m->author_identity = keyring_find_identity(keyring, &m->author);
+	  if (!m->author_identity){
+	    m->authorship = AUTHOR_UNKNOWN;
+	    break;
+	  }
+	}
+
+	keypair *kp = get_secret(m->author_identity);
+	if (!kp){
+	  m->authorship = AUTHENTICATION_ERROR;
+	  break;
+	}
+
+	rhizome_bk_t bkey;
+	if (rhizome_secret2bk(&m->cryptoSignPublic, kp->private_key, kp->private_key_len, bkey.binary, m->cryptoSignSecret) != 0) {
+	  m->authorship = AUTHENTICATION_ERROR;
+	  break;
+	}
+
+	rhizome_manifest_set_bundle_key(m, &bkey);
+	m->authorship = AUTHOR_AUTHENTIC;
+	RETURN(1);
+      }
+      break;
+    default:
+      FATALF("m->authorship = %d", (int)m->authorship);
+  }
+  rhizome_manifest_del_bundle_key(m);
+  switch (m->authorship) {
+    case AUTHOR_UNKNOWN:
+      INFOF("Cannot set BK because author=%s is not in keyring", alloca_tohex_sid_t(m->author));
+      break;
+    case AUTHENTICATION_ERROR:
+      WHY("Cannot set BK due to error");
+      break;
+    default:
+      break;
+  }
+  RETURN(0);
+}
+
 /* If the given bundle secret key corresponds to the bundle's ID (public key) then store it in the
  * manifest structure and mark the secret key as known.  Return 1 if the secret key was assigned,
  * 0 if not.
@@ -305,122 +404,6 @@ int rhizome_apply_bundle_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
   OUT();
 }
 
-/* Return true if the bundle's BK field combined with the given Rhizome Secret produces the bundle's
- * secret key.
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-static int rhizome_secret_yields_bundle_secret(rhizome_manifest *m, const unsigned char *rs, size_t rs_len) {
-  assert(m->has_bundle_key);
-  if (rs_len < 16 || rs_len > 1024) {
-    // should a bad key be fatal??
-    WARNF("invalid Rhizome Secret: length=%zu", rs_len);
-    return 0;
-  }
-  unsigned char *secretp = m->haveSecret ? alloca(sizeof m->cryptoSignSecret) : m->cryptoSignSecret;
-  if (rhizome_bk2secret(&m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, secretp) == 0) {
-    if (m->haveSecret && memcmp(secretp, m->cryptoSignSecret, sizeof m->cryptoSignSecret) != 0)
-      FATALF("Bundle secret does not match derived secret");
-    return 1; // success
-  }
-  return 0;
-}
-
-/* Discover if the given manifest was created (signed) by any unlocked identity currently in the
- * keyring.
- *
- * If the authorship is already known (ie, not ANONYMOUS) then returns without changing anything.
- * That means this function can be called several times on the same manifest, but will only perform
- * any work the first time.
- *
- * If the manifest has no bundle key (BK) field, then it is anonymous, so leaves 'authorship'
- * unchanged and returns.
- *
- * If an identity is found in the keyring with permission to alter the bundle, then sets the
- * manifest 'authorship' field to AUTHOR_AUTHENTIC, the 'author' field to the SID of the identity,
- * the manifest 'cryptoSignSecret' field to the bundle secret key and the 'haveSecret' field to
- * EXISTING_BUNDLE_ID.
- *
- * If no identity is found in the keyring that combines with the bundle key (BK) field to yield
- * the bundle's secret key, then leaves the manifest 'authorship' field as ANONYMOUS.
- *
- * If an error occurs, eg, the keyring contains an invalid Rhizome Secret or a cryptographic
- * operation fails, then sets the 'authorship' field to AUTHENTICATION_ERROR and leaves the
- * 'author', 'haveSecret' and 'cryptoSignSecret' fields unchanged.
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-void rhizome_find_bundle_author_and_secret(rhizome_manifest *m)
-{
-  IN();
-  DEBUGF(rhizome, "Finding author and secret for bid=%s", m->has_id ? alloca_tohex_rhizome_bid_t(m->cryptoSignPublic) : "(none)");
-  if (m->authorship != ANONYMOUS) {
-    DEBUGF(rhizome, "   bundle author already found");
-    RETURNVOID;
-  }
-  assert(is_sid_t_any(m->author));
-  if (!m->has_bundle_key) {
-    DEBUGF(rhizome, "   bundle has no BK field");
-    RETURNVOID;
-  }
-  // Optimisation: try 'sender' SID first, if present.
-  const sid_t *author_sidp = NULL;
-  const unsigned char *sender_rs = NULL;
-  if (m->has_sender) {
-    size_t rs_len;
-    enum rhizome_secret_disposition d = find_rhizome_secret(&m->sender, &rs_len, &sender_rs);
-    switch (d) {
-      case FOUND_RHIZOME_SECRET:
-	DEBUGF(rhizome, "   sender has Rhizome secret");
-	if (rhizome_secret_yields_bundle_secret(m, sender_rs, rs_len)) {
-	  DEBUGF(rhizome, "   ... that matches!");
-	  author_sidp = &m->sender;
-	}
-	break;
-      case IDENTITY_NOT_FOUND:
-	DEBUGF(rhizome, "   sender not found");
-	break;
-      case IDENTITY_HAS_NO_RHIZOME_SECRET:
-	DEBUGF(rhizome, "   sender has no Rhizome secret");
-	break;
-    }
-  }
-  // If 'sender' SID does not work, try all the other identities in the keyring.
-  if (!author_sidp) {
-    keyring_iterator it;
-    keyring_iterator_start(keyring, &it);
-    keypair *kp;
-    while ((kp = keyring_next_keytype(&it, KEYTYPE_RHIZOME))) {
-      if (kp->private_key == sender_rs)
-	continue; // don't try the same identity again
-      if (rhizome_secret_yields_bundle_secret(m, kp->private_key, kp->private_key_len)) {
-	DEBUGF(rhizome, "   found matching Rhizome secret!");
-	author_sidp = keyring_identity_sid(it.identity);
-	if (author_sidp)
-	  break;
-	DEBUGF(rhizome, "   ... but its identity has no SID");
-      }
-    }
-  }
-  if (author_sidp) {
-    m->haveSecret = EXISTING_BUNDLE_ID;
-    DEBUGF(rhizome, "   found bundle author sid=%s", alloca_tohex_sid_t(*author_sidp));
-    rhizome_manifest_set_author(m, author_sidp);
-    m->authorship = AUTHOR_AUTHENTIC;
-    // if this bundle is already in the database, update the author.
-    if (m->rowid)
-      sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
-	  "UPDATE MANIFESTS SET author = ? WHERE rowid = ?;",
-	  SID_T, &m->author,
-	  INT64, m->rowid,
-	  END);
-  } else {
-    DEBUG(rhizome, "   bundle author not found");
-    assert(m->authorship == ANONYMOUS);
-  }
-  OUT();
-}
-
 /* Verify the validity of a given secret manifest key.  Return 1 if valid, 0 if not.
  */
 int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned char *pkin)
@@ -582,31 +565,48 @@ int rhizome_derive_payload_key(rhizome_manifest *m)
 {
   assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
   unsigned char hash[crypto_hash_sha512_BYTES];
-  if (m->has_sender && m->has_recipient) {
-    unsigned char *nm_bytes=NULL;
-    keyring_iterator it;
-    keyring_iterator_start(keyring, &it);
 
-    if (!keyring_find_sid(&it, &m->sender)){
+  if(m->has_recipient){
-      keyring_iterator_start(keyring, &it);
+    sid_t scratch;
-      if (!keyring_find_sid(&it, &m->recipient)){
+    const sid_t *other_pk = &m->recipient;
-	WARNF("Neither sender=%s nor recipient=%s is in keyring",
+    const sid_t *box_pk;
-	    alloca_tohex_sid_t(m->sender),
+    const uint8_t *box_sk = NULL;
-	    alloca_tohex_sid_t(m->recipient));
+
+    {
+      const keyring_identity *id=NULL;
+      id = keyring_find_identity(keyring, &m->recipient);
+      if (id){
+	if (m->has_sender){
+	  other_pk = &m->sender;
+	}else{
+	  // derive other_pk from BID
+	  other_pk = &scratch;
+	  if (crypto_sign_ed25519_pk_to_curve25519(scratch.binary, m->cryptoSignPublic.binary))
+	    other_pk = NULL;
+	}
+      } else if (m->has_sender){
+	id = keyring_find_identity(keyring, &m->sender);
+	// TODO error if sender != author?
+      } else if (m->haveSecret){
+	id = m->author_identity;
+      }
+      if (id){
+	box_pk = id->box_pk;
+	box_sk = id->box_sk;
+      }
+    }
+
+    if (!box_sk || !other_pk){
+      WARNF("Could not find known crypto secret for bundle");
       return 0;
     }
-      nm_bytes = keyring_get_nm_bytes(&m->recipient, &m->sender);
+
-      DEBUGF(rhizome, "derived payload key from recipient=%s* to sender=%s*",
+    unsigned char *nm_bytes=NULL;
-	     alloca_tohex_sid_t_trunc(m->recipient, 7),
+    nm_bytes = keyring_get_nm_bytes(box_sk, box_pk, other_pk);
-	     alloca_tohex_sid_t_trunc(m->sender, 7)
+    DEBUGF(rhizome, "derived payload key from known=%s*, unknown=%s*",
+	   alloca_tohex_sid_t_trunc(*box_pk, 7),
+	   alloca_tohex_sid_t_trunc(*other_pk, 7)
     );
-    }else{
-      nm_bytes = keyring_get_nm_bytes(&m->sender, &m->recipient);
-      DEBUGF(rhizome, "derived payload key from sender=%s* to recipient=%s*",
-	     alloca_tohex_sid_t_trunc(m->sender, 7),
-	     alloca_tohex_sid_t_trunc(m->recipient, 7)
-	    );
-    }
     assert(nm_bytes != NULL);
     crypto_hash_sha512(hash, nm_bytes, crypto_box_BEFORENMBYTES);
     

rhizome_direct_http.c

@@ -248,8 +248,9 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
     // If manifest template did not specify a service field, then by default it is "file".
     if (m->service == NULL)
       rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
-    const sid_t *author = is_sid_t_any(config.rhizome.api.addfile.default_author) ? NULL : &config.rhizome.api.addfile.default_author;
+    if (!is_sid_t_any(config.rhizome.api.addfile.default_author))
-    struct rhizome_bundle_result result = rhizome_fill_manifest(m, r->u.direct_import.data_file_name, author);
+      rhizome_manifest_set_author(m, &config.rhizome.api.addfile.default_author);
+    struct rhizome_bundle_result result = rhizome_fill_manifest(m, r->u.direct_import.data_file_name);
     rhizome_manifest *mout = NULL;
     if (result.status == RHIZOME_BUNDLE_STATUS_NEW) {
       rhizome_bundle_result_free(&result);

rhizome_fetch.c

@@ -1020,7 +1020,7 @@ static int rhizome_fetch_mdp_touch_timeout(struct rhizome_fetch_slot *slot)
   // For now, we will just make the timeout 1 second from the time of the last
   // received block.
   unschedule(&slot->alarm);
-  slot->alarm.alarm=gettime_ms()+config.rhizome.mdp_stall_timeout; 
+  slot->alarm.alarm=gettime_ms()+config.rhizome.mdp.stall_timeout;
   slot->alarm.deadline=slot->alarm.alarm+500;
   schedule(&slot->alarm);
   return 0;
@@ -1186,7 +1186,7 @@ static enum rhizome_start_fetch_result rhizome_fetch_switch_to_mdp(struct rhizom
   if (q)
     slot->mdpIdleTimeout *= 1+(q - rhizome_fetch_queues);
   
-  slot->mdpRXBlockLength = config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
+  slot->mdpRXBlockLength = config.rhizome.mdp.block_size; // Rhizome over MDP block size
   rhizome_fetch_mdp_requestblocks(slot);
 
   RETURN(STARTED);

tests/rhizomeops

@@ -966,29 +966,47 @@ test_EncryptedPayload() {
    assert ! diff file1 file1y
 }
 
-doc_RecipientIsEncrypted="Sender & recipient triggers encryption by default"
+doc_RecipientIsEncrypted="Setting recipient triggers encryption by default"
 setup_RecipientIsEncrypted() {
    A_IDENTITY_COUNT=2
    setup_servald
    setup_rhizome
-   echo "Clear Text" >file1
+   echo "Clear Text1" >file1
    echo -e "service=file\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
+   echo "Clear Text2" >file2
+   echo -e "service=file\nrecipient=$SIDA2" >file2.manifest
 }
 test_RecipientIsEncrypted() {
+# add file with sender & recipient
    executeOk_servald rhizome add file $SIDA1 file1 file1.manifest
    tfw_cat --stdout --stderr
    assert_stdout_add_file file1
    assert_manifest_complete file1.manifest
-   executeOk_servald rhizome list
+   assertGrep file1.manifest "crypt=1"
-   assert_rhizome_list --fromhere=1 file1
+# add file sith only recipient
-   extract_manifest_id BID file1.manifest
+   executeOk_servald rhizome add file $SIDA1 file2 file2.manifest
-   executeOk_servald rhizome extract file $BID file1x
+   tfw_cat --stdout --stderr
+   assert_stdout_add_file file2
+   assert_manifest_complete file2.manifest
+   assertGrep file2.manifest "crypt=1"
+# test that both files can be decrypted
+   extract_manifest_id BID1 file1.manifest
+   extract_manifest_filehash filehash1 file1.manifest
+   extract_manifest_id BID2 file2.manifest
+   extract_manifest_filehash filehash2 file2.manifest
+   executeOk_servald rhizome extract file $BID1 file1x
    tfw_cat --stdout --stderr
    assert diff file1 file1x
-   extract_manifest_filehash filehash file1.manifest
+   executeOk_servald rhizome extract file $BID2 file2x
-   executeOk_servald rhizome export file $filehash file1y
+   tfw_cat --stdout --stderr
+   assert diff file2 file2x
+# and that the stored files are obscured
+   executeOk_servald rhizome export file $filehash1 file1y
    tfw_cat file1 -v file1y
    assert ! diff file1 file1y
+   executeOk_servald rhizome export file $filehash2 file2y
+   tfw_cat file2 -v file2y
+   assert ! diff file2 file2y
 }
 
 doc_BroadcastNotEncrypted="Broadcast recipients are not encrypted by default"

tests/rhizomeprotocol

@@ -779,7 +779,7 @@ start_radio_instance() {
       set debug.rhizome_rx on \
       set debug.radio_link on \
       set rhizome.advertise.interval 5000 \
-      set rhizome.rhizome_mdp_block_size 375 \
+      set rhizome.mdp.block_size 375 \
       set log.console.level debug \
       set log.console.show_pid on \
       set log.console.show_time on \