Add MeshMB cli commands to follow and ignore feeds

meshmb.c

@@ -1,13 +1,455 @@
 #include "serval.h"
 #include "serval_types.h"
 #include "dataformats.h"
-#include "cli.h"
 #include "log.h"
 #include "debug.h"
-#include "instance.h"
 #include "conf.h"
-#include "commandline.h"
 #include "overlay_buffer.h"
+#include "keyring.h"
+#include "crypto.h"
+#include "mem.h"
+#include "meshmb.h"
+
+struct feed_metadata{
+  size_t tree_depth;
+  struct message_ply ply; // (ply starts with a rhizome_bid_t, so this is consistent with a nibble tree)
+  struct meshmb_feed_details details;
+  // what is the offset of their last message
+  uint64_t last_message_offset;
+  // what is the last message we processed?
+  uint64_t last_seen;
+  // our cached value for the last known size of their ply
+  uint64_t size;
+};
+
+struct meshmb_feeds{
+  struct tree_root root;
+  keyring_identity *id;
+  sign_keypair_t bundle_keypair;
+  bool_t dirty;
+};
+
+// only remember this many bytes of ply names & last messages
+#define MAX_NAME_LEN (256)  // ??
+#define MAX_MSG_LEN (256)  // ??
+
+static void update_stats(struct meshmb_feeds *feeds, struct feed_metadata *metadata, struct message_ply_read *reader)
+{
+  if (!metadata->ply.found){
+    // get the current size from the db
+    sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+    if (sqlite_exec_uint64_retry(&retry, &metadata->ply.size,
+      "SELECT filesize FROM manifests WHERE id = ?",
+      RHIZOME_BID_T, &metadata->ply.bundle_id,
+      END) == SQLITE_ROW)
+	metadata->ply.found = 1;
+    else
+      return;
+  }
+
+  if (metadata->size == metadata->ply.size)
+    return;
+
+  if (!message_ply_is_open(reader)
+    && message_ply_read_open(reader, &metadata->ply.bundle_id)!=0)
+    return;
+
+  // TODO remember if user has overridden the name?
+  if (metadata->details.name){
+    free((void*)metadata->details.name);
+    metadata->details.name = NULL;
+  }
+
+  if (reader->name){
+    size_t len = strlen(reader->name);
+    if (len >= MAX_NAME_LEN)
+      len = MAX_NAME_LEN -1;
+    metadata->details.name = strn_edup(reader->name, len);
+  }
+
+  reader->read.offset = reader->read.length;
+  time_s_t timestamp = 0;
+  while (message_ply_read_prev(reader) == 0){
+    if (reader->type == MESSAGE_BLOCK_TYPE_TIME){
+      if (reader->record_length<4){
+	WARN("Malformed ply, expected 4 byte timestamp");
+	continue;
+      }
+      timestamp = read_uint32(reader->record);
+
+    }else if(reader->type == MESSAGE_BLOCK_TYPE_MESSAGE){
+      if (metadata->last_message_offset == reader->record_end_offset)
+	break;
+
+      metadata->last_message_offset = reader->record_end_offset;
+
+      if (metadata->details.last_message)
+	free((void*)metadata->details.last_message);
+      size_t len = reader->record_length;
+      if (len >= MAX_MSG_LEN)
+	len = MAX_MSG_LEN -1;
+      metadata->details.last_message = strn_edup((const char *)reader->record, len);
+      metadata->details.timestamp = timestamp;
+      break;
+    }
+  }
+
+  metadata->size = metadata->ply.size;
+
+  // TODO assemble ACK list for unified reading....?
+
+  feeds->dirty=1;
+
+  return;
+}
+
+// TODO, might be quicker to fetch all meshmb bundles and test if they are in the feed list
+static int update_stats_tree(void **record, void *context)
+{
+  struct feed_metadata *metadata = (struct feed_metadata *)*record;
+  struct meshmb_feeds *feeds = (struct meshmb_feeds *)context;
+  struct message_ply_read reader;
+  bzero(&reader, sizeof(reader));
+  update_stats(feeds, metadata, &reader);
+  message_ply_read_close(&reader);
+  return 0;
+}
+
+// eg, if a bundle_add trigger occurs while the feed list is open
+void meshmb_bundle_update(struct meshmb_feeds *feeds, rhizome_manifest *m, struct message_ply_read *reader)
+{
+  struct feed_metadata *metadata;
+  if (strcmp(m->service, RHIZOME_SERVICE_MESHMB) == 0
+    && tree_find(&feeds->root, (void**)&metadata, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, NULL, NULL)==0){
+
+    metadata->ply.found = 1;
+    metadata->ply.size = m->filesize;
+
+    update_stats(feeds, metadata, reader);
+  }
+}
+
+int meshmb_update(struct meshmb_feeds *feeds)
+{
+  return tree_walk(&feeds->root, NULL, 0, update_stats_tree, feeds);
+}
+
+static int write_metadata(void **record, void *context)
+{
+  struct feed_metadata *metadata = (struct feed_metadata *)*record;
+  struct rhizome_write *write = (struct rhizome_write *)context;
+
+  assert(metadata->size >= metadata->last_message_offset);
+  assert(metadata->size >= metadata->last_seen);
+  unsigned name_len = (metadata->details.name ? strlen(metadata->details.name) : 0) + 1;
+  assert(name_len <= MAX_NAME_LEN);
+  unsigned msg_len = (metadata->details.last_message ? strlen(metadata->details.last_message) : 0) + 1;
+  assert(msg_len <= MAX_MSG_LEN);
+
+  uint8_t buffer[sizeof (rhizome_bid_t) + 1 + 12*4 + name_len + msg_len];
+  bcopy(metadata->ply.bundle_id.binary, buffer, sizeof (rhizome_bid_t));
+  size_t len = sizeof (rhizome_bid_t);
+  buffer[len++]=0;// flags?
+  len+=pack_uint(&buffer[len], metadata->size);
+  len+=pack_uint(&buffer[len], metadata->size - metadata->last_message_offset);
+  len+=pack_uint(&buffer[len], metadata->size - metadata->last_seen);
+  len+=pack_uint(&buffer[len], metadata->details.timestamp);
+  if (name_len > 1)
+    strncpy_nul((char *)&buffer[len], metadata->details.name, name_len);
+  else
+    buffer[len]=0;
+  len+=name_len;
+  if (msg_len > 1)
+    strncpy_nul((char *)&buffer[len], metadata->details.last_message, msg_len);
+  else
+    buffer[len]=0;
+  len+=msg_len;
+  assert(len < sizeof buffer);
+  DEBUGF(meshmb, "Write %u bytes of metadata for %s", len, alloca_tohex_rhizome_bid_t(metadata->ply.bundle_id));
+  return rhizome_write_buffer(write, buffer, len);
+}
+
+#define CURRENT_VERSION 0
+
+int meshmb_flush(struct meshmb_feeds *feeds)
+{
+  if (!feeds->dirty){
+    DEBUGF(meshmb, "Ignoring flush, not dirty");
+    return 0;
+  }
+
+  rhizome_manifest *mout = NULL;
+  rhizome_manifest *m = rhizome_new_manifest();
+  if (!m)
+    return -1;
+
+  int ret =-1;
+  struct rhizome_bundle_result result = rhizome_private_bundle(m, &feeds->bundle_keypair);
+
+  switch(result.status){
+    case RHIZOME_BUNDLE_STATUS_SAME:
+      rhizome_manifest_set_version(m, m->version + 1);
+      rhizome_manifest_set_filesize(m, RHIZOME_SIZE_UNSET);
+      rhizome_manifest_set_filehash(m, NULL);
+      // fallthrough
+    case RHIZOME_BUNDLE_STATUS_NEW:
+    {
+      struct rhizome_write write;
+      bzero(&write, sizeof(write));
+
+      enum rhizome_payload_status pstatus = rhizome_write_open_manifest(&write, m);
+      if (pstatus==RHIZOME_PAYLOAD_STATUS_NEW){
+	uint8_t version = CURRENT_VERSION;
+	rhizome_write_buffer(&write, &version, 1);
+
+	if (tree_walk(&feeds->root, NULL, 0, write_metadata, &write)==0){
+	  pstatus = rhizome_finish_write(&write);
+	  if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW){
+
+	    rhizome_manifest_set_filehash(m, &write.id);
+	    rhizome_manifest_set_filesize(m, write.file_length);
+	    struct rhizome_bundle_result result = rhizome_manifest_finalise(m, &mout, 1);
+	    if (result.status == RHIZOME_BUNDLE_STATUS_NEW){
+	      ret = 0;
+	      feeds->dirty = 0;
+	    }
+	    rhizome_bundle_result_free(&result);
+	  }
+	}
+      }
+      if (ret!=0)
+	rhizome_fail_write(&write);
+      break;
+    }
+    default:
+      break;
+  }
+
+  rhizome_manifest_free(m);
+  return ret;
+}
+
+static int free_feed(void **record, void *context)
+{
+  struct meshmb_feeds *feeds = (struct meshmb_feeds *)context;
+  struct feed_metadata *f = *record;
+  if (f->details.name)
+    free((void *)f->details.name);
+  if (f->details.last_message)
+    free((void *)f->details.last_message);
+  free(f);
+  DEBUGF(meshmb, "free feed");
+  *record = NULL;
+  feeds->dirty = 1;
+  return 0;
+}
+
+void meshmb_close(struct meshmb_feeds *feeds)
+{
+  tree_walk(&feeds->root, NULL, 0, free_feed, feeds);
+  free(feeds);
+}
+
+static void* alloc_feed(void *context, const uint8_t *binary, size_t UNUSED(bin_length))
+{
+  struct meshmb_feeds *feeds = (struct meshmb_feeds *)context;
+  struct feed_metadata *feed = emalloc_zero(sizeof(struct feed_metadata));
+  if (feed){
+    feed->ply.bundle_id = *(rhizome_bid_t *)binary;
+    feeds->dirty = 1;
+    DEBUGF(meshmb, "Allocated feed");
+  }
+  return feed;
+}
+
+static int read_metadata(struct meshmb_feeds *feeds, struct rhizome_read *read)
+{
+  struct rhizome_read_buffer buff;
+  bzero(&buff, sizeof(buff));
+  uint8_t buffer[sizeof (rhizome_bid_t) + 12*3 + MAX_NAME_LEN + MAX_MSG_LEN];
+
+  uint8_t version=0xFF;
+  if (rhizome_read_buffered(read, &buff, &version, 1)==-1)
+    return -1;
+
+  if (version > CURRENT_VERSION)
+    return WHYF("Unknown file format version (got 0x%02x)", version);
+
+  while(1){
+    ssize_t bytes = rhizome_read_buffered(read, &buff, buffer, sizeof buffer);
+    if (bytes==0)
+      break;
+
+    uint64_t delta=0;
+    uint64_t size;
+    uint64_t last_message_offset;
+    uint64_t last_seen;
+    uint64_t timestamp;
+
+    int unpacked;
+    const rhizome_bid_t *bid = (const rhizome_bid_t *)&buffer[0];
+    unsigned offset = sizeof(rhizome_bid_t);
+    if (offset >= (unsigned)bytes)
+      return WHY("Buffer overflow while parsing metadata");
+    //uint8_t flags = buffer[offset++];
+    offset++;
+    if (offset >= (unsigned)bytes)
+      return WHY("Buffer overflow while parsing metadata");
+
+    if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &size)) == -1)
+      return WHY("Buffer overflow while parsing metadata");
+    offset += unpacked;
+
+    if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &delta)) == -1)
+      return WHY("Buffer overflow while parsing metadata");
+    offset += unpacked;
+    last_message_offset = size - delta;
+
+    if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &delta)) == -1)
+      return WHY("Buffer overflow while parsing metadata");
+    offset += unpacked;
+    last_seen = size - delta;
+
+    if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &timestamp)) == -1)
+      return WHY("Buffer overflow while parsing metadata");
+    offset += unpacked;
+
+    const char *name = (const char *)&buffer[offset];
+    while(buffer[offset++]){
+      if (offset >= (unsigned)bytes)
+	return WHY("Buffer overflow while parsing metadata");
+    }
+
+    const char *msg = (const char *)&buffer[offset];
+    while(buffer[offset++]){
+      if (offset >= (unsigned)bytes)
+	return WHY("Buffer overflow while parsing metadata");
+    }
+
+    read->offset += offset - bytes;
+    struct feed_metadata *result;
+    if (tree_find(&feeds->root, (void**)&result, bid->binary, sizeof *bid, alloc_feed, feeds)<0)
+      return WHY("Failed to allocate metadata");
+
+    result->last_message_offset = last_message_offset;
+    result->last_seen = last_seen;
+    result->size = size;
+    result->details.bundle_id = *bid;
+    result->details.name = (name && *name) ? str_edup(name) : NULL;
+    result->details.last_message = (msg && *msg) ? str_edup(msg) : NULL;
+    result->details.timestamp = timestamp;
+
+    DEBUGF(meshmb, "Processed %u bytes of metadata for %s", offset, alloca_tohex_rhizome_bid_t(result->ply.bundle_id));
+  }
+  feeds->dirty = 0;
+  return 0;
+}
+
+int meshmb_open(keyring_identity *id, struct meshmb_feeds **feeds)
+{
+  int ret = -1;
+
+  *feeds = emalloc_zero(sizeof(struct meshmb_feeds));
+  if (*feeds){
+    (*feeds)->root.binary_length = sizeof(rhizome_bid_t);
+    (*feeds)->id = id;
+    rhizome_manifest *m = rhizome_new_manifest();
+    if (m){
+      crypto_seed_keypair(&(*feeds)->bundle_keypair,
+	"91656c3d62e9fe2678a1a81fabe3f413%s5a37120ca55d911634560e4d4dc1283f",
+	alloca_tohex(id->sign_keypair->private_key.binary, sizeof id->sign_keypair->private_key));
+      struct rhizome_bundle_result result = rhizome_private_bundle(m, &(*feeds)->bundle_keypair);
+      DEBUGF(meshmb, "Private bundle %s, %s",
+	alloca_tohex_identity_t(&(*feeds)->bundle_keypair.public_key),
+	alloca_rhizome_bundle_result(result));
+      switch(result.status){
+	case RHIZOME_BUNDLE_STATUS_SAME:{
+	  struct rhizome_read read;
+	  bzero(&read, sizeof(read));
+
+	  enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &read);
+	  if (pstatus == RHIZOME_PAYLOAD_STATUS_STORED){
+	    if (read_metadata(*feeds, &read)==-1)
+	      WHYF("Failed to read metadata");
+	    else
+	      ret = 0;
+	  }else
+	    WHYF("Failed to read metadata: %s", rhizome_payload_status_message(pstatus));
+
+	  rhizome_read_close(&read);
+	}break;
+
+	case RHIZOME_BUNDLE_STATUS_NEW:
+	  ret = 0;
+	  break;
+
+	case RHIZOME_BUNDLE_STATUS_BUSY:
+	  break;
+
+	default:
+	  // everything else should be impossible.
+	  FATALF("Cannot create manifest: %s", alloca_rhizome_bundle_result(result));
+      }
+
+      rhizome_bundle_result_free(&result);
+    }
+
+    rhizome_manifest_free(m);
+  }
+
+  if (ret!=0){
+    meshmb_close(*feeds);
+    *feeds=NULL;
+  }
+  return ret;
+}
+
+int meshmb_follow(struct meshmb_feeds *feeds, rhizome_bid_t *bid)
+{
+  struct feed_metadata *metadata;
+  DEBUGF(meshmb, "Attempting to follow %s", alloca_tohex_rhizome_bid_t(*bid));
+
+  // TODO load the manifest and check the service!
+
+  if (tree_find(&feeds->root, (void**)&metadata, bid->binary, sizeof *bid, alloc_feed, feeds)!=TREE_FOUND)
+    return WHYF("Failed to follow feed");
+
+  struct message_ply_read reader;
+  bzero(&reader, sizeof(reader));
+  update_stats(feeds, metadata, &reader);
+  message_ply_read_close(&reader);
+  return 0;
+}
+
+int meshmb_ignore(struct meshmb_feeds *feeds, rhizome_bid_t *bid)
+{
+  DEBUGF(meshmb, "Attempting to ignore %s", alloca_tohex_rhizome_bid_t(*bid));
+  tree_walk_prefix(&feeds->root, bid->binary, sizeof *bid, free_feed, feeds);
+  return 0;
+}
+
+struct enum_context{
+  meshmb_callback callback;
+  void *context;
+};
+
+static int enum_callback(void **record, void *context)
+{
+  struct feed_metadata *feed = *record;
+  struct enum_context *enum_context = context;
+  return enum_context->callback(&feed->details, enum_context->context);
+}
+
+int meshmb_enum(struct meshmb_feeds *feeds, rhizome_bid_t *restart_from, meshmb_callback callback, void *context)
+{
+  DEBUGF(meshmb, "Enumerating feeds from %s",
+    restart_from?alloca_tohex_rhizome_bid_t(*restart_from):"the beginning");
+  struct enum_context enum_context = {
+    .callback = callback,
+    .context = context
+  };
+  return tree_walk(&feeds->root, restart_from ? restart_from->binary : NULL, sizeof *restart_from, enum_callback, &enum_context);
+}
 
 int meshmb_send(const keyring_identity *id, const char *message, size_t message_len,
   unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments){
@@ -30,184 +472,3 @@ int meshmb_send(const keyring_identity *id, const char *message, size_t message_
 
   return ret;
 }
-
-DEFINE_FEATURE(cli_meshmb);
-
-DEFINE_CMD(app_meshmb_send, 0,
-  "Append a public broadcast message to your feed",
-  "meshmb", "send" KEYRING_PIN_OPTIONS, "<id>", "<message>", "...");
-static int app_meshmb_send(const struct cli_parsed *parsed, struct cli_context *UNUSED(context))
-{
-  const char *idhex, *message;
-  if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1
-    || cli_arg(parsed, "message", &message, NULL, "") == -1)
-    return -1;
-
-  unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi;
-  struct rhizome_manifest_field_assignment fields[nfields];
-
-  if (nfields){
-    if (rhizome_parse_field_assignments(fields, nfields, parsed->args + parsed->varargi)==-1)
-      return -1;
-  }
-
-  identity_t identity;
-  if (str_to_identity_t(&identity, idhex) == -1)
-    return WHY("Invalid identity");
-
-  if (create_serval_instance_dir() == -1)
-    return -1;
-  if (rhizome_opendb() == -1)
-    return -1;
-  assert(keyring == NULL);
-  if (!(keyring = keyring_open_instance_cli(parsed)))
-    return -1;
-
-  keyring_identity *id = keyring_find_identity(keyring, &identity);
-  if (!id)
-    return WHY("Invalid identity");
-
-  return meshmb_send(id, message, strlen(message)+1, nfields, fields);
-}
-
-DEFINE_CMD(app_meshmb_read, 0,
-  "Read a broadcast message feed.",
-  "meshmb", "read", "<id>");
-static int app_meshmb_read(const struct cli_parsed *parsed, struct cli_context *context)
-{
-  const char *hex_id;
-  if (cli_arg(parsed, "id", &hex_id, str_is_identity, "") == -1)
-    return -1;
-
-  rhizome_bid_t bid;
-  if (str_to_rhizome_bid_t(&bid, hex_id) == -1)
-    return WHY("Invalid Identity");
-
-  /* Ensure the Rhizome database exists and is open */
-  if (create_serval_instance_dir() == -1)
-    return -1;
-  if (rhizome_opendb() == -1)
-    return -1;
-
-  struct message_ply_read read;
-  bzero(&read, sizeof read);
-
-  if (message_ply_read_open(&read, &bid)==-1)
-    return -1;
-
-  int ret=0;
-  size_t row_id = 0;
-  const char *names[]={
-    "_id","offset","age","message"
-  };
-  cli_start_table(context, NELS(names), names);
-  time_s_t timestamp = 0;
-  time_s_t now = gettime();
-
-  while(message_ply_read_prev(&read)==0){
-    switch(read.type){
-      case MESSAGE_BLOCK_TYPE_TIME:
-	if (read.record_length<4){
-	  WARN("Malformed ply, expected 4 byte timestamp");
-	  continue;
-	}
-	timestamp = read_uint32(read.record);
-	break;
-
-      case MESSAGE_BLOCK_TYPE_MESSAGE:
-	cli_put_long(context, row_id++, ":");
-	cli_put_long(context, read.record_end_offset, ":");
-	cli_put_long(context, timestamp ? (long)(now - timestamp) : (long)-1, ":");
-	cli_put_string(context, (const char *)read.record, "\n");
-
-	break;
-
-      case MESSAGE_BLOCK_TYPE_ACK:
-	// TODO, link to some other ply?
-	break;
-
-      default:
-	//ignore unknown types
-	break;
-    }
-  }
-  cli_end_table(context, row_id);
-
-  message_ply_read_close(&read);
-  return ret;
-}
-
-DEFINE_CMD(app_meshmb_find, 0,
-  "Browse available broadcast message feeds",
-  "meshmb", "find", "[<search>]");
-static int app_meshmb_find(const struct cli_parsed *parsed, struct cli_context *context)
-{
-  const char *search=NULL;
-  cli_arg(parsed, "search", &search, NULL, "");
-  // Ensure the Rhizome database exists and is open
-  if (create_serval_instance_dir() == -1)
-    return -1;
-  if (rhizome_opendb() == -1)
-    return -1;
-
-  struct rhizome_list_cursor cursor;
-  bzero(&cursor, sizeof cursor);
-  cursor.service = RHIZOME_SERVICE_MESHMB;
-  cursor.name = search && search[0] ? search : NULL;
-
-  //TODO hide feeds that have been blocked
-
-  if (rhizome_list_open(&cursor) == -1)
-    return -1;
-
-  const char *names[]={
-    "_id",
-    "id",
-    "version",
-    "date",
-    "name"
-  };
-  cli_start_table(context, NELS(names), names);
-
-  unsigned rowcount=0;
-  int n;
-
-  while ((n = rhizome_list_next(&cursor)) == 1) {
-    rowcount++;
-    rhizome_manifest *m = cursor.manifest;
-    cli_put_long(context, m->rowid, ":");
-    cli_put_hexvalue(context, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, ":");
-    cli_put_long(context, m->version, ":");
-    cli_put_long(context, m->has_date ? m->date : 0, ":");
-    cli_put_string(context, m->name, "\n");
-  }
-  rhizome_list_release(&cursor);
-  cli_end_table(context, rowcount);
-  return 0;
-}
-
-/*
-DEFINE_CMD(app_meshmb_follow, 0,
-  "",
-  "meshmb", "follow|ignore|block" KEYRING_PIN_OPTIONS, "<id>", "<peer>");
-static int app_meshmb_follow(const struct cli_parsed *parsed, struct cli_context *context)
-{
-  return 0;
-}
-
-DEFINE_CMD(app_meshmb_list, 0,
-  "",
-  "meshmb", "list", "following|blocked" KEYRING_PIN_OPTIONS, "--last-message", "<id>");
-static int app_meshmb_list(const struct cli_parsed *parsed, struct cli_context *context)
-{
-  return 0;
-}
-
-DEFINE_CMD(app_meshmb_news, 0,
-  "",
-  "meshmb", "news" KEYRING_PIN_OPTIONS, "<sid>");
-static int app_meshmb_news(const struct cli_parsed *parsed, struct cli_context *context)
-{
-  return 0;
-}
-*/

meshmb.h

@@ -1,7 +1,44 @@
 #ifndef __SERVAL_DNA__MESHMB_H
 #define __SERVAL_DNA__MESHMB_H
 
+struct meshmb_feeds;
+
+enum meshmb_send_status{
+  MESHMB_ERROR = -1,
+  MESHMB_OK = 0,
+};
+
+// details of a feed that you are following
+struct meshmb_feed_details{
+  rhizome_bid_t bundle_id;
+  const char *name;
+  const char *last_message;
+  time_s_t timestamp;
+};
+
+struct rhizome_manifest_field_assignment;
 int meshmb_send(const keyring_identity *id, const char *message, size_t message_len,
   unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments);
 
+// feed tracking
+int meshmb_open(keyring_identity *id, struct meshmb_feeds **feeds);
+void meshmb_close(struct meshmb_feeds *feeds);
+
+// re-write metadata if required
+int meshmb_flush(struct meshmb_feeds *feeds);
+
+// set / clear follow flag for this feed
+int meshmb_follow(struct meshmb_feeds *feeds, rhizome_bid_t *bid);
+int meshmb_ignore(struct meshmb_feeds *feeds, rhizome_bid_t *bid);
+
+// enumerate feeds, starting from restart_from
+typedef int (*meshmb_callback) (struct meshmb_feed_details *details, void *context);
+int meshmb_enum(struct meshmb_feeds *feeds, rhizome_bid_t *restart_from, meshmb_callback callback, void *context);
+
+// update metadata of all feeds based on current rhizome contents (optionally call after opening)
+int meshmb_update(struct meshmb_feeds *feeds);
+// update metadata of a single feed, eg because of a new bundle or when about to read a single ply.
+// it is the callers reponsibility to supply a reader and close it
+void meshmb_bundle_update(struct meshmb_feeds *feeds, rhizome_manifest *m, struct message_ply_read *reader);
+
 #endif

meshmb_cli.c

@@ -0,0 +1,305 @@
+#include "serval.h"
+#include "serval_types.h"
+#include "dataformats.h"
+#include "cli.h"
+#include "log.h"
+#include "debug.h"
+#include "instance.h"
+#include "commandline.h"
+#include "keyring.h"
+#include "rhizome.h"
+#include "message_ply.h"
+#include "meshmb.h"
+#include "feature.h"
+
+DEFINE_FEATURE(cli_meshmb);
+
+DEFINE_CMD(app_meshmb_send, 0,
+  "Append a public broadcast message to your feed",
+  "meshmb", "send" KEYRING_PIN_OPTIONS, "<id>", "<message>", "...");
+static int app_meshmb_send(const struct cli_parsed *parsed, struct cli_context *UNUSED(context))
+{
+  const char *idhex, *message;
+  if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1
+    || cli_arg(parsed, "message", &message, NULL, "") == -1)
+    return -1;
+
+  unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi;
+  struct rhizome_manifest_field_assignment fields[nfields];
+
+  if (nfields){
+    if (rhizome_parse_field_assignments(fields, nfields, parsed->args + parsed->varargi)==-1)
+      return -1;
+  }
+
+  identity_t identity;
+  if (str_to_identity_t(&identity, idhex) == -1)
+    return WHY("Invalid identity");
+
+  if (create_serval_instance_dir() == -1
+    || rhizome_opendb() == -1
+    || !(keyring = keyring_open_instance_cli(parsed)))
+    return -1;
+
+  keyring_identity *id = keyring_find_identity(keyring, &identity);
+  if (!id)
+    return WHY("Invalid identity");
+
+  return meshmb_send(id, message, strlen(message)+1, nfields, fields);
+}
+
+// TODO from offset....?
+DEFINE_CMD(app_meshmb_read, 0,
+  "Read all messages in a broadcast message feed.",
+  "meshmb", "read", "<id>");
+static int app_meshmb_read(const struct cli_parsed *parsed, struct cli_context *context)
+{
+  const char *hex_id;
+  if (cli_arg(parsed, "id", &hex_id, str_is_identity, "") == -1)
+    return -1;
+
+  rhizome_bid_t bid;
+  if (str_to_rhizome_bid_t(&bid, hex_id) == -1)
+    return WHY("Invalid Identity");
+
+  /* Ensure the Rhizome database exists and is open */
+  if (create_serval_instance_dir() == -1
+    || rhizome_opendb() == -1)
+    return -1;
+
+  struct message_ply_read read;
+  bzero(&read, sizeof read);
+
+  if (message_ply_read_open(&read, &bid)==-1)
+    return -1;
+
+  int ret=0;
+  size_t row_id = 0;
+  const char *names[]={
+    "_id","offset","age","message"
+  };
+  cli_start_table(context, NELS(names), names);
+  time_s_t timestamp = 0;
+  time_s_t now = gettime();
+
+  while(message_ply_read_prev(&read)==0){
+    switch(read.type){
+      case MESSAGE_BLOCK_TYPE_TIME:
+	if (read.record_length<4){
+	  WARN("Malformed ply, expected 4 byte timestamp");
+	  continue;
+	}
+	timestamp = read_uint32(read.record);
+	break;
+
+      case MESSAGE_BLOCK_TYPE_MESSAGE:
+	cli_put_long(context, row_id++, ":");
+	cli_put_long(context, read.record_end_offset, ":");
+	cli_put_long(context, timestamp ? (long)(now - timestamp) : (long)-1, ":");
+	cli_put_string(context, (const char *)read.record, "\n");
+
+	break;
+
+      case MESSAGE_BLOCK_TYPE_ACK:
+	// TODO, link to some other ply?
+	break;
+
+      default:
+	//ignore unknown types
+	break;
+    }
+  }
+  cli_end_table(context, row_id);
+
+  message_ply_read_close(&read);
+  return ret;
+}
+
+DEFINE_CMD(app_meshmb_find, 0,
+  "Browse available broadcast message feeds",
+  "meshmb", "find", "[<search>]");
+static int app_meshmb_find(const struct cli_parsed *parsed, struct cli_context *context)
+{
+  const char *search=NULL;
+  cli_arg(parsed, "search", &search, NULL, "");
+  // Ensure the Rhizome database exists and is open
+  if (create_serval_instance_dir() == -1
+    || rhizome_opendb() == -1)
+    return -1;
+
+  struct rhizome_list_cursor cursor;
+  bzero(&cursor, sizeof cursor);
+  cursor.service = RHIZOME_SERVICE_MESHMB;
+  cursor.name = search && search[0] ? search : NULL;
+
+  //TODO hide feeds that have been blocked
+
+  if (rhizome_list_open(&cursor) == -1)
+    return -1;
+
+  const char *names[]={
+    "_id",
+    "id",
+    "version",
+    "date",
+    "name"
+  };
+  cli_start_table(context, NELS(names), names);
+
+  unsigned rowcount=0;
+  int n;
+
+  while ((n = rhizome_list_next(&cursor)) == 1) {
+    rowcount++;
+    rhizome_manifest *m = cursor.manifest;
+    cli_put_long(context, m->rowid, ":");
+    cli_put_hexvalue(context, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, ":");
+    cli_put_long(context, m->version, ":");
+    cli_put_long(context, m->has_date ? m->date : 0, ":");
+    cli_put_string(context, m->name, "\n");
+  }
+  rhizome_list_release(&cursor);
+  cli_end_table(context, rowcount);
+  return 0;
+}
+
+
+DEFINE_CMD(app_meshmb_follow, 0,
+  "Start or stop following a broadcast feed",
+  "meshmb", "follow|ignore" KEYRING_PIN_OPTIONS, "<id>", "<peer>");
+static int app_meshmb_follow(const struct cli_parsed *parsed, struct cli_context *UNUSED(context))
+{
+  const char *idhex, *peerhex;
+  if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1
+    ||cli_arg(parsed, "peer", &peerhex, str_is_identity, "") == -1)
+    return -1;
+
+  int follow = cli_arg(parsed, "follow", NULL, NULL, NULL) == 0;
+
+  identity_t identity;
+  identity_t peer;
+  if (str_to_identity_t(&identity, idhex) == -1
+    ||str_to_identity_t(&peer, peerhex) == -1)
+    return WHY("Invalid identity");
+
+  if (create_serval_instance_dir() == -1
+    || rhizome_opendb() == -1
+    || !(keyring = keyring_open_instance_cli(parsed)))
+    return -1;
+
+  int ret = -1;
+  struct meshmb_feeds *feeds = NULL;
+
+  keyring_identity *id = keyring_find_identity(keyring, &identity);
+  if (!id){
+    WHY("Invalid identity");
+    goto end;
+  }
+
+  if (meshmb_open(id, &feeds)==-1)
+    goto end;
+
+  if (follow){
+    if (meshmb_follow(feeds, &peer)==-1)
+      goto end;
+  }else{
+    if (meshmb_ignore(feeds, &peer)==-1)
+      goto end;
+  }
+
+  if (meshmb_flush(feeds)==-1)
+    goto end;
+
+  ret = 0;
+
+end:
+  if (feeds)
+    meshmb_close(feeds);
+  if (keyring)
+    keyring_free(keyring);
+  keyring = NULL;
+  return ret;
+}
+
+struct cli_enum_context{
+  unsigned rowcount;
+  struct cli_context *context;
+};
+
+static int list_callback(struct meshmb_feed_details *details, void *context)
+{
+  struct cli_enum_context *enum_context = context;
+  enum_context->rowcount++;
+  cli_put_long(enum_context->context, enum_context->rowcount, ":");
+  cli_put_string(enum_context->context, alloca_tohex_rhizome_bid_t(details->bundle_id), ":");
+  cli_put_string(enum_context->context, details->name, ":");
+  cli_put_long(enum_context->context, details->timestamp ? (long)(gettime() - details->timestamp) : (long)-1, ":");
+  cli_put_string(enum_context->context, details->last_message, "\n");
+  return 0;
+}
+
+DEFINE_CMD(app_meshmb_list, 0,
+  "List the feeds that you are currently following",
+  "meshmb", "list", "following" KEYRING_PIN_OPTIONS, "<id>");
+static int app_meshmb_list(const struct cli_parsed *parsed, struct cli_context *context)
+{
+  const char *idhex;
+  if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1)
+    return -1;
+
+  identity_t identity;
+  if (str_to_identity_t(&identity, idhex) == -1)
+    return WHY("Invalid identity");
+
+  if (create_serval_instance_dir() == -1
+    || rhizome_opendb() == -1
+    || !(keyring = keyring_open_instance_cli(parsed)))
+    return -1;
+
+  int ret = -1;
+  struct meshmb_feeds *feeds = NULL;
+
+  keyring_identity *id = keyring_find_identity(keyring, &identity);
+  if (!id){
+    WHY("Invalid identity");
+    goto end;
+  }
+
+  if (meshmb_open(id, &feeds)==-1)
+    goto end;
+
+  const char *names[]={
+    "_id",
+    "id",
+    "name",
+    "age",
+    "last_message"
+  };
+  cli_start_table(context, NELS(names), names);
+  struct cli_enum_context enum_context = {
+    .rowcount = 0,
+    .context = context,
+  };
+
+  meshmb_enum(feeds, NULL, list_callback, &enum_context);
+
+  cli_end_table(context, enum_context.rowcount);
+  ret = 0;
+
+end:
+  if (feeds)
+    meshmb_close(feeds);
+  keyring_free(keyring);
+  keyring = NULL;
+  return 0;
+}
+
+/*
+DEFINE_CMD(app_meshmb_news, 0,
+  "",
+  "meshmb", "news" KEYRING_PIN_OPTIONS, "<id>");
+static int app_meshmb_news(const struct cli_parsed *parsed, struct cli_context *context)
+{
+  return 0;
+}
+*/

meshmb_restful.c

@@ -6,6 +6,7 @@
 #include "numeric_str.h"
 #include "base64.h"
 #include "strbuf_helpers.h"
+#include "keyring.h"
 #include "meshmb.h"
 
 DEFINE_FEATURE(http_rest_meshmb);
@@ -344,6 +345,47 @@ static int restful_meshmb_newsince_list(httpd_request *r, const char *remainder)
   return ret;
 }
 
+/*
+static char *find_token_to_str(char *buf, uint64_t rowid)
+{
+  struct iovec iov[2];
+  iov[0].iov_base = rhizome_db_uuid.u.binary;
+  iov[0].iov_len = sizeof rhizome_db_uuid.u.binary;
+  iov[1].iov_base = &rowid;
+  iov[1].iov_len = sizeof rowid;
+  size_t n = base64url_encodev(buf, iov, 2);
+  assert(n == LIST_TOKEN_STRLEN);
+  buf[n] = '\0';
+  return buf;
+}
+
+static int strn_to_find_token(const char *str, uint64_t *rowidp, const char **afterp)
+{
+  unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp];
+  if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token
+    && cmp_uuid_t(&rhizome_db_uuid, (serval_uuid_t *) &token) == 0
+    && **afterp=='/'){
+    memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp);
+    (*afterp)++;
+  }else{
+    // don't skip the token
+    *afterp=str;
+    *rowidp=1;
+  }
+  return 1;
+}
+
+static int restful_meshmb_find(httpd_request *r, const char *remainder)
+{
+  return -1;
+}
+
+static int restful_meshmb_newsince_find(httpd_request *r, const char *remainder)
+{
+  return -1;
+}
+*/
+
 DECLARE_HANDLER("/restful/meshmb/", restful_meshmb_);
 static int restful_meshmb_(httpd_request *r, const char *remainder)
 {
@@ -374,6 +416,16 @@ static int restful_meshmb_(httpd_request *r, const char *remainder)
       handler = restful_meshmb_newsince_list;
       remainder = "";
     }
+/*
+  } else if(strcmp(remainder, "/find.json") == 0) {
+    handler = restful_meshmb_find;
+    remainder = "";
+  } else if (   str_startswith(remainder, "/newsince/", &end) {
+	     && strn_to_find_token(end, &r->ui64, &end)
+	     && strcmp(end, "find.json") == 0) {
+    handler = restful_meshmb_newsince_find;
+    remainder = "";
+    */
   }
 
   if (handler == NULL)

nibble_tree.c

@@ -61,7 +61,7 @@ enum tree_error_reason tree_find(struct tree_root *root, void **result, const ui
 	if (!node_ptr)
 	  return TREE_ERROR;
 	struct tree_record *tree_record = (struct tree_record *)node_ptr;
-	// TODO assert(memcmp(tree_record->binary, binary, bin_length) == 0))?
+	assert(memcmp(tree_record->binary, binary, bin_length) == 0);
 	tree_record ->tree_depth = pos*4;
 	if (result)
 	  *result = node_ptr;
@@ -134,13 +134,15 @@ static int walk(struct tree_node *node, unsigned pos,
       return ret;
     if (node->tree_nodes[i])
       *empty=0;
-    // stop comparing the start sid after looking at the first branch of the tree
+    // stop comparing the start binary after looking at the first branch of the tree
     binary=NULL;
   }
 
   return ret;
 }
 
+// start enumerating the tree from binary, and continue until the end
+// callback is allowed to free any nodes while the walk is in progress
 int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context)
 {
   assert(!binary || bin_length <= root->binary_length);
@@ -151,12 +153,15 @@ int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length,
 int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context)
 {
   assert(bin_length <= root->binary_length);
+  //TODO if callback free's nodes, collapse parent tree nodes too without needing to walk again?
   struct tree_node *node = &root->_root_node;
   unsigned pos=0;
+  // look for a branch of the tree with a partial match
   for (; node && pos<bin_length*2; pos++){
     uint8_t i=get_nibble(binary, pos);
     if ((node->is_tree & (1<<i))==0){
       struct tree_record *tree_record = (struct tree_record *)node->tree_nodes[i];
+      // only one match?
       if (tree_record && memcmp(tree_record->binary, binary, bin_length)==0){
 	return callback(&node->tree_nodes[i], context);
       }
@@ -164,6 +169,7 @@ int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_l
     }
     node = node->tree_nodes[i];
   }
+  // walk the whole branch
   uint8_t ignore;
   return walk(node, pos+1, &ignore, NULL, 0, callback, context);
 }

nibble_tree.h

@@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 struct tree_record{
   // number of bits of the binary value, to uniquely identify this record within the tree's current contents
-  unsigned tree_depth;
+  size_t tree_depth;
   uint8_t binary[0];
 };
 

overlay_address.h

@@ -56,7 +56,7 @@ struct overlay_buffer;
 struct subscriber{
   // minimum abbreviation length, in bits.
   // Note this must be here to match the memory layout of struct tree_record
-  unsigned tree_depth;
+  size_t tree_depth;
   sid_t sid;
 
   int max_packet_version;

rhizome.c

@@ -97,19 +97,14 @@ int rhizome_fetch_delay_ms()
   return config.rhizome.fetch_delay_ms;
 }
 
-int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args)
+int rhizome_add_field_assignment(struct rhizome_manifest_field_assignment *field, const char *arg, size_t arglen)
 {
-  unsigned i;
-  for (i = 0; i < argc; ++i) {
-    struct rhizome_manifest_field_assignment *field = &fields[i];
-    const char *arg = args[i];
-    size_t arglen = strlen(arg);
   const char *eq;
   if (arglen > 0 && arg[0] == '!') {
     field->label = arg + 1;
     field->labellen = arglen - 1;
     field->value = NULL;
-    } else if ((eq = strchr(arg, '='))) {
+  } else if ((eq = strnchr(arg, arglen, '='))) {
     field->label = arg;
     field->labellen = eq - arg;
     field->value = eq + 1;
@@ -120,6 +115,18 @@ int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fi
     return WHYF("invalid manifest field label: %s", alloca_toprint(-1, field->label, field->labellen));
   if (field->value && !rhizome_manifest_field_value_is_valid(field->value, field->valuelen))
     return WHYF("invalid manifest field value: %s", alloca_toprint(-1, field->value, field->valuelen));
+  return 0;
+}
+
+int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args)
+{
+  unsigned i;
+  for (i = 0; i < argc; ++i) {
+    struct rhizome_manifest_field_assignment *field = &fields[i];
+    const char *arg = args[i];
+    size_t arglen = strlen(arg);
+    if (rhizome_add_field_assignment(field, arg, arglen)==-1)
+      return -1;
   }
   return argc;
 }

rhizome.h

@@ -473,6 +473,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
 
 int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
 
+int rhizome_add_field_assignment(struct rhizome_manifest_field_assignment *field, const char *arg, size_t arglen);
 int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args);
 struct rhizome_bundle_result rhizome_apply_assignments(rhizome_manifest *m,
   unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments);

serval_types.h

@@ -25,6 +25,50 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <inttypes.h>
 #include <sodium.h>
 
+// all of the response codes we might want to return
+// with well defined semantics
+enum status_codes{
+  // non-specific conditions
+  // - error
+  CODE_ERROR = -1,
+  // - success
+  CODE_OK = 0,
+
+  // For the requested item;
+  // - we (already) have it
+  CODE_FOUND = 1,
+  // - we don't have it
+  CODE_NOT_FOUND = 2,
+  // - we have a newer version
+  CODE_SUPERSEDED = 3,
+  // - we have too many other things we need to keep
+  CODE_EVICTED = 4,
+
+  // - will never fit
+  CODE_TOO_BIG = 5,
+
+  // Something about the supplied data is incorrect.
+  // Anything from syntax errors, to semantic errors or missing required values
+  // Should always be acompanied by a formatted result string
+  CODE_INVALID_ARGUMENT = 6,
+
+  // Environmental issues;
+  // - our back end was locked
+  CODE_BUSY = 7,
+  // - we ran out of ram
+  CODE_OUT_OF_MEMORY = 8,
+
+  // Server state
+  CODE_NOT_RUNNING = 9,
+  CODE_NOT_RESPONDING = 10,
+
+  // ?? or Id not found?
+  //CODE_READONLY = 11,
+  //CODE_CRYPTO_ERROR = 12,
+};
+
+
+
 /* Conveniences to assist readability
  */
 

sourcefiles.mk

@@ -64,6 +64,7 @@ SERVAL_DAEMON_SOURCES = \
 	radio_link.c \
 	meshms.c \
 	meshmb.c \
+	meshmb_cli.c \
 	message_ply.c \
 	meshms_cli.c \
 	meshms_restful.c \

strbuf.h

@@ -366,7 +366,7 @@ __STRBUF_INLINE strbuf __strbuf_init_chk(strbuf sb, char *buffer, ssize_t size,
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 __STRBUF_INLINE strbuf strbuf_make(char *buffer, size_t size) {
-  return size < SIZEOF_STRBUF ? NULL : strbuf_init((strbuf) buffer, buffer + SIZEOF_STRBUF, size - SIZEOF_STRBUF);
+  return size < SIZEOF_STRBUF ? NULL : strbuf_init((strbuf) buffer, buffer + SIZEOF_STRBUF, (ssize_t)(size - SIZEOF_STRBUF));
 }
 
 /** Reset a strbuf.  The current position is set to the start of the buffer, so

tests/meshmb

@@ -10,10 +10,7 @@ setup_identities() {
    setup_servald
    set_instance +A
    executeOk_servald config \
-	set debug.meshms on \
+	set debug.meshmb on \
-	set debug.rhizome on \
-        set debug.rhizome_manifest on \
-        set debug.rhizome_store on \
         set log.console.level debug \
         set log.console.show_time on
    create_identities $1
@@ -75,5 +72,30 @@ test_meshmbListFeeds() {
    assertStdoutLineCount '==' 5
 }
 
+doc_meshmbFollow="Follow another feed"
+setup_meshmbFollow() {
+   setup_identities 3
+   executeOk_servald keyring set did $SIDA1 "" "Feed A"
+   executeOk_servald keyring set did $SIDA2 "" "Feed B"
+   executeOk_servald keyring set did $SIDA3 "" "Feed C"
+   executeOk_servald meshmb send $IDA1 "Message 1"
+   executeOk_servald meshmb send $IDA2 "Message 2"
+   executeOk_servald meshmb send $IDA3 "Message 3"
+}
+test_meshmbFollow() {
+   executeOk_servald meshmb follow $IDA1 $IDA2
+   tfw_cat --stderr
+   executeOk_servald meshmb list following $IDA1
+   tfw_cat --stdout --stderr
+   executeOk_servald meshmb follow $IDA1 $IDA3
+   tfw_cat --stderr
+   executeOk_servald meshmb list following $IDA1
+   tfw_cat --stdout --stderr
+   executeOk_servald meshmb ignore $IDA1 $IDA2
+   tfw_cat --stderr
+   executeOk_servald meshmb list following $IDA1
+   tfw_cat --stdout --stderr
+#TODO list....
+}
 
 runTests "$@"