diff --git a/meshmb.c b/meshmb.c index 390d16c6..cf9330e6 100644 --- a/meshmb.c +++ b/meshmb.c @@ -1,13 +1,455 @@ #include "serval.h" #include "serval_types.h" #include "dataformats.h" -#include "cli.h" #include "log.h" #include "debug.h" -#include "instance.h" #include "conf.h" -#include "commandline.h" #include "overlay_buffer.h" +#include "keyring.h" +#include "crypto.h" +#include "mem.h" +#include "meshmb.h" + +struct feed_metadata{ + size_t tree_depth; + struct message_ply ply; // (ply starts with a rhizome_bid_t, so this is consistent with a nibble tree) + struct meshmb_feed_details details; + // what is the offset of their last message + uint64_t last_message_offset; + // what is the last message we processed? + uint64_t last_seen; + // our cached value for the last known size of their ply + uint64_t size; +}; + +struct meshmb_feeds{ + struct tree_root root; + keyring_identity *id; + sign_keypair_t bundle_keypair; + bool_t dirty; +}; + +// only remember this many bytes of ply names & last messages +#define MAX_NAME_LEN (256) // ?? +#define MAX_MSG_LEN (256) // ?? + +static void update_stats(struct meshmb_feeds *feeds, struct feed_metadata *metadata, struct message_ply_read *reader) +{ + if (!metadata->ply.found){ + // get the current size from the db + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + if (sqlite_exec_uint64_retry(&retry, &metadata->ply.size, + "SELECT filesize FROM manifests WHERE id = ?", + RHIZOME_BID_T, &metadata->ply.bundle_id, + END) == SQLITE_ROW) + metadata->ply.found = 1; + else + return; + } + + if (metadata->size == metadata->ply.size) + return; + + if (!message_ply_is_open(reader) + && message_ply_read_open(reader, &metadata->ply.bundle_id)!=0) + return; + + // TODO remember if user has overridden the name? + if (metadata->details.name){ + free((void*)metadata->details.name); + metadata->details.name = NULL; + } + + if (reader->name){ + size_t len = strlen(reader->name); + if (len >= MAX_NAME_LEN) + len = MAX_NAME_LEN -1; + metadata->details.name = strn_edup(reader->name, len); + } + + reader->read.offset = reader->read.length; + time_s_t timestamp = 0; + while (message_ply_read_prev(reader) == 0){ + if (reader->type == MESSAGE_BLOCK_TYPE_TIME){ + if (reader->record_length<4){ + WARN("Malformed ply, expected 4 byte timestamp"); + continue; + } + timestamp = read_uint32(reader->record); + + }else if(reader->type == MESSAGE_BLOCK_TYPE_MESSAGE){ + if (metadata->last_message_offset == reader->record_end_offset) + break; + + metadata->last_message_offset = reader->record_end_offset; + + if (metadata->details.last_message) + free((void*)metadata->details.last_message); + size_t len = reader->record_length; + if (len >= MAX_MSG_LEN) + len = MAX_MSG_LEN -1; + metadata->details.last_message = strn_edup((const char *)reader->record, len); + metadata->details.timestamp = timestamp; + break; + } + } + + metadata->size = metadata->ply.size; + + // TODO assemble ACK list for unified reading....? + + feeds->dirty=1; + + return; +} + +// TODO, might be quicker to fetch all meshmb bundles and test if they are in the feed list +static int update_stats_tree(void **record, void *context) +{ + struct feed_metadata *metadata = (struct feed_metadata *)*record; + struct meshmb_feeds *feeds = (struct meshmb_feeds *)context; + struct message_ply_read reader; + bzero(&reader, sizeof(reader)); + update_stats(feeds, metadata, &reader); + message_ply_read_close(&reader); + return 0; +} + +// eg, if a bundle_add trigger occurs while the feed list is open +void meshmb_bundle_update(struct meshmb_feeds *feeds, rhizome_manifest *m, struct message_ply_read *reader) +{ + struct feed_metadata *metadata; + if (strcmp(m->service, RHIZOME_SERVICE_MESHMB) == 0 + && tree_find(&feeds->root, (void**)&metadata, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, NULL, NULL)==0){ + + metadata->ply.found = 1; + metadata->ply.size = m->filesize; + + update_stats(feeds, metadata, reader); + } +} + +int meshmb_update(struct meshmb_feeds *feeds) +{ + return tree_walk(&feeds->root, NULL, 0, update_stats_tree, feeds); +} + +static int write_metadata(void **record, void *context) +{ + struct feed_metadata *metadata = (struct feed_metadata *)*record; + struct rhizome_write *write = (struct rhizome_write *)context; + + assert(metadata->size >= metadata->last_message_offset); + assert(metadata->size >= metadata->last_seen); + unsigned name_len = (metadata->details.name ? strlen(metadata->details.name) : 0) + 1; + assert(name_len <= MAX_NAME_LEN); + unsigned msg_len = (metadata->details.last_message ? strlen(metadata->details.last_message) : 0) + 1; + assert(msg_len <= MAX_MSG_LEN); + + uint8_t buffer[sizeof (rhizome_bid_t) + 1 + 12*4 + name_len + msg_len]; + bcopy(metadata->ply.bundle_id.binary, buffer, sizeof (rhizome_bid_t)); + size_t len = sizeof (rhizome_bid_t); + buffer[len++]=0;// flags? + len+=pack_uint(&buffer[len], metadata->size); + len+=pack_uint(&buffer[len], metadata->size - metadata->last_message_offset); + len+=pack_uint(&buffer[len], metadata->size - metadata->last_seen); + len+=pack_uint(&buffer[len], metadata->details.timestamp); + if (name_len > 1) + strncpy_nul((char *)&buffer[len], metadata->details.name, name_len); + else + buffer[len]=0; + len+=name_len; + if (msg_len > 1) + strncpy_nul((char *)&buffer[len], metadata->details.last_message, msg_len); + else + buffer[len]=0; + len+=msg_len; + assert(len < sizeof buffer); + DEBUGF(meshmb, "Write %u bytes of metadata for %s", len, alloca_tohex_rhizome_bid_t(metadata->ply.bundle_id)); + return rhizome_write_buffer(write, buffer, len); +} + +#define CURRENT_VERSION 0 + +int meshmb_flush(struct meshmb_feeds *feeds) +{ + if (!feeds->dirty){ + DEBUGF(meshmb, "Ignoring flush, not dirty"); + return 0; + } + + rhizome_manifest *mout = NULL; + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) + return -1; + + int ret =-1; + struct rhizome_bundle_result result = rhizome_private_bundle(m, &feeds->bundle_keypair); + + switch(result.status){ + case RHIZOME_BUNDLE_STATUS_SAME: + rhizome_manifest_set_version(m, m->version + 1); + rhizome_manifest_set_filesize(m, RHIZOME_SIZE_UNSET); + rhizome_manifest_set_filehash(m, NULL); + // fallthrough + case RHIZOME_BUNDLE_STATUS_NEW: + { + struct rhizome_write write; + bzero(&write, sizeof(write)); + + enum rhizome_payload_status pstatus = rhizome_write_open_manifest(&write, m); + if (pstatus==RHIZOME_PAYLOAD_STATUS_NEW){ + uint8_t version = CURRENT_VERSION; + rhizome_write_buffer(&write, &version, 1); + + if (tree_walk(&feeds->root, NULL, 0, write_metadata, &write)==0){ + pstatus = rhizome_finish_write(&write); + if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW){ + + rhizome_manifest_set_filehash(m, &write.id); + rhizome_manifest_set_filesize(m, write.file_length); + struct rhizome_bundle_result result = rhizome_manifest_finalise(m, &mout, 1); + if (result.status == RHIZOME_BUNDLE_STATUS_NEW){ + ret = 0; + feeds->dirty = 0; + } + rhizome_bundle_result_free(&result); + } + } + } + if (ret!=0) + rhizome_fail_write(&write); + break; + } + default: + break; + } + + rhizome_manifest_free(m); + return ret; +} + +static int free_feed(void **record, void *context) +{ + struct meshmb_feeds *feeds = (struct meshmb_feeds *)context; + struct feed_metadata *f = *record; + if (f->details.name) + free((void *)f->details.name); + if (f->details.last_message) + free((void *)f->details.last_message); + free(f); + DEBUGF(meshmb, "free feed"); + *record = NULL; + feeds->dirty = 1; + return 0; +} + +void meshmb_close(struct meshmb_feeds *feeds) +{ + tree_walk(&feeds->root, NULL, 0, free_feed, feeds); + free(feeds); +} + +static void* alloc_feed(void *context, const uint8_t *binary, size_t UNUSED(bin_length)) +{ + struct meshmb_feeds *feeds = (struct meshmb_feeds *)context; + struct feed_metadata *feed = emalloc_zero(sizeof(struct feed_metadata)); + if (feed){ + feed->ply.bundle_id = *(rhizome_bid_t *)binary; + feeds->dirty = 1; + DEBUGF(meshmb, "Allocated feed"); + } + return feed; +} + +static int read_metadata(struct meshmb_feeds *feeds, struct rhizome_read *read) +{ + struct rhizome_read_buffer buff; + bzero(&buff, sizeof(buff)); + uint8_t buffer[sizeof (rhizome_bid_t) + 12*3 + MAX_NAME_LEN + MAX_MSG_LEN]; + + uint8_t version=0xFF; + if (rhizome_read_buffered(read, &buff, &version, 1)==-1) + return -1; + + if (version > CURRENT_VERSION) + return WHYF("Unknown file format version (got 0x%02x)", version); + + while(1){ + ssize_t bytes = rhizome_read_buffered(read, &buff, buffer, sizeof buffer); + if (bytes==0) + break; + + uint64_t delta=0; + uint64_t size; + uint64_t last_message_offset; + uint64_t last_seen; + uint64_t timestamp; + + int unpacked; + const rhizome_bid_t *bid = (const rhizome_bid_t *)&buffer[0]; + unsigned offset = sizeof(rhizome_bid_t); + if (offset >= (unsigned)bytes) + return WHY("Buffer overflow while parsing metadata"); + //uint8_t flags = buffer[offset++]; + offset++; + if (offset >= (unsigned)bytes) + return WHY("Buffer overflow while parsing metadata"); + + if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &size)) == -1) + return WHY("Buffer overflow while parsing metadata"); + offset += unpacked; + + if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &delta)) == -1) + return WHY("Buffer overflow while parsing metadata"); + offset += unpacked; + last_message_offset = size - delta; + + if ((unpacked = unpack_uint(buffer+offset, bytes-offset, &delta)) == -1) + return WHY("Buffer overflow while parsing metadata"); + offset += unpacked; + last_seen = size - delta; + + if ((unpacked = unpack_uint(buffer+offset, bytes-offset, ×tamp)) == -1) + return WHY("Buffer overflow while parsing metadata"); + offset += unpacked; + + const char *name = (const char *)&buffer[offset]; + while(buffer[offset++]){ + if (offset >= (unsigned)bytes) + return WHY("Buffer overflow while parsing metadata"); + } + + const char *msg = (const char *)&buffer[offset]; + while(buffer[offset++]){ + if (offset >= (unsigned)bytes) + return WHY("Buffer overflow while parsing metadata"); + } + + read->offset += offset - bytes; + struct feed_metadata *result; + if (tree_find(&feeds->root, (void**)&result, bid->binary, sizeof *bid, alloc_feed, feeds)<0) + return WHY("Failed to allocate metadata"); + + result->last_message_offset = last_message_offset; + result->last_seen = last_seen; + result->size = size; + result->details.bundle_id = *bid; + result->details.name = (name && *name) ? str_edup(name) : NULL; + result->details.last_message = (msg && *msg) ? str_edup(msg) : NULL; + result->details.timestamp = timestamp; + + DEBUGF(meshmb, "Processed %u bytes of metadata for %s", offset, alloca_tohex_rhizome_bid_t(result->ply.bundle_id)); + } + feeds->dirty = 0; + return 0; +} + +int meshmb_open(keyring_identity *id, struct meshmb_feeds **feeds) +{ + int ret = -1; + + *feeds = emalloc_zero(sizeof(struct meshmb_feeds)); + if (*feeds){ + (*feeds)->root.binary_length = sizeof(rhizome_bid_t); + (*feeds)->id = id; + rhizome_manifest *m = rhizome_new_manifest(); + if (m){ + crypto_seed_keypair(&(*feeds)->bundle_keypair, + "91656c3d62e9fe2678a1a81fabe3f413%s5a37120ca55d911634560e4d4dc1283f", + alloca_tohex(id->sign_keypair->private_key.binary, sizeof id->sign_keypair->private_key)); + struct rhizome_bundle_result result = rhizome_private_bundle(m, &(*feeds)->bundle_keypair); + DEBUGF(meshmb, "Private bundle %s, %s", + alloca_tohex_identity_t(&(*feeds)->bundle_keypair.public_key), + alloca_rhizome_bundle_result(result)); + switch(result.status){ + case RHIZOME_BUNDLE_STATUS_SAME:{ + struct rhizome_read read; + bzero(&read, sizeof(read)); + + enum rhizome_payload_status pstatus = rhizome_open_decrypt_read(m, &read); + if (pstatus == RHIZOME_PAYLOAD_STATUS_STORED){ + if (read_metadata(*feeds, &read)==-1) + WHYF("Failed to read metadata"); + else + ret = 0; + }else + WHYF("Failed to read metadata: %s", rhizome_payload_status_message(pstatus)); + + rhizome_read_close(&read); + }break; + + case RHIZOME_BUNDLE_STATUS_NEW: + ret = 0; + break; + + case RHIZOME_BUNDLE_STATUS_BUSY: + break; + + default: + // everything else should be impossible. + FATALF("Cannot create manifest: %s", alloca_rhizome_bundle_result(result)); + } + + rhizome_bundle_result_free(&result); + } + + rhizome_manifest_free(m); + } + + if (ret!=0){ + meshmb_close(*feeds); + *feeds=NULL; + } + return ret; +} + +int meshmb_follow(struct meshmb_feeds *feeds, rhizome_bid_t *bid) +{ + struct feed_metadata *metadata; + DEBUGF(meshmb, "Attempting to follow %s", alloca_tohex_rhizome_bid_t(*bid)); + + // TODO load the manifest and check the service! + + if (tree_find(&feeds->root, (void**)&metadata, bid->binary, sizeof *bid, alloc_feed, feeds)!=TREE_FOUND) + return WHYF("Failed to follow feed"); + + struct message_ply_read reader; + bzero(&reader, sizeof(reader)); + update_stats(feeds, metadata, &reader); + message_ply_read_close(&reader); + return 0; +} + +int meshmb_ignore(struct meshmb_feeds *feeds, rhizome_bid_t *bid) +{ + DEBUGF(meshmb, "Attempting to ignore %s", alloca_tohex_rhizome_bid_t(*bid)); + tree_walk_prefix(&feeds->root, bid->binary, sizeof *bid, free_feed, feeds); + return 0; +} + +struct enum_context{ + meshmb_callback callback; + void *context; +}; + +static int enum_callback(void **record, void *context) +{ + struct feed_metadata *feed = *record; + struct enum_context *enum_context = context; + return enum_context->callback(&feed->details, enum_context->context); +} + +int meshmb_enum(struct meshmb_feeds *feeds, rhizome_bid_t *restart_from, meshmb_callback callback, void *context) +{ + DEBUGF(meshmb, "Enumerating feeds from %s", + restart_from?alloca_tohex_rhizome_bid_t(*restart_from):"the beginning"); + struct enum_context enum_context = { + .callback = callback, + .context = context + }; + return tree_walk(&feeds->root, restart_from ? restart_from->binary : NULL, sizeof *restart_from, enum_callback, &enum_context); +} int meshmb_send(const keyring_identity *id, const char *message, size_t message_len, unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments){ @@ -30,184 +472,3 @@ int meshmb_send(const keyring_identity *id, const char *message, size_t message_ return ret; } - -DEFINE_FEATURE(cli_meshmb); - -DEFINE_CMD(app_meshmb_send, 0, - "Append a public broadcast message to your feed", - "meshmb", "send" KEYRING_PIN_OPTIONS, "", "", "..."); -static int app_meshmb_send(const struct cli_parsed *parsed, struct cli_context *UNUSED(context)) -{ - const char *idhex, *message; - if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1 - || cli_arg(parsed, "message", &message, NULL, "") == -1) - return -1; - - unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi; - struct rhizome_manifest_field_assignment fields[nfields]; - - if (nfields){ - if (rhizome_parse_field_assignments(fields, nfields, parsed->args + parsed->varargi)==-1) - return -1; - } - - identity_t identity; - if (str_to_identity_t(&identity, idhex) == -1) - return WHY("Invalid identity"); - - if (create_serval_instance_dir() == -1) - return -1; - if (rhizome_opendb() == -1) - return -1; - assert(keyring == NULL); - if (!(keyring = keyring_open_instance_cli(parsed))) - return -1; - - keyring_identity *id = keyring_find_identity(keyring, &identity); - if (!id) - return WHY("Invalid identity"); - - return meshmb_send(id, message, strlen(message)+1, nfields, fields); -} - -DEFINE_CMD(app_meshmb_read, 0, - "Read a broadcast message feed.", - "meshmb", "read", ""); -static int app_meshmb_read(const struct cli_parsed *parsed, struct cli_context *context) -{ - const char *hex_id; - if (cli_arg(parsed, "id", &hex_id, str_is_identity, "") == -1) - return -1; - - rhizome_bid_t bid; - if (str_to_rhizome_bid_t(&bid, hex_id) == -1) - return WHY("Invalid Identity"); - - /* Ensure the Rhizome database exists and is open */ - if (create_serval_instance_dir() == -1) - return -1; - if (rhizome_opendb() == -1) - return -1; - - struct message_ply_read read; - bzero(&read, sizeof read); - - if (message_ply_read_open(&read, &bid)==-1) - return -1; - - int ret=0; - size_t row_id = 0; - const char *names[]={ - "_id","offset","age","message" - }; - cli_start_table(context, NELS(names), names); - time_s_t timestamp = 0; - time_s_t now = gettime(); - - while(message_ply_read_prev(&read)==0){ - switch(read.type){ - case MESSAGE_BLOCK_TYPE_TIME: - if (read.record_length<4){ - WARN("Malformed ply, expected 4 byte timestamp"); - continue; - } - timestamp = read_uint32(read.record); - break; - - case MESSAGE_BLOCK_TYPE_MESSAGE: - cli_put_long(context, row_id++, ":"); - cli_put_long(context, read.record_end_offset, ":"); - cli_put_long(context, timestamp ? (long)(now - timestamp) : (long)-1, ":"); - cli_put_string(context, (const char *)read.record, "\n"); - - break; - - case MESSAGE_BLOCK_TYPE_ACK: - // TODO, link to some other ply? - break; - - default: - //ignore unknown types - break; - } - } - cli_end_table(context, row_id); - - message_ply_read_close(&read); - return ret; -} - -DEFINE_CMD(app_meshmb_find, 0, - "Browse available broadcast message feeds", - "meshmb", "find", "[]"); -static int app_meshmb_find(const struct cli_parsed *parsed, struct cli_context *context) -{ - const char *search=NULL; - cli_arg(parsed, "search", &search, NULL, ""); - // Ensure the Rhizome database exists and is open - if (create_serval_instance_dir() == -1) - return -1; - if (rhizome_opendb() == -1) - return -1; - - struct rhizome_list_cursor cursor; - bzero(&cursor, sizeof cursor); - cursor.service = RHIZOME_SERVICE_MESHMB; - cursor.name = search && search[0] ? search : NULL; - - //TODO hide feeds that have been blocked - - if (rhizome_list_open(&cursor) == -1) - return -1; - - const char *names[]={ - "_id", - "id", - "version", - "date", - "name" - }; - cli_start_table(context, NELS(names), names); - - unsigned rowcount=0; - int n; - - while ((n = rhizome_list_next(&cursor)) == 1) { - rowcount++; - rhizome_manifest *m = cursor.manifest; - cli_put_long(context, m->rowid, ":"); - cli_put_hexvalue(context, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, ":"); - cli_put_long(context, m->version, ":"); - cli_put_long(context, m->has_date ? m->date : 0, ":"); - cli_put_string(context, m->name, "\n"); - } - rhizome_list_release(&cursor); - cli_end_table(context, rowcount); - return 0; -} - -/* -DEFINE_CMD(app_meshmb_follow, 0, - "", - "meshmb", "follow|ignore|block" KEYRING_PIN_OPTIONS, "", ""); -static int app_meshmb_follow(const struct cli_parsed *parsed, struct cli_context *context) -{ - return 0; -} - -DEFINE_CMD(app_meshmb_list, 0, - "", - "meshmb", "list", "following|blocked" KEYRING_PIN_OPTIONS, "--last-message", ""); -static int app_meshmb_list(const struct cli_parsed *parsed, struct cli_context *context) -{ - return 0; -} - -DEFINE_CMD(app_meshmb_news, 0, - "", - "meshmb", "news" KEYRING_PIN_OPTIONS, ""); -static int app_meshmb_news(const struct cli_parsed *parsed, struct cli_context *context) -{ - return 0; -} -*/ diff --git a/meshmb.h b/meshmb.h index 77cbfcc5..a9888a4f 100644 --- a/meshmb.h +++ b/meshmb.h @@ -1,7 +1,44 @@ #ifndef __SERVAL_DNA__MESHMB_H #define __SERVAL_DNA__MESHMB_H +struct meshmb_feeds; + +enum meshmb_send_status{ + MESHMB_ERROR = -1, + MESHMB_OK = 0, +}; + +// details of a feed that you are following +struct meshmb_feed_details{ + rhizome_bid_t bundle_id; + const char *name; + const char *last_message; + time_s_t timestamp; +}; + +struct rhizome_manifest_field_assignment; int meshmb_send(const keyring_identity *id, const char *message, size_t message_len, unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments); +// feed tracking +int meshmb_open(keyring_identity *id, struct meshmb_feeds **feeds); +void meshmb_close(struct meshmb_feeds *feeds); + +// re-write metadata if required +int meshmb_flush(struct meshmb_feeds *feeds); + +// set / clear follow flag for this feed +int meshmb_follow(struct meshmb_feeds *feeds, rhizome_bid_t *bid); +int meshmb_ignore(struct meshmb_feeds *feeds, rhizome_bid_t *bid); + +// enumerate feeds, starting from restart_from +typedef int (*meshmb_callback) (struct meshmb_feed_details *details, void *context); +int meshmb_enum(struct meshmb_feeds *feeds, rhizome_bid_t *restart_from, meshmb_callback callback, void *context); + +// update metadata of all feeds based on current rhizome contents (optionally call after opening) +int meshmb_update(struct meshmb_feeds *feeds); +// update metadata of a single feed, eg because of a new bundle or when about to read a single ply. +// it is the callers reponsibility to supply a reader and close it +void meshmb_bundle_update(struct meshmb_feeds *feeds, rhizome_manifest *m, struct message_ply_read *reader); + #endif diff --git a/meshmb_cli.c b/meshmb_cli.c new file mode 100644 index 00000000..ee0c3e51 --- /dev/null +++ b/meshmb_cli.c @@ -0,0 +1,305 @@ +#include "serval.h" +#include "serval_types.h" +#include "dataformats.h" +#include "cli.h" +#include "log.h" +#include "debug.h" +#include "instance.h" +#include "commandline.h" +#include "keyring.h" +#include "rhizome.h" +#include "message_ply.h" +#include "meshmb.h" +#include "feature.h" + +DEFINE_FEATURE(cli_meshmb); + +DEFINE_CMD(app_meshmb_send, 0, + "Append a public broadcast message to your feed", + "meshmb", "send" KEYRING_PIN_OPTIONS, "", "", "..."); +static int app_meshmb_send(const struct cli_parsed *parsed, struct cli_context *UNUSED(context)) +{ + const char *idhex, *message; + if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1 + || cli_arg(parsed, "message", &message, NULL, "") == -1) + return -1; + + unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi; + struct rhizome_manifest_field_assignment fields[nfields]; + + if (nfields){ + if (rhizome_parse_field_assignments(fields, nfields, parsed->args + parsed->varargi)==-1) + return -1; + } + + identity_t identity; + if (str_to_identity_t(&identity, idhex) == -1) + return WHY("Invalid identity"); + + if (create_serval_instance_dir() == -1 + || rhizome_opendb() == -1 + || !(keyring = keyring_open_instance_cli(parsed))) + return -1; + + keyring_identity *id = keyring_find_identity(keyring, &identity); + if (!id) + return WHY("Invalid identity"); + + return meshmb_send(id, message, strlen(message)+1, nfields, fields); +} + +// TODO from offset....? +DEFINE_CMD(app_meshmb_read, 0, + "Read all messages in a broadcast message feed.", + "meshmb", "read", ""); +static int app_meshmb_read(const struct cli_parsed *parsed, struct cli_context *context) +{ + const char *hex_id; + if (cli_arg(parsed, "id", &hex_id, str_is_identity, "") == -1) + return -1; + + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, hex_id) == -1) + return WHY("Invalid Identity"); + + /* Ensure the Rhizome database exists and is open */ + if (create_serval_instance_dir() == -1 + || rhizome_opendb() == -1) + return -1; + + struct message_ply_read read; + bzero(&read, sizeof read); + + if (message_ply_read_open(&read, &bid)==-1) + return -1; + + int ret=0; + size_t row_id = 0; + const char *names[]={ + "_id","offset","age","message" + }; + cli_start_table(context, NELS(names), names); + time_s_t timestamp = 0; + time_s_t now = gettime(); + + while(message_ply_read_prev(&read)==0){ + switch(read.type){ + case MESSAGE_BLOCK_TYPE_TIME: + if (read.record_length<4){ + WARN("Malformed ply, expected 4 byte timestamp"); + continue; + } + timestamp = read_uint32(read.record); + break; + + case MESSAGE_BLOCK_TYPE_MESSAGE: + cli_put_long(context, row_id++, ":"); + cli_put_long(context, read.record_end_offset, ":"); + cli_put_long(context, timestamp ? (long)(now - timestamp) : (long)-1, ":"); + cli_put_string(context, (const char *)read.record, "\n"); + + break; + + case MESSAGE_BLOCK_TYPE_ACK: + // TODO, link to some other ply? + break; + + default: + //ignore unknown types + break; + } + } + cli_end_table(context, row_id); + + message_ply_read_close(&read); + return ret; +} + +DEFINE_CMD(app_meshmb_find, 0, + "Browse available broadcast message feeds", + "meshmb", "find", "[]"); +static int app_meshmb_find(const struct cli_parsed *parsed, struct cli_context *context) +{ + const char *search=NULL; + cli_arg(parsed, "search", &search, NULL, ""); + // Ensure the Rhizome database exists and is open + if (create_serval_instance_dir() == -1 + || rhizome_opendb() == -1) + return -1; + + struct rhizome_list_cursor cursor; + bzero(&cursor, sizeof cursor); + cursor.service = RHIZOME_SERVICE_MESHMB; + cursor.name = search && search[0] ? search : NULL; + + //TODO hide feeds that have been blocked + + if (rhizome_list_open(&cursor) == -1) + return -1; + + const char *names[]={ + "_id", + "id", + "version", + "date", + "name" + }; + cli_start_table(context, NELS(names), names); + + unsigned rowcount=0; + int n; + + while ((n = rhizome_list_next(&cursor)) == 1) { + rowcount++; + rhizome_manifest *m = cursor.manifest; + cli_put_long(context, m->rowid, ":"); + cli_put_hexvalue(context, m->keypair.public_key.binary, sizeof m->keypair.public_key.binary, ":"); + cli_put_long(context, m->version, ":"); + cli_put_long(context, m->has_date ? m->date : 0, ":"); + cli_put_string(context, m->name, "\n"); + } + rhizome_list_release(&cursor); + cli_end_table(context, rowcount); + return 0; +} + + +DEFINE_CMD(app_meshmb_follow, 0, + "Start or stop following a broadcast feed", + "meshmb", "follow|ignore" KEYRING_PIN_OPTIONS, "", ""); +static int app_meshmb_follow(const struct cli_parsed *parsed, struct cli_context *UNUSED(context)) +{ + const char *idhex, *peerhex; + if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1 + ||cli_arg(parsed, "peer", &peerhex, str_is_identity, "") == -1) + return -1; + + int follow = cli_arg(parsed, "follow", NULL, NULL, NULL) == 0; + + identity_t identity; + identity_t peer; + if (str_to_identity_t(&identity, idhex) == -1 + ||str_to_identity_t(&peer, peerhex) == -1) + return WHY("Invalid identity"); + + if (create_serval_instance_dir() == -1 + || rhizome_opendb() == -1 + || !(keyring = keyring_open_instance_cli(parsed))) + return -1; + + int ret = -1; + struct meshmb_feeds *feeds = NULL; + + keyring_identity *id = keyring_find_identity(keyring, &identity); + if (!id){ + WHY("Invalid identity"); + goto end; + } + + if (meshmb_open(id, &feeds)==-1) + goto end; + + if (follow){ + if (meshmb_follow(feeds, &peer)==-1) + goto end; + }else{ + if (meshmb_ignore(feeds, &peer)==-1) + goto end; + } + + if (meshmb_flush(feeds)==-1) + goto end; + + ret = 0; + +end: + if (feeds) + meshmb_close(feeds); + if (keyring) + keyring_free(keyring); + keyring = NULL; + return ret; +} + +struct cli_enum_context{ + unsigned rowcount; + struct cli_context *context; +}; + +static int list_callback(struct meshmb_feed_details *details, void *context) +{ + struct cli_enum_context *enum_context = context; + enum_context->rowcount++; + cli_put_long(enum_context->context, enum_context->rowcount, ":"); + cli_put_string(enum_context->context, alloca_tohex_rhizome_bid_t(details->bundle_id), ":"); + cli_put_string(enum_context->context, details->name, ":"); + cli_put_long(enum_context->context, details->timestamp ? (long)(gettime() - details->timestamp) : (long)-1, ":"); + cli_put_string(enum_context->context, details->last_message, "\n"); + return 0; +} + +DEFINE_CMD(app_meshmb_list, 0, + "List the feeds that you are currently following", + "meshmb", "list", "following" KEYRING_PIN_OPTIONS, ""); +static int app_meshmb_list(const struct cli_parsed *parsed, struct cli_context *context) +{ + const char *idhex; + if (cli_arg(parsed, "id", &idhex, str_is_identity, "") == -1) + return -1; + + identity_t identity; + if (str_to_identity_t(&identity, idhex) == -1) + return WHY("Invalid identity"); + + if (create_serval_instance_dir() == -1 + || rhizome_opendb() == -1 + || !(keyring = keyring_open_instance_cli(parsed))) + return -1; + + int ret = -1; + struct meshmb_feeds *feeds = NULL; + + keyring_identity *id = keyring_find_identity(keyring, &identity); + if (!id){ + WHY("Invalid identity"); + goto end; + } + + if (meshmb_open(id, &feeds)==-1) + goto end; + + const char *names[]={ + "_id", + "id", + "name", + "age", + "last_message" + }; + cli_start_table(context, NELS(names), names); + struct cli_enum_context enum_context = { + .rowcount = 0, + .context = context, + }; + + meshmb_enum(feeds, NULL, list_callback, &enum_context); + + cli_end_table(context, enum_context.rowcount); + ret = 0; + +end: + if (feeds) + meshmb_close(feeds); + keyring_free(keyring); + keyring = NULL; + return 0; +} + +/* +DEFINE_CMD(app_meshmb_news, 0, + "", + "meshmb", "news" KEYRING_PIN_OPTIONS, ""); +static int app_meshmb_news(const struct cli_parsed *parsed, struct cli_context *context) +{ + return 0; +} +*/ diff --git a/meshmb_restful.c b/meshmb_restful.c index 910dee68..2192207e 100644 --- a/meshmb_restful.c +++ b/meshmb_restful.c @@ -6,6 +6,7 @@ #include "numeric_str.h" #include "base64.h" #include "strbuf_helpers.h" +#include "keyring.h" #include "meshmb.h" DEFINE_FEATURE(http_rest_meshmb); @@ -344,6 +345,47 @@ static int restful_meshmb_newsince_list(httpd_request *r, const char *remainder) return ret; } +/* +static char *find_token_to_str(char *buf, uint64_t rowid) +{ + struct iovec iov[2]; + iov[0].iov_base = rhizome_db_uuid.u.binary; + iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; + iov[1].iov_base = &rowid; + iov[1].iov_len = sizeof rowid; + size_t n = base64url_encodev(buf, iov, 2); + assert(n == LIST_TOKEN_STRLEN); + buf[n] = '\0'; + return buf; +} + +static int strn_to_find_token(const char *str, uint64_t *rowidp, const char **afterp) +{ + unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; + if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token + && cmp_uuid_t(&rhizome_db_uuid, (serval_uuid_t *) &token) == 0 + && **afterp=='/'){ + memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); + (*afterp)++; + }else{ + // don't skip the token + *afterp=str; + *rowidp=1; + } + return 1; +} + +static int restful_meshmb_find(httpd_request *r, const char *remainder) +{ + return -1; +} + +static int restful_meshmb_newsince_find(httpd_request *r, const char *remainder) +{ + return -1; +} +*/ + DECLARE_HANDLER("/restful/meshmb/", restful_meshmb_); static int restful_meshmb_(httpd_request *r, const char *remainder) { @@ -374,6 +416,16 @@ static int restful_meshmb_(httpd_request *r, const char *remainder) handler = restful_meshmb_newsince_list; remainder = ""; } +/* + } else if(strcmp(remainder, "/find.json") == 0) { + handler = restful_meshmb_find; + remainder = ""; + } else if ( str_startswith(remainder, "/newsince/", &end) { + && strn_to_find_token(end, &r->ui64, &end) + && strcmp(end, "find.json") == 0) { + handler = restful_meshmb_newsince_find; + remainder = ""; + */ } if (handler == NULL) diff --git a/nibble_tree.c b/nibble_tree.c index 87ab725d..9a1e92f7 100644 --- a/nibble_tree.c +++ b/nibble_tree.c @@ -61,7 +61,7 @@ enum tree_error_reason tree_find(struct tree_root *root, void **result, const ui if (!node_ptr) return TREE_ERROR; struct tree_record *tree_record = (struct tree_record *)node_ptr; - // TODO assert(memcmp(tree_record->binary, binary, bin_length) == 0))? + assert(memcmp(tree_record->binary, binary, bin_length) == 0); tree_record ->tree_depth = pos*4; if (result) *result = node_ptr; @@ -134,13 +134,15 @@ static int walk(struct tree_node *node, unsigned pos, return ret; if (node->tree_nodes[i]) *empty=0; - // stop comparing the start sid after looking at the first branch of the tree + // stop comparing the start binary after looking at the first branch of the tree binary=NULL; } return ret; } +// start enumerating the tree from binary, and continue until the end +// callback is allowed to free any nodes while the walk is in progress int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context) { assert(!binary || bin_length <= root->binary_length); @@ -151,12 +153,15 @@ int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length, int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context) { assert(bin_length <= root->binary_length); + //TODO if callback free's nodes, collapse parent tree nodes too without needing to walk again? struct tree_node *node = &root->_root_node; unsigned pos=0; + // look for a branch of the tree with a partial match for (; node && posis_tree & (1<tree_nodes[i]; + // only one match? if (tree_record && memcmp(tree_record->binary, binary, bin_length)==0){ return callback(&node->tree_nodes[i], context); } @@ -164,6 +169,7 @@ int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_l } node = node->tree_nodes[i]; } + // walk the whole branch uint8_t ignore; return walk(node, pos+1, &ignore, NULL, 0, callback, context); } diff --git a/nibble_tree.h b/nibble_tree.h index 634eba50..22178090 100644 --- a/nibble_tree.h +++ b/nibble_tree.h @@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. struct tree_record{ // number of bits of the binary value, to uniquely identify this record within the tree's current contents - unsigned tree_depth; + size_t tree_depth; uint8_t binary[0]; }; diff --git a/overlay_address.h b/overlay_address.h index ef0ad170..5f6227d6 100644 --- a/overlay_address.h +++ b/overlay_address.h @@ -56,7 +56,7 @@ struct overlay_buffer; struct subscriber{ // minimum abbreviation length, in bits. // Note this must be here to match the memory layout of struct tree_record - unsigned tree_depth; + size_t tree_depth; sid_t sid; int max_packet_version; diff --git a/rhizome.c b/rhizome.c index 583c6175..7a57e9af 100644 --- a/rhizome.c +++ b/rhizome.c @@ -97,6 +97,27 @@ int rhizome_fetch_delay_ms() return config.rhizome.fetch_delay_ms; } +int rhizome_add_field_assignment(struct rhizome_manifest_field_assignment *field, const char *arg, size_t arglen) +{ + const char *eq; + if (arglen > 0 && arg[0] == '!') { + field->label = arg + 1; + field->labellen = arglen - 1; + field->value = NULL; + } else if ((eq = strnchr(arg, arglen, '='))) { + field->label = arg; + field->labellen = eq - arg; + field->value = eq + 1; + field->valuelen = (arg + arglen) - field->value; + } else + return WHYF("invalid manifest field argument: %s", alloca_str_toprint(arg)); + if (!rhizome_manifest_field_label_is_valid(field->label, field->labellen)) + return WHYF("invalid manifest field label: %s", alloca_toprint(-1, field->label, field->labellen)); + if (field->value && !rhizome_manifest_field_value_is_valid(field->value, field->valuelen)) + return WHYF("invalid manifest field value: %s", alloca_toprint(-1, field->value, field->valuelen)); + return 0; +} + int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args) { unsigned i; @@ -104,22 +125,8 @@ int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fi struct rhizome_manifest_field_assignment *field = &fields[i]; const char *arg = args[i]; size_t arglen = strlen(arg); - const char *eq; - if (arglen > 0 && arg[0] == '!') { - field->label = arg + 1; - field->labellen = arglen - 1; - field->value = NULL; - } else if ((eq = strchr(arg, '='))) { - field->label = arg; - field->labellen = eq - arg; - field->value = eq + 1; - field->valuelen = (arg + arglen) - field->value; - } else - return WHYF("invalid manifest field argument: %s", alloca_str_toprint(arg)); - if (!rhizome_manifest_field_label_is_valid(field->label, field->labellen)) - return WHYF("invalid manifest field label: %s", alloca_toprint(-1, field->label, field->labellen)); - if (field->value && !rhizome_manifest_field_value_is_valid(field->value, field->valuelen)) - return WHYF("invalid manifest field value: %s", alloca_toprint(-1, field->value, field->valuelen)); + if (rhizome_add_field_assignment(field, arg, arglen)==-1) + return -1; } return argc; } diff --git a/rhizome.h b/rhizome.h index fa4dc629..af36e400 100644 --- a/rhizome.h +++ b/rhizome.h @@ -473,6 +473,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc); int rhizome_store_file(rhizome_manifest *m,const unsigned char *key); +int rhizome_add_field_assignment(struct rhizome_manifest_field_assignment *field, const char *arg, size_t arglen); int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args); struct rhizome_bundle_result rhizome_apply_assignments(rhizome_manifest *m, unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments); diff --git a/serval_types.h b/serval_types.h index 6ed3b217..034bb7fa 100644 --- a/serval_types.h +++ b/serval_types.h @@ -25,6 +25,50 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include +// all of the response codes we might want to return +// with well defined semantics +enum status_codes{ + // non-specific conditions + // - error + CODE_ERROR = -1, + // - success + CODE_OK = 0, + + // For the requested item; + // - we (already) have it + CODE_FOUND = 1, + // - we don't have it + CODE_NOT_FOUND = 2, + // - we have a newer version + CODE_SUPERSEDED = 3, + // - we have too many other things we need to keep + CODE_EVICTED = 4, + + // - will never fit + CODE_TOO_BIG = 5, + + // Something about the supplied data is incorrect. + // Anything from syntax errors, to semantic errors or missing required values + // Should always be acompanied by a formatted result string + CODE_INVALID_ARGUMENT = 6, + + // Environmental issues; + // - our back end was locked + CODE_BUSY = 7, + // - we ran out of ram + CODE_OUT_OF_MEMORY = 8, + + // Server state + CODE_NOT_RUNNING = 9, + CODE_NOT_RESPONDING = 10, + + // ?? or Id not found? + //CODE_READONLY = 11, + //CODE_CRYPTO_ERROR = 12, +}; + + + /* Conveniences to assist readability */ diff --git a/sourcefiles.mk b/sourcefiles.mk index 9b659ae0..22d279bf 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -64,6 +64,7 @@ SERVAL_DAEMON_SOURCES = \ radio_link.c \ meshms.c \ meshmb.c \ + meshmb_cli.c \ message_ply.c \ meshms_cli.c \ meshms_restful.c \ diff --git a/strbuf.h b/strbuf.h index c990e89a..60aedf50 100644 --- a/strbuf.h +++ b/strbuf.h @@ -366,7 +366,7 @@ __STRBUF_INLINE strbuf __strbuf_init_chk(strbuf sb, char *buffer, ssize_t size, * @author Andrew Bettison */ __STRBUF_INLINE strbuf strbuf_make(char *buffer, size_t size) { - return size < SIZEOF_STRBUF ? NULL : strbuf_init((strbuf) buffer, buffer + SIZEOF_STRBUF, size - SIZEOF_STRBUF); + return size < SIZEOF_STRBUF ? NULL : strbuf_init((strbuf) buffer, buffer + SIZEOF_STRBUF, (ssize_t)(size - SIZEOF_STRBUF)); } /** Reset a strbuf. The current position is set to the start of the buffer, so diff --git a/tests/meshmb b/tests/meshmb index c1b0ef0a..e1cd6d60 100755 --- a/tests/meshmb +++ b/tests/meshmb @@ -10,10 +10,7 @@ setup_identities() { setup_servald set_instance +A executeOk_servald config \ - set debug.meshms on \ - set debug.rhizome on \ - set debug.rhizome_manifest on \ - set debug.rhizome_store on \ + set debug.meshmb on \ set log.console.level debug \ set log.console.show_time on create_identities $1 @@ -75,5 +72,30 @@ test_meshmbListFeeds() { assertStdoutLineCount '==' 5 } +doc_meshmbFollow="Follow another feed" +setup_meshmbFollow() { + setup_identities 3 + executeOk_servald keyring set did $SIDA1 "" "Feed A" + executeOk_servald keyring set did $SIDA2 "" "Feed B" + executeOk_servald keyring set did $SIDA3 "" "Feed C" + executeOk_servald meshmb send $IDA1 "Message 1" + executeOk_servald meshmb send $IDA2 "Message 2" + executeOk_servald meshmb send $IDA3 "Message 3" +} +test_meshmbFollow() { + executeOk_servald meshmb follow $IDA1 $IDA2 + tfw_cat --stderr + executeOk_servald meshmb list following $IDA1 + tfw_cat --stdout --stderr + executeOk_servald meshmb follow $IDA1 $IDA3 + tfw_cat --stderr + executeOk_servald meshmb list following $IDA1 + tfw_cat --stdout --stderr + executeOk_servald meshmb ignore $IDA1 $IDA2 + tfw_cat --stderr + executeOk_servald meshmb list following $IDA1 + tfw_cat --stdout --stderr +#TODO list.... +} runTests "$@"