diff --git a/commandline.c b/commandline.c index a1172041..5767f6e9 100644 --- a/commandline.c +++ b/commandline.c @@ -1336,10 +1336,15 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co rhizome_manifest_free(m); return WHY("Manifest file could not be loaded -- not added to rhizome"); } - } else if(manifestid && *manifestid) { + } else if (manifestid && *manifestid) { if (config.debug.rhizome) DEBUGF("Reading manifest from database"); - if (rhizome_retrieve_manifest(manifestid, m)){ + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, manifestid) == -1) { + rhizome_manifest_free(m); + return WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid)); + } + if (rhizome_retrieve_manifest(&bid, m)){ rhizome_manifest_free(m); return WHY("Existing manifest could not be loaded -- not added to rhizome"); } @@ -1398,10 +1403,8 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co cli_put_string(context, service, "\n"); } { - char bid[RHIZOME_MANIFEST_ID_STRLEN + 1]; - rhizome_bytes_to_hex_upper(mout->cryptoSignPublic, bid, RHIZOME_MANIFEST_ID_BYTES); cli_field_name(context, "manifestid", ":"); - cli_put_string(context, bid, "\n"); + cli_put_string(context, alloca_tohex_rhizome_bid_t(mout->cryptoSignPublic), "\n"); } { char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; @@ -1508,7 +1511,7 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex } { cli_field_name(context, "manifestid", ":"); - cli_put_string(context, alloca_tohex(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES), "\n"); + cli_put_string(context, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), "\n"); } { char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; @@ -1596,17 +1599,15 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont } else { if (!manifestid) return WHY("missing argument"); - unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES]; - if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1) + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, manifestid) == -1) return WHY("Invalid manifest ID"); - char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1]; - tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES); if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0) - ret = rhizome_delete_bundle(manifestIdUpper); + ret = rhizome_delete_bundle(&bid); else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0) - ret = rhizome_delete_manifest(manifestIdUpper); + ret = rhizome_delete_manifest(&bid); else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0) - ret = rhizome_delete_payload(manifestIdUpper); + ret = rhizome_delete_payload(&bid); else return WHY("unrecognised command"); } @@ -1656,13 +1657,10 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con int ret=0; - unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES]; - if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1) + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, manifestid) == -1) return WHY("Invalid manifest ID"); - char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1]; - tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES); - // treat empty string the same as null if (bskhex && !*bskhex) bskhex=NULL; @@ -1675,7 +1673,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con if (m==NULL) return WHY("Out of manifests"); - ret = rhizome_retrieve_manifest(manifestIdUpper, m); + ret = rhizome_retrieve_manifest(&bid, m); if (ret==0){ // ignore errors @@ -1683,7 +1681,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0); cli_field_name(context, "service", ":"); cli_put_string(context, blob_service, "\n"); - cli_field_name(context, "manifestid", ":"); cli_put_string(context, manifestIdUpper, "\n"); + cli_field_name(context, "manifestid", ":"); cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n"); cli_field_name(context, "version", ":"); cli_put_long(context, m->version, "\n"); cli_field_name(context, "inserttime", ":"); cli_put_long(context, m->inserttime, "\n"); if (m->haveSecret) { diff --git a/conf_schema.h b/conf_schema.h index e33a57e0..6cd3b776 100644 --- a/conf_schema.h +++ b/conf_schema.h @@ -260,6 +260,7 @@ ATOM(bool_t, slipdecode, 0, boolean,, "") ATOM(bool_t, slipbytestream, 0, boolean,, "") ATOM(bool_t, packetconstruction, 0, boolean,, "") ATOM(bool_t, rhizome, 0, boolean,, "") +ATOM(bool_t, rhizome_bind, 0, boolean,, "") ATOM(bool_t, rhizome_tx, 0, boolean,, "") ATOM(bool_t, rhizome_rx, 0, boolean,, "") ATOM(bool_t, rhizome_ads, 0, boolean,, "") diff --git a/dataformats.c b/dataformats.c index e898fd50..a6830b47 100644 --- a/dataformats.c +++ b/dataformats.c @@ -22,8 +22,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "str.h" #include -int str_to_sid_t(sid_t *sid, const char *hex) +int cmp_sid_t(const sid_t *a, const sid_t *b) { + return memcmp(a, b, sizeof a->binary); +} + +int str_to_sid_t(sid_t *sid, const char *hex) { if (strcmp(hex, "broadcast") == 0) { *sid = SID_BROADCAST; return 0; @@ -68,6 +72,28 @@ int strn_is_subscriber_id(const char *sid, size_t *lenp) return 0; } +int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b) +{ + return memcmp(a, b, sizeof a->binary); +} + +int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex) +{ + return fromhexstr(bid->binary, hex, sizeof bid->binary); +} + +int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp) +{ + rhizome_bid_t tmp; + int n = fromhex(tmp.binary, hex, sizeof tmp.binary); + if (n != sizeof tmp.binary) + return -1; + *bid = tmp; + if (endp) + *endp = hex + sizeof bid->binary * 2; + return 0; +} + int rhizome_strn_is_manifest_id(const char *id) { return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN); diff --git a/meshms.c b/meshms.c index 6d05d0bd..17afd0d7 100644 --- a/meshms.c +++ b/meshms.c @@ -11,7 +11,7 @@ // the manifest details for one half of a conversation struct ply{ - char bundle_id[RHIZOME_MANIFEST_ID_STRLEN+1]; + rhizome_bid_t bundle_id; uint64_t version; uint64_t tail; uint64_t size; @@ -90,13 +90,14 @@ static int get_my_conversation_bundle(const sid_t *my_sid, rhizome_manifest *m) return 0; } -static struct conversations *add_conv(struct conversations **conv, const sid_t *them){ - struct conversations **ptr=conv; +static struct conversations *add_conv(struct conversations **conv, const sid_t *them) +{ + struct conversations **ptr = conv; while(*ptr){ - int cmp = memcmp((*ptr)->them.binary, them, sizeof((*ptr)->them)); - if (cmp==0) + int cmp = cmp_sid_t(&(*ptr)->them, them); + if (cmp == 0) break; - if (cmp<0) + if (cmp < 0) ptr = &(*ptr)->_left; else ptr = &(*ptr)->_right; @@ -104,60 +105,64 @@ static struct conversations *add_conv(struct conversations **conv, const sid_t * if (!*ptr){ *ptr = emalloc_zero(sizeof(struct conversations)); if (*ptr) - memcpy((*ptr)->them.binary, them->binary, sizeof((*ptr)->them)); + (*ptr)->them = *them; } return *ptr; } // find matching conversations -// if their_sid_hex == my_sid_hex, return all conversations with any recipient -static int get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv){ - const char *my_sid_hex = alloca_tohex_sid(my_sid->binary); - const char *their_sid_hex = alloca_tohex_sid(their_sid?their_sid->binary:my_sid->binary); - +// if their_sid == my_sid, return all conversations with any recipient +static int get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv) +{ sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, - "SELECT id, version, filesize, tail, sender, recipient " - "FROM manifests " - "WHERE service = '"RHIZOME_SERVICE_MESHMS2"' " - "AND (sender=?1 or recipient=?1) " - "AND (sender=?2 or recipient=?2)"); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT id, version, filesize, tail, sender, recipient" + " FROM manifests" + " WHERE service = ?3" + " AND (sender=?1 or recipient=?1)" + " AND (sender=?2 or recipient=?2)", + SID_T, my_sid, + SID_T, their_sid ? their_sid : my_sid, + STATIC_TEXT, RHIZOME_SERVICE_MESHMS2, + END + ); if (!statement) return -1; - - int ret = sqlite3_bind_text(statement, 1, my_sid_hex, -1, SQLITE_STATIC); - if (ret!=SQLITE_OK) - goto end; - - ret = sqlite3_bind_text(statement, 2, their_sid_hex, -1, SQLITE_STATIC); - if (ret!=SQLITE_OK) - goto end; - - if (config.debug.meshms) + if (config.debug.meshms) { + const char *my_sid_hex = alloca_tohex_sid_t(*my_sid); + const char *their_sid_hex = alloca_tohex_sid_t(*(their_sid ? their_sid : my_sid)); DEBUGF("Looking for conversations for %s, %s", my_sid_hex, their_sid_hex); - + } while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { - const char *id = (const char *)sqlite3_column_text(statement, 0); + const char *id_hex = (const char *)sqlite3_column_text(statement, 0); long long version = sqlite3_column_int64(statement, 1); long long size = sqlite3_column_int64(statement, 2); long long tail = sqlite3_column_int64(statement, 3); const char *sender = (const char *)sqlite3_column_text(statement, 4); const char *recipient = (const char *)sqlite3_column_text(statement, 5); - const char *them = recipient; - - if (strcasecmp(them, my_sid_hex)==0) - them=sender; - - sid_t their_sid; - fromhex(their_sid.binary, them, sizeof(their_sid)); - if (config.debug.meshms) - DEBUGF("found id %s, sender %s, recipient %s", id, sender, recipient); - + DEBUGF("found id %s, sender %s, recipient %s", id_hex, sender, recipient); + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, id_hex) == -1) { + WHYF("invalid Bundle ID hex: %s -- skipping", alloca_str_toprint(id_hex)); + continue; + } + const char *them = recipient; + sid_t their_sid; + if (str_to_sid_t(&their_sid, them) == -1) { + WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them)); + continue; + } + if (cmp_sid_t(&their_sid, my_sid) == 0) { + them = sender; + if (str_to_sid_t(&their_sid, them) == -1) { + WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them)); + continue; + } + } struct conversations *ptr = add_conv(conv, &their_sid); if (!ptr) - goto end; - + break; struct ply *p; if (them==sender){ ptr->found_their_ply=1; @@ -166,20 +171,13 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si ptr->found_my_ply=1; p=&ptr->my_ply; } - strncpy(p->bundle_id, id, RHIZOME_MANIFEST_ID_STRLEN+1); + p->bundle_id = bid; p->version = version; p->tail = tail; p->size = size; } - -end: - if (ret!=SQLITE_OK){ - WHYF("Query failed: %s", sqlite3_errmsg(rhizome_db)); - free_conversations(*conv); - *conv=NULL; - } sqlite3_finalize(statement); - return (ret==SQLITE_OK)?0:-1; + return 0; } static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid){ @@ -201,12 +199,10 @@ static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_m rhizome_manifest_set(m, "sender", my_sidhex); rhizome_manifest_set(m, "recipient", their_sidhex); rhizome_manifest_set_ll(m, "tail", m->journalTail); - if (rhizome_fill_manifest(m, NULL, my_sid, NULL)) return -1; - - rhizome_manifest_get(m, "id", conv->my_ply.bundle_id, sizeof(conv->my_ply.bundle_id)); - conv->found_my_ply=1; + conv->my_ply.bundle_id = m->cryptoSignPublic; + conv->found_my_ply = 1; return 0; } @@ -216,14 +212,15 @@ static int append_footer(unsigned char *buffer, char type, int payload_len){ return 2; } -static int ply_read_open(struct ply_read *ply, const char *id, rhizome_manifest *m){ +static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome_manifest *m) +{ if (config.debug.meshms) - DEBUGF("Opening ply %s", id); - if (rhizome_retrieve_manifest(id, m)) + DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid)); + if (rhizome_retrieve_manifest(bid, m)) return -1; int ret = rhizome_open_decrypt_read(m, NULL, &ply->read); if (ret>0) - WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version); + WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); if (ret) return ret; ply->read.offset = ply->read.length = m->fileLength; @@ -304,7 +301,7 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, goto end; if (conv->found_my_ply){ - if (rhizome_retrieve_manifest(conv->my_ply.bundle_id, m)) + if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m)) goto end; // set the author of the manifest as we should already know that bcopy(my_sid->binary, m->author, sizeof(m->author)); @@ -354,10 +351,10 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){ DEBUG("Locating their last message"); // find the offset of their last message - if (rhizome_retrieve_manifest(conv->their_ply.bundle_id, m_theirs)) + if (rhizome_retrieve_manifest(&conv->their_ply.bundle_id, m_theirs)) goto end; - if (ply_read_open(&ply, conv->their_ply.bundle_id, m_theirs)) + if (ply_read_open(&ply, &conv->their_ply.bundle_id, m_theirs)) goto end; ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_MESSAGE); @@ -389,10 +386,10 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){ m_ours = rhizome_new_manifest(); if (!m_ours) goto end; - if (rhizome_retrieve_manifest(conv->my_ply.bundle_id, m_ours)) + if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m_ours)) goto end; - if (ply_read_open(&ply, conv->my_ply.bundle_id, m_ours)) + if (ply_read_open(&ply, &conv->my_ply.bundle_id, m_ours)) goto end; ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK); @@ -464,7 +461,8 @@ static int update_conversations(const sid_t *my_sid, struct conversations *conv) // read our cached conversation list from our rhizome payload // if we can't load the existing data correctly, just ignore it. -static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct conversations **conv){ +static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct conversations **conv) +{ if (m->haveSecret==NEW_BUNDLE_ID) return 0; @@ -491,7 +489,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, break; if (config.debug.meshms) DEBUGF("Reading existing conversation for %s", alloca_tohex_sid(sid.binary)); - if (their_sid && memcmp(sid.binary, their_sid->binary, sizeof(sid))) + if (their_sid && cmp_sid_t(&sid, their_sid) != 0) continue; struct conversations *ptr = add_conv(conv, &sid); if (!ptr) @@ -773,7 +771,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context m_ours = rhizome_new_manifest(); if (!m_ours) goto end; - if (ply_read_open(&read_ours, conv->my_ply.bundle_id, m_ours)) + if (ply_read_open(&read_ours, &conv->my_ply.bundle_id, m_ours)) goto end; uint64_t their_last_ack=0; @@ -784,7 +782,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context m_theirs = rhizome_new_manifest(); if (!m_theirs) goto end; - if (ply_read_open(&read_theirs, conv->their_ply.bundle_id, m_theirs)) + if (ply_read_open(&read_theirs, &conv->their_ply.bundle_id, m_theirs)) goto end; // find their last ACK so we know if messages have been received @@ -881,7 +879,7 @@ end: static int mark_read(struct conversations *conv, const sid_t *their_sid, const char *offset_str){ int ret=0; if (conv){ - int cmp = their_sid?memcmp(conv->them.binary, their_sid->binary, sizeof(sid_t)):0; + int cmp = their_sid ? cmp_sid_t(&conv->them, their_sid) : 0; if (!their_sid || cmp<0){ ret+=mark_read(conv->_left, their_sid, offset_str); } diff --git a/monitor.c b/monitor.c index 5981b2a5..355c0416 100644 --- a/monitor.c +++ b/monitor.c @@ -577,7 +577,7 @@ int monitor_announce_bundle(rhizome_manifest *m) char msg[1024]; int len = snprintf(msg,1024,"\n*%d:BUNDLE:%s\n", m->manifest_all_bytes, - alloca_tohex_bid(m->cryptoSignPublic)); + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); bcopy(m->manifestdata, &msg[len], m->manifest_all_bytes); len+=m->manifest_all_bytes; msg[len++]='\n'; diff --git a/overlay_mdp_services.c b/overlay_mdp_services.c index 3c0ac40d..7f870ab4 100644 --- a/overlay_mdp_services.c +++ b/overlay_mdp_services.c @@ -29,7 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "crypto.h" #include "log.h" -int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength) +int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength) { IN(); if (!is_rhizome_mdp_server_running()) @@ -38,7 +38,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t RETURN(WHYF("Invalid block length %d", blockLength)); if (config.debug.rhizome_tx) - DEBUGF("Requested blocks for %s @%"PRIx64" bitmap %x", alloca_tohex_bid(id), fileOffset, bitmap); + DEBUGF("Requested blocks for %s @%"PRIx64" bitmap %x", alloca_tohex_rhizome_bid_t(*bid), fileOffset, bitmap); overlay_mdp_frame reply; bzero(&reply,sizeof(reply)); @@ -69,7 +69,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t reply.out.queue=OQ_OPPORTUNISTIC; reply.out.payload[0]='B'; // reply contains blocks // include 16 bytes of BID prefix for identification - bcopy(id, &reply.out.payload[1], 16); + bcopy(bid->binary, &reply.out.payload[1], 16); // and version of manifest (in the correct byte order) // bcopy(&version, &reply.out.payload[1+16], sizeof(uint64_t)); write_uint64(&reply.out.payload[1+16],version); @@ -87,7 +87,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t write_uint64(&reply.out.payload[1+16+8], offset); - int bytes_read = rhizome_read_cached(id, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength); + int bytes_read = rhizome_read_cached(bid, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength); if (bytes_read<=0) break; @@ -108,16 +108,12 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t int overlay_mdp_service_rhizomerequest(struct overlay_frame *frame, overlay_mdp_frame *mdp) { - uint64_t version= - read_uint64(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES]); - uint64_t fileOffset= - read_uint64(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8]); - uint32_t bitmap= - read_uint32(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8]); - uint16_t blockLength= - read_uint16(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8+4]); - - return rhizome_mdp_send_block(frame->source, &mdp->out.payload[0], version, fileOffset, bitmap, blockLength); + const rhizome_bid_t *bidp = (const rhizome_bid_t *) &mdp->out.payload[0]; + uint64_t version = read_uint64(&mdp->out.payload[sizeof bidp->binary]); + uint64_t fileOffset = read_uint64(&mdp->out.payload[sizeof bidp->binary + 8]); + uint32_t bitmap = read_uint32(&mdp->out.payload[sizeof bidp->binary + 8 + 8]); + uint16_t blockLength = read_uint16(&mdp->out.payload[sizeof bidp->binary + 8 + 8 + 4]); + return rhizome_mdp_send_block(frame->source, bidp, version, fileOffset, bitmap, blockLength); } int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp) @@ -353,30 +349,23 @@ end: RETURN(ret); } -static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp){ +static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp) +{ int offset=0; - char id_hex[RHIZOME_MANIFEST_ID_STRLEN]; - - while (offsetout.payload_length){ - unsigned char *bar=&mdp->out.payload[offset]; - tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES); - strcat(id_hex, "%"); + while (offsetout.payload_length) { rhizome_manifest *m = rhizome_new_manifest(); if (!m) return WHY("Unable to allocate manifest"); - if (!rhizome_retrieve_manifest(id_hex, m)){ + unsigned char *bar = &mdp->out.payload[offset]; + if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){ rhizome_advertise_manifest(frame->source, m); - // pre-emptively send the payload if it will fit in a single packet - if (m->fileLength > 0 && m->fileLength <= 1024){ - rhizome_mdp_send_block(frame->source, m->cryptoSignPublic, m->version, - 0, 0, m->fileLength); - } + if (m->fileLength > 0 && m->fileLength <= 1024) + rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->fileLength); } rhizome_manifest_free(m); offset+=RHIZOME_BAR_BYTES; } - return 0; } diff --git a/rhizome.c b/rhizome.c index e2a0b498..576ddfbc 100644 --- a/rhizome.c +++ b/rhizome.c @@ -139,8 +139,7 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, /* Do we already have this manifest or newer? */ int64_t dbVersion = -1; - const char *id=rhizome_manifest_get(m, "id", NULL, 0); - if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id='%s';", id) == -1) + if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1) return WHY("Select failure"); if (dbVersion>=m->version) @@ -229,7 +228,7 @@ int rhizome_manifest_bind_id(rhizome_manifest *m_in) return WHYF("Failed to obtain RS for %s to calculate BK", alloca_tohex_sid(m_in->author)); } - if (!rhizome_secret2bk(m_in->cryptoSignPublic,rs,rs_len,bkbytes,m_in->cryptoSignSecret)) { + if (!rhizome_secret2bk(&m_in->cryptoSignPublic, rs, rs_len, bkbytes, m_in->cryptoSignSecret)) { char bkhex[RHIZOME_BUNDLE_KEY_STRLEN + 1]; (void) tohex(bkhex, bkbytes, RHIZOME_BUNDLE_KEY_BYTES); if (config.debug.rhizome) DEBUGF("set BK=%s", bkhex); @@ -261,15 +260,12 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) } /* If the manifest already has an ID */ - char id[SID_STRLEN + 1]; - if (!rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) - /* no manifest ID */ + if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic)) return WHY("Manifest does not have an ID"); - str_toupper_inplace(id); /* Discard the new manifest unless it is newer than the most recent known version with the same ID */ int64_t storedversion = -1; - switch (sqlite_exec_int64(&storedversion, "SELECT version from manifests where id='%s';", id)) { + switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m_in->cryptoSignPublic, END)) { case -1: return WHY("Select failed"); case 0: @@ -281,7 +277,7 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) if (m_in->version < storedversion) return WHY("Newer version exists"); if (m_in->version == storedversion) - return WHYF("Already have %s:%"PRId64", not adding", id, m_in->version); + return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m_in->cryptoSignPublic), m_in->version); break; default: return WHY("Select found too many rows!"); @@ -291,12 +287,6 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) return rhizome_store_bundle(m_in); } -/* Update an existing Rhizome bundle */ -int rhizome_bundle_push_update(char *id,long long version,unsigned char *data,int appendP) -{ - return WHY("Not implemented"); -} - /* When voice traffic is being carried, we need to throttle Rhizome down to a more sensible level. Or possibly even supress it entirely. */ diff --git a/rhizome.h b/rhizome.h index 8247f056..ac240cae 100644 --- a/rhizome.h +++ b/rhizome.h @@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # endif #endif +// TODO Rename MANIFEST_ID to BUNDLE_ID #define RHIZOME_MANIFEST_ID_BYTES crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES #define RHIZOME_MANIFEST_ID_STRLEN (RHIZOME_MANIFEST_ID_BYTES * 2) #define RHIZOME_BUNDLE_KEY_BYTES (crypto_sign_edwards25519sha512batch_SECRETKEYBYTES-crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES) @@ -51,6 +52,19 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define RHIZOME_HTTP_PORT 4110 #define RHIZOME_HTTP_PORT_MAX 4150 +typedef struct rhizome_bid_binary { + unsigned char binary[RHIZOME_MANIFEST_ID_BYTES]; +} rhizome_bid_t; + +#define RHIZOME_BID_ZERO ((rhizome_bid_t){{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}}) +#define RHIZOME_BID_MAX ((rhizome_bid_t){{0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff}}) +#define rhizome_bid_t_is_zero(bid) is_all_matching((bid).binary, sizeof (*(rhizome_bid_t*)0).binary, 0) +#define rhizome_bid_t_is_max(bid) is_all_matching((bid).binary, sizeof (*(rhizome_bid_t*)0).binary, 0xff) +#define alloca_tohex_rhizome_bid_t(bid) alloca_tohex((bid).binary, sizeof (*(rhizome_bid_t*)0).binary) +int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b); +int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex); +int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp); + typedef struct rhizome_bk_binary { unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES]; } rhizome_bk_t; @@ -106,7 +120,7 @@ typedef struct rhizome_manifest { The filename as distributed on Rhizome will be the public key of this pair, thus ensuring that noone can tamper with a bundle except the creator. */ - unsigned char cryptoSignPublic[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES]; + rhizome_bid_t cryptoSignPublic; unsigned char cryptoSignSecret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]; /* Whether we have the secret for this manifest on hand */ int haveSecret; @@ -227,8 +241,6 @@ int rhizome_str_is_bundle_crypt_key(const char *text); int rhizome_strn_is_file_hash(const char *text); int rhizome_str_is_file_hash(const char *text); -#define alloca_tohex_bid(bid) alloca_tohex((bid), RHIZOME_MANIFEST_ID_BYTES) - int http_header_complete(const char *buf, size_t len, size_t read_since_last_call); typedef struct sqlite_retry_state { @@ -247,7 +259,7 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append); int rhizome_manifest_selfsign(rhizome_manifest *m); int rhizome_drop_stored_file(const char *id,int maximum_priority); -int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id); +int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp); int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize); int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out); char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen); @@ -294,35 +306,71 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))(); int is_debug_rhizome(); int is_debug_rhizome_ads(); -sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...); -sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt); +enum sqlbind_type { + END = 0xbabecafe, + NUL = 0xbeef, // (no arg) + INT, // int value + INT_TOSTR, // int value + UINT_TOSTR, // unsigned value + INT64, // int64_t value + INT64_TOSTR, // int64_t value + UINT64_TOSTR, // uint64_t value + TEXT, // const char *text, + TEXT_LEN, // const char *text, int bytes + STATIC_TEXT, // const char *text, + STATIC_TEXT_LEN, // const char *text, int bytes + STATIC_BLOB, // const void *blob, int bytes + ZEROBLOB, // int bytes + SID_T, // const sid_t *sidp + RHIZOME_BID_T, // const rhizome_bid_t *bidp + FILEHASH_T, // const unsigned char hash_binary[RHIZOME_FILEHASH_BYTES] + TOHEX, // const unsigned char *binary, unsigned bytes + TEXT_TOUPPER, // const char *text, + TEXT_LEN_TOUPPER, // const char *text, unsigned bytes + INDEX = 0xfade0000, // INDEX|INT, int index, ... + NAMED = 0xdead0000 // NAMED|INT, const char *label, ... +}; + +sqlite3_stmt *_sqlite_prepare(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext); +int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...); +int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap); +sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...); int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action); void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action); -int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement); -int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...); -int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...); -int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...); -int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...); -int _sqlite_exec_int64(struct __sourceloc, int64_t *result, const char *sqlformat, ...); -int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, int64_t *result, const char *sqlformat, ...); -int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...); -int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...); -int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap); +int _sqlite_step(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement); +int _sqlite_exec_void(struct __sourceloc, int log_level, const char *sqltext, ...); +int _sqlite_exec_void_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...); +int _sqlite_exec_int64(struct __sourceloc, int64_t *result, const char *sqltext, ...); +int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, int64_t *result, const char *sqltext, ...); +int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqltext, ...); +int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, ...); +int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap); -#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) -#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb)) -#define sqlite_retry(rs,action) _sqlite_retry(__WHENCE__, (rs), (action)) -#define sqlite_retry_done(rs,action) _sqlite_retry_done(__WHENCE__, (rs), (action)) -#define sqlite_step(stmt) _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt)) -#define sqlite_step_retry(rs,stmt) _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt)) -#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__) -#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__) -#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) -#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__) -#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__) -#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__) -#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__) -#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__) +// The 'arg' arguments in the following macros appear to be unnecessary, but +// they serve a very useful purpose, so don't remove them! They ensure that +// programmers do not forget the bind args, of which there must be at least +// one, even if it is only 'END' to make no bindings at all. +#define sqlite_prepare(rs,sql) _sqlite_prepare(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql)) +#define sqlite_prepare_loglevel(ll,rs,sql) _sqlite_prepare(__WHENCE__, (ll), (rs), (sql)) +#define sqlite_prepare_bind(rs,sql,arg,...) _sqlite_prepare_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__) +#define sqlite_prepare_bind_loglevel(ll,rs,sql,arg,...) _sqlite_prepare_bind(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__) +#define sqlite_bind(rs,stmt,arg,...) _sqlite_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt), arg, ##__VA_ARGS__) +#define sqlite_bind_loglevel(ll,rs,stmt,arg,...) _sqlite_bind(__WHENCE__, (ll), (rs), (stmt), arg, ##__VA_ARGS__) +#define sqlite_retry(rs,action) _sqlite_retry(__WHENCE__, (rs), (action)) +#define sqlite_retry_done(rs,action) _sqlite_retry_done(__WHENCE__, (rs), (action)) +#define sqlite_exec(stmt) _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt)) +#define sqlite_exec_retry(rs,stmt) _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt)) +#define sqlite_exec_retry_loglevel(ll,rs,stmt) _sqlite_exec(__WHENCE__, (ll), (rs), (stmt)) +#define sqlite_step(stmt) _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt)) +#define sqlite_step_retry(rs,stmt) _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt)) +#define sqlite_exec_void(sql,arg,...) _sqlite_exec_void(__WHENCE__, LOG_LEVEL_ERROR, (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_void_loglevel(ll,sql,arg,...) _sqlite_exec_void(__WHENCE__, (ll), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_void_retry(rs,sql,arg,...) _sqlite_exec_void_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_void_retry_loglevel(ll,rs,sql,arg,...) _sqlite_exec_void_retry(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_int64(res,sql,arg,...) _sqlite_exec_int64(__WHENCE__, (res), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_int64_retry(rs,res,sql,arg,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_strbuf(sb,sql,arg,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (sql), arg, ##__VA_ARGS__) +#define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__) double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); @@ -336,11 +384,12 @@ int rhizome_is_manifest_interesting(rhizome_manifest *m); int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, const char *sender_sid, const char *recipient_sid, int limit, int offset, char count_rows); -int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m); +int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m); +int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m); int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m); -int rhizome_delete_bundle(const char *manifestid); -int rhizome_delete_manifest(const char *manifestid); -int rhizome_delete_payload(const char *manifestid); +int rhizome_delete_bundle(const rhizome_bid_t *bidp); +int rhizome_delete_manifest(const rhizome_bid_t *bidp); +int rhizome_delete_payload(const rhizome_bid_t *bidp); int rhizome_delete_file(const char *fileid); #define RHIZOME_DONTVERIFY 0 @@ -350,25 +399,25 @@ int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax); int monitor_announce_bundle(rhizome_manifest *m); int rhizome_find_secret(const unsigned char *authorSid, int *rs_len, const unsigned char **rs); int rhizome_bk_xor_stream( - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, unsigned char *xor_stream, int xor_stream_byte_count); int rhizome_bk2secret(rhizome_manifest *m, - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, /* The BK need only be the length of the secret half of the secret key */ const unsigned char bkin[RHIZOME_BUNDLE_KEY_BYTES], unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES] ); int rhizome_secret2bk( - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, /* The BK need only be the length of the secret half of the secret key */ unsigned char bkout[RHIZOME_BUNDLE_KEY_BYTES], const unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES] - ); +); unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m); int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk); int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk); @@ -536,7 +585,7 @@ struct http_response { const char * body; }; -int rhizome_received_content(unsigned char *bidprefix,uint64_t version, +int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, uint64_t offset,int count,unsigned char *bytes, int type); int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid, @@ -563,16 +612,16 @@ int is_rhizome_http_server_running(); typedef struct rhizome_direct_bundle_cursor { /* Where the current fill started */ int64_t start_size_high; - unsigned char start_bid_low[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t start_bid_low; /* Limit of where this cursor may traverse */ int64_t limit_size_high; - unsigned char limit_bid_high[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t limit_bid_high; int64_t size_low; int64_t size_high; - unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES]; - unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t bid_low; + rhizome_bid_t bid_high; unsigned char *buffer; int buffer_size; int buffer_used; @@ -591,10 +640,10 @@ int rhizome_direct_bundle_iterator_unpickle_range(rhizome_direct_bundle_cursor * int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c, int max_bars); void rhizome_direct_bundle_iterator_free(rhizome_direct_bundle_cursor **c); -int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES], - unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES], +int rhizome_direct_get_bars(const rhizome_bid_t *bid_low, + rhizome_bid_t *bid_high, int64_t size_low, int64_t size_high, - const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES], + const rhizome_bid_t *bid_max, unsigned char *bars_out, int bars_requested); int rhizome_direct_process_post_multipart_bytes @@ -720,11 +769,11 @@ int rhizome_read_close(struct rhizome_read *read); int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); int rhizome_dump_file(const char *id, const char *filepath, int64_t *length); -int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout, +int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length); int rhizome_cache_close(); -int rhizome_database_filehash_from_id(const char *id, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]); +int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]); int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 450010f3..b8017175 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -51,15 +51,15 @@ int rhizome_manifest_verify(rhizome_manifest *m) /* Make sure that id variable is correct */ { - unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t bid; char *id = rhizome_manifest_get(m,"id",NULL,0); if (!id) { WARN("Manifest lacks 'id' field"); m->errors++; - } else if (fromhexstr(manifest_id, id, RHIZOME_MANIFEST_ID_BYTES) == -1) { + } else if (str_to_rhizome_bid_t(&bid, id) == -1) { WARN("Invalid manifest 'id' field"); m->errors++; - } else if (m->sig_count == 0 || memcmp(m->signatories[0], manifest_id, RHIZOME_MANIFEST_ID_BYTES) != 0) { + } else if (m->sig_count == 0 || memcmp(m->signatories[0], bid.binary, sizeof bid.binary) != 0) { if (config.debug.rhizome) { if (m->sig_count>0) { DEBUGF("Manifest id variable does not match first signature block (signature key is %s)", @@ -159,7 +159,7 @@ int rhizome_manifest_parse(rhizome_manifest *m) if (strcasecmp(var, "id") == 0) { have_id = 1; - if (fromhexstr(m->cryptoSignPublic, value, RHIZOME_MANIFEST_ID_BYTES) == -1) { + if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) == -1) { if (config.debug.rejecteddata) WARNF("Invalid manifest id: %s", value); m->errors++; diff --git a/rhizome_crypto.c b/rhizome_crypto.c index 57c50623..74577a5f 100644 --- a/rhizome_crypto.c +++ b/rhizome_crypto.c @@ -38,16 +38,16 @@ unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m) int rhizome_manifest_createid(rhizome_manifest *m) { - if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic,m->cryptoSignSecret)) + if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic.binary, m->cryptoSignSecret)) return WHY("Failed to create keypair for manifest ID."); - rhizome_manifest_set(m, "id", alloca_tohex_bid(m->cryptoSignPublic)); + rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); m->haveSecret = NEW_BUNDLE_ID; return 0; } struct signing_key{ unsigned char Private[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]; - unsigned char Public[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES]; + rhizome_bid_t Public; }; /* generate a keypair from a given seed string */ @@ -58,7 +58,7 @@ static int generate_keypair(const char *seed, struct signing_key *key) // The first 256 bits of the hash will be used as the private key of the BID. bcopy(hash, key->Private, sizeof(key->Private)); - if (crypto_sign_compute_public_key(key->Private, key->Public)) + if (crypto_sign_compute_public_key(key->Private, key->Public.binary)) return WHY("Could not generate public key"); return 0; } @@ -71,17 +71,15 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed) if (generate_keypair(seed, &key)) return -1; - char *id = alloca_tohex_bid(key.Public); - - int ret=rhizome_retrieve_manifest(id, m); + int ret=rhizome_retrieve_manifest(&key.Public, m); if (ret<0) return -1; m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID; - bcopy(key.Public, m->cryptoSignPublic, sizeof(m->cryptoSignPublic)); - bcopy(key.Private, m->cryptoSignSecret, sizeof(m->cryptoSignSecret)); + m->cryptoSignPublic = key.Public; + bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret); if (ret>0) - rhizome_manifest_set(m, "id", alloca_tohex_bid(m->cryptoSignPublic)); + rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); return ret; } @@ -93,7 +91,7 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed) * @author Paul Gardner-Stephen */ int rhizome_bk_xor_stream( - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, unsigned char *xor_stream, @@ -107,7 +105,7 @@ int rhizome_bk_xor_stream( int combined_len = rs_len + crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES; unsigned char buffer[combined_len]; bcopy(&rs[0], &buffer[0], rs_len); - bcopy(&bid[0], &buffer[rs_len], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); + bcopy(&bidp->binary[0], &buffer[rs_len], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); unsigned char hash[crypto_hash_sha512_BYTES]; crypto_hash_sha512(hash,buffer,combined_len); bcopy(hash,xor_stream,xor_stream_byte_count); @@ -121,7 +119,7 @@ int rhizome_bk_xor_stream( second half of the secret key. The public key is the BID, so this simplifies the BK<-->SECRET conversion processes. */ int rhizome_bk2secret(rhizome_manifest *m, - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, /* The BK need only be the length of the secret half of the secret key */ const unsigned char bkin[RHIZOME_BUNDLE_KEY_BYTES], @@ -130,7 +128,7 @@ int rhizome_bk2secret(rhizome_manifest *m, { IN(); unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES]; - if (rhizome_bk_xor_stream(bid,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES)) + if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES)) RETURN(WHY("rhizome_bk_xor_stream() failed")); int i; @@ -140,16 +138,16 @@ int rhizome_bk2secret(rhizome_manifest *m, secret[i] = bkin[i] ^ xor_stream[i]; /* Copy BID as public-key part of secret key */ for(;i!=crypto_sign_edwards25519sha512batch_SECRETKEYBYTES;++i) - secret[i]=bid[i-RHIZOME_BUNDLE_KEY_BYTES]; + secret[i] = bidp->binary[i - RHIZOME_BUNDLE_KEY_BYTES]; bzero(xor_stream, sizeof xor_stream); - RETURN(rhizome_verify_bundle_privatekey(m,secret,bid)); + RETURN(rhizome_verify_bundle_privatekey(m, secret, bidp->binary)); OUT(); } int rhizome_secret2bk( - const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES], + const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, /* The BK need only be the length of the secret half of the secret key */ unsigned char bkout[RHIZOME_BUNDLE_KEY_BYTES], @@ -158,7 +156,7 @@ int rhizome_secret2bk( { IN(); unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES]; - if (rhizome_bk_xor_stream(bid,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES)) + if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES)) RETURN(WHY("rhizome_bk_xor_stream() failed")); int i; @@ -268,8 +266,7 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk) const unsigned char *rs; result = rhizome_find_secret(m->author, &rs_len, &rs); if (result==0) - result = rhizome_bk2secret(m,m->cryptoSignPublic,rs,rs_len, - bkBytes,m->cryptoSignSecret); + result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret); } if (result == 0 && bsk && !rhizome_is_bk_none(bsk)){ @@ -280,10 +277,9 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk) } }else if(bsk && !rhizome_is_bk_none(bsk)){ - bcopy(m->cryptoSignPublic, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof(m->cryptoSignPublic)); + bcopy(m->cryptoSignPublic.binary, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof m->cryptoSignPublic.binary); bcopy(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES); - if (rhizome_verify_bundle_privatekey(m,m->cryptoSignSecret, - m->cryptoSignPublic)) + if (rhizome_verify_bundle_privatekey(m, m->cryptoSignSecret, m->cryptoSignPublic.binary)) result=5; else result=0; @@ -363,8 +359,7 @@ int rhizome_find_bundle_author(rhizome_manifest *m) RETURN(WHYF("invalid Rhizome Secret: length=%d", rs_len)); const unsigned char *rs = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key; - if (!rhizome_bk2secret(m,m->cryptoSignPublic,rs,rs_len, - bkBytes,m->cryptoSignSecret)) { + if (!rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret)) { m->haveSecret=EXISTING_BUNDLE_ID; if (memcmp(m->author, authorSid, sizeof m->author)){ @@ -373,11 +368,12 @@ int rhizome_find_bundle_author(rhizome_manifest *m) DEBUGF("found bundle author sid=%s", alloca_tohex_sid(m->author)); // if this bundle is already in the database, update the author. - if (m->inserttime){ - const char *id = rhizome_manifest_get(m, "id", NULL, 0); - if (sqlite_exec_void("UPDATE MANIFESTS SET author='%s' WHERE id='%s';", alloca_tohex_sid(m->author), id) == -1) - WARN("Error updating MANIFESTS author column"); - } + if (m->inserttime) + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, + "UPDATE MANIFESTS SET author = ? WHERE id = ?;", + SID_T, (sid_t*)m->author, + RHIZOME_BID_T, &m->cryptoSignPublic, + END); } RETURN(0); // bingo @@ -407,11 +403,11 @@ int rhizome_verify_bundle_privatekey(rhizome_manifest *m, crypto_sign_compute_public_key(sk,pk); for (i = 0;i < 32;++i) if (pkin[i] != pk[i]) { - if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic) + if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) m->haveSecret=0; RETURN(-1); } - if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic) { + if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) { if (config.debug.rhizome) DEBUGF("We have the private key for this bundle."); m->haveSecret=EXISTING_BUNDLE_ID; @@ -427,7 +423,7 @@ int rhizome_sign_hash(rhizome_manifest *m, if (!m->haveSecret && rhizome_extract_privatekey_required(m, NULL)) RETURN(-1); - int ret=rhizome_sign_hash_with_key(m,m->cryptoSignSecret,m->cryptoSignPublic,out); + int ret=rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out); RETURN(ret); OUT(); } @@ -692,10 +688,10 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk) // journal bundles must always have the same nonce, regardless of version. // otherwise, generate nonce from version#bundle id#version; - unsigned char raw_nonce[8+8+sizeof(m->cryptoSignPublic)]; + unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary]; write_uint64(&raw_nonce[0], m->journalTail>=0?0:m->version); - bcopy(m->cryptoSignPublic, &raw_nonce[8], sizeof(m->cryptoSignPublic)); - write_uint64(&raw_nonce[8+sizeof(m->cryptoSignPublic)], m->journalTail>=0?0:m->version); + bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary); + write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->journalTail>=0?0:m->version); unsigned char hash[crypto_hash_sha512_BYTES]; diff --git a/rhizome_database.c b/rhizome_database.c index 3b28ec44..2848e7f4 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -20,6 +20,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define __RHIZOME_INLINE #include #include +#include +#include #include "serval.h" #include "conf.h" #include "rhizome.h" @@ -29,9 +31,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. static char rhizome_thisdatastore_path[256]; -static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid); +static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid); -static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid); +static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); const char *rhizome_datastore_path() { @@ -76,15 +78,16 @@ int create_rhizome_datastore_dir() sqlite3 *rhizome_db=NULL; /* XXX Requires a messy join that might be slow. */ -int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id) +int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp) { int64_t result = 0; if (sqlite_exec_int64_retry(retry, &result, - "select max(grouplist.priorty) from grouplist,manifests,groupmemberships" - " where manifests.id='%s'" - " and grouplist.id=groupmemberships.groupid" - " and groupmemberships.manifestid=manifests.id;", - id + "SELECT max(grouplist.priorty) FROM GROUPLIST,MANIFESTS,GROUPMEMBERSHIPS" + " WHERE MANIFESTS.id = ?" + " AND GROUPLIST.id = GROUPMEMBERSHIPS.groupid" + " AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id;", + RHIZOME_BID_T, bidp, + END ) == -1 ) return -1; @@ -133,7 +136,8 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))() return oldfunc; } -void sqlite_log(void *ignored, int result, const char *msg){ +void sqlite_log(void *ignored, int result, const char *msg) +{ WARNF("Sqlite: %d %s", result, msg); } @@ -164,7 +168,7 @@ void verify_bundles(){ } if (ret!=0){ DEBUGF("Removing invalid manifest entry @%lld", rowid); - sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID=%lld;", rowid); + sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END); } rhizome_manifest_free(m); } @@ -176,7 +180,7 @@ void verify_bundles(){ * that has write permission on the bundle, ie, possesses the Rhizome secret key that generated the * BID, and hence can derive the Bundle Secret from the bundle's BK field: * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created - * locally bu a non-secret identity, so no verification need ever be performed for one's own + * locally by a non-secret identity, so no verification need be performed for one's own * bundles while they remain in the Rhizome store. * - When a bundle is imported, the 'author' column is set to NULL to indicate that no * verification has passed yet. This includes one's own bundles that have been purged from @@ -232,47 +236,47 @@ int rhizome_opendb() sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; int64_t version; - if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;") == -1) + if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;", END) == -1) RETURN(-1); if (version<1){ /* Create tables as required */ - sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;"); - if ( sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);") == -1 - || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, version integer,inserttime integer, filesize integer, filehash text, author text, bar blob, manifest blob);") == -1 - || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, length integer, highestpriority integer, datavalid integer, inserttime integer);") == -1 - || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILEBLOBS(id text not null primary key, data blob);") == -1 - || sqlite_exec_void_retry(&retry, "DROP TABLE IF EXISTS FILEMANIFESTS;") == -1 - || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);") == -1 - || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);") == -1 + sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;", END); + if ( sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);", END) == -1 + || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, version integer,inserttime integer, filesize integer, filehash text, author text, bar blob, manifest blob);", END) == -1 + || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, length integer, highestpriority integer, datavalid integer, inserttime integer);", END) == -1 + || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILEBLOBS(id text not null primary key, data blob);", END) == -1 + || sqlite_exec_void_retry(&retry, "DROP TABLE IF EXISTS FILEMANIFESTS;", END) == -1 + || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);", END) == -1 + || sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);", END) == -1 ) { RETURN(WHY("Failed to create schema")); } /* Create indexes if they don't already exist */ - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END); } if (version<2){ - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;", END); // if more bundle verification is required in later upgrades, move this to the end, don't run it more than once. verify_bundles(); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END); } if (version<3){ - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END); } if (version<4){ - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;"); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;"); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END); } // TODO recreate tables with collate nocase on hex columns @@ -296,7 +300,7 @@ int rhizome_close_db() if (!sqlite3_get_autocommit(rhizome_db)){ WHY("Uncommitted transaction!"); - sqlite_exec_void("ROLLBACK;"); + sqlite_exec_void("ROLLBACK;", END); } sqlite3_stmt *stmt = NULL; while ((stmt = sqlite3_next_stmt(rhizome_db, stmt))) { @@ -409,47 +413,286 @@ void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, retry->start = -1; } -/* - Convenience wrapper for preparing an SQL command. - Returns -1 if an error occurs (logged as an error), otherwise zero with the prepared - statement in *statement. +/* Prepare an SQL command from a simple string. Returns NULL if an error occurs (logged as an + * error), otherwise returns a pointer to the prepared SQLite statement. + * + * IMPORTANT! Do not form statement strings using sprintf(3) or strbuf_sprintf() or similar + * methods, because those are susceptible to SQL injection attacks. Instead, use bound parameters + * and bind them using the _sqlite_bind() function below. + * + * IMPORTANT! Do not add sprintf(3)-like functionality to this method. It used to take + * sprintf(3)-style varargs and these were deliberately removed. It is vital to discourage bad + * practice, and adding sprintf(3)-style args to this function would be a step in the wrong + * direction. + * + * See GitHub issue #69. + * + * @author Andrew Bettison */ -sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...) -{ - strbuf sql = strbuf_alloca(8192); - strbuf_va_printf(sql, sqlformat); - return _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, sql); -} - -sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, strbuf stmt) +sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext) { IN(); sqlite3_stmt *statement = NULL; - if (strbuf_overrun(stmt)) { - WHYF("SQL overrun: %s", strbuf_str(stmt)); - RETURN(NULL); - } if (!rhizome_db && rhizome_opendb() == -1) RETURN(NULL); while (1) { - switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement, NULL)) { + switch (sqlite3_prepare_v2(rhizome_db, sqltext, -1, &statement, NULL)) { case SQLITE_OK: RETURN(statement); case SQLITE_BUSY: case SQLITE_LOCKED: - if (retry && _sqlite_retry(__whence, retry, strbuf_str(stmt))) { + if (retry && _sqlite_retry(__whence, retry, sqltext)) { break; // back to sqlite3_prepare_v2() } // fall through... default: - LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_db), strbuf_str(stmt)); + LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_db), sqltext); sqlite3_finalize(statement); RETURN(NULL); } } } -int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) +/* Bind some parameters to a prepared SQL statement. Returns -1 if an error occurs (logged as an + * error), otherwise zero with the prepared statement in *statement. + * + * Developed as part of GitHub issue #69. + * + * @author Andrew Bettison + */ +int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap) +{ + const int index_limit = sqlite3_limit(rhizome_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1); + enum sqlbind_type typ; + int index_counter = 0; + strbuf ext = NULL; + do { + typ = va_arg(ap, int); + int index; + const char *name = NULL; + if ((typ & 0xffff0000) == INDEX) { + typ &= 0xffff; + index = va_arg(ap, int); + if (index < 1 || index > index_limit) { + LOGF(log_level, "illegal index %d: %s", index, sqlite3_sql(statement)); + return -1; + } + if (config.debug.rhizome) + strbuf_sprintf((ext = strbuf_alloca(25)), "|INDEX index=%d", index); + } else if ((typ & 0xffff0000) == NAMED) { + typ &= 0xffff; + name = va_arg(ap, const char *); + index = sqlite3_bind_parameter_index(statement, name); + if (index == 0) { + LOGF(log_level, "no parameter %s in query: %s", alloca_str_toprint(name), sqlite3_sql(statement)); + return -1; + } + if (config.debug.rhizome) { + ext = strbuf_alloca(20 + toprint_str_len(name, "\"\"")); + strbuf_puts(ext, "|NAMED name="); + strbuf_toprint_quoted(ext, "\"\"", name); + } + } else { + index = ++index_counter; + if (config.debug.rhizome) + ext = strbuf_alloca(1); + } +#define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \ + if (config.debug.rhizome_bind) \ + DEBUGF("%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement)) +#define BIND_RETRY(FUNC, ...) \ + do { \ + switch (FUNC(statement, index, ##__VA_ARGS__)) { \ + case SQLITE_OK: \ + break; \ + case SQLITE_BUSY: \ + case SQLITE_LOCKED: \ + if (retry && _sqlite_retry(__whence, retry, #FUNC "()")) \ + continue; \ + default: \ + LOGF(log_level, #FUNC "(%d) failed, %s: %s", index, sqlite3_errmsg(rhizome_db), sqlite3_sql(statement)); \ + sqlite3_finalize(statement); \ + return -1; \ + } \ + break; \ + } while (1) + switch (typ) { + case END: + break; + case NUL: + BIND_DEBUG(NUL, sqlite3_bind_null, ""); + BIND_RETRY(sqlite3_bind_null); + break; + case INT: { + int value = va_arg(ap, int); + BIND_DEBUG(INT, sqlite3_bind_int, "%d", value); + BIND_RETRY(sqlite3_bind_int, value); + } + break; + case INT_TOSTR: { + int value = va_arg(ap, int); + char str[25]; + sprintf(str, "%d", value); + BIND_DEBUG(INT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str)); + BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT); + } + break; + case UINT_TOSTR: { + unsigned value = va_arg(ap, unsigned); + char str[25]; + sprintf(str, "%u", value); + BIND_DEBUG(UINT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str)); + BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT); + } + break; + case INT64: { + sqlite3_int64 value = va_arg(ap, int64_t); + BIND_DEBUG(INT64, sqlite3_bind_int64, "%"PRId64, (int64_t)value); + BIND_RETRY(sqlite3_bind_int64, value); + } + break; + case INT64_TOSTR: { + int64_t value = va_arg(ap, int64_t); + char str[35]; + sprintf(str, "%"PRId64, value); + BIND_DEBUG(INT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str)); + BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT); + } + break; + case UINT64_TOSTR: { + uint64_t value = va_arg(ap, uint64_t); + char str[35]; + sprintf(str, "%"PRIu64, value); + BIND_DEBUG(UINT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str)); + BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT); + } + break; + case TEXT: { + const char *text = va_arg(ap, const char *); + BIND_DEBUG(TEXT, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(text)); + BIND_RETRY(sqlite3_bind_text, text, -1, SQLITE_TRANSIENT); + } + break; + case TEXT_LEN: { + const char *text = va_arg(ap, const char *); + int bytes = va_arg(ap, int); + BIND_DEBUG(TEXT_LEN, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_str_toprint(text), bytes); + BIND_RETRY(sqlite3_bind_text, text, bytes, SQLITE_TRANSIENT); + } + break; + case STATIC_TEXT: { + const char *text = va_arg(ap, const char *); + BIND_DEBUG(STATIC_TEXT, sqlite3_bind_text, "%s,-1,SQLITE_STATIC", alloca_str_toprint(text)); + BIND_RETRY(sqlite3_bind_text, text, -1, SQLITE_STATIC); + } + break; + case STATIC_TEXT_LEN: { + const char *text = va_arg(ap, const char *); + int bytes = va_arg(ap, int); + BIND_DEBUG(STATIC_TEXT_LEN, sqlite3_bind_text, "%s,%d,SQLITE_STATIC", alloca_str_toprint(text), bytes); + BIND_RETRY(sqlite3_bind_text, text, bytes, SQLITE_STATIC); + } + break; + case STATIC_BLOB: { + const void *blob = va_arg(ap, const void *); + int bytes = va_arg(ap, int); + BIND_DEBUG(STATIC_BLOB, sqlite3_bind_blob, "%s,%d,SQLITE_STATIC", alloca_toprint(20, blob, bytes), bytes); + BIND_RETRY(sqlite3_bind_blob, blob, bytes, SQLITE_STATIC); + }; + break; + case ZEROBLOB: { + int bytes = va_arg(ap, int); + BIND_DEBUG(ZEROBLOB, sqlite3_bind_zeroblob, "%d,SQLITE_STATIC", bytes); + BIND_RETRY(sqlite3_bind_zeroblob, bytes); + }; + break; + case SID_T: { + const sid_t *sidp = va_arg(ap, const sid_t *); + const char *sid_hex = alloca_tohex_sid_t(*sidp); + BIND_DEBUG(SID_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", sid_hex, SID_STRLEN); + BIND_RETRY(sqlite3_bind_text, sid_hex, SID_STRLEN, SQLITE_TRANSIENT); + } + break; + case RHIZOME_BID_T: { + const rhizome_bid_t *bidp = va_arg(ap, const rhizome_bid_t *); + const char *bid_hex = alloca_tohex_rhizome_bid_t(*bidp); + BIND_DEBUG(RHIZOME_BID_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", bid_hex, RHIZOME_MANIFEST_ID_STRLEN); + BIND_RETRY(sqlite3_bind_text, bid_hex, RHIZOME_MANIFEST_ID_STRLEN, SQLITE_TRANSIENT); + } + break; + case FILEHASH_T: { + const char *hash_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_FILEHASH_BYTES); + BIND_DEBUG(FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN); + BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT); + } + break; + case TOHEX: { + const unsigned char *binary = va_arg(ap, const unsigned char *); + unsigned bytes = va_arg(ap, unsigned); + const char *hex = alloca_tohex(binary, bytes); + BIND_DEBUG(TOHEX, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hex, bytes * 2); + BIND_RETRY(sqlite3_bind_text, hex, bytes * 2, SQLITE_TRANSIENT); + } + break; + case TEXT_TOUPPER: { + const char *text = va_arg(ap, const char *); + unsigned bytes = strlen(text); + char upper[bytes + 1]; + str_toupper_inplace(strcpy(upper, text)); + BIND_DEBUG(TEXT_TOUPPER, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes); + BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT); + } + break; + case TEXT_LEN_TOUPPER: { + const char *text = va_arg(ap, const char *); + unsigned bytes = va_arg(ap, unsigned); + char upper[bytes]; + unsigned i; + for (i = 0; i != bytes; ++i) + upper[i] = toupper(text[i]); + BIND_DEBUG(TEXT_LEN_TOUPPER, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes); + BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT); + } + break; +#undef BIND_RETRY + default: + FATALF("unsupported bind code, index=%d typ=0x%08x: %s", index, typ, sqlite3_sql(statement)); + } + } while (typ != END); + return 0; +} + +int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...) +{ + va_list ap; + va_start(ap, statement); + int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap); + va_end(ap); + return ret; +} + +/* Prepare an SQL statement and bind some parameters. Returns a pointer to the SQLite statement if + * successful or NULL if an error occurs (which is logged at the given log level). + * + * @author Andrew Bettison + */ +sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...) +{ + sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext); + if (statement != NULL) { + va_list ap; + va_start(ap, sqltext); + int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap); + va_end(ap); + if (ret == -1) { + sqlite3_finalize(statement); + statement = NULL; + } + } + return statement; +} + +int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) { IN(); int ret = -1; @@ -499,13 +742,13 @@ int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_ * * @author Andrew Bettison */ -static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) +static int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) { if (!statement) return -1; int rowcount = 0; int stepcode; - while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW) + while ((stepcode = _sqlite_step(__whence, log_level, retry, statement)) == SQLITE_ROW) ++rowcount; sqlite3_finalize(statement); if (sqlite_trace_func()) @@ -518,11 +761,14 @@ static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sql * * @author Andrew Bettison */ -static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap) +static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, va_list ap) { - strbuf stmt = strbuf_alloca(8192); - strbuf_vsprintf(stmt, sqlformat, ap); - int rowcount = _sqlite_exec_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt)); + sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext); + if (!statement) + return -1; + if (_sqlite_vbind(__whence, log_level, retry, statement, ap) == -1) + return -1; + int rowcount = _sqlite_exec(__whence, log_level, retry, statement); if (rowcount == -1) return -1; if (rowcount) @@ -536,26 +782,12 @@ static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite * * @author Andrew Bettison */ -int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...) +int _sqlite_exec_void(struct __sourceloc __whence, int log_level, const char *sqltext, ...) { va_list ap; - va_start(ap, sqlformat); + va_start(ap, sqltext); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - int ret = _sqlite_vexec_void(__whence, LOG_LEVEL_ERROR, &retry, sqlformat, ap); - va_end(ap); - return ret; -} - -/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR. - * - * @author Andrew Bettison - */ -int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...) -{ - va_list ap; - va_start(ap, sqlformat); - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - int ret = _sqlite_vexec_void(__whence, log_level, &retry, sqlformat, ap); + int ret = _sqlite_vexec_void(__whence, log_level, &retry, sqltext, ap); va_end(ap); return ret; } @@ -567,39 +799,26 @@ int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const * * @author Andrew Bettison */ -int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...) +int _sqlite_exec_void_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...) { va_list ap; - va_start(ap, sqlformat); - int ret = _sqlite_vexec_void(__whence, LOG_LEVEL_ERROR, retry, sqlformat, ap); + va_start(ap, sqltext); + int ret = _sqlite_vexec_void(__whence, log_level, retry, sqltext, ap); va_end(ap); return ret; } -/* Same as sqlite_exec_void_retry(), but logs any error at the given level instead of ERROR. - * - * @author Andrew Bettison - */ -int _sqlite_exec_void_retry_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...) +static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, int64_t *result, const char *sqltext, va_list ap) { - va_list ap; - va_start(ap, sqlformat); - int ret = _sqlite_vexec_void(__whence, log_level, retry, sqlformat, ap); - va_end(ap); - return ret; -} - -static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, int64_t *result, const char *sqlformat, va_list ap) -{ - strbuf stmt = strbuf_alloca(8192); - strbuf_vsprintf(stmt, sqlformat, ap); - sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt); + sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext); if (!statement) return -1; + if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1) + return -1; int ret = 0; int rowcount = 0; int stepcode; - while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) { + while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) { int columncount = sqlite3_column_count(statement); if (columncount != 1) ret = WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement)); @@ -679,17 +898,17 @@ int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *r return ret; } -int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap) +int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap) { - strbuf stmt = strbuf_alloca(8192); - strbuf_vsprintf(stmt, sqlformat, ap); - sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt); + sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext); if (!statement) return -1; + if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1) + return -1; int ret = 0; int rowcount = 0; int stepcode; - while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) { + while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) { int columncount = sqlite3_column_count(statement); if (columncount != 1) ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement)); @@ -707,20 +926,20 @@ int64_t rhizome_database_used_bytes() int64_t db_page_size; int64_t db_page_count; int64_t db_free_page_count; - if ( sqlite_exec_int64(&db_page_size, "PRAGMA page_size;") == -1LL - || sqlite_exec_int64(&db_page_count, "PRAGMA page_count;") == -1LL - || sqlite_exec_int64(&db_free_page_count, "PRAGMA free_count;") == -1LL + if ( sqlite_exec_int64(&db_page_size, "PRAGMA page_size;", END) == -1LL + || sqlite_exec_int64(&db_page_count, "PRAGMA page_count;", END) == -1LL + || sqlite_exec_int64(&db_free_page_count, "PRAGMA free_count;", END) == -1LL ) return WHY("Cannot measure database used bytes"); return db_page_size * (db_page_count - db_free_page_count); } -int rhizome_database_filehash_from_id(const char *id, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]) +int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]) { IN(); strbuf hash_sb = strbuf_local(hash, SHA512_DIGEST_STRING_LENGTH); - RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE manifests.version=%lld AND manifests.id='%s';", - version,id)); + RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;", + INT64, version, RHIZOME_BID_T, bidp, END)); OUT(); } @@ -735,15 +954,25 @@ static int rhizome_delete_external(const char *fileid) static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry) { - return sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );"); + return sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, + "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );", + END); } int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid) { int ret = 0; - if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, "DELETE FROM FILES WHERE id='%s' and datavalid=0;", fileid) == -1) + if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, + "DELETE FROM FILES WHERE id = ? and datavalid = 0;", + TEXT_TOUPPER, fileid, END + ) == -1 + ) ret = -1; - if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, "DELETE FROM FILEBLOBS WHERE id='%s' AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id=FILEBLOBS.id );", fileid) == -1) + if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, + "DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );", + TEXT_TOUPPER, fileid, END + ) == -1 + ) ret = -1; return ret; } @@ -761,7 +990,9 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) int candidates=0; if (report) report->deleted_orphan_fileblobs = 0; - sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=0;", insert_horizon); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 0;", + INT64, insert_horizon, END); while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { candidates++; const char *id = (const char *) sqlite3_column_text(statement, 0); @@ -770,7 +1001,9 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) } sqlite3_finalize(statement); - statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", insert_horizon_no_manifest); + statement = sqlite_prepare_bind(&retry, + "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", + INT64, insert_horizon_no_manifest, END); while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { candidates++; const char *id = (const char *) sqlite3_column_text(statement, 0); @@ -782,10 +1015,14 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) int ret; if (candidates) { // clean out unreferenced files - ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", insert_horizon); + ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, + "DELETE FROM FILES WHERE inserttime < ? AND datavalid = 0;", + INT64, insert_horizon, END); if (report) report->deleted_stale_incoming_files = ret; - ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", insert_horizon_no_manifest); + ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, + "DELETE FROM FILES WHERE inserttime < ? AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", + INT64, insert_horizon_no_manifest, END); if (report) report->deleted_orphan_files = ret; } @@ -815,7 +1052,9 @@ int rhizome_make_space(int group_priority, long long bytes) /* Okay, not enough space, so free up some. */ sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, "select id,length from files where highestpriority < %d order by descending length", group_priority); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT id,length FROM FILES WHERE highestpriority < ? ORDER BY DESCENDING LENGTH", + INT, group_priority, END); if (!statement) return -1; while (bytes > (config.rhizome.database_size - 65536 - rhizome_database_used_bytes()) @@ -853,15 +1092,15 @@ int rhizome_make_space(int group_priority, long long bytes) return WHY("Incomplete"); } -/* Drop the specified file from storage, and any manifests that reference it, - provided that none of those manifests are being retained at a higher priority - than the maximum specified here. */ -int rhizome_drop_stored_file(const char *id,int maximum_priority) +/* Drop the specified file from storage, and any manifests that reference it, provided that none of + * those manifests are being retained at a higher priority than the maximum specified here. + */ +int rhizome_drop_stored_file(const char *id, int maximum_priority) { if (!rhizome_str_is_file_hash(id)) - return WHYF("invalid file hash id=%s", alloca_toprint(-1, id, strlen(id))); + return WHYF("invalid file hash id=%s", alloca_str_toprint(id)); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, "select id from manifests where filehash='%s'", id); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", TEXT_TOUPPER, id, END); if (!statement) return WHYF("Could not drop stored file id=%s", id); int can_drop = 1; @@ -871,23 +1110,28 @@ int rhizome_drop_stored_file(const char *id,int maximum_priority) WHYF("Incorrect type in id column of manifests table"); break; } - const char *manifestId = (char *) sqlite3_column_text(statement, 0); + const char *q_id = (char *) sqlite3_column_text(statement, 0); + rhizome_bid_t bid; + if (str_to_rhizome_bid_t(&bid, q_id) == -1) { + WARNF("malformed column value MANIFESTS.id = %s -- skipped", alloca_str_toprint(q_id)); + continue; + } /* Check that manifest is not part of a higher priority group. If so, we cannot drop the manifest or the file. However, we will keep iterating, as we can still drop any other manifests pointing to this file that are lower priority, and thus free up a little space. */ - int priority = rhizome_manifest_priority(&retry, manifestId); + int priority = rhizome_manifest_priority(&retry, &bid); if (priority == -1) - WHYF("Cannot drop fileid=%s due to error, manifestId=%s", id, manifestId); + WHYF("Cannot drop fileid=%s due to error, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); else if (priority > maximum_priority) { - WHYF("Cannot drop fileid=%s due to manifest priority, manifestId=%s", id, manifestId); + WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); can_drop = 0; } else { if (config.debug.rhizome) DEBUGF("removing stale manifests, groupmemberships"); - sqlite_exec_void_retry(&retry, "delete from manifests where id='%s';", manifestId); - sqlite_exec_void_retry(&retry, "delete from keypairs where public='%s';", manifestId); - sqlite_exec_void_retry(&retry, "delete from groupmemberships where manifestid='%s';", manifestId); + sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &bid, END); + sqlite_exec_void_retry(&retry, "DELETE FROM KEYPAIRS WHERE public = ?;", RHIZOME_BID_T, &bid, END); + sqlite_exec_void_retry(&retry, "DELETE FROM GROUPMEMBERSHIPS WHERE manifestid = ?;", RHIZOME_BID_T, &bid, END); } } sqlite3_finalize(statement); @@ -958,7 +1202,7 @@ int rhizome_store_bundle(rhizome_manifest *m) const char *service = rhizome_manifest_get(m, "service", NULL, 0); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1) + if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) return WHY("Failed to begin transaction"); sqlite3_stmt *stmt; @@ -1047,12 +1291,12 @@ int rhizome_store_bundle(rhizome_manifest *m) sqlite3_finalize(stmt); stmt = NULL; } - if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1){ + if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){ // This message used in tests; do not modify or remove. const char *service = rhizome_manifest_get(m, "service", NULL, 0); INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRId64, service ? service : "NULL", - alloca_tohex_sid(m->cryptoSignPublic), + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version ); monitor_announce_bundle(m); @@ -1064,7 +1308,7 @@ rollback: if (stmt) sqlite3_finalize(stmt); WHYF("Failed to store bundle bid=%s", manifestid); - sqlite_exec_void_retry(&retry, "ROLLBACK;"); + sqlite_exec_void_retry(&retry, "ROLLBACK;", END); return -1; } @@ -1094,7 +1338,7 @@ int rhizome_list_manifests(struct cli_context *context, const char *service, con RETURN(WHYF("SQL command too long: %s", strbuf_str(b))); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, "%s", strbuf_str(b)); + sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b)); if (!statement) RETURN(-1); @@ -1212,7 +1456,7 @@ int rhizome_list_manifests(struct cli_context *context, const char *service, con cli_put_long(context, rowid, ":"); cli_put_string(context, blob_service, ":"); - cli_put_hexvalue(context, m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES, ":"); + cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":"); cli_put_long(context, blob_version, ":"); cli_put_long(context, blob_date, ":"); cli_put_long(context, q_inserttime, ":"); @@ -1257,13 +1501,18 @@ int rhizome_update_file_priority(const char *fileid) int64_t highestPriority = -1; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; if (sqlite_exec_int64_retry(&retry, &highestPriority, - "SELECT max(grouplist.priority) FROM MANIFESTS,GROUPMEMBERSHIPS,GROUPLIST" - " where manifests.filehash='%s'" - " AND groupmemberships.manifestid=manifests.id" - " AND groupmemberships.groupid=grouplist.id;", - fileid) == -1) + "SELECT max(grouplist.priority) FROM MANIFESTS, GROUPMEMBERSHIPS, GROUPLIST" + " WHERE MANIFESTS.filehash = ?" + " AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id" + " AND GROUPMEMBERSHIPS.groupid = GROUPLIST.id;", + TEXT_TOUPPER, fileid, END) == -1) return -1; - if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) == -1) + if ( highestPriority >= 0 + && sqlite_exec_void_retry(&retry, + "UPDATE files SET highestPriority = ? WHERE id = ?;", + INT, highestPriority, TEXT_TOUPPER, fileid, END + ) == -1 + ) return WHYF("cannot update priority for fileid=%s", fileid); return 0; } @@ -1303,7 +1552,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) int ret = 0; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, "%s", strbuf_str(b)); + sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b)); if (!statement) return -1; @@ -1372,62 +1621,89 @@ next: return ret; } -/* Retrieve a manifest from the database, given its manifest ID. +static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement) +{ + const char *q_id = (const char *) sqlite3_column_text(statement, 0); + const char *q_blob = (char *) sqlite3_column_blob(statement, 1); + int64_t q_version = sqlite3_column_int64(statement, 2); + int64_t q_inserttime = sqlite3_column_int64(statement, 3); + const char *q_author = (const char *) sqlite3_column_text(statement, 4); + size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() + if (rhizome_read_manifest_file(m, q_blob, q_blobsize)) + return WHYF("Manifest %s exists but is invalid", q_id); + if (q_author) { + if (stowSid(m->author, 0, q_author) == -1) + WARNF("manifest id=%s contains invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author)); + } + if (m->version != q_version) + WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version); + m->inserttime = q_inserttime; + return 0; +} + +/* Retrieve a manifest from the database, given its Bundle ID. * * Returns 0 if manifest is found * Returns 1 if manifest is not found * Returns -1 on error * Caller is responsible for allocating and freeing rhizome_manifest */ -int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m) +int rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m) { - int ret=0; - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - - sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT manifest, version, inserttime, author FROM manifests WHERE id like ?"); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id = ?", + RHIZOME_BID_T, bidp, + END); if (!statement) return -1; - - sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC); - if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){ - const char *manifestblob = (char *) sqlite3_column_blob(statement, 0); - int64_t q_version = sqlite3_column_int64(statement, 1); - int64_t q_inserttime = sqlite3_column_int64(statement, 2); - const char *q_author = (const char *) sqlite3_column_text(statement, 3); - size_t manifestblobsize = sqlite3_column_bytes(statement, 0); // must call after sqlite3_column_blob() - - if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize)){ - ret=WHYF("Manifest %s exists but is invalid", manifestid); - goto done; - } - - if (q_author){ - if (stowSid(m->author, 0, q_author) == -1) - WARNF("Manifest %s contains invalid author=%s -- ignored", manifestid, alloca_str_toprint(q_author)); - } - - if (m->version!=q_version) - WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version); - - m->inserttime = q_inserttime; - }else{ - INFOF("Manifest %s was not found", manifestid); - ret=1; - } - -done: + int ret = 1; + if (sqlite_step_retry(&retry, statement) == SQLITE_ROW) + ret = unpack_manifest_row(m, statement); + else + INFOF("Manifest id=%s not found", alloca_tohex_rhizome_bid_t(*bidp)); sqlite3_finalize(statement); - return ret; + return ret; } -static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid) +/* Retrieve any manifest from the database whose Bundle ID starts with the given prefix. + * + * Returns 0 if a manifest is found + * Returns 1 if no manifest is found + * Returns -1 on error + * Caller is responsible for allocating and freeing rhizome_manifest + */ +int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m) { - sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?"); + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + char like[prefix_len * 2 + 2]; + tohex(like, prefix, prefix_len); + like[prefix_len * 2] = '%'; + like[prefix_len * 2 + 1] = '\0'; + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id like ?", + TEXT, like, + END); if (!statement) return -1; - sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC); - if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1) + int ret = 1; + if (sqlite_step_retry(&retry, statement) == SQLITE_ROW) + ret = unpack_manifest_row(m, statement); + else + INFOF("Manifest with id prefix=`%s` not found", like); + sqlite3_finalize(statement); + return ret; +} + +static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp) +{ + sqlite3_stmt *statement = sqlite_prepare_bind(retry, + "DELETE FROM manifests WHERE id = ?", + RHIZOME_BID_T, bidp, + END); + if (!statement) + return -1; + if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1) return -1; return sqlite3_changes(rhizome_db) ? 0 : 1; } @@ -1435,32 +1711,20 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char * static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid) { int ret = 0; - rhizome_delete_external(fileid); - - sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?"); - if (!statement) + sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", TEXT_TOUPPER, fileid, END); + if (!statement || sqlite_exec_retry(retry, statement) == -1) ret = -1; - else { - sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC); - if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1) - ret = -1; - } - statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?"); - if (!statement) + statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", TEXT_TOUPPER, fileid, END); + if (!statement || sqlite_exec_retry(retry, statement) == -1) ret = -1; - else { - sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC); - if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1) - ret = -1; - } return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1; } -static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid) +static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp) { strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1); - int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = '%s'", manifestid); + int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END); if (rows == -1) return -1; if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1) @@ -1476,12 +1740,12 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *m * * @author Andrew Bettison */ -int rhizome_delete_bundle(const char *manifestid) +int rhizome_delete_bundle(const rhizome_bid_t *bidp) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - if (rhizome_delete_payload_retry(&retry, manifestid) == -1) + if (rhizome_delete_payload_retry(&retry, bidp) == -1) return -1; - if (rhizome_delete_manifest_retry(&retry, manifestid) == -1) + if (rhizome_delete_manifest_retry(&retry, bidp) == -1) return -1; return 0; } @@ -1495,10 +1759,10 @@ int rhizome_delete_bundle(const char *manifestid) * * @author Andrew Bettison */ -int rhizome_delete_manifest(const char *manifestid) +int rhizome_delete_manifest(const rhizome_bid_t *bidp) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - return rhizome_delete_manifest_retry(&retry, manifestid); + return rhizome_delete_manifest_retry(&retry, bidp); } /* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest @@ -1510,10 +1774,10 @@ int rhizome_delete_manifest(const char *manifestid) * * @author Andrew Bettison */ -int rhizome_delete_payload(const char *manifestid) +int rhizome_delete_payload(const rhizome_bid_t *bidp) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - return rhizome_delete_payload_retry(&retry, manifestid); + return rhizome_delete_payload_retry(&retry, bidp); } /* Remove a file from the database, given its file hash. @@ -1537,15 +1801,13 @@ static int is_interesting(const char *id_hex, int64_t version) // do we have this bundle [or later]? sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, - "SELECT filehash FROM manifests WHERE id like ? and version >= ?"); - + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT filehash FROM manifests WHERE id like ? and version >= ?", + TEXT_TOUPPER, id_hex, + INT64, version, + END); if (!statement) RETURN(-1); - - sqlite3_bind_text(statement, 1, id_hex, -1, SQLITE_STATIC); - sqlite3_bind_int64(statement, 2, version); - if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){ const char *q_filehash = (const char *) sqlite3_column_text(statement, 0); ret=0; @@ -1553,7 +1815,6 @@ static int is_interesting(const char *id_hex, int64_t version) ret=1; } sqlite3_finalize(statement); - RETURN(ret); OUT(); } diff --git a/rhizome_direct.c b/rhizome_direct.c index 1d3294ed..d816fc00 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -191,22 +191,19 @@ int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r) if (r->cursor->size_high>=r->cursor->limit_size_high) { DEBUG("Out of bins"); - if (memcmp(r->cursor->bid_low,r->cursor->limit_bid_high, - RHIZOME_MANIFEST_ID_BYTES)>=0) - { - DEBUG("out of BIDs"); - /* Sync has finished. - The transport may have initiated one or more transfers, so - we cannot declare the sync complete until we know the transport - has finished transferring. */ - if (!r->bundle_transfers_in_progress) - { - /* seems that all is done */ - DEBUG("All done"); - return rhizome_direct_conclude_sync_request(r); - } else - DEBUG("Stuck on in-progress transfers"); - } else + if (cmp_rhizome_bid_t(&r->cursor->bid_low, &r->cursor->limit_bid_high) >= 0) { + DEBUG("out of BIDs"); + /* Sync has finished. + The transport may have initiated one or more transfers, so + we cannot declare the sync complete until we know the transport + has finished transferring. */ + if (!r->bundle_transfers_in_progress) { + /* seems that all is done */ + DEBUG("All done"); + return rhizome_direct_conclude_sync_request(r); + } else + DEBUG("Stuck on in-progress transfers"); + } else DEBUGF("bid_low=0); - assert(prefix_length<=RHIZOME_MANIFEST_ID_BYTES); - unsigned char low[RHIZOME_MANIFEST_ID_BYTES]; - unsigned char high[RHIZOME_MANIFEST_ID_BYTES]; - - memset(low,0x00,RHIZOME_MANIFEST_ID_BYTES); - memset(high,0xff,RHIZOME_MANIFEST_ID_BYTES); - bcopy(bid_prefix,low,prefix_length); - bcopy(bid_prefix,high,prefix_length); - - char query[1024]; - snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS WHERE ID>='%s' AND ID<='%s'", - alloca_tohex(low,RHIZOME_MANIFEST_ID_BYTES), - alloca_tohex(high,RHIZOME_MANIFEST_ID_BYTES)); - + rhizome_bid_t low = RHIZOME_BID_ZERO; + rhizome_bid_t high = RHIZOME_BID_MAX; + assert(prefix_length >= 0); + assert(prefix_length <= sizeof(rhizome_bid_t)); + bcopy(bid_prefix, low.binary, prefix_length); + bcopy(bid_prefix, high.binary, prefix_length); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, query); - sqlite3_blob *blob=NULL; + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT manifest, rowid FROM MANIFESTS WHERE id >= ? AND id <= ?", + RHIZOME_BID_T, &low, + RHIZOME_BID_T, &high, + END); + sqlite3_blob *blob=NULL; if (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { int ret; @@ -564,9 +556,8 @@ rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size) void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r) { assert(r!=NULL); - r->limit_size_high=1LL<<48LL; - memset(r->limit_bid_high,0xff,RHIZOME_MANIFEST_ID_BYTES); + r->limit_bid_high = RHIZOME_BID_MAX; return; } @@ -599,13 +590,13 @@ int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r, v=r->start_size_high; while(v>1) { ltwov++; v=v>>1; } pickled[0]=ltwov; - for(v=0;v<4;v++) pickled[1+v]=r->start_bid_low[v]; + for(v=0;v<4;v++) pickled[1+v]=r->start_bid_low.binary[v]; v=r->size_high; DEBUGF("pickling size_high=%"PRId64,r->size_high); ltwov=0; while(v>1) { ltwov++; v=v>>1; } pickled[1+4]=ltwov; - for(v=0;v<4;v++) pickled[1+4+1+v]=r->bid_high[v]; + for(v=0;v<4;v++) pickled[1+4+1+v]=r->bid_high.binary[v]; return 1+4+1+4; } @@ -626,13 +617,13 @@ int rhizome_direct_bundle_iterator_unpickle_range(rhizome_direct_bundle_cursor * r->size_high=1LL<size_low=(r->size_high/2)+1; if (r->size_high<=1024) r->size_low=0; - for(v=0;v<4;v++) r->bid_low[v]=pickled[1+v]; - for(;vbid_low[v]=0x00; + r->bid_low = RHIZOME_BID_ZERO; + for (v=0;v<4;v++) r->bid_low.binary[v]=pickled[1+v]; /* Get end of range */ r->limit_size_high=1LL<limit_bid_high[v]=pickled[1+4+1+v]; - for(;vlimit_bid_high[v]=0xff; + r->limit_bid_high = RHIZOME_BID_MAX; + for (v=0;v<4;v++) r->limit_bid_high.binary[v]=pickled[1+4+1+v]; return 0; } @@ -654,7 +645,7 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_ */ /* This is the only information required to remember where we started: */ c->start_size_high=c->size_high; - bcopy(c->bid_low,c->start_bid_low,RHIZOME_MANIFEST_ID_BYTES); + c->start_bid_low = c->bid_low; c->buffer_offset_bytes=1+4+1+4; /* space for pickled cursor range */ /* -1 is magic value for fill right up */ @@ -675,17 +666,15 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_ If we are not yet at the bundle data size limit, then any bundle is okay. If we are at the bundle data size limit, then we need to honour c->limit_bid_high. */ - unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES]; - if (c->size_high==c->limit_size_high) - bcopy(c->limit_bid_high,bid_max,RHIZOME_MANIFEST_ID_BYTES); + rhizome_bid_t bid_max; + if (c->size_high == c->limit_size_high) + bid_max = c->limit_bid_high; else - memset(bid_max,0xff,RHIZOME_MANIFEST_ID_BYTES); - - int stuffed_now=rhizome_direct_get_bars(c->bid_low,c->bid_high, - c->size_low,c->size_high, - bid_max, - &c->buffer[c->buffer_used - +c->buffer_offset_bytes], + bid_max = RHIZOME_BID_MAX; + int stuffed_now=rhizome_direct_get_bars(&c->bid_low, &c->bid_high, + c->size_low, c->size_high, + &bid_max, + &c->buffer[c->buffer_used + c->buffer_offset_bytes], stuffable); bundles_stuffed+=stuffed_now; c->buffer_used+=RHIZOME_BAR_BYTES*stuffed_now; @@ -696,20 +685,19 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_ if (c->size_high<=1024) c->size_low=0; DEBUGF("size=%"PRId64"..%"PRId64,c->size_low,c->size_high); /* Record that we covered to the end of that size bin */ - memset(c->bid_high,0xff,RHIZOME_MANIFEST_ID_BYTES); + c->bid_high = RHIZOME_BID_MAX; if (c->size_high>c->limit_size_high) - memset(c->bid_low,0xff,RHIZOME_MANIFEST_ID_BYTES); + c->bid_low = RHIZOME_BID_MAX; else - memset(c->bid_low,0x00,RHIZOME_MANIFEST_ID_BYTES); + c->bid_low = RHIZOME_BID_ZERO; } else { /* Continue from next BID */ - bcopy(c->bid_high,c->bid_low,RHIZOME_MANIFEST_ID_BYTES); + c->bid_low = c->bid_high; int i; - for(i=RHIZOME_BAR_BYTES-1;i>=0;i--) - { - c->bid_low[i]++; - if (c->bid_low[i]) break; - } + for(i=RHIZOME_BAR_BYTES-1;i>=0;i--) { + if (++c->bid_low.binary[i]) + break; + } if (i<0) break; } } @@ -743,31 +731,28 @@ void rhizome_direct_bundle_iterator_free(rhizome_direct_bundle_cursor **c) it is possible to make provably complete comparison of the contents of the respective rhizome databases. */ -int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES], - unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES], +int rhizome_direct_get_bars(const rhizome_bid_t *bidp_low, + rhizome_bid_t *bidp_high, int64_t size_low, int64_t size_high, - const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES], + const rhizome_bid_t *bidp_max, unsigned char *bars_out, int bars_requested) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - char query[1024]; - snprintf(query,1024, - "SELECT BAR,ROWID,ID,FILESIZE FROM MANIFESTS" - " WHERE" - " FILESIZE BETWEEN %" PRId64 " AND %" PRId64 - " AND ID>='%s' AND ID<='%s'" - // The following formulation doesn't remove the weird returning of - // bundles with out of range filesize values - // " WHERE ID>='%s' AND ID<='%s' AND FILESIZE > %lld AND FILESIZE < %lld" - " ORDER BY BAR LIMIT %d;", - size_low, size_high, - alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES), - alloca_tohex(bid_max,RHIZOME_MANIFEST_ID_BYTES), - bars_requested); - - sqlite3_stmt *statement=sqlite_prepare(&retry, query); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, + "SELECT bar, rowid, id, filesize FROM MANIFESTS" + " WHERE filesize BETWEEN ? AND ? AND id >= ? AND id <= ?" + " ORDER BY bar LIMIT ?;", + INT64, size_low, + INT64, size_high, + RHIZOME_BID_T, bidp_low, + RHIZOME_BID_T, bidp_max, + INT, bars_requested, + // The following formulation doesn't remove the weird returning of + // bundles with out of range filesize values + // " WHERE id >= ? AND id <= ? AND filesize > ? AND filesize < ?" + END); sqlite3_blob *blob=NULL; int bars_written=0; @@ -784,8 +769,7 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE int ret; int64_t filesize = sqlite3_column_int64(statement, 3); if (filesizesize_high) { - DEBUGF("WEIRDNESS ALERT: filesize=%"PRId64", but query was: %s", - filesize,query); + DEBUGF("WEIRDNESS ALERT: filesize=%"PRId64", but query was: %s", filesize, sqlite3_sql(statement)); break; } int64_t rowid = sqlite3_column_int64(statement, 1); @@ -813,9 +797,7 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE /* Remember the BID so that we cant write it into bid_high so that the caller knows how far we got. */ - fromhex(bid_high, - (const char *)sqlite3_column_text(statement, 2), - RHIZOME_MANIFEST_ID_BYTES); + str_to_rhizome_bid_t(bidp_high, (const char *)sqlite3_column_text(statement, 2)); bars_written++; break; @@ -831,4 +813,4 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE return bars_written; } - + diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index 7cfbac81..99de479f 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -884,17 +884,14 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* Start by getting the manifest, which is the main thing we need, and also gives us the information we need for sending any associated file. */ - rhizome_manifest - *m=rhizome_direct_get_manifest(&actionlist[i+1], - RHIZOME_BAR_PREFIX_BYTES); - if (!m) { + rhizome_manifest *m = rhizome_direct_get_manifest(&actionlist[i+1], RHIZOME_BAR_PREFIX_BYTES); + if (m == NULL) { WHY("This should never happen. The manifest exists, but when I went looking for it, it doesn't appear to be there."); goto next_item; } /* Get filehash and size from manifest if present */ - const char *id = rhizome_manifest_get(m, "id", NULL, 0); - DEBUGF("bundle id = '%s'",id); + DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0); DEBUGF("bundle file hash = '%s'",hash); long long filesize = rhizome_manifest_get_ll(m, "filesize"); @@ -965,7 +962,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* send file contents */ { char filehash[SHA512_DIGEST_STRING_LENGTH]; - if (rhizome_database_filehash_from_id(id, version, filehash)<=0) + if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, filehash) == -1) goto closeit; struct rhizome_read read; @@ -1046,10 +1043,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) c->size_high,c->limit_size_high); DEBUGF("c->buffer_size=%d",c->buffer_size); r->cursor->size_low=c->limit_size_high; - bcopy(c->limit_bid_high,r->cursor->bid_low,4); /* Set tail of BID to all high, as we assume the far end has returned all BIDs with the specified prefix. */ - memset(&r->cursor->bid_low[4],0xff,RHIZOME_MANIFEST_ID_BYTES); + r->cursor->bid_low = RHIZOME_BID_MAX; + bcopy(c->limit_bid_high.binary, r->cursor->bid_low.binary, 4); } rhizome_direct_bundle_iterator_free(&c); #endif diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 91d81a7e..3a056fb1 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -76,7 +76,7 @@ struct rhizome_fetch_slot { int manifest_bytes; /* MDP transport specific elements */ - unsigned char bid[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t bid; int64_t bidVersion; int prefix_length; int mdpIdleTimeout; @@ -263,7 +263,7 @@ static struct rhizome_fetch_slot *fetch_search_slot(const unsigned char *id, int struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; if (q->active.state != RHIZOME_FETCH_FREE && - memcmp(id, q->active.manifest->cryptoSignPublic, prefix_length) == 0) + memcmp(id, q->active.manifest->cryptoSignPublic.binary, prefix_length) == 0) return &q->active; } return NULL; @@ -279,7 +279,7 @@ static struct rhizome_fetch_candidate *fetch_search_candidate(const unsigned cha struct rhizome_fetch_candidate *c = &q->candidate_queue[j]; if (!c->manifest) continue; - if (memcmp(c->manifest->cryptoSignPublic, id, prefix_length)) + if (memcmp(c->manifest->cryptoSignPublic.binary, id, prefix_length)) continue; return c; } @@ -478,8 +478,8 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) slot->write_state.blob_rowid=-1; if (slot->manifest) { - bcopy(slot->manifest->cryptoSignPublic,slot->bid,RHIZOME_MANIFEST_ID_BYTES); - slot->prefix_length=RHIZOME_MANIFEST_ID_BYTES; + slot->bid = slot->manifest->cryptoSignPublic; + slot->prefix_length = sizeof slot->bid.binary; slot->bidVersion=slot->manifest->version; /* Don't provide a filename, because we will stream the file straight into the database. */ @@ -493,8 +493,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) // if we're fetching a journal bundle, work out how many bytes we have of a previous version // and therefore what range of bytes we should ask for slot->previous = rhizome_new_manifest(); - const char *id = rhizome_manifest_get(slot->manifest, "id", NULL, 0); - if (rhizome_retrieve_manifest(id, slot->previous)){ + if (rhizome_retrieve_manifest(&slot->manifest->cryptoSignPublic, slot->previous)){ rhizome_manifest_free(slot->previous); slot->previous=NULL; @@ -519,7 +518,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) RETURN(-1); } else { strbuf r = strbuf_local(slot->request, sizeof slot->request); - strbuf_sprintf(r, "GET /rhizome/manifestbyprefix/%s HTTP/1.0\r\n\r\n", alloca_tohex(slot->bid, slot->prefix_length)); + strbuf_sprintf(r, "GET /rhizome/manifestbyprefix/%s HTTP/1.0\r\n\r\n", alloca_tohex(slot->bid.binary, slot->prefix_length)); if (strbuf_overrun(r)) RETURN(WHY("request overrun")); slot->request_len = strbuf_len(r); @@ -631,8 +630,6 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct if (slot->state != RHIZOME_FETCH_FREE) RETURN(SLOTBUSY); - const char *bid = alloca_tohex_bid(m->cryptoSignPublic); - /* Do the quick rejection tests first, before the more expensive ones, like querying the database for manifests. @@ -650,7 +647,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct if (config.debug.rhizome_rx) DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRId64" peerip=%s", slotno(slot), - bid, + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, alloca_sockaddr(peerip, sizeof(struct sockaddr_in)) @@ -671,7 +668,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct * being published faster than we can fetch them. */ { - struct rhizome_fetch_slot *as = fetch_search_slot(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES); + struct rhizome_fetch_slot *as = fetch_search_slot(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); if (as){ const rhizome_manifest *am = as->manifest; if (am->version < m->version) { @@ -746,7 +743,7 @@ rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip, slot->peer_ipandport = *peerip; slot->manifest = NULL; bcopy(peersid,slot->peer_sid,SID_SIZE); - bcopy(prefix,slot->bid,prefix_length); + bcopy(prefix, slot->bid.binary, prefix_length); slot->prefix_length=prefix_length; /* Don't stream into a file blob in the database, because it is a manifest. @@ -857,11 +854,11 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock RETURN(0); } - const char *bid = alloca_tohex_bid(m->cryptoSignPublic); int priority=100; /* normal priority */ if (config.debug.rhizome_rx) - DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", bid, m->version, m->fileLength, priority); + DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, priority); if (!rhizome_is_manifest_interesting(m)) { if (config.debug.rhizome_rx) @@ -872,7 +869,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (config.debug.rhizome_rx) { int64_t stored_version; - if (sqlite_exec_int64(&stored_version, "select version from manifests where id='%s'", bid) > 0) + if (sqlite_exec_int64(&stored_version, "SELECT version FROM MANIFESTS WHERE id = ?", RHIZOME_BID_T, &m->cryptoSignPublic, END) > 0) DEBUGF(" is new (have version %"PRId64")", stored_version); } @@ -880,8 +877,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (rhizome_manifest_verify(m) != 0) { WHY("Error verifying manifest when considering for import"); /* Don't waste time looking at this manifest again for a while */ - rhizome_queue_ignore_manifest(m->cryptoSignPublic, - crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000); + rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); rhizome_manifest_free(m); RETURN(-1); } @@ -908,7 +904,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock for (j = 0; j < q->candidate_queue_size; ) { struct rhizome_fetch_candidate *c = &q->candidate_queue[j]; if (c->manifest) { - if (memcmp(m->cryptoSignPublic, c->manifest->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES) == 0) { + if (cmp_rhizome_bid_t(&m->cryptoSignPublic, &c->manifest->cryptoSignPublic) == 0) { if (c->manifest->version >= m->version) { rhizome_manifest_free(m); RETURN(0); @@ -916,8 +912,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (!m->selfSigned && rhizome_manifest_verify(m)) { WHY("Error verifying manifest when considering queuing for import"); /* Don't waste time looking at this manifest again for a while */ - rhizome_queue_ignore_manifest(m->cryptoSignPublic, - crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000); + rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); rhizome_manifest_free(m); RETURN(-1); } @@ -943,8 +938,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (!m->selfSigned && rhizome_manifest_verify(m)) { WHY("Error verifying manifest when considering queuing for import"); /* Don't waste time looking at this manifest again for a while */ - rhizome_queue_ignore_manifest(m->cryptoSignPublic, - crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000); + rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); rhizome_manifest_free(m); RETURN(-1); } @@ -966,7 +960,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock break; DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%lld", i, j, c->manifest, - alloca_tohex_bid(c->manifest->cryptoSignPublic), + alloca_tohex_rhizome_bid_t(c->manifest->cryptoSignPublic), c->priority, (long long) c->manifest->fileLength ); @@ -1080,8 +1074,8 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot) mdp.packetTypeAndFlags=MDP_TX; mdp.out.queue=OQ_ORDINARY; - mdp.out.payload_length=RHIZOME_MANIFEST_ID_BYTES+8+8+4+2; - bcopy(slot->bid,&mdp.out.payload[0],RHIZOME_MANIFEST_ID_BYTES); + mdp.out.payload_length= sizeof slot->bid.binary + 8 + 8 + 4 + 2; + bcopy(slot->bid.binary, &mdp.out.payload[0], sizeof slot->bid.binary); uint32_t bitmap=0; int requests=32; @@ -1099,11 +1093,11 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot) } offset+=slot->mdpRXBlockLength; } - - write_uint64(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES], slot->bidVersion); - write_uint64(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8], slot->write_state.file_offset); - write_uint32(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8], bitmap); - write_uint16(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8+4], slot->mdpRXBlockLength); + + write_uint64(&mdp.out.payload[sizeof slot->bid.binary], slot->bidVersion); + write_uint64(&mdp.out.payload[sizeof slot->bid.binary + 8], slot->write_state.file_offset); + write_uint32(&mdp.out.payload[sizeof slot->bid.binary + 8 + 8], bitmap); + write_uint16(&mdp.out.payload[sizeof slot->bid.binary + 8 + 8 + 4], slot->mdpRXBlockLength); if (config.debug.rhizome_tx) DEBUGF("src sid=%s, dst sid=%s, mdpRXWindowStart=0x%"PRIx64", slot->bidVersion=0x%"PRIx64, @@ -1297,7 +1291,7 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) rhizome_manifest_free(m); } else { if (config.debug.rhizome_rx){ - DEBUGF("All looks good for importing manifest id=%s", alloca_tohex_bid(m->cryptoSignPublic)); + DEBUGF("All looks good for importing manifest id=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); dump("slot->peerip",&slot->peer_ipandport,sizeof(slot->peer_ipandport)); dump("slot->peersid",&slot->peer_sid,sizeof(slot->peer_sid)); } @@ -1307,11 +1301,16 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) } } - if (config.debug.rhizome_rx) + if (config.debug.rhizome_rx) { + time_ms_t now = gettime_ms(); + time_ms_t interval = now - slot->start_time; + if (interval <= 0) + interval = 1; DEBUGF("Closing rhizome fetch slot = 0x%p. Received %lld bytes in %lldms (%lldKB/sec).", slot,(long long)slot->write_state.file_offset, - (long long)gettime_ms()-slot->start_time, - (long long)(slot->write_state.file_offset)/(gettime_ms()-slot->start_time)); + (long long)interval, + (long long)((slot->write_state.file_offset) / interval)); + } rhizome_fetch_close(slot); RETURN(-1); @@ -1355,7 +1354,7 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer OUT(); } -int rhizome_received_content(unsigned char *bidprefix, +int rhizome_received_content(const unsigned char *bidprefix, uint64_t version, uint64_t offset, int count,unsigned char *bytes,int type) { diff --git a/rhizome_http.c b/rhizome_http.c index 81ac3f6b..1514e11b 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -497,13 +497,13 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde { if (!is_rhizome_http_enabled()) return 1; - - char id_hex[RHIZOME_MANIFEST_ID_STRLEN+1]; - strncpy(id_hex, remainder, sizeof id_hex -1); - str_toupper_inplace(id_hex); - strcat(id_hex, "%"); + rhizome_bid_t prefix; + const char *endp = NULL; + unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp); + if (endp == NULL || *endp != '\0' || prefix_len < 1) + return 1; // not found rhizome_manifest *m = rhizome_new_manifest(); - int ret = rhizome_retrieve_manifest(id_hex, m); + int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m); if (ret==0) rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes); rhizome_manifest_free(m); diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c index 6cac9139..ff669468 100644 --- a/rhizome_packetformats.c +++ b/rhizome_packetformats.c @@ -84,7 +84,7 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar) /* Manifest prefix */ for(i=0;icryptoSignPublic[i]; + bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic.binary[i]; /* file length */ bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->fileLength); /* Version */ @@ -134,38 +134,43 @@ uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar) return bidprefix; } -static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, const char *sql, int64_t *last_rowid){ - int count=0; - - sqlite3_stmt *statement=sqlite_prepare(retry, sql, *last_rowid); - +static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, const char *sql, int64_t *last_rowid) +{ + sqlite3_stmt *statement = sqlite_prepare(retry, sql); + if (statement == NULL) + return -1; + int params = sqlite3_bind_parameter_count(statement); + switch (params) { + case 0: break; + case 1: + if (sqlite_bind(retry, statement, INT64, *last_rowid, END) == -1) + return -1; + break; + default: + return WHYF("query has invalid number of parameters (%d): %s", params, sqlite3_sql(statement)); + } + int count = 0; while(sqlite_step_retry(retry, statement) == SQLITE_ROW) { count++; if (sqlite3_column_type(statement, 0)!=SQLITE_BLOB) continue; - const void *data = sqlite3_column_blob(statement, 0); int blob_bytes = sqlite3_column_bytes(statement, 0); int64_t rowid = sqlite3_column_int64(statement, 1); - if (blob_bytes!=RHIZOME_BAR_BYTES) { if (config.debug.rhizome_ads) DEBUG("Found a BAR that is the wrong size - ignoring"); continue; } - if (ob_append_bytes(e, (unsigned char *)data, blob_bytes)){ // out of room count--; break; } - *last_rowid=rowid; } - if (statement) sqlite3_finalize(statement); - return count; } @@ -191,7 +196,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){ goto end; /* Get number of bundles available */ - if (sqlite_exec_int64_retry(&retry, &bundles_available, "SELECT COUNT(BAR) FROM MANIFESTS;") != 1){ + if (sqlite_exec_int64_retry(&retry, &bundles_available, "SELECT COUNT(BAR) FROM MANIFESTS;", END) != 1){ WHY("Could not count BARs for advertisement"); goto end; } @@ -221,7 +226,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){ bundle_last_rowid=rowid; count = append_bars(frame->payload, &retry, - "SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < %lld ORDER BY ROWID DESC LIMIT 17", + "SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < ? ORDER BY ROWID DESC LIMIT 17", &bundle_last_rowid); if (count<17) bundle_last_rowid=INT64_MAX; @@ -263,7 +268,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){ if (overlay_payload_enqueue(frame)) goto error; if (config.debug.rhizome_ads) DEBUGF("Advertising manifest %s %"PRId64" to %s", - alloca_tohex_bid(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid(dest->sid):"broadcast"); + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid(dest->sid):"broadcast"); return 0; error: @@ -334,7 +339,7 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st (that is the only use of this */ if (config.debug.rhizome_ads){ long long version = rhizome_manifest_get_ll(m, "version"); - DEBUGF("manifest id=%s version=%lld", alloca_tohex_bid(m->cryptoSignPublic), version); + DEBUGF("manifest id=%s version=%lld", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), version); } /* Crude signature presence test */ @@ -347,10 +352,10 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st goto next; } - if (rhizome_ignore_manifest_check(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES)){ + if (rhizome_ignore_manifest_check(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary)){ /* Ignoring manifest that has caused us problems recently */ if (config.debug.rhizome_ads) - DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_bid(m->cryptoSignPublic)); + DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); goto next; } @@ -359,14 +364,13 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st DEBUG("Unverified manifest has errors - so not processing any further."); /* Don't waste any time on this manifest in future attempts for at least a minute. */ - rhizome_queue_ignore_manifest(m->cryptoSignPublic, - RHIZOME_MANIFEST_ID_BYTES, 60000); + rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); goto next; } /* Manifest is okay, so see if it is worth storing */ // are we already fetching this bundle [or later]? - rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES); + rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); if (mf && mf->version >= m->version) goto next; diff --git a/rhizome_store.c b/rhizome_store.c index e4a96ccd..7d7c0807 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -8,12 +8,13 @@ int rhizome_exists(const char *fileHash) { int64_t gotfile = 0; - if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE ID='%s' and datavalid=1;", fileHash) != 1) + if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", TEXT_TOUPPER, fileHash, END) != 1) return 0; return gotfile; } -int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority){ +int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority) +{ write->blob_fd=-1; if (expectedFileHash){ @@ -24,15 +25,16 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6 }else{ write->id_known=0; } + time_ms_t now = gettime_ms(); static uint64_t last_id=0; - write->temp_id = gettime_ms(); - if (write->temp_idtemp_id=last_id+1; - last_id=write->temp_id; + write->temp_id = now; + if (write->temp_id < last_id) + write->temp_id = last_id + 1; + last_id = write->temp_id; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1) + if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) return WHY("Failed to begin transaction"); /* @@ -45,10 +47,15 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6 */ sqlite3_stmt *statement = NULL; - int ret=sqlite_exec_void_retry(&retry, - "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) " - "VALUES('%"PRId64"',%"PRId64",%d,0,%"PRId64");", - write->temp_id, file_length, priority, gettime_ms()); + int ret = sqlite_exec_void_retry( + &retry, + "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES(?,?,?,0,?);", + UINT64_TOSTR, write->temp_id, + INT64, file_length, + INT, priority, + INT64, now, + END + ); if (ret==-1) goto insert_row_fail; @@ -61,8 +68,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6 } if (config.debug.externalblobs) - DEBUGF("Attempting to put blob for %"PRId64" in %s", - write->temp_id,blob_path); + DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path); write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664); if (write->blob_fd<0) @@ -72,45 +78,39 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6 DEBUGF("Writing to new blob file %s (fd=%d)", blob_path, write->blob_fd); }else{ - statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%"PRId64"',?)",write->temp_id); - if (!statement) { - WHYF("Failed to insert into fileblobs: %s", sqlite3_errmsg(rhizome_db)); + statement = sqlite_prepare_bind( + &retry, + "INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES(?,?)", + UINT64_TOSTR, write->temp_id, + ZEROBLOB, (int)file_length, + END); + if (statement == NULL) goto insert_row_fail; - } - - /* Bind appropriate sized zero-filled blob to data field */ - if (sqlite3_bind_zeroblob(statement, 1, file_length) != SQLITE_OK) { - WHYF("sqlite3_bind_zeroblob() failed: %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement)); - goto insert_row_fail; - } - /* Do actual insert, and abort if it fails */ int rowcount = 0; int stepcode; - while ((stepcode = _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) + while ((stepcode = sqlite_step_retry(&retry, statement)) == SQLITE_ROW) ++rowcount; if (rowcount) WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s"); - if (!sqlite_code_ok(stepcode)){ insert_row_fail: - WHYF("Failed to insert row for fileid=%"PRId64, write->temp_id); + WHYF("Failed to insert row for id='%"PRId64"'", write->temp_id); if (statement) sqlite3_finalize(statement); - sqlite_exec_void_retry(&retry, "ROLLBACK;"); + sqlite_exec_void_retry(&retry, "ROLLBACK;", END); return -1; } - sqlite3_finalize(statement); statement=NULL; /* Get rowid for inserted row, so that we can modify the blob */ write->blob_rowid = sqlite3_last_insert_rowid(rhizome_db); if (config.debug.rhizome_rx) - DEBUGF("Got rowid %"PRId64" for %"PRId64, write->blob_rowid, write->temp_id); + DEBUGF("Got rowid=%"PRId64" for id='%"PRId64"'", write->blob_rowid, write->temp_id); } - if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1){ + if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){ if (write->blob_fd>=0){ if (config.debug.externalblobs) DEBUGF("Cancel write to fd %d", write->blob_fd); @@ -167,7 +167,7 @@ static int write_get_lock(struct rhizome_write *write_state){ sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; // use an explicit transaction so we can delay I/O failures until COMMIT so they can be retried. - if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1) + if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) return -1; while(1){ @@ -243,7 +243,7 @@ static int write_release_lock(struct rhizome_write *write_state){ sqlite3_errmsg(rhizome_db)); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1) + if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1) ret=-1; } write_state->sql_blob=NULL; @@ -477,21 +477,27 @@ int rhizome_finish_write(struct rhizome_write *write) rhizome_remove_file_datainvalid(&retry, write->id); if (rhizome_exists(write->id)) { // we've already got that payload, delete the new copy - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%"PRId64"';", write->temp_id); - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%"PRId64"';", write->temp_id); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); if (config.debug.rhizome) DEBUGF("File id='%s' already present, removed id='%"PRId64"'", write->id, write->temp_id); } else { - if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1) + if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) goto dbfailure; // delete any half finished records - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out); - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", STATIC_TEXT, write->id, END); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", STATIC_TEXT, write->id, END); - if (sqlite_exec_void_retry(&retry, - "UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%"PRId64"'", - hash_out, gettime_ms(), write->temp_id) == -1) + if (sqlite_exec_void_retry( + &retry, + "UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?", + STATIC_TEXT, write->id, + INT64, gettime_ms(), + UINT64_TOSTR, write->temp_id, + END + ) == -1 + ) goto dbfailure; if (fd>=0){ @@ -501,7 +507,7 @@ int rhizome_finish_write(struct rhizome_write *write) WHYF("Failed to generate file path"); goto dbfailure; } - if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, hash_out)){ + if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, write->id)){ WHYF("Failed to generate file path"); goto dbfailure; } @@ -512,13 +518,17 @@ int rhizome_finish_write(struct rhizome_write *write) } }else{ - if (sqlite_exec_void_retry(&retry, - "UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld", - hash_out, write->blob_rowid) == -1){ - goto dbfailure; - } + if (sqlite_exec_void_retry( + &retry, + "UPDATE FILEBLOBS SET id = ? WHERE rowid = ?", + STATIC_TEXT, write->id, + INT64, write->blob_rowid, + END + ) == -1 + ) + goto dbfailure; } - if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1) + if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1) goto dbfailure; if (config.debug.rhizome) DEBUGF("Stored file %s", write->id); @@ -527,7 +537,7 @@ int rhizome_finish_write(struct rhizome_write *write) return 0; dbfailure: - sqlite_exec_void_retry(&retry, "ROLLBACK;"); + sqlite_exec_void_retry(&retry, "ROLLBACK;", END); failure: rhizome_fail_write(write); return -1; @@ -630,7 +640,7 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru return -1; if (config.debug.rhizome) - DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version); + DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); write->crypt=1; if (m->journalTail>0) @@ -687,9 +697,9 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid) if (sqlite_exec_int64(&read->blob_rowid, "SELECT FILEBLOBS.rowid " "FROM FILEBLOBS, FILES " - "WHERE FILEBLOBS.id = FILES.id " - "AND FILES.id = '%s' " - "AND FILES.datavalid != 0", read->id) == -1) + "WHERE FILEBLOBS.id = FILES.id" + " AND FILES.id = ?" + " AND FILES.datavalid != 0", STATIC_TEXT, read->id, END) == -1) return -1; if (read->blob_rowid != -1) { read->length = -1; // discover the length on opening the db BLOB @@ -872,18 +882,18 @@ int rhizome_read_close(struct rhizome_read *read) struct cache_entry{ struct cache_entry *_left; struct cache_entry *_right; - unsigned char bundle_id[RHIZOME_MANIFEST_ID_BYTES]; + rhizome_bid_t bundle_id; uint64_t version; struct rhizome_read read_state; time_ms_t expires; }; struct cache_entry *root; -static struct cache_entry ** find_entry_location(struct cache_entry **ptr, unsigned char *bundle_id, uint64_t version) +static struct cache_entry ** find_entry_location(struct cache_entry **ptr, const rhizome_bid_t *bundle_id, uint64_t version) { while(*ptr){ struct cache_entry *entry = *ptr; - int cmp = memcmp(bundle_id, entry->bundle_id, sizeof entry->bundle_id); + int cmp = cmp_rhizome_bid_t(bundle_id, &entry->bundle_id); if (cmp==0){ if (entry->version==version) break; @@ -921,7 +931,7 @@ static time_ms_t close_entries(struct cache_entry **entry, time_ms_t timeout) // re-add both children to the tree *entry=left; if (right){ - entry = find_entry_location(entry, right->bundle_id, right->version); + entry = find_entry_location(entry, &right->bundle_id, right->version); *entry=right; } }else{ @@ -970,19 +980,17 @@ int rhizome_cache_count() } // read a block of data, caching meta data for reuse -int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout, +int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length) { // look for a cached entry - struct cache_entry **ptr = find_entry_location(&root, bundle_id, version); + struct cache_entry **ptr = find_entry_location(&root, bidp, version); struct cache_entry *entry = *ptr; // if we don't have one yet, create one and open it if (!entry){ - char *id_str = alloca_tohex_bid(bundle_id); - char filehash[SHA512_DIGEST_STRING_LENGTH]; - if (rhizome_database_filehash_from_id(id_str, version, filehash)<=0) + if (rhizome_database_filehash_from_id(bidp, version, filehash) == -1) return -1; entry = emalloc_zero(sizeof(struct cache_entry)); @@ -991,7 +999,7 @@ int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t ti free(entry); return WHYF("Payload %s not found", filehash); } - bcopy(bundle_id, entry->bundle_id, sizeof(entry->bundle_id)); + entry->bundle_id = *bidp; entry->version = version; *ptr = entry; } @@ -1055,7 +1063,7 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom return WHY("Unable to decrypt bundle, valid key not found"); } if (config.debug.rhizome) - DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version); + DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); if (m->journalTail>0) read_state->tail = m->journalTail; diff --git a/serval.h b/serval.h index 1214897c..402038b4 100644 --- a/serval.h +++ b/serval.h @@ -552,6 +552,7 @@ typedef struct sid_binary { #define alloca_tohex_sid_t(sid) alloca_tohex((sid).binary, sizeof (*(sid_t*)0).binary) +int cmp_sid_t(const sid_t *a, const sid_t *b); int str_to_sid_t(sid_t *sid, const char *hex); int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp); diff --git a/tests/rhizomeops b/tests/rhizomeops index a8c38cab..b8ca6976 100755 --- a/tests/rhizomeops +++ b/tests/rhizomeops @@ -577,7 +577,7 @@ test_AddMismatched() { doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails" setup_AddUpdateSameVersion() { - setup_AddDuplicate + setup_AddDeDuplicate cp file1.manifest file1_2.manifest strip_signatures file1_2.manifest $SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d' file1_2.manifest diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol index 86f575ce..9a5a6786 100755 --- a/tests/rhizomeprotocol +++ b/tests/rhizomeprotocol @@ -710,7 +710,7 @@ test_DirectPull() { set_instance +B executeOk_servald rhizome direct pull tfw_cat --stdout --stderr - assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr + assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr set_instance +A executeOk_servald rhizome list assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 @@ -732,7 +732,7 @@ test_DirectSync() { set_instance +B executeOk_servald rhizome direct sync tfw_cat --stdout --stderr - assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr + assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr set_instance +A executeOk_servald rhizome list assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3