diff --git a/cli.c b/cli.c index 9d1ea003..2013f262 100644 --- a/cli.c +++ b/cli.c @@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "rhizome.h" #include "strbuf_helpers.h" +#include "dataformats.h" int cli_usage(const struct cli_schema *commands, XPRINTF xpf) { diff --git a/commandline.c b/commandline.c index c26336fc..11be1a4a 100644 --- a/commandline.c +++ b/commandline.c @@ -45,6 +45,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "cli.h" #include "overlay_address.h" #include "overlay_buffer.h" +#include "keyring.h" +#include "dataformats.h" extern struct cli_schema command_line_options[]; @@ -250,6 +252,7 @@ int parseCommandLine(struct cli_context *context, const char *argv0, int argc, c /* clean up after ourselves */ rhizome_close_db(); + free_subscribers(); OUT(); if (config.debug.timing) @@ -929,9 +932,6 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context) int64_t interval_ms = 1000; str_to_uint64_interval_ms(opt_interval, &interval_ms, NULL); - overlay_mdp_frame mdp; - bzero(&mdp, sizeof(overlay_mdp_frame)); - /* First sequence number in the echo frames */ unsigned int firstSeq=random(); unsigned int sequence_number=firstSeq; @@ -939,22 +939,23 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context) int broadcast = is_sid_t_broadcast(ping_sid); /* Bind to MDP socket and await confirmation */ - if ((mdp_sockfd = overlay_mdp_client_socket()) < 0) + if ((mdp_sockfd = mdp_socket()) < 0) return WHY("Cannot create MDP socket"); - sid_t srcsid; - mdp_port_t port=32768+(random()&32767); - if (overlay_mdp_getmyaddr(mdp_sockfd, 0, &srcsid)) { - overlay_mdp_client_close(mdp_sockfd); - return WHY("Could not get local address"); - } - if (overlay_mdp_bind(mdp_sockfd, &srcsid, port)) { - overlay_mdp_client_close(mdp_sockfd); - return WHY("Could not bind to MDP socket"); - } + struct mdp_header mdp_header; + bzero(&mdp_header, sizeof(mdp_header)); + + mdp_header.local.sid = BIND_PRIMARY; + mdp_header.remote.sid = ping_sid; + mdp_header.remote.port = MDP_PORT_ECHO; + mdp_header.qos = OQ_MESH_MANAGEMENT; + mdp_header.ttl = PAYLOAD_TTL_DEFAULT; + mdp_header.flags = MDP_FLAG_BIND; + if (broadcast) + mdp_header.flags |= MDP_FLAG_NO_CRYPT; /* TODO Eventually we should try to resolve SID to phone number and vice versa */ - cli_printf(context, "MDP PING %s (%s): 12 data bytes", alloca_tohex_sid_t(ping_sid), alloca_tohex_sid_t(ping_sid)); + cli_printf(context, "MDP PING %s: 12 data bytes", alloca_tohex_sid_t(ping_sid)); cli_delim(context, "\n"); cli_flush(context); @@ -966,81 +967,91 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context) if (broadcast) WARN("broadcast ping packets will not be encrypted"); + for (; icount==0 || tx_count0) { - int ttl=-1; - if (overlay_mdp_recv(mdp_sockfd, &mdp, port, &ttl)==0) { - switch(mdp.packetTypeAndFlags&MDP_TYPE_MASK) { - case MDP_ERROR: - WHYF("mdpping: overlay_mdp_recv: %s (code %d)", mdp.error.message, mdp.error.error); - break; - case MDP_TX: - { - int *rxseq=(int *)&mdp.in.payload; - time_ms_t txtime = read_uint64(&mdp.in.payload[4]); - int hop_count = 64 - mdp.in.ttl; - time_ms_t delay = gettime_ms() - txtime; - cli_printf(context, "%s: seq=%d time=%"PRId64"ms hops=%d %s%s", - alloca_tohex_sid_t(mdp.in.src.sid), - (*rxseq)-firstSeq+1, - (int64_t)delay, - hop_count, - mdp.packetTypeAndFlags&MDP_NOCRYPT?"":" ENCRYPTED", - mdp.packetTypeAndFlags&MDP_NOSIGN?"":" SIGNED"); - cli_delim(context, "\n"); - cli_flush(context); - // TODO Put duplicate pong detection here so that stats work properly. - rx_count++; - ret=0; - rx_ms+=delay; - if (rx_mintime>delay||rx_mintime==-1) rx_mintime=delay; - if (delay>rx_maxtime) rx_maxtime=delay; - rx_times[rx_count%1024]=delay; - } - break; - default: - WHYF("mdpping: overlay_mdp_recv: Unexpected MDP frame type 0x%x", mdp.packetTypeAndFlags); - break; - } - } + + if (mdp_poll(mdp_sockfd, poll_timeout_ms)<=0) + continue; + + struct mdp_header mdp_recv_header; + uint8_t recv_payload[12]; + ssize_t len = mdp_recv(mdp_sockfd, &mdp_recv_header, recv_payload, sizeof(recv_payload)); + + if (len<0){ + WHY_perror("mdp_recv"); + break; } + + if (mdp_recv_header.flags & MDP_FLAG_ERROR){ + WHY("Serval daemon reported an error, please check the log for more information"); + break; + } + + if (mdp_recv_header.flags & MDP_FLAG_BIND){ + // received port binding confirmation + mdp_header.local = mdp_recv_header.local; + mdp_header.flags &= ~MDP_FLAG_BIND; + DEBUGF("Bound to %s:%d", alloca_tohex_sid_t(mdp_header.local.sid), mdp_header.local.port); + continue; + } + + if (lendelay||rx_mintime==-1) rx_mintime=delay; + if (delay>rx_maxtime) rx_maxtime=delay; + rx_times[rx_count%1024]=delay; } } - overlay_mdp_client_close(mdp_sockfd); + mdp_close(mdp_sockfd); { float rx_stddev=0; @@ -1099,24 +1110,25 @@ int app_trace(const struct cli_parsed *parsed, struct cli_context *context) mdp.out.dst.port=MDP_PORT_TRACE; mdp.packetTypeAndFlags=MDP_TX; struct overlay_buffer *b = ob_static(mdp.out.payload, sizeof(mdp.out.payload)); - ob_append_byte(b, SID_SIZE); ob_append_bytes(b, srcsid.binary, SID_SIZE); - ob_append_byte(b, SID_SIZE); ob_append_bytes(b, dstsid.binary, SID_SIZE); - - mdp.out.payload_length = ob_position(b); - cli_printf(context, "Tracing the network path from %s to %s", - alloca_tohex_sid_t(srcsid), alloca_tohex_sid_t(dstsid)); - cli_delim(context, "\n"); - cli_flush(context); - - int ret=overlay_mdp_send(mdp_sockfd, &mdp, MDP_AWAITREPLY, 5000); + int ret; + if (ob_overrun(b)) + ret = WHY("overlay buffer overrun"); + else { + mdp.out.payload_length = ob_position(b); + cli_printf(context, "Tracing the network path from %s to %s", + alloca_tohex_sid_t(srcsid), alloca_tohex_sid_t(dstsid)); + cli_delim(context, "\n"); + cli_flush(context); + ret = overlay_mdp_send(mdp_sockfd, &mdp, MDP_AWAITREPLY, 5000); + if (ret) + WHYF("overlay_mdp_send returned %d", ret); + } ob_free(b); - if (ret) - DEBUGF("overlay_mdp_send returned %d", ret); - else{ + if (ret == 0) { int offset=0; { // skip the first two sid's @@ -1314,25 +1326,32 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co if (create_serval_instance_dir() == -1) return -1; + if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + + if (rhizome_opendb() == -1){ + keyring_free(keyring); return -1; + } /* Create a new manifest that will represent the file. If a manifest file was supplied, then read * it, otherwise create a blank manifest. */ rhizome_manifest *m = rhizome_new_manifest(); - if (!m) + if (!m){ + keyring_free(keyring); return WHY("Manifest struct could not be allocated -- not added to rhizome"); - + } if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) { if (config.debug.rhizome) DEBUGF("reading manifest from %s", manifestpath); /* Don't verify the manifest, because it will fail if it is incomplete. This is okay, because we fill in any missing bits and sanity check before - trying to write it out. */ - if (rhizome_read_manifest_file(m, manifestpath, 0) == -1) { + trying to write it out. However, we do insist that whatever we load is + valid and not malformed. */ + if (rhizome_read_manifest_file(m, manifestpath, 0) == -1 || m->malformed) { rhizome_manifest_free(m); + keyring_free(keyring); return WHY("Manifest file could not be loaded -- not added to rhizome"); } } else if (manifestid && *manifestid) { @@ -1341,104 +1360,124 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co rhizome_bid_t bid; if (str_to_rhizome_bid_t(&bid, manifestid) == -1) { rhizome_manifest_free(m); + keyring_free(keyring); return WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid)); } if (rhizome_retrieve_manifest(&bid, m)){ rhizome_manifest_free(m); + keyring_free(keyring); return WHY("Existing manifest could not be loaded -- not added to rhizome"); } } else { if (config.debug.rhizome) DEBUGF("Creating new manifest"); - if (journal){ - m->journalTail = 0; - rhizome_manifest_set_ll(m,"tail",m->journalTail); + if (journal) { + rhizome_manifest_set_filesize(m, 0); + rhizome_manifest_set_tail(m, 0); } } - if (journal && m->journalTail==-1) - return WHY("Existing manifest is not a journal"); - - if ((!journal) && m->journalTail>=0) - return WHY("Existing manifest is a journal"); - - if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); - - if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL, bskhex ? &bsk : NULL)){ + if (journal && !m->is_journal){ rhizome_manifest_free(m); + keyring_free(keyring); + return WHY("Existing manifest is not a journal"); + } + if (!journal && m->is_journal) { + rhizome_manifest_free(m); + keyring_free(keyring); + return WHY("Existing manifest is a journal"); + } + + if (bskhex) + rhizome_apply_bundle_secret(m, &bsk); + if (m->service == NULL) + rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE); + if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL)) { + rhizome_manifest_free(m); + keyring_free(keyring); return -1; } if (journal){ - if (rhizome_append_journal_file(m, bskhex?&bsk:NULL, 0, filepath)){ + if (rhizome_append_journal_file(m, 0, filepath)){ rhizome_manifest_free(m); + keyring_free(keyring); return -1; } }else{ - if (rhizome_stat_file(m, filepath)){ + if (rhizome_stat_file(m, filepath) == -1) { rhizome_manifest_free(m); + keyring_free(keyring); return -1; } - - if (m->fileLength){ - if (rhizome_add_file(m, filepath)){ + if (m->filesize) { + if (rhizome_add_file(m, filepath) == -1) { rhizome_manifest_free(m); + keyring_free(keyring); return -1; } } } rhizome_manifest *mout = NULL; - int ret=rhizome_manifest_finalise(m, &mout, !force_new); + int ret = rhizome_manifest_finalise(m, &mout, !force_new); if (ret<0){ rhizome_manifest_free(m); + keyring_free(keyring); return -1; } if (manifestpath && *manifestpath && rhizome_write_manifest_file(mout, manifestpath, 0) == -1) ret = WHY("Could not overwrite manifest file."); - const char *service = rhizome_manifest_get(mout, "service", NULL, 0); - if (service) { + if (mout->service) { cli_field_name(context, "service", ":"); - cli_put_string(context, service, "\n"); + cli_put_string(context, mout->service, "\n"); } { cli_field_name(context, "manifestid", ":"); cli_put_string(context, alloca_tohex_rhizome_bid_t(mout->cryptoSignPublic), "\n"); } + assert(m->haveSecret); { char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; rhizome_bytes_to_hex_upper(mout->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES); cli_field_name(context, ".secret", ":"); cli_put_string(context, secret, "\n"); } - if (*authorSidHex) { + assert(mout->authorship != AUTHOR_LOCAL); + if (mout->authorship == AUTHOR_AUTHENTIC) { cli_field_name(context, ".author", ":"); - cli_put_string(context, alloca_tohex_sid_t(authorSid), "\n"); + cli_put_string(context, alloca_tohex_sid_t(mout->author), "\n"); } - const char *bk = rhizome_manifest_get(mout, "BK", NULL, 0); - if (bk) { + cli_field_name(context, ".rowid", ":"); + cli_put_long(context, m->rowid, "\n"); + cli_field_name(context, ".inserttime", ":"); + cli_put_long(context, m->inserttime, "\n"); + if (mout->has_bundle_key) { cli_field_name(context, "BK", ":"); - cli_put_string(context, bk, "\n"); + cli_put_string(context, alloca_tohex_rhizome_bk_t(mout->bundle_key), "\n"); + } + if (mout->has_date) { + cli_field_name(context, "date", ":"); + cli_put_long(context, mout->date, "\n"); } cli_field_name(context, "version", ":"); - cli_put_long(context, m->version, "\n"); + cli_put_long(context, mout->version, "\n"); cli_field_name(context, "filesize", ":"); - cli_put_long(context, mout->fileLength, "\n"); - if (mout->fileLength != 0) { + cli_put_long(context, mout->filesize, "\n"); + if (mout->filesize != 0) { cli_field_name(context, "filehash", ":"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n"); } - const char *name = rhizome_manifest_get(mout, "name", NULL, 0); - if (name) { + if (mout->name) { cli_field_name(context, "name", ":"); - cli_put_string(context, name, "\n"); + cli_put_string(context, mout->name, "\n"); } if (mout != m) rhizome_manifest_free(mout); rhizome_manifest_free(m); + keyring_free(keyring); return ret; } @@ -1506,10 +1545,9 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex // TODO generalise the way we dump manifest details from add, import & export // so callers can also generalise their parsing - const char *service = rhizome_manifest_get(m, "service", NULL, 0); - if (service) { + if (m->service) { cli_field_name(context, "service", ":"); - cli_put_string(context, service, "\n"); + cli_put_string(context, m->service, "\n"); } { cli_field_name(context, "manifestid", ":"); @@ -1521,23 +1559,26 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex cli_field_name(context, ".secret", ":"); cli_put_string(context, secret, "\n"); } - const char *bk = rhizome_manifest_get(m, "BK", NULL, 0); - if (bk) { + if (m->has_bundle_key) { cli_field_name(context, "BK", ":"); - cli_put_string(context, bk, "\n"); + cli_put_string(context, alloca_tohex_rhizome_bk_t(m->bundle_key), "\n"); } cli_field_name(context, "version", ":"); cli_put_long(context, m->version, "\n"); + if (m->has_date) { + cli_field_name(context, "date", ":"); + cli_put_long(context, m->date, "\n"); + } cli_field_name(context, "filesize", ":"); - cli_put_long(context, m->fileLength, "\n"); - if (m->fileLength != 0) { + cli_put_long(context, m->filesize, "\n"); + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize != 0) { cli_field_name(context, "filehash", ":"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n"); } - const char *name = rhizome_manifest_get(m, "name", NULL, 0); - if (name) { + if (m->name) { cli_field_name(context, "name", ":"); - cli_put_string(context, name, "\n"); + cli_put_string(context, m->name, "\n"); } cleanup: @@ -1553,22 +1594,19 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, struct cli_cont if ( cli_arg(parsed, "manifestpath", &manifestpath, NULL, "") == -1 || cli_arg(parsed, "filepath", &filepath, NULL, "") == -1) return -1; - rhizome_manifest *m = rhizome_new_manifest(); if (!m) return WHY("Out of manifests."); - - int ret=0; - if (rhizome_read_manifest_file(m, manifestpath, 0) == -1) - ret=-1; - // TODO why doesn't read manifest file set finalised??? - m->finalised=1; - - if (ret==0 && rhizome_write_manifest_file(m, filepath, 1) == -1) - ret = -1; - - if (m) - rhizome_manifest_free(m); + int ret = -1; + if ( rhizome_read_manifest_file(m, manifestpath, 0) != -1 + && rhizome_manifest_validate(m) + && rhizome_manifest_verify(m) + ) { + assert(m->finalised); + if (rhizome_write_manifest_file(m, filepath, 1) != -1) + ret = 0; + } + rhizome_manifest_free(m); return ret; } @@ -1590,27 +1628,38 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont return -1; int ret=0; if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) { - if (!fileid) + if (!fileid){ + keyring_free(keyring); return WHY("missing argument"); + } rhizome_filehash_t hash; - if (str_to_rhizome_filehash_t(&hash, fileid) == -1) + if (str_to_rhizome_filehash_t(&hash, fileid) == -1){ + keyring_free(keyring); return WHYF("invalid argument: %s", alloca_str_toprint(fileid)); + } ret = rhizome_delete_file(&hash); } else { - if (!manifestid) + if (!manifestid){ + keyring_free(keyring); return WHY("missing argument"); + } rhizome_bid_t bid; - if (str_to_rhizome_bid_t(&bid, manifestid) == -1) + if (str_to_rhizome_bid_t(&bid, manifestid) == -1){ + keyring_free(keyring); return WHY("Invalid manifest ID"); + } if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0) ret = rhizome_delete_bundle(&bid); else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0) ret = rhizome_delete_manifest(&bid); else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0) ret = rhizome_delete_payload(&bid); - else + else{ + keyring_free(keyring); return WHY("unrecognised command"); + } } + keyring_free(keyring); return ret; } @@ -1658,50 +1707,77 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con int ret=0; rhizome_bid_t bid; - if (str_to_rhizome_bid_t(&bid, manifestid) == -1) + if (str_to_rhizome_bid_t(&bid, manifestid) == -1){ + keyring_free(keyring); return WHY("Invalid manifest ID"); + } // treat empty string the same as null if (bskhex && !*bskhex) bskhex=NULL; rhizome_bk_t bsk; - if (bskhex && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1) + if (bskhex && str_to_rhizome_bk_t(&bsk, bskhex) == -1){ + keyring_free(keyring); return WHYF("invalid bsk: \"%s\"", bskhex); + } rhizome_manifest *m = rhizome_new_manifest(); - if (m==NULL) + if (m==NULL){ + keyring_free(keyring); return WHY("Out of manifests"); - + } ret = rhizome_retrieve_manifest(&bid, m); if (ret==0){ - // ignore errors - rhizome_extract_privatekey(m, NULL); - const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0); - - cli_field_name(context, "service", ":"); cli_put_string(context, blob_service, "\n"); - cli_field_name(context, "manifestid", ":"); cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n"); - cli_field_name(context, "version", ":"); cli_put_long(context, m->version, "\n"); - cli_field_name(context, "inserttime", ":"); cli_put_long(context, m->inserttime, "\n"); - if (m->haveSecret) { - cli_field_name(context, ".author", ":"); cli_put_string(context, alloca_tohex_sid_t(m->author), "\n"); + if (bskhex) + rhizome_apply_bundle_secret(m, &bsk); + rhizome_authenticate_author(m); + + if (m->service) { + cli_field_name(context, "service", ":"); + cli_put_string(context, m->service, "\n"); } - cli_field_name(context, ".readonly", ":"); cli_put_long(context, m->haveSecret?0:1, "\n"); - cli_field_name(context, "filesize", ":"); cli_put_long(context, m->fileLength, "\n"); - if (m->fileLength != 0) { + cli_field_name(context, "manifestid", ":"); + cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n"); + cli_field_name(context, "version", ":"); + cli_put_long(context, m->version, "\n"); + if (m->has_date) { + cli_field_name(context, "date", ":"); + cli_put_long(context, m->date, "\n"); + } + cli_field_name(context, "filesize", ":"); + cli_put_long(context, m->filesize, "\n"); + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize != 0) { cli_field_name(context, "filehash", ":"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n"); } + if (m->haveSecret) { + char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; + rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES); + cli_field_name(context, ".secret", ":"); + cli_put_string(context, secret, "\n"); + } + assert(m->authorship != AUTHOR_LOCAL); + if (m->authorship == AUTHOR_AUTHENTIC) { + cli_field_name(context, ".author", ":"); + cli_put_string(context, alloca_tohex_sid_t(m->author), "\n"); + } + cli_field_name(context, ".rowid", ":"); + cli_put_long(context, m->rowid, "\n"); + cli_field_name(context, ".inserttime", ":"); + cli_put_long(context, m->inserttime, "\n"); + cli_field_name(context, ".readonly", ":"); + cli_put_long(context, m->haveSecret?0:1, "\n"); } int retfile=0; - if (ret==0 && m->fileLength != 0 && filepath && *filepath){ + if (ret==0 && m->filesize != 0 && filepath && *filepath){ if (extract){ // Save the file, implicitly decrypting if required. - // TODO, this may cause us to search for an author a second time if the above call to rhizome_extract_privatekey failed - retfile = rhizome_extract_file(m, filepath, bskhex?&bsk:NULL); + retfile = rhizome_extract_file(m, filepath); }else{ // Save the file without attempting to decrypt int64_t length; @@ -1733,6 +1809,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con ret = retfile == -1 ? -1 : 1; if (m) rhizome_manifest_free(m); + keyring_free(keyring); return ret; } @@ -1768,21 +1845,99 @@ int app_rhizome_list(const struct cli_parsed *parsed, struct cli_context *contex { if (config.debug.verbose) DEBUG_cli_parsed(parsed); - const char *service, *name, *sender_sid, *recipient_sid, *offset, *limit; + const char *service = NULL, *name = NULL, *sender_hex = NULL, *recipient_hex = NULL, *offset_ascii = NULL, *limit_ascii = NULL; cli_arg(parsed, "service", &service, NULL, ""); cli_arg(parsed, "name", &name, NULL, ""); - cli_arg(parsed, "sender_sid", &sender_sid, cli_optional_sid, ""); - cli_arg(parsed, "recipient_sid", &recipient_sid, cli_optional_sid, ""); - cli_arg(parsed, "offset", &offset, cli_uint, "0"); - cli_arg(parsed, "limit", &limit, cli_uint, "0"); + cli_arg(parsed, "sender_sid", &sender_hex, cli_optional_sid, ""); + cli_arg(parsed, "recipient_sid", &recipient_hex, cli_optional_sid, ""); + cli_arg(parsed, "offset", &offset_ascii, cli_uint, "0"); + cli_arg(parsed, "limit", &limit_ascii, cli_uint, "0"); /* Create the instance directory if it does not yet exist */ if (create_serval_instance_dir() == -1) return -1; if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + if (rhizome_opendb() == -1) { + keyring_free(keyring); return -1; - return rhizome_list_manifests(context, service, name, sender_sid, recipient_sid, atoi(offset), atoi(limit), 0); + } + size_t rowlimit = atoi(limit_ascii); + size_t rowoffset = atoi(offset_ascii); + struct rhizome_list_cursor cursor; + bzero(&cursor, sizeof cursor); + cursor.service = service && service[0] ? service : NULL; + cursor.name = name && name[0] ? name : NULL; + if (sender_hex && sender_hex[0]) { + if (str_to_sid_t(&cursor.sender, sender_hex) == -1) + return WHYF("Invalid : %s", sender_hex); + cursor.is_sender_set = 1; + } + if (recipient_hex && recipient_hex[0]) { + if (str_to_sid_t(&cursor.recipient, recipient_hex) == -1) + return WHYF("Invalid filesize != RHIZOME_SIZE_UNSET); + rhizome_lookup_author(m); + cli_put_long(context, m->rowid, ":"); + cli_put_string(context, m->service, ":"); + cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":"); + cli_put_long(context, m->version, ":"); + cli_put_long(context, m->has_date ? m->date : 0, ":"); + cli_put_long(context, m->inserttime, ":"); + switch (m->authorship) { + case AUTHOR_LOCAL: + case AUTHOR_AUTHENTIC: + cli_put_hexvalue(context, m->author.binary, sizeof m->author.binary, ":"); + cli_put_long(context, 1, ":"); + break; + default: + cli_put_string(context, NULL, ":"); + cli_put_long(context, 0, ":"); + break; + } + cli_put_long(context, m->filesize, ":"); + cli_put_hexvalue(context, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary, ":"); + cli_put_hexvalue(context, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary, ":"); + cli_put_hexvalue(context, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary, ":"); + cli_put_string(context, m->name, "\n"); + } + } + rhizome_list_release(&cursor); + keyring_free(keyring); + if (n == -1) + return -1; + cli_row_count(context, rowcount); + return 0; } int app_keyring_create(const struct cli_parsed *parsed, struct cli_context *context) @@ -1888,7 +2043,48 @@ int app_keyring_list(const struct cli_parsed *parsed, struct cli_context *contex } keyring_free(k); return 0; - } +} + +static void cli_output_identity(struct cli_context *context, const keyring_identity *id) +{ + int i; + for (i=0;ikeypair_count;i++){ + keypair *kp=id->keypairs[i]; + switch(kp->type){ + case KEYTYPE_CRYPTOBOX: + cli_field_name(context, "sid", ":"); + cli_put_string(context, alloca_tohex(kp->public_key, kp->public_key_len), "\n"); + break; + case KEYTYPE_DID: + { + char *str = (char*)kp->private_key; + int l = strlen(str); + if (l){ + cli_field_name(context, "did", ":"); + cli_put_string(context, str, "\n"); + } + str = (char*)kp->public_key; + l=strlen(str); + if (l){ + cli_field_name(context, "name", ":"); + cli_put_string(context, str, "\n"); + } + } + break; + case KEYTYPE_PUBLIC_TAG: + { + const char *name; + const unsigned char *value; + size_t length; + if (keyring_unpack_tag(kp->public_key, kp->public_key_len, &name, &value, &length)==0){ + cli_field_name(context, name, ":"); + cli_put_string(context, alloca_toprint_quoted(-1, value, length, NULL), "\n"); + } + } + break; + } + } +} int app_keyring_add(const struct cli_parsed *parsed, struct cli_context *context) { @@ -1917,16 +2113,7 @@ int app_keyring_add(const struct cli_parsed *parsed, struct cli_context *context keyring_free(k); return WHY("Could not write new identity"); } - cli_field_name(context, "sid", ":"); - cli_put_string(context, alloca_tohex_sid_t(*sidp), "\n"); - if (did) { - cli_field_name(context, "did", ":"); - cli_put_string(context, did, "\n"); - } - if (name) { - cli_field_name(context, "name", ":"); - cli_put_string(context, name, "\n"); - } + cli_output_identity(context, id); keyring_free(k); return 0; } @@ -1936,12 +2123,52 @@ int app_keyring_set_did(const struct cli_parsed *parsed, struct cli_context *con if (config.debug.verbose) DEBUG_cli_parsed(parsed); const char *sidhex, *did, *name; - cli_arg(parsed, "sid", &sidhex, str_is_subscriber_id, ""); - cli_arg(parsed, "did", &did, cli_optional_did, ""); - cli_arg(parsed, "name", &name, NULL, ""); + + if (cli_arg(parsed, "sid", &sidhex, str_is_subscriber_id, "") == -1 || + cli_arg(parsed, "did", &did, cli_optional_did, "") == -1 || + cli_arg(parsed, "name", &name, NULL, "") == -1) + return -1; - if (strlen(name)>63) return WHY("Name too long (31 char max)"); + if (strlen(name)>63) + return WHY("Name too long (31 char max)"); + sid_t sid; + if (str_to_sid_t(&sid, sidhex) == -1){ + keyring_free(keyring); + return WHY("str_to_sid_t() failed"); + } + + if (!(keyring = keyring_open_instance_cli(parsed))) + return -1; + + int cn=0,in=0,kp=0; + int r=0; + if (!keyring_find_sid(keyring, &cn, &in, &kp, &sid)) + r=WHY("No matching SID"); + else{ + if (keyring_set_did(keyring->contexts[cn]->identities[in], did, name)) + r=WHY("Could not set DID"); + else{ + if (keyring_commit(keyring)) + r=WHY("Could not write updated keyring record"); + else{ + cli_output_identity(context, keyring->contexts[cn]->identities[in]); + } + } + } + + keyring_free(keyring); + return r; +} + +static int app_keyring_set_tag(const struct cli_parsed *parsed, struct cli_context *context) +{ + const char *sidhex, *tag, *value; + if (cli_arg(parsed, "sid", &sidhex, str_is_subscriber_id, "") == -1 || + cli_arg(parsed, "tag", &tag, NULL, "") == -1 || + cli_arg(parsed, "value", &value, NULL, "") == -1 ) + return -1; + if (!(keyring = keyring_open_instance_cli(parsed))) return -1; @@ -1950,32 +2177,50 @@ int app_keyring_set_did(const struct cli_parsed *parsed, struct cli_context *con return WHY("str_to_sid_t() failed"); int cn=0,in=0,kp=0; - int r=keyring_find_sid(keyring, &cn, &in, &kp, &sid); - if (!r) return WHY("No matching SID"); - if (keyring_set_did(keyring->contexts[cn]->identities[in], did, name)) - return WHY("Could not set DID"); - if (keyring_commit(keyring)) - return WHY("Could not write updated keyring record"); - - cli_field_name(context, "sid", ":"); - cli_put_string(context, alloca_tohex_sid_t(sid), "\n"); - if (did) { - cli_field_name(context, "did", ":"); - cli_put_string(context, did, "\n"); - } - if (name) { - cli_field_name(context, "name", ":"); - cli_put_string(context, name, "\n"); + int r=0; + if (!keyring_find_sid(keyring, &cn, &in, &kp, &sid)) + r=WHY("No matching SID"); + else{ + int length = strlen(value); + if (keyring_set_public_tag(keyring->contexts[cn]->identities[in], tag, (const unsigned char*)value, length)) + r=WHY("Could not set tag value"); + else{ + if (keyring_commit(keyring)) + r=WHY("Could not write updated keyring record"); + else{ + cli_output_identity(context, keyring->contexts[cn]->identities[in]); + } + } } + keyring_free(keyring); - return 0; + return r; +} + +ssize_t mdp_poll_recv(int mdp_sock, time_ms_t timeout, struct mdp_header *rev_header, unsigned char *payload, size_t buffer_size) +{ + time_ms_t now = gettime_ms(); + if (now>timeout) + return -2; + int p=mdp_poll(mdp_sock, timeout - now); + if (p<0) + return WHY_perror("mdp_poll"); + if (p==0) + return -2; + ssize_t len = mdp_recv(mdp_sock, rev_header, payload, buffer_size); + if (len<0) + return WHY_perror("mdp_recv"); + if (rev_header->flags & MDP_FLAG_ERROR) + return WHY("Operation failed, check the log for more information"); + return len; } static int handle_pins(const struct cli_parsed *parsed, struct cli_context *context, int revoke) { const char *pin, *sid_hex; - cli_arg(parsed, "entry-pin", &pin, NULL, ""); - cli_arg(parsed, "sid", &sid_hex, str_is_subscriber_id, ""); + if (cli_arg(parsed, "entry-pin", &pin, NULL, "") == -1 || + cli_arg(parsed, "sid", &sid_hex, str_is_subscriber_id, "") == -1) + return -1; int ret=1; struct mdp_header header={ @@ -1984,8 +2229,8 @@ static int handle_pins(const struct cli_parsed *parsed, struct cli_context *cont int mdp_sock = mdp_socket(); set_nonblock(mdp_sock); - unsigned char payload[1200]; - struct mdp_identity_request *request = (struct mdp_identity_request *)payload; + unsigned char request_payload[1200]; + struct mdp_identity_request *request = (struct mdp_identity_request *)request_payload; if (revoke){ request->action=ACTION_LOCK; @@ -1997,50 +2242,39 @@ static int handle_pins(const struct cli_parsed *parsed, struct cli_context *cont if (pin && *pin){ request->type=TYPE_PIN; int pin_len = strlen(pin)+1; - if (pin_len+len > sizeof(payload)) + if (pin_len+len > sizeof(request_payload)) return WHY("Supplied pin is too long"); - bcopy(pin, &payload[len], pin_len); + bcopy(pin, &request_payload[len], pin_len); len+=pin_len; }else if(sid_hex && *sid_hex){ request->type=TYPE_SID; sid_t sid; if (str_to_sid_t(&sid, sid_hex) == -1) return WHY("str_to_sid_t() failed"); - bcopy(sid.binary, &payload[len], sizeof(sid)); + bcopy(sid.binary, &request_payload[len], sizeof(sid)); len+=sizeof(sid); } - if (!mdp_send(mdp_sock, &header, payload, len)){ + if (!mdp_send(mdp_sock, &header, request_payload, len)){ WHY_perror("mdp_send"); goto end; } time_ms_t timeout=gettime_ms()+500; while(1){ - time_ms_t now = gettime_ms(); - if (now>timeout) + struct mdp_header rev_header; + unsigned char response_payload[1600]; + ssize_t len = mdp_poll_recv(mdp_sock, timeout, &rev_header, response_payload, sizeof(response_payload)); + if (len==-1) break; - int p=mdp_poll(mdp_sock, timeout - now); - if (p<0){ - WHY_perror("mdp_poll"); - break; - } - if (p==0){ + if (len==-2){ WHYF("Timeout while waiting for response"); break; } - struct mdp_header rev_header; - unsigned char payload[1600]; - ssize_t len = mdp_recv(mdp_sock, &rev_header, payload, sizeof(payload)); - if (len<0){ - WHY_perror("mdp_recv"); - continue; - } - if (rev_header.flags & MDP_FLAG_OK) + if (rev_header.flags & MDP_FLAG_CLOSE){ ret=0; - if (rev_header.flags & MDP_FLAG_ERROR) - WHY("Operation failed, check the log for more information"); - break; + break; + } } end: mdp_close(mdp_sock); @@ -2057,6 +2291,67 @@ int app_id_pin(const struct cli_parsed *parsed, struct cli_context *context) return handle_pins(parsed, context, 0); } +int app_id_list(const struct cli_parsed *parsed, struct cli_context *context) +{ + const char *tag, *value; + if (cli_arg(parsed, "tag", &tag, NULL, "") == -1 || + cli_arg(parsed, "value", &value, NULL, "") == -1 ) + return -1; + + int ret=-1; + struct mdp_header header={ + .remote.port=MDP_SEARCH_IDS, + }; + int mdp_sock = mdp_socket(); + set_nonblock(mdp_sock); + + unsigned char request_payload[1200]; + size_t len=0; + + if (tag && *tag){ + size_t value_len=0; + if (value && *value) + value_len = strlen(value); + len = sizeof(request_payload); + if (keyring_pack_tag(request_payload, &len, tag, (unsigned char*)value, value_len)) + goto end; + } + + if (!mdp_send(mdp_sock, &header, request_payload, len)){ + WHY_perror("mdp_send"); + goto end; + } + + time_ms_t timeout=gettime_ms()+500; + while(1){ + struct mdp_header rev_header; + unsigned char response_payload[1600]; + ssize_t len = mdp_poll_recv(mdp_sock, timeout, &rev_header, response_payload, sizeof(response_payload)); + if (len==-1) + break; + if (len==-2){ + WHYF("Timeout while waiting for response"); + break; + } + + if (len>=SID_SIZE){ + sid_t *id = (sid_t*)response_payload; + cli_field_name(context, "sid", ":"); + cli_put_hexvalue(context, id->binary, sizeof(sid_t), "\n"); + // TODO receive and decode other details about this identity + } + + if (rev_header.flags & MDP_FLAG_CLOSE){ + ret=0; + break; + } + } + +end: + mdp_close(mdp_sock); + return ret; +} + int app_id_self(const struct cli_parsed *parsed, struct cli_context *context) { int mdp_sockfd; @@ -2471,7 +2766,7 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont /* Got a good DNA reply, copy it into place and stop polling */ cli_field_name(context, "sid", ":"); - cli_put_string(context, sidhex, ":"); + cli_put_string(context, sidhex, "\n"); cli_field_name(context, "did", ":"); cli_put_string(context, did, "\n"); cli_field_name(context, "name", ":"); @@ -2484,6 +2779,58 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont return 1; } +void context_switch_test(int); +int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context) +{ + size_t mem_size; + size_t addr; + uint64_t count; + + + // First test context switch speed + context_switch_test(1); + + for(mem_size=1024;mem_size<=(128*1024*1024);mem_size*=2) { + uint8_t *mem=malloc(mem_size); + if (!mem) { + fprintf(stderr,"Could not allocate %zdKB memory -- stopping test.\n",mem_size/1024); + return -1; + } + + // Fill memory with random stuff so that we don't have memory page-in + // delays when doing the reads + for(addr=0;addraddr)) return WHY("Unable to parse the address"); }else - DEBUGF("Scanning local networks"); + INFO("Scanning local networks"); if ((mdp_sockfd = overlay_mdp_client_socket()) < 0) return WHY("Cannot create MDP socket"); @@ -2610,6 +2956,10 @@ struct cli_schema command_line_options[]={ "Create a new identity in the keyring protected by the supplied PIN (empty PIN if not given)"}, {app_keyring_set_did,{"keyring", "set","did" KEYRING_PIN_OPTIONS,"","","",NULL}, 0, "Set the DID for the specified SID (must supply PIN to unlock the SID record in the keyring)"}, + {app_keyring_set_tag,{"keyring", "set","tag" KEYRING_PIN_OPTIONS,"","","",NULL}, 0, + "Set a named tag for the specified SID (must supply PIN to unlock the SID record in the keyring)"}, + {app_id_list, {"id", "list", "[]", "[]", NULL}, 0, + "Search unlocked identities based on an optional tag and value"}, {app_id_self,{"id","self|peers|allpeers",NULL}, 0, "Return identity(s) as URIs of own node, or of known routable peers, or all known peers"}, {app_id_pin, {"id", "enter", "pin", "", NULL}, 0, @@ -2634,6 +2984,8 @@ struct cli_schema command_line_options[]={ "Run cryptography speed test"}, {app_nonce_test,{"test","nonce",NULL}, 0, "Run nonce generation test"}, + {app_mem_test,{"test","memory",NULL}, 0, + "Run memory speed test"}, {app_byteorder_test,{"test","byteorder",NULL}, 0, "Run byte order handling test"}, {app_slip_test,{"test","slip","[--seed=]","[--duration=|--iterations=]",NULL}, 0, diff --git a/conf_schema.c b/conf_schema.c index 02348b8a..c53c74f5 100644 --- a/conf_schema.c +++ b/conf_schema.c @@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "strbuf.h" #include "strbuf_helpers.h" #include "conf.h" +#include "dataformats.h" int cf_opt_boolean(bool_t *booleanp, const char *text) { diff --git a/conf_schema.h b/conf_schema.h index a44adc67..bab3c5fd 100644 --- a/conf_schema.h +++ b/conf_schema.h @@ -242,10 +242,9 @@ ATOM(bool_t, gateway, 0, boolean,, "") ATOM(bool_t, keyring, 0, boolean,, "") ATOM(bool_t, security, 0, boolean,, "") ATOM(bool_t, mdprequests, 0, boolean,, "") -ATOM(bool_t, mavlink, 0, boolean,, "") -ATOM(bool_t, mavlink_payloads, 0, boolean,, "") -ATOM(bool_t, mavlinkfsm, 0, boolean,, "") +ATOM(bool_t, radio_link, 0, boolean,, "") ATOM(bool_t, peers, 0, boolean,, "") +ATOM(bool_t, overlaybuffer, 0, boolean,, "") ATOM(bool_t, overlayframes, 0, boolean,, "") ATOM(bool_t, overlayabbreviations, 0, boolean,, "") ATOM(bool_t, overlayrouting, 0, boolean,, "") @@ -261,13 +260,15 @@ ATOM(bool_t, slipdecode, 0, boolean,, "") ATOM(bool_t, slipbytestream, 0, boolean,, "") ATOM(bool_t, packetconstruction, 0, boolean,, "") ATOM(bool_t, rhizome, 0, boolean,, "") -ATOM(bool_t, rhizome_bind, 0, boolean,, "") +ATOM(bool_t, rhizome_manifest, 0, boolean,, "") +ATOM(bool_t, rhizome_sql_bind, 0, boolean,, "") ATOM(bool_t, rhizome_httpd, 0, boolean,, "") ATOM(bool_t, rhizome_tx, 0, boolean,, "") ATOM(bool_t, rhizome_rx, 0, boolean,, "") ATOM(bool_t, rhizome_ads, 0, boolean,, "") ATOM(bool_t, rhizome_nohttptx, 0, boolean,, "") ATOM(bool_t, rhizome_mdp_rx, 0, boolean,, "") +ATOM(bool_t, subscriber, 0, boolean,, "") ATOM(bool_t, throttling, 0, boolean,, "") ATOM(bool_t, meshms, 0, boolean,, "") ATOM(bool_t, manifests, 0, boolean,, "") @@ -396,6 +397,8 @@ END_STRUCT STRUCT(rhizome_api_restful) SUB_STRUCT(userlist, users,) +ATOM(uint32_t, newsince_timeout, 60, uint32_time_interval,, "Time to block while reporting new bundles") +ATOM(uint32_t, newsince_poll_ms, 2000, uint32_nonzero,, "Database poll interval while blocked reporting new bundles") END_STRUCT STRUCT(rhizome_api) @@ -427,6 +430,7 @@ ATOM(bool_t, external_blobs, 0, boolean,, "Store rhizome bundles ATOM(uint64_t, rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.") ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.") +ATOM(uint64_t, mdp_stall_timeout, 1000, uint64_scaled,, "Timeout to request more data via mdp.") ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch") SUB_STRUCT(rhizome_direct, direct,) SUB_STRUCT(rhizome_api, api,) @@ -472,8 +476,6 @@ ATOM(bool_t, debug, 0, boolean,, "If true, log details ATOM(bool_t, point_to_point, 0, boolean,, "If true, assume there will only be two devices on this interface") ATOM(bool_t, ctsrts, 0, boolean,, "If true, enable CTS/RTS hardware handshaking") ATOM(int32_t, uartbps, 57600, int32_rs232baudrate,, "Speed of serial UART link speed (which may be different to serial device link speed)") -ATOM(int32_t, throttle, 0, int32_nonneg,, "Limit transmit speed of serial interface (bytes per second)") -ATOM(int32_t, burst_size, 0, int32_nonneg,, "Write no more than this many bytes at a time to a serial interface") END_STRUCT ARRAY(interface_list, NO_DUPLICATES) diff --git a/context1.c b/context1.c new file mode 100644 index 00000000..93402c36 --- /dev/null +++ b/context1.c @@ -0,0 +1,103 @@ + +/******************************************************************************* + * The BYTE UNIX Benchmarks - Release 3 + * Module: context1.c SID: 3.3 5/15/91 19:30:18 + * + ******************************************************************************* + * Bug reports, patches, comments, suggestions should be sent to: + * + * Ben Smith, Rick Grehan or Tom Yager + * ben@bytepb.byte.com rick_g@bytepb.byte.com tyager@bytepb.byte.com + * + ******************************************************************************* + * Modification Log: + * $Header: context1.c,v 3.4 87/06/22 14:22:59 kjmcdonell Beta $ + * August 28, 1990 - changed timing routines--now returns total number of + * iterations in specified time period + * October 22, 1997 - code cleanup to remove ANSI C compiler warnings + * Andy Kahn + * + ******************************************************************************/ +char SCCSid[] = "@(#) @(#)context1.c:3.3 -- 5/15/91 19:30:18"; +/* + * Context switching via synchronized unbuffered pipe i/o + * + */ + +#include +#include +#include +#include "timeit.c" + +unsigned long iter; +int stop_timing=0; + +void report() +{ + fprintf(stderr, "%lu context switches per second.\n", iter); + stop_timing=1; +} + +void context_switch_test(int duration) +{ + unsigned long check; + int p1[2], p2[2]; + + + /* set up alarm call */ + iter = 0; + wake_me(duration, report); + + if (pipe(p1) || pipe(p2)) { + perror("pipe create failed"); + exit(1); + } + + if (fork()) { /* parent process */ + /* master, write p1 & read p2 */ + close(p1[0]); close(p2[1]); + while (!stop_timing) { + if (write(p1[1], (char *)&iter, sizeof(iter)) != sizeof(iter)) { + if ((errno != 0) && (errno != EINTR)) + perror("master write failed"); + exit(1); + } + if (read(p2[0], (char *)&check, sizeof(check)) != sizeof(check)) { + if ((errno != 0) && (errno != EINTR)) + perror("master read failed"); + exit(1); + } + if (check != iter) { + fprintf(stderr, "Master sync error: expect %lu, got %lu\n", + iter, check); + exit(2); + } + iter++; + } + } + else { /* child process */ + unsigned long iter1; + + iter1 = 0; + /* slave, read p1 & write p2 */ + close(p1[1]); close(p2[0]); + while (!stop_timing) { + if (read(p1[0], (char *)&check, sizeof(check)) != sizeof(check)) { + if ((errno != 0) && (errno != EINTR)) + perror("slave read failed"); + exit(1); + } + if (check != iter1) { + fprintf(stderr, "Slave sync error: expect %lu, got %lu\n", + iter, check); + exit(2); + } + if (write(p2[1], (char *)&iter1, sizeof(iter1)) != sizeof(check)) { + if ((errno != 0) && (errno != EINTR)) + perror("slave write failed"); + exit(1); + } + iter1++; + } + } +} diff --git a/crypto.c b/crypto.c index ce048c67..550a7aec 100644 --- a/crypto.c +++ b/crypto.c @@ -3,6 +3,7 @@ #include "serval.h" #include "overlay_address.h" #include "crypto.h" +#include "keyring.h" // verify a signature against a public sas key. int crypto_verify_signature(unsigned char *sas_key, diff --git a/dataformats.c b/dataformats.c index 48396ef7..ed841ca2 100644 --- a/dataformats.c +++ b/dataformats.c @@ -17,10 +17,11 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include "serval.h" #include "rhizome.h" #include "str.h" -#include +#include "dataformats.h" int cmp_sid_t(const sid_t *a, const sid_t *b) { @@ -116,6 +117,11 @@ int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const return 0; } +int str_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex) +{ + return fromhexstr(bkp->binary, hex, sizeof bkp->binary); +} + int rhizome_strn_is_manifest_id(const char *id) { return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN); @@ -156,6 +162,15 @@ int rhizome_str_is_file_hash(const char *hash) return is_xstring(hash, RHIZOME_FILEHASH_STRLEN); } +int rhizome_str_is_manifest_service(const char *text) +{ + if (text[0] == '\0') + return 0; + while (*text && (isalnum(*text) || *text == '_' || *text == '.')) + ++text; + return *text == '\0'; +} + int str_is_did(const char *did) { size_t len = 0; @@ -200,7 +215,7 @@ void write_uint16(unsigned char *o,uint16_t v) { *(o++)=v&0xff; v=v>>8; } } -uint64_t read_uint64(unsigned char *o) +uint64_t read_uint64(const unsigned char *o) { int i; uint64_t v=0; @@ -208,7 +223,7 @@ uint64_t read_uint64(unsigned char *o) return v; } -uint32_t read_uint32(unsigned char *o) +uint32_t read_uint32(const unsigned char *o) { int i; uint32_t v=0; @@ -216,7 +231,7 @@ uint32_t read_uint32(unsigned char *o) return v; } -uint16_t read_uint16(unsigned char *o) +uint16_t read_uint16(const unsigned char *o) { int i; uint16_t v=0; diff --git a/dataformats.h b/dataformats.h new file mode 100644 index 00000000..67afb746 --- /dev/null +++ b/dataformats.h @@ -0,0 +1,26 @@ +#ifndef __SERVALD_DATA_FORMATS_H +#define __SERVALD_DATA_FORMATS_H + +int str_is_subscriber_id(const char *sid); +int strn_is_subscriber_id(const char *sid, size_t *lenp); +int str_is_did(const char *did); +int strn_is_did(const char *did, size_t *lenp); + +int rhizome_strn_is_manifest_id(const char *text); +int rhizome_str_is_manifest_id(const char *text); +int rhizome_strn_is_bundle_key(const char *text); +int rhizome_str_is_bundle_key(const char *text); +int rhizome_strn_is_bundle_crypt_key(const char *text); +int rhizome_str_is_bundle_crypt_key(const char *text); +int rhizome_strn_is_file_hash(const char *text); +int rhizome_str_is_file_hash(const char *text); +int rhizome_str_is_manifest_service(const char *text); + +void write_uint64(unsigned char *o,uint64_t v); +void write_uint16(unsigned char *o,uint16_t v); +void write_uint32(unsigned char *o,uint32_t v); +uint64_t read_uint64(const unsigned char *o); +uint32_t read_uint32(const unsigned char *o); +uint16_t read_uint16(const unsigned char *o); + +#endif diff --git a/directory_client.c b/directory_client.c index 4513afc3..1f542464 100644 --- a/directory_client.c +++ b/directory_client.c @@ -16,6 +16,7 @@ #include "str.h" #include "overlay_address.h" #include "conf.h" +#include "keyring.h" struct subscriber *directory_service; @@ -51,7 +52,7 @@ static void directory_send(struct subscriber *directory_service, const sid_t *si // Used by tests INFOF("Sending directory registration for %s*, %s, %s to %s*", alloca_tohex_sid_t_trunc(*sidp, 14), did, name, alloca_tohex_sid_t_trunc(directory_service->sid, 14)); - overlay_mdp_dispatch(&request, 0, NULL, 0); + overlay_mdp_dispatch(&request, NULL); } // send a registration packet for each unlocked identity diff --git a/dna_helper.c b/dna_helper.c index 026c7f7e..3d6d26b7 100644 --- a/dna_helper.c +++ b/dna_helper.c @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "strbuf.h" #include "strbuf_helpers.h" #include "overlay_address.h" +#include "dataformats.h" /* The challenge with making an interface for calling an external program to diff --git a/fakeradio.c b/fakeradio.c index 0361c177..2ed5441f 100644 --- a/fakeradio.c +++ b/fakeradio.c @@ -339,42 +339,107 @@ int transfer_bytes(struct radio_state *radios) log_time(); fprintf(stderr, "Transferring %d byte packet from %s to %s\n", bytes, t->name, r->name); } + int i, j; - for (i=0;irxb_lenrxbuffer);i++){ - char byte = t->txbuffer[i]; - // introduce bit errors - for(j=0;j<8;j++) { - if (random()rxb_lenrxbuffer);i++){ + char byte = t->txbuffer[i]; + // introduce bit errors + for(j=0;j<8;j++) { + if (random()rxbuffer[r->rxb_len++]=byte; } - r->rxbuffer[r->rxb_len++]=byte; } if (bytes>0 && bytes < t->txb_len) bcopy(&t->txbuffer[bytes], t->txbuffer, t->txb_len - bytes); t->txb_len-=bytes; - if (bytes==0 || --t->tx_count<=0){ - // swap who's turn it is to transmit - transmitter = receiver; - r->tx_count=6; - } // set the wait time for the next transmission - next_transmit_time = gettime_ms() + (bytes+10)/chars_per_ms; + next_transmit_time = gettime_ms() + 5 + bytes/chars_per_ms; + + if (bytes==0 || --t->tx_count<=0){ + // swap who's turn it is to transmit after sending 3 packets or running out of data. + transmitter = receiver; + r->tx_count=3; + // add Tx->Rx change time (it's about 40ms between receiving empty packets) + next_transmit_time+=15; + } return bytes; } +int calc_ber(double target_packet_fraction) +{ + int byte_count=220+32; + int max_error_bytes=16; + + int ber; + int p; + int byte; + int bit; + + // 9,000,000 gives a packet delivery rate of ~99% + // so no point starting smaller than that. + // Only ~30,000,000 reduces packet delivery rate to + // ~1%, so the search range is fairly narrow. + ber=0; + if (target_packet_fraction<=0.9) ber=6900000; + if (target_packet_fraction<=0.5) ber=16900000; + if (target_packet_fraction<=0.25) ber=20600000; + if (target_packet_fraction<=0.1) ber=23400000; + if (target_packet_fraction<=0.05) ber=28600000; + + for(;ber<0x70ffffff;ber+=100000) + { + int packet_errors=0; + for(p=0;p<1000;p++) { + int byte_errors=0; + int dropped = 0; + for (byte=0;bytemax_error_bytes) { dropped=1; break; } + } + } + if (dropped) + packet_errors++; + } + if (packet_errors>=((1.0-target_packet_fraction)*1000)) break; + } + fprintf(stderr,"ber magic value=%d\n",ber); + return ber; +} + int main(int argc,char **argv) { - if (1 < argc) { + if (argc>=1) { chars_per_ms=atol(argv[1]); - if (2 < argc) - ber=atol(argv[2]); + if (argc>=2) + ber=calc_ber(atof(argv[2])); } - fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms); - fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF); struct pollfd fds[2]; struct radio_state radios[2]; @@ -382,18 +447,21 @@ int main(int argc,char **argv) bzero(&radios,sizeof radios); int i; + radios[0].name="left"; + radios[1].name="right"; for (i=0;i<2;i++){ radios[i].fd=posix_openpt(O_RDWR|O_NOCTTY); grantpt(radios[i].fd); unlockpt(radios[i].fd); fcntl(radios[i].fd,F_SETFL,fcntl(radios[i].fd, F_GETFL, NULL)|O_NONBLOCK); - fprintf(stdout,"%s\n",ptsname(radios[i].fd)); + fprintf(stdout,"%s:%s\n", radios[i].name, ptsname(radios[i].fd)); fds[i].fd = radios[i].fd; } - radios[0].name="left"; - radios[1].name="right"; fflush(stdout); + fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms); + fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF); + while(1) { // what events do we need to poll for? how long can we block? int64_t now = gettime_ms(); diff --git a/fdqueue.c b/fdqueue.c index 27a2e523..81b594d7 100644 --- a/fdqueue.c +++ b/fdqueue.c @@ -85,13 +85,15 @@ int is_scheduled(const struct sched_ent *alarm) // on calling .poll.revents will be zero. int _schedule(struct __sourceloc __whence, struct sched_ent *alarm) { + time_ms_t now = gettime_ms(); if (config.debug.io) - DEBUGF("schedule(alarm=%s) called from %s() %s:%d", - alloca_alarm_name(alarm), - __whence.function,__whence.file,__whence.line); + DEBUGF("schedule(alarm=%s) alarm=%.3f deadline=%.3f", + alloca_alarm_name(alarm), + (double)(alarm->alarm - now) / 1000, + (double)(alarm->deadline - now) / 1000 + ); if (!alarm->stats) - WARNF("schedule() called from %s() %s:%d without supplying an alarm name", - __whence.function,__whence.file,__whence.line); + WARN("schedule() called without supplying an alarm name"); struct sched_ent *node = next_alarm, *last = NULL; @@ -100,18 +102,16 @@ int _schedule(struct __sourceloc __whence, struct sched_ent *alarm) if (!alarm->function) return WHY("Can't schedule if you haven't set the function pointer"); - - time_ms_t now = gettime_ms(); if (alarm->deadline < alarm->alarm) alarm->deadline = alarm->alarm; if (now - alarm->deadline > 1000){ // 1000ms ago? thats silly, if you keep doing it noone else will get a turn. - WHYF("Alarm %s tried to schedule a deadline %"PRId64"ms ago, from %s() %s:%d", + FATALF("Alarm %s tried to schedule a deadline %"PRId64"ms ago", alloca_alarm_name(alarm), - (now - alarm->deadline), - __whence.function,__whence.file,__whence.line); + (now - alarm->deadline) + ); } // if the alarm has already expired, move straight to the deadline queue @@ -168,8 +168,7 @@ int _watch(struct __sourceloc __whence, struct sched_ent *alarm) if (config.debug.io) DEBUGF("watch(alarm=%s)", alloca_alarm_name(alarm)); if (!alarm->stats) - WARNF("watch() called from %s() %s:%d without supplying an alarm name", - __whence.function,__whence.file,__whence.line); + WARN("watch() called without supplying an alarm name"); if (!alarm->function) return WHY("Can't watch if you haven't set the function pointer"); @@ -177,10 +176,10 @@ int _watch(struct __sourceloc __whence, struct sched_ent *alarm) if (alarm->_poll_index>=0 && fd_callbacks[alarm->_poll_index]==alarm){ // updating event flags if (config.debug.io) - DEBUGF("Updating watch %s, #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events); + DEBUGF("Updating watch %s, #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events)); }else{ if (config.debug.io) - DEBUGF("Adding watch %s, #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events); + DEBUGF("Adding watch %s, #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events)); if (fdcount>=MAX_WATCHED_FDS) return WHY("Too many file handles to watch"); fd_callbacks[fdcount]=alarm; @@ -213,7 +212,7 @@ int _unwatch(struct __sourceloc __whence, struct sched_ent *alarm) fd_callbacks[fdcount]=NULL; alarm->_poll_index=-1; if (config.debug.io) - DEBUGF("%s stopped watching #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events); + DEBUGF("%s stopped watching #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events)); return 0; } @@ -225,8 +224,8 @@ static void call_alarm(struct sched_ent *alarm, int revents) struct call_stats call_stats; call_stats.totals = alarm->stats; - if (config.debug.io) DEBUGF("Calling alarm/callback %p ('%s')", - alarm, alloca_alarm_name(alarm)); + if (config.debug.io) + DEBUGF("Calling alarm/callback %p %s", alarm, alloca_alarm_name(alarm)); if (call_stats.totals) fd_func_enter(__HERE__, &call_stats); @@ -237,7 +236,8 @@ static void call_alarm(struct sched_ent *alarm, int revents) if (call_stats.totals) fd_func_exit(__HERE__, &call_stats); - if (config.debug.io) DEBUGF("Alarm %p returned",alarm); + if (config.debug.io) + DEBUGF("Alarm %p returned",alarm); OUT(); } diff --git a/fdqueue.h b/fdqueue.h index 044ed064..df09e5b3 100644 --- a/fdqueue.h +++ b/fdqueue.h @@ -80,13 +80,15 @@ int fd_checkalarms(); int fd_func_enter(struct __sourceloc, struct call_stats *this_call); int fd_func_exit(struct __sourceloc, struct call_stats *this_call); void dump_stack(int log_level); +unsigned fd_depth(); #define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0,0}; \ struct call_stats _this_call={.totals=&_aggregate_stats}; \ fd_func_enter(__HERE__, &_this_call); #define OUT() fd_func_exit(__HERE__, &_this_call) -#define RETURN(X) do { OUT(); return (X); } while (0); -#define RETURNNULL do { OUT(); return (NULL); } while (0); +#define RETURN(X) do { OUT(); return (X); } while (0) +#define RETURNNULL(X) do { X; OUT(); return (NULL); } while (0) +#define RETURNVOID do { OUT(); return; } while (0) #endif // __SERVALDNA__FDQUEUE_H diff --git a/golay.c b/golay.c index 1f973814..119cdae5 100644 --- a/golay.c +++ b/golay.c @@ -1,13 +1,14 @@ #define POLY 0xAE3 /* or use the other polynomial, 0xC75 */ +#include -static unsigned long golay(unsigned long cw) +static uint32_t golay(uint32_t cw) /* This function calculates [23,12] Golay codewords. The format of the returned longint is [checkbits(11),data(12)]. */ { int i; - unsigned long c; + uint32_t c; cw&=0xfffl; c=cw; /* save original codeword */ for (i=1; i<=12; i++) /* examine each data bit */ @@ -19,16 +20,16 @@ static unsigned long golay(unsigned long cw) return((cw<<12)|c); /* assemble codeword */ } -static int parity(unsigned long cw) +static int parity(uint32_t cw) /* This function checks the overall parity of codeword cw. If parity is even, 0 is returned, else 1. */ { - unsigned char p; - + uint8_t p; + /* XOR the bytes of the codeword */ - p=*(unsigned char*)&cw; - p^=*((unsigned char*)&cw+1); - p^=*((unsigned char*)&cw+2); + p=cw & 0xFF; + p^=(cw>>8) & 0xFF; + p^=(cw>>16) & 0xFF; /* XOR the halves of the intermediate result */ p=p ^ (p>>4); @@ -39,9 +40,9 @@ static int parity(unsigned long cw) return(p & 1); } -int golay_encode(unsigned char *data) +int golay_encode(uint8_t *data) { - unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16); + uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16); cw = golay(cw); if (parity(cw)) cw|=0x800000l; @@ -51,7 +52,7 @@ int golay_encode(unsigned char *data) return 0; } -static unsigned long syndrome(unsigned long cw) +static uint32_t syndrome(uint32_t cw) /* This function calculates and returns the syndrome of a [23,12] Golay codeword. */ { @@ -66,7 +67,7 @@ static unsigned long syndrome(unsigned long cw) return(cw<<12); /* value pairs with upper bits of cw */ } -static int weight(unsigned long cw) +static int weight(uint32_t cw) /* This function calculates the weight of 23 bit codeword cw. */ { @@ -88,7 +89,7 @@ static int weight(unsigned long cw) return(bits); } -static unsigned long rotate_left(unsigned long cw, int n) +static uint32_t rotate_left(uint32_t cw, int n) /* This function rotates 23 bit codeword cw left by n bits. */ { int i; @@ -107,7 +108,7 @@ static unsigned long rotate_left(unsigned long cw, int n) return(cw & 0x7fffffl); } -static unsigned long rotate_right(unsigned long cw, int n) +static uint32_t rotate_right(uint32_t cw, int n) /* This function rotates 23 bit codeword cw right by n bits. */ { int i; @@ -126,20 +127,20 @@ static unsigned long rotate_right(unsigned long cw, int n) return(cw & 0x7fffffl); } -static unsigned long correct(unsigned long cw, int *errs) +static uint32_t correct(uint32_t cw, int *errs) /* This function corrects Golay [23,12] codeword cw, returning the corrected codeword. This function will produce the corrected codeword for three or fewer errors. It will produce some other valid Golay codeword for four or more errors, possibly not the intended one. *errs is set to the number of bit errors corrected. */ { - unsigned char + uint8_t w; /* current syndrome limit weight, 2 or 3 */ - unsigned long + uint32_t mask; /* mask for bit flipping */ int i,j; /* index */ - unsigned long + uint32_t s, /* calculated syndrome */ cwsaver; /* saves initial value of cw */ @@ -187,17 +188,17 @@ static unsigned long correct(unsigned long cw, int *errs) return(cwsaver); /* return original if no corrections */ } /* correct */ -int golay_decode(int *errs, const unsigned char *data) +int golay_decode(int *errs, const uint8_t *data) /* This function decodes codeword *cw , error correction is attempted, with *errs set to the number of bits corrected, and returning 0 if no errors exist, or 1 if parity errors exist. */ { - unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16); - unsigned long parity_bit=cw & 0x800000l; + uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16); + uint32_t parity_bit=cw & 0x800000l; cw&=~0x800000l; /* remove parity bit for correction */ cw=correct(cw, errs); /* correct up to three bits */ cw|=parity_bit; if (parity(cw)) - return -1; + *errs++; return cw&0xFFF; } /* decode */ diff --git a/golay.h b/golay.h index 44ecfd88..506eeacc 100644 --- a/golay.h +++ b/golay.h @@ -1,2 +1,8 @@ -int golay_encode(unsigned char *data); -int golay_decode(int *errs, unsigned char *data); + +#ifndef __SERVALD_GOLAY_H +#define __SERVALD_GOLAY_H + +int golay_encode(uint8_t *data); +int golay_decode(int *errs, uint8_t *data); + +#endif \ No newline at end of file diff --git a/headerfiles.mk b/headerfiles.mk index 20fae731..fc9ae543 100644 --- a/headerfiles.mk +++ b/headerfiles.mk @@ -5,17 +5,21 @@ HDRS= fifo.h \ overlay_packet.h \ rhizome.h \ serval.h \ + keyring.h \ + socket.h \ cli.h \ str.h \ rotbuf.h \ mem.h \ os.h \ + uuid.h \ strbuf.h \ strbuf_helpers.h \ sha2.h \ conf.h \ conf_schema.h \ crypto.h \ + dataformats.h \ log.h \ net.h \ fdqueue.h \ @@ -24,4 +28,5 @@ HDRS= fifo.h \ constants.h \ monitor-client.h \ mdp_client.h \ + radio_link.h \ sqlite-amalgamation-3070900/sqlite3.h diff --git a/http_server.c b/http_server.c index 9209a897..814f0919 100644 --- a/http_server.c +++ b/http_server.c @@ -75,15 +75,16 @@ static struct profile_total http_server_stats = { #define DEBUG_DUMP_PARSER(r) do { \ if (config.debug.httpd) \ - DEBUGF("parsed %"PRIhttp_size_t" %s cursor %"PRIhttp_size_t" %s end %"PRIhttp_size_t" remain %"PRIhttp_size_t, \ - r->parsed - r->received, alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \ - r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor), \ - r->end - r->received, \ + DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \ + (int)(r->parsed - r->received), alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \ + (int)(r->cursor - r->received), alloca_toprint(50, r->cursor, r->end - r->cursor), \ + (int)(r->end - r->received), \ r->request_content_remaining \ ); \ } while (0) static void http_server_poll(struct sched_ent *); +static void http_request_set_idle_timeout(struct http_request *r); static int http_request_parse_verb(struct http_request *r); static int http_request_parse_path(struct http_request *r); static int http_request_parse_http_version(struct http_request *r); @@ -98,24 +99,35 @@ void http_request_init(struct http_request *r, int sockfd) assert(sockfd != -1); r->request_header.content_length = CONTENT_LENGTH_UNKNOWN; r->request_content_remaining = CONTENT_LENGTH_UNKNOWN; + r->response.header.content_length = CONTENT_LENGTH_UNKNOWN; + r->response.header.resource_length = CONTENT_LENGTH_UNKNOWN; r->alarm.stats = &http_server_stats; r->alarm.function = http_server_poll; if (r->idle_timeout == 0) r->idle_timeout = 10000; // 10 seconds - r->alarm.alarm = gettime_ms() + r->idle_timeout; - r->alarm.deadline = r->alarm.alarm + r->idle_timeout; r->alarm.poll.fd = sockfd; r->alarm.poll.events = POLLIN; r->phase = RECEIVE; r->received = r->end = r->parsed = r->cursor = r->buffer; r->parser = http_request_parse_verb; watch(&r->alarm); + http_request_set_idle_timeout(r); +} + +static void http_request_set_idle_timeout(struct http_request *r) +{ + assert(r->phase == RECEIVE || r->phase == TRANSMIT); + r->alarm.alarm = gettime_ms() + r->idle_timeout; + r->alarm.deadline = r->alarm.alarm + r->idle_timeout; + unschedule(&r->alarm); schedule(&r->alarm); } void http_request_free_response_buffer(struct http_request *r) { if (r->response_free_buffer) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Free response buffer of %zu bytes", r->response_buffer_size); r->response_free_buffer(r->response_buffer); r->response_free_buffer = NULL; } @@ -125,6 +137,8 @@ void http_request_free_response_buffer(struct http_request *r) int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz) { + // Don't allocate a new buffer if the existing one contains content. + assert(r->response_buffer_sent == r->response_buffer_length); const char *const bufe = r->buffer + sizeof r->buffer; assert(r->received < bufe); size_t rbufsiz = bufe - r->received; @@ -132,6 +146,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz) http_request_free_response_buffer(r); r->response_buffer = (char *) r->received; r->response_buffer_size = rbufsiz; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Static response buffer %zu bytes", r->response_buffer_size); return 0; } if (bufsiz != r->response_buffer_size) { @@ -140,6 +156,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz) return -1; r->response_free_buffer = free; r->response_buffer_size = bufsiz; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Allocated response buffer %zu bytes", r->response_buffer_size); } assert(r->response_buffer_size >= bufsiz); assert(r->response_buffer != NULL); @@ -150,9 +168,10 @@ void http_request_finalise(struct http_request *r) { if (r->phase == DONE) return; - assert(r->phase == RECEIVE || r->phase == TRANSMIT); + assert(r->phase == RECEIVE || r->phase == TRANSMIT || r->phase == PAUSE); unschedule(&r->alarm); - unwatch(&r->alarm); + if (r->phase != PAUSE) + unwatch(&r->alarm); close(r->alarm.poll.fd); r->alarm.poll.fd = -1; if (r->finalise) @@ -162,82 +181,6 @@ void http_request_finalise(struct http_request *r) r->phase = DONE; } -#define _SEP (1 << 0) -#define _BND (1 << 1) - -uint8_t http_ctype[256] = { - ['0'] = _BND, ['1'] = _BND, ['2'] = _BND, ['3'] = _BND, ['4'] = _BND, - ['5'] = _BND, ['6'] = _BND, ['7'] = _BND, ['8'] = _BND, ['9'] = _BND, - ['A'] = _BND, ['B'] = _BND, ['C'] = _BND, ['D'] = _BND, ['E'] = _BND, - ['F'] = _BND, ['G'] = _BND, ['H'] = _BND, ['I'] = _BND, ['J'] = _BND, - ['K'] = _BND, ['L'] = _BND, ['M'] = _BND, ['N'] = _BND, ['O'] = _BND, - ['P'] = _BND, ['Q'] = _BND, ['R'] = _BND, ['S'] = _BND, ['T'] = _BND, - ['U'] = _BND, ['V'] = _BND, ['W'] = _BND, ['X'] = _BND, ['Y'] = _BND, - ['Z'] = _BND, - ['a'] = _BND, ['b'] = _BND, ['c'] = _BND, ['d'] = _BND, ['e'] = _BND, - ['f'] = _BND, ['g'] = _BND, ['h'] = _BND, ['i'] = _BND, ['j'] = _BND, - ['k'] = _BND, ['l'] = _BND, ['m'] = _BND, ['n'] = _BND, ['o'] = _BND, - ['p'] = _BND, ['q'] = _BND, ['r'] = _BND, ['s'] = _BND, ['t'] = _BND, - ['u'] = _BND, ['v'] = _BND, ['w'] = _BND, ['x'] = _BND, ['y'] = _BND, - ['z'] = _BND, - ['+'] = _BND, ['-'] = _BND, ['.'] = _BND, - ['_'] = _BND, - ['('] = _SEP | _BND, - [')'] = _SEP | _BND, - [','] = _SEP | _BND, - ['?'] = _SEP | _BND, - ['='] = _SEP | _BND, - [' '] = _SEP | _BND, - ['\t'] = _SEP, - ['<'] = _SEP, - ['>'] = _SEP, - ['@'] = _SEP, - [';'] = _SEP, - [':'] = _SEP, - ['\\'] = _SEP, - ['"'] = _SEP, - ['/'] = _SEP, - ['['] = _SEP, - [']'] = _SEP, - ['{'] = _SEP, - ['}'] = _SEP, -}; - -inline int is_http_char(char c) -{ - return c >= 0; -} - -inline int is_http_ctl(char c) -{ - return iscntrl(c); -} - -inline int is_http_separator(char c) -{ - return (http_ctype[(unsigned char) c] & _SEP) != 0; -} - -inline int is_http_boundary(char c) -{ - return (http_ctype[(unsigned char) c] & _BND) != 0; -} - -inline int is_http_token(char c) -{ - return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c); -} - -inline int is_valid_http_boundary_string(const char *s) -{ - if (s[0] == '\0') - return 0; - for (; *s; ++s) - if (!is_http_boundary(*s)) - return 0; - return s[-1] != ' '; -} - struct substring { const char *start; const char *end; @@ -410,9 +353,14 @@ static int _skip_literal_nocase(struct http_request *r, const char *literal) return *literal == '\0'; } +static int is_http_space(char c) +{ + return c == ' ' || c == '\t'; +} + static int _skip_optional_space(struct http_request *r) { - while (!_run_out(r) && (*r->cursor == ' ' || *r->cursor == '\t')) + while (!_run_out(r) && is_http_space(*r->cursor)) ++r->cursor; return 1; } @@ -556,6 +504,114 @@ static unsigned _parse_ranges(struct http_request *r, struct http_range *range, return i; } +static int _parse_content_type(struct http_request *r, struct mime_content_type *ct) +{ + size_t n = _parse_token(r, ct->type, sizeof ct->type); + if (n == 0) + return 0; + if (n >= sizeof ct->type) { + WARNF("HTTP Content-Type type truncated: %s", alloca_str_toprint(ct->type)); + return 0; + } + if (!_skip_literal(r, "/")) + return 0; + n = _parse_token(r, ct->subtype, sizeof ct->subtype); + if (n == 0) + return 0; + if (n >= sizeof ct->subtype) { + WARNF("HTTP Content-Type subtype truncated: %s", alloca_str_toprint(ct->subtype)); + return 0; + } + while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) { + const char *start = r->cursor; + if (_skip_literal(r, "charset=")) { + size_t n = _parse_token_or_quoted_string(r, ct->charset, sizeof ct->charset); + if (n == 0) + return 0; + if (n >= sizeof ct->charset) { + WARNF("HTTP Content-Type charset truncated: %s", alloca_str_toprint(ct->charset)); + return 0; + } + continue; + } + r->cursor = start; + if (_skip_literal(r, "boundary=")) { + size_t n = _parse_token_or_quoted_string(r, ct->multipart_boundary, sizeof ct->multipart_boundary); + if (n == 0) + return 0; + if (n >= sizeof ct->multipart_boundary) { + WARNF("HTTP Content-Type boundary truncated: %s", alloca_str_toprint(ct->multipart_boundary)); + return 0; + } + continue; + } + r->cursor = start; + struct substring param; + if (_skip_token(r, ¶m) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping HTTP Content-Type parameter: %s", alloca_substring_toprint(param)); + continue; + } + WARNF("Malformed HTTP Content-Type: %s", alloca_toprint(50, r->cursor, r->end - r->cursor)); + return 0; + } + return 1; +} + +static size_t _parse_base64(struct http_request *r, char *bin, size_t binsize) +{ + return base64_decode((unsigned char *)bin, binsize, r->cursor, r->end - r->cursor, &r->cursor, B64_CONSUME_ALL, is_http_space); +} + +static int _parse_authorization_credentials_basic(struct http_request *r, struct http_client_credentials_basic *cred, char *buf, size_t bufsz) +{ + size_t n = _parse_base64(r, buf, bufsz - 1); // leave room for NUL terminator on password + assert(n < bufsz); // buffer must be big enough + char *pw = (char *) strnchr(buf, n, ':'); + if (pw == NULL) + return 0; // malformed + cred->user = buf; + *pw++ = '\0'; // NUL terminate user + cred->password = pw; + buf[n] = '\0'; // NUL terminate password + return 1; +} + +static int _parse_authorization(struct http_request *r, struct http_client_authorization *auth, size_t header_bytes) +{ + const char *start = r->cursor; + if (_skip_literal(r, "Basic") && _skip_space(r)) { + size_t bufsz = 5 + header_bytes * 3 / 4; // enough for base64 decoding + char buf[bufsz]; + if (_parse_authorization_credentials_basic(r, &auth->credentials.basic, buf, bufsz)) { + auth->scheme = BASIC; + if ( (auth->credentials.basic.user = _reserve_str(r, auth->credentials.basic.user)) == NULL + || (auth->credentials.basic.password = _reserve_str(r, auth->credentials.basic.password)) == NULL + ) + return 0; // error + return 1; + } + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP header: Authorization: %s", alloca_toprint(50, start, header_bytes)); + return 0; + } + if (_skip_literal(r, "Digest") && _skip_space(r)) { + if (r->debug_flag && *r->debug_flag) + DEBUG("Ignoring unsupported HTTP Authorization scheme: Digest"); + r->cursor += header_bytes; + return 1; + } + struct substring scheme; + if (_skip_token(r, &scheme) && _skip_space(r)) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Unrecognised HTTP Authorization scheme: %s", alloca_toprint(-1, scheme.start, scheme.end - scheme.start)); + return 0; + } + if (r->debug_flag && *r->debug_flag) + DEBUGF("Malformed HTTP Authorization header: %s", alloca_toprint(50, r->parsed, r->end - r->parsed)); + return 0; +} + static int _parse_quoted_rfc822_time(struct http_request *r, time_t *timep) { char datestr[40]; @@ -743,6 +799,13 @@ static int http_request_parse_header(struct http_request *r) _rewind(r); const char *const sol = r->cursor; if (_skip_literal_nocase(r, "Content-Length:")) { + if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP header Content-Length: %s", alloca_toprint(50, sol, r->end - sol)); + r->cursor = nextline; + _commit(r); + return 0; + } _skip_optional_space(r); http_size_t length; if (_parse_http_size_t(r, &length) && _skip_optional_space(r) && r->cursor == eol) { @@ -757,56 +820,41 @@ static int http_request_parse_header(struct http_request *r) } _rewind(r); if (_skip_literal_nocase(r, "Content-Type:")) { + if (r->request_header.content_type.type[0]) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP header Content-Type: %s", alloca_toprint(50, sol, r->end - sol)); + r->cursor = nextline; + _commit(r); + return 0; + } _skip_optional_space(r); - struct substring type = substring_NULL; - struct substring subtype = substring_NULL; - char boundary[BOUNDARY_STRING_MAXLEN + 1]; - boundary[0] = '\0'; - if (_skip_token(r, &type) && _skip_literal(r, "/") && _skip_token(r, &subtype)) { - // Parse zero or more content-type parameters. - for (_skip_optional_space(r); r->cursor < eol && _skip_literal(r, ";"); _skip_optional_space(r)) { - _skip_optional_space(r); - const char *startparam = r->cursor; - if (_skip_literal(r, "boundary=")) { - size_t n = _parse_token_or_quoted_string(r, boundary, sizeof boundary); - if (n == 0 || n >= sizeof boundary || !is_valid_http_boundary_string(boundary)) - goto malformed; - continue; - } - // Silently ignore unrecognised parameters (eg, charset=) if they are well formed. - r->cursor = startparam; // partial rewind - if (_skip_token(r, NULL) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) - continue; - break; - } - if (r->cursor == eol) { - r->cursor = nextline; - _commit(r); - if ( (r->request_header.content_type = _reserve(r, type)) == NULL - || (r->request_header.content_subtype = _reserve(r, subtype)) == NULL - || (boundary[0] && (r->request_header.boundary = _reserve_str(r, boundary)) == NULL) - ) - return 0; // error - if (r->debug_flag && *r->debug_flag) - DEBUGF("Parsed HTTP request Content-type: %s/%s%s%s", - r->request_header.content_type, - r->request_header.content_subtype, - r->request_header.boundary ? "; boundary=" : "", - r->request_header.boundary ? alloca_str_toprint(r->request_header.boundary) : "" - ); - return 0; - } + if ( _parse_content_type(r, &r->request_header.content_type) + && _skip_optional_space(r) + && r->cursor == eol + ) { + r->cursor = nextline; + _commit(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP request Content-type: %s", alloca_mime_content_type(&r->request_header.content_type)); + return 0; } goto malformed; } _rewind(r); if (_skip_literal_nocase(r, "Range:")) { + if (r->request_header.content_range_count) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP header Range: %s", alloca_toprint(50, sol, r->end - sol)); + r->cursor = nextline; + _commit(r); + return 0; + } _skip_optional_space(r); unsigned int n; if ( _skip_literal(r, "bytes=") && (n = _parse_ranges(r, r->request_header.content_ranges, NELS(r->request_header.content_ranges))) && _skip_optional_space(r) - && (r->cursor == eol) + && r->cursor == eol ) { r->cursor = nextline; _commit(r); @@ -826,6 +874,27 @@ static int http_request_parse_header(struct http_request *r) goto malformed; } _rewind(r); + if (_skip_literal_nocase(r, "Authorization:")) { + if (r->request_header.authorization.scheme != NOAUTH) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP header Authorization: %s", alloca_toprint(50, sol, r->end - sol)); + r->cursor = nextline; + _commit(r); + return 0; + } + _skip_optional_space(r); + if ( _parse_authorization(r, &r->request_header.authorization, eol - r->cursor) + && _skip_optional_space(r) + && r->cursor == eol + ) { + assert(r->request_header.authorization.scheme != NOAUTH); + r->cursor = nextline; + _commit(r); + return 0; + } + goto malformed; + } + _rewind(r); if (r->debug_flag && *r->debug_flag) DEBUGF("Skipped HTTP request header: %s", alloca_toprint(-1, sol, eol - sol)); r->cursor = nextline; @@ -858,7 +927,7 @@ static int http_request_start_body(struct http_request *r) DEBUGF("Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb); return 400; } - if (r->request_header.content_type) { + if (r->request_header.content_type.type) { if (r->debug_flag && *r->debug_flag) DEBUGF("Malformed HTTP %s request: Content-Type not allowed", r->verb); return 400; @@ -871,26 +940,28 @@ static int http_request_start_body(struct http_request *r) DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb); return 411; } - if (r->request_header.content_type == NULL) { + if (r->request_header.content_type.type == NULL) { if (r->debug_flag && *r->debug_flag) DEBUGF("Malformed HTTP %s request: missing Content-Type header", r->verb); return 400; } - if ( strcmp(r->request_header.content_type, "multipart") == 0 - && strcmp(r->request_header.content_subtype, "form-data") == 0 + if ( strcmp(r->request_header.content_type.type, "multipart") == 0 + && strcmp(r->request_header.content_type.subtype, "form-data") == 0 ) { - if (r->request_header.boundary == NULL || r->request_header.boundary[0] == '\0') { + if ( r->request_header.content_type.multipart_boundary == NULL + || r->request_header.content_type.multipart_boundary[0] == '\0' + ) { if (r->debug_flag && *r->debug_flag) DEBUGF("Malformed HTTP %s request: Content-Type %s/%s missing boundary parameter", - r->verb, r->request_header.content_type, r->request_header.content_subtype); + r->verb, r->request_header.content_type.type, r->request_header.content_type.subtype); return 400; } r->parser = http_request_parse_body_form_data; r->form_data_state = START; } else { if (r->debug_flag && *r->debug_flag) - DEBUGF("Unsupported HTTP %s request: Content-Type %s/%s not supported", - r->verb, r->request_header.content_type, r->request_header.content_subtype); + DEBUGF("Unsupported HTTP %s request: Content-Type %s not supported", + r->verb, alloca_mime_content_type(&r->request_header.content_type)); return 415; } } @@ -909,7 +980,7 @@ static int http_request_start_body(struct http_request *r) */ static int _skip_mime_boundary(struct http_request *r) { - if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.boundary)) + if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.content_type.multipart_boundary)) return 0; if (_skip_literal(r, "--") && _skip_crlf(r)) return 2; @@ -988,6 +1059,43 @@ malformed: return 1; } +static void http_request_form_data_start_part(struct http_request *r, int b) +{ + switch (r->form_data_state) { + case BODY: + if ( r->part_header.content_length != CONTENT_LENGTH_UNKNOWN + && r->part_body_length != r->part_header.content_length + ) { + WARNF("HTTP multipart part body length (%"PRIhttp_size_t") does not match Content-Length header (%"PRIhttp_size_t")", + r->part_body_length, + r->part_header.content_length + ); + } + // fall through... + case HEADER: + if (r->form_data.handle_mime_part_end) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_end()"); + r->form_data.handle_mime_part_end(r); + } + break; + default: + break; + } + if (b == 1) { + r->form_data_state = HEADER; + bzero(&r->part_header, sizeof r->part_header); + r->part_body_length = 0; + r->part_header.content_length = CONTENT_LENGTH_UNKNOWN; + if (r->form_data.handle_mime_part_start) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_start()"); + r->form_data.handle_mime_part_start(r); + } + } else + r->form_data_state = EPILOGUE; +} + /* If parsing completes (ie, parsed to end of epilogue), then sets r->parser to NULL and returns 0, * so this function will not be called again. If parsing cannot complete due to running out of * data, returns 100, so this function will not be called again until more data has been read. @@ -1029,15 +1137,7 @@ static int http_request_parse_body_form_data(struct http_request *r) } _rewind_crlf(r); _commit(r); - if (b == 1) { - r->form_data_state = HEADER; - if (r->form_data.handle_mime_part_start) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_part_start()"); - r->form_data.handle_mime_part_start(r); - } - } else - r->form_data_state = EPILOGUE; + http_request_form_data_start_part(r, b); return 0; } } @@ -1074,6 +1174,16 @@ static int http_request_parse_body_form_data(struct http_request *r) // A blank line finishes the headers. The CRLF does not form part of the body. if (_skip_crlf(r)) { _commit(r); + if (r->form_data.handle_mime_part_header) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("handle_mime_part_header(Content-Length: %"PRIhttp_size_t", Content-Type: %s, Content-Disposition: %s)", + r->part_header.content_length, + alloca_mime_content_type(&r->part_header.content_type), + alloca_mime_content_disposition(&r->part_header.content_disposition) + ); + r->form_data.handle_mime_part_header(r, &r->part_header); + } + r->form_data_state = BODY; return 0; } @@ -1086,23 +1196,9 @@ static int http_request_parse_body_form_data(struct http_request *r) if ((b = _skip_mime_boundary(r))) { _rewind_crlf(r); _commit(r); - if (r->form_data.handle_mime_part_end) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_part_end()"); - r->form_data.handle_mime_part_end(r); - } // A boundary in the middle of headers finishes the current part and starts a new part. // An end boundary terminates the current part and starts the epilogue. - if (b == 1) { - r->form_data_state = HEADER; - if (r->form_data.handle_mime_part_start) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_part_start()"); - r->form_data.handle_mime_part_start(r); - } - } - else - r->form_data_state = EPILOGUE; + http_request_form_data_start_part(r, b); return 0; } if (_run_out(r)) @@ -1115,27 +1211,54 @@ static int http_request_parse_body_form_data(struct http_request *r) strncpy(labelstr, label.start, labellen)[labellen] = '\0'; str_tolower_inplace(labelstr); const char *value = r->cursor; - if (strcmp(labelstr, "content-disposition") == 0) { - struct mime_content_disposition cd; - bzero(&cd, sizeof cd); - if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) { + if (strcmp(labelstr, "content-length") == 0) { + if (r->part_header.content_length != CONTENT_LENGTH_UNKNOWN) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP multipart header Content-Length: %s", alloca_toprint(50, sol, r->end - sol)); + return 400; + } + http_size_t length; + if (_parse_http_size_t(r, &length) && _skip_optional_space(r) && _skip_crlf(r)) { _rewind_crlf(r); _commit(r); - if (r->form_data.handle_mime_content_disposition) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_content_disposition(%s)", alloca_mime_content_disposition(&cd)); - r->form_data.handle_mime_content_disposition(r, &cd); - } + r->part_header.content_length = length; + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP multipart header Content-Length: %"PRIhttp_size_t, r->part_header.content_length); + return 0; + } + } + else if (strcmp(labelstr, "content-type") == 0) { + if (r->part_header.content_type.type[0]) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP multipart header Content-Type: %s", alloca_toprint(50, sol, r->end - sol)); + return 400; + } + if (_parse_content_type(r, &r->part_header.content_type) && _skip_optional_space(r) && _skip_crlf(r)) { + _rewind_crlf(r); + _commit(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP multipart header Content-Type: %s", alloca_mime_content_type(&r->part_header.content_type)); + return 0; + } + } + else if (strcmp(labelstr, "content-disposition") == 0) { + if (r->part_header.content_disposition.type[0]) { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skipping duplicate HTTP multipart header Content-Disposition: %s", alloca_toprint(50, sol, r->end - sol)); + return 400; + } + if (_parse_content_disposition(r, &r->part_header.content_disposition) && _skip_optional_space(r) && _skip_crlf(r)) { + _rewind_crlf(r); + _commit(r); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Parsed HTTP multipart header Content-Disposition: %s", alloca_mime_content_disposition(&r->part_header.content_disposition)); return 0; } } else if (_skip_to_crlf(r)) { _commit(r); - if (r->form_data.handle_mime_header) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_header(%s, %s)", alloca_str_toprint(labelstr), alloca_toprint(-1, value, value - r->cursor)); - r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end - } + if (r->debug_flag && *r->debug_flag) + DEBUGF("Skip HTTP multipart header: %s: %s", alloca_str_toprint(labelstr), alloca_toprint(-1, value, value - r->cursor)); return 0; } } @@ -1169,25 +1292,14 @@ static int http_request_parse_body_form_data(struct http_request *r) if ((b = _skip_mime_boundary(r))) { _rewind_crlf(r); _commit(r); + assert(end_body >= start); + r->part_body_length += end_body - start; if (end_body > start && r->form_data.handle_mime_body) { if (r->debug_flag && *r->debug_flag) DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, end_body - start), end_body - start); r->form_data.handle_mime_body(r, start, end_body - start); // excluding CRLF at end } - if (r->form_data.handle_mime_part_end) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_part_end()"); - r->form_data.handle_mime_part_end(r); - } - r->form_data_state = EPILOGUE; - if (b == 1) { - r->form_data_state = HEADER; - if (r->form_data.handle_mime_part_start) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("handle_mime_part_start()"); - r->form_data.handle_mime_part_start(r); - } - } + http_request_form_data_start_part(r, b); return 0; } } @@ -1198,6 +1310,8 @@ static int http_request_parse_body_form_data(struct http_request *r) } _rewind_optional_cr(r); _commit(r); + assert(r->parsed >= start); + r->part_body_length += r->parsed - start; if (r->parsed > start && r->form_data.handle_mime_body) { if (r->debug_flag && *r->debug_flag) DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, r->parsed - start), r->parsed - start); @@ -1294,10 +1408,7 @@ static void http_request_receive(struct http_request *r) r->request_content_remaining -= (size_t) bytes; // We got some data, so reset the inactivity timer and invoke the parsing state machine to process // it. The state machine invokes the caller-supplied callback functions. - r->alarm.alarm = gettime_ms() + r->idle_timeout; - r->alarm.deadline = r->alarm.alarm + r->idle_timeout; - unschedule(&r->alarm); - schedule(&r->alarm); + http_request_set_idle_timeout(r); // Parse the unparsed and received data. while (r->phase == RECEIVE) { int result; @@ -1369,45 +1480,93 @@ static void http_request_receive(struct http_request *r) */ static void http_request_send_response(struct http_request *r) { - assert(r->response_sent <= r->response_length); - while (r->response_sent < r->response_length) { + assert(r->phase == TRANSMIT); + while (1) { + if (r->response_length != CONTENT_LENGTH_UNKNOWN) + assert(r->response_sent <= r->response_length); assert(r->response_buffer_sent <= r->response_buffer_length); - if (r->response_buffer_sent == r->response_buffer_length) { - if (r->response.content_generator) { - // Content generator must fill or partly fill response_buffer and set response_buffer_sent - // and response_buffer_length. May also malloc() a bigger buffer and set response_buffer to - // point to it. - r->response_buffer_sent = r->response_buffer_length = 0; - if (r->response.content_generator(r) == -1) { - if (r->debug_flag && *r->debug_flag) - DEBUG("Content generation error, closing connection"); - http_request_finalise(r); - return; - } - assert(r->response_buffer_sent <= r->response_buffer_length); - if (r->response_buffer_sent == r->response_buffer_length) { - WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)", - r->response_sent, r->response_length, r->response_length - r->response_sent); - http_request_finalise(r); - return; - } - } else { - WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes", - r->response_length, r->response_length - r->response_sent); - http_request_finalise(r); - return; - } + uint64_t remaining = CONTENT_LENGTH_UNKNOWN; + size_t unsent = r->response_buffer_length - r->response_buffer_sent; + if (r->debug_flag && *r->debug_flag) + DEBUGF("HTTP response buffer contains %zu bytes unsent", unsent); + if (r->response_length != CONTENT_LENGTH_UNKNOWN) { + remaining = r->response_length - r->response_sent; + assert(unsent <= remaining); + assert(r->response_buffer_need <= remaining); + if (remaining == 0) + break; // no more to generate } - assert(r->response_buffer_sent < r->response_buffer_length); - size_t bytes = r->response_buffer_length - r->response_buffer_sent; - if (r->response_sent + bytes > r->response_length) { - WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating", - r->response_length, - r->response_sent + bytes - r->response_length); - bytes = r->response_length - r->response_sent; + if (unsent == 0) + r->response_buffer_sent = r->response_buffer_length = 0; + if (r->phase == PAUSE) { + if (unsent == 0) + return; // nothing to send + } else if (r->response.content_generator) { + // If the buffer is smaller than the content generator needs, and it contains no unsent + // content, then allocate a larger buffer. + if (r->response_buffer_need > r->response_buffer_size && unsent == 0) { + if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1) { + WHYF("HTTP response truncated at offset=%"PRIhttp_size_t" due to insufficient buffer space", + r->response_sent); + http_request_finalise(r); + return; + } + } + // If there are some sent bytes at the start of the buffer and only a few unsent bytes, then + // move the unsent content to the start of the buffer to make more room. + if (r->response_buffer_sent > 0 && unsent < 128) { + memmove(r->response_buffer, r->response_buffer + r->response_buffer_sent, unsent); + r->response_buffer_length -= r->response_buffer_sent; + r->response_buffer_sent = 0; + } + // If there is enough unfilled room at the end of the buffer, then fill the buffer with some + // more content. + assert(r->response_buffer_length <= r->response_buffer_size); + size_t unfilled = r->response_buffer_size - r->response_buffer_length; + if (unfilled > 0 && unfilled >= r->response_buffer_need) { + // The content generator must fill or partly fill the part of the buffer we indicate and + // return the number of bytes appended. If it returns zero, it means it has no more + // content (EOF), and must not be called again. If the return value exceeds the buffer size + // we supply, it gives the amount of free space the generator needs in order to append; the + // generator will not append any bytes until that much free space is available. If returns + // -1, it means an unrecoverable error occurred, and the generator must not be called again. + struct http_content_generator_result result; + bzero(&result, sizeof result); + int ret = r->response.content_generator(r, (unsigned char *) r->response_buffer + r->response_buffer_length, unfilled, &result); + if (ret == -1) { + WHY("Content generation error, closing connection"); + http_request_finalise(r); + return; + } + assert(result.generated <= unfilled); + r->response_buffer_length += result.generated; + r->response_buffer_need = result.need; + if (result.generated == 0 && result.need <= unfilled && r->phase != PAUSE) { + WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t" (ret=%d)", r->response_sent, ret); + http_request_finalise(r); + return; + } + if (r->debug_flag && *r->debug_flag) + DEBUGF("Generated HTTP %zu bytes of content, need %zu bytes of buffer (ret=%d)", result.generated, result.need, ret); + if (r->phase != PAUSE && ret == 0) + r->response.content_generator = NULL; // ensure we never invoke again + continue; + } + } else if (remaining != CONTENT_LENGTH_UNKNOWN && unsent < remaining) { + WHYF("HTTP response generator finished prematurely at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)", + r->response_sent, r->response_length, remaining); + http_request_finalise(r); + return; + } else if (unsent == 0) + break; + assert(unsent > 0); + if (remaining != CONTENT_LENGTH_UNKNOWN && unsent > remaining) { + WHYF("HTTP response overruns Content-Length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes -- truncating", + r->response_length, unsent - remaining); + unsent = remaining; } sigPipeFlag = 0; - ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes); + ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, unsent); if (written == -1) { if (r->debug_flag && *r->debug_flag) DEBUG("HTTP socket write error, closing connection"); @@ -1427,13 +1586,14 @@ static void http_request_send_response(struct http_request *r) r->response_buffer_sent += (size_t) written; assert(r->response_sent <= r->response_length); assert(r->response_buffer_sent <= r->response_buffer_length); + if (r->debug_flag && *r->debug_flag) + DEBUGF("Wrote %zu bytes to HTTP socket, total %"PRIhttp_size_t", remaining=%"PRIhttp_size_t, + (size_t) written, r->response_sent, r->response_length - r->response_sent); // Reset inactivity timer. - r->alarm.alarm = gettime_ms() + r->idle_timeout; - r->alarm.deadline = r->alarm.alarm + r->idle_timeout; - unschedule(&r->alarm); - schedule(&r->alarm); - // If we wrote less than we tried, then go back to polling. - if (written < (size_t) bytes) + if (r->phase != PAUSE) + http_request_set_idle_timeout(r); + // If we wrote less than we tried, then go back to polling, otherwise keep generating content. + if (written < (size_t) unsent) return; } if (r->debug_flag && *r->debug_flag) @@ -1445,9 +1605,16 @@ static void http_server_poll(struct sched_ent *alarm) { struct http_request *r = (struct http_request *) alarm; if (alarm->poll.revents == 0) { - if (r->debug_flag && *r->debug_flag) - DEBUGF("Timeout, closing connection"); - http_request_finalise(r); + if (r->phase == PAUSE) { + r->phase = TRANSMIT; + r->alarm.poll.events = POLLOUT; + watch(&r->alarm); + http_request_set_idle_timeout(r); + } else { + if (r->debug_flag && *r->debug_flag) + DEBUGF("Timeout, closing connection"); + http_request_finalise(r); + } } else if (alarm->poll.revents & (POLLHUP | POLLERR)) { if (r->debug_flag && *r->debug_flag) @@ -1538,18 +1705,6 @@ static const char *httpResultString(int response_code) } } -static strbuf strbuf_append_quoted_string(strbuf sb, const char *str) -{ - strbuf_putc(sb, '"'); - for (; *str; ++str) { - if (*str == '"' || *str == '\\') - strbuf_putc(sb, '\\'); - strbuf_putc(sb, *str); - } - strbuf_putc(sb, '"'); - return sb; -} - /* Render the HTTP response into the current response buffer. Return 1 if it fits, 0 if it does * not. The buffer response_pointer may be NULL, in which case no response is rendered, but the * content_length is still computed @@ -1559,23 +1714,55 @@ static strbuf strbuf_append_quoted_string(strbuf sb, const char *str) static int _render_response(struct http_request *r) { struct http_response hr = r->response; - assert(hr.result_code != 0); - assert(hr.header.content_range_start <= hr.header.resource_length); - assert(hr.header.content_length <= hr.header.resource_length); - // To save page handlers having to decide between 200 (OK) and 206 (Partial Content), they can - // just send 200 and the content range fields, and this logic will detect if it should be 206. - if (hr.header.content_length > 0 && hr.header.content_length < hr.header.resource_length && hr.result_code == 200) - hr.result_code = 206; // Partial Content + assert(hr.result_code >= 100); + assert(hr.result_code < 600); + // Status code 401 must be accompanied by a WWW-Authenticate header. + if (hr.result_code == 401) + assert(hr.header.www_authenticate.scheme != NOAUTH); const char *result_string = httpResultString(hr.result_code); strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size); - if (hr.content == NULL && hr.content_generator == NULL) { + // Cannot specify both static (pre-rendered) content AND generated content. + assert(!(hr.content && hr.content_generator)); + if (hr.content || hr.content_generator) { + // With static (pre-rendered) content, the content length is mandatory (so we know how much data + // follows the 'hr.content' pointer. Generated content will generally not send a Content-Length + // header, nor send partial content, but they might. + if (hr.content) + assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN); + // Ensure that all partial content fields are consistent. If content length or resource length + // are unknown, there can be no range field. + if ( hr.header.content_length != CONTENT_LENGTH_UNKNOWN + && hr.header.resource_length != CONTENT_LENGTH_UNKNOWN + ) { + assert(hr.header.content_length <= hr.header.resource_length); + assert(hr.header.content_range_start + hr.header.content_length <= hr.header.resource_length); + } else { + assert(hr.header.content_range_start == 0); + } + // Convert a 200 status code into 206 if only partial content is being sent. This saves page + // handlers having to decide between 200 (OK) and 206 (Partial Content), they can just set the + // content and resource length fields and pass 200 to http_request_response_static(), and this + // logic will change it to 206 if appropriate. + if ( hr.header.content_length != CONTENT_LENGTH_UNKNOWN + && hr.header.resource_length != CONTENT_LENGTH_UNKNOWN + && hr.header.content_length > 0 + && hr.header.content_length < hr.header.resource_length + ) { + if (hr.result_code == 200) + hr.result_code = 206; // Partial Content + } + } else { + // If no content is supplied at all, then render a standard, short body based solely on result + // code. + assert(hr.header.content_length == CONTENT_LENGTH_UNKNOWN); + assert(hr.header.resource_length == CONTENT_LENGTH_UNKNOWN); + assert(hr.header.content_range_start == 0); + assert(hr.result_code != 206); strbuf cb = strbuf_alloca(100 + strlen(result_string)); - strbuf_puts(cb, "

"); - strbuf_puts(cb, result_string); - strbuf_puts(cb, "

\r\n"); + strbuf_sprintf(cb, "

%03u %s

", hr.result_code, result_string); hr.content = strbuf_str(cb); - hr.header.resource_length = hr.header.content_length = strbuf_len(cb); hr.header.content_type = "text/html"; + hr.header.resource_length = hr.header.content_length = strbuf_len(cb); hr.header.content_range_start = 0; } assert(hr.header.content_type != NULL); @@ -1593,6 +1780,8 @@ static int _render_response(struct http_request *r) if (hr.result_code == 206) { // Must only use result code 206 (Partial Content) if the content is in fact less than the whole // resource length. + assert(hr.header.content_length != CONTENT_LENGTH_UNKNOWN); + assert(hr.header.resource_length != CONTENT_LENGTH_UNKNOWN); assert(hr.header.content_length > 0); assert(hr.header.content_length < hr.header.resource_length); strbuf_sprintf(sb, @@ -1602,18 +1791,39 @@ static int _render_response(struct http_request *r) hr.header.resource_length ); } - strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length); + if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN) + strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length); + const char *scheme = NULL; + switch (hr.header.www_authenticate.scheme) { + case NOAUTH: break; + case BASIC: scheme = "Basic"; break; + } + if (scheme) { + assert(hr.result_code == 401); + strbuf_sprintf(sb, "WWW-Authenticate: %s realm=", scheme); + strbuf_append_quoted_string(sb, hr.header.www_authenticate.realm); + strbuf_puts(sb, "\r\n"); + } strbuf_puts(sb, "\r\n"); - if (strbuf_overrun(sb)) - return 0; - r->response_length = strbuf_len(sb) + hr.header.content_length; + if (hr.header.content_length != CONTENT_LENGTH_UNKNOWN) + r->response_length = strbuf_count(sb) + hr.header.content_length; + else + r->response_length = CONTENT_LENGTH_UNKNOWN; + r->response_buffer_need = strbuf_count(sb) + 1; // the header and the strbuf terminating NUL + if (hr.content) { + assert(r->response_length != CONTENT_LENGTH_UNKNOWN); + if (r->response_buffer_need < r->response_length) + r->response_buffer_need = r->response_length; + } else + assert(hr.content_generator); + if (r->response_buffer_size < r->response_buffer_need) + return 0; // doesn't fit + assert(!strbuf_overrun(sb)); if (hr.content) { - if (r->response_buffer_size < r->response_length) - return 0; bcopy(hr.content, strbuf_end(sb), hr.header.content_length); r->response_buffer_length = r->response_length; } else { - r->response_buffer_length = strbuf_len(sb); + r->response_buffer_length = strbuf_count(sb); } r->response_buffer_sent = 0; return 1; @@ -1633,9 +1843,9 @@ static void http_request_render_response(struct http_request *r) // rendered headers, so after this step, whether or not the buffer was overrun, we know the total // length of the response. if (!_render_response(r)) { - // If the response did not fit into the existing buffer, then allocate a large buffer from the - // heap and try rendering again. - if (http_request_set_response_bufsize(r, r->response_length + 1) == -1) + // If the static response did not fit into the existing buffer, then allocate a large buffer + // from the heap and try rendering again. + if (http_request_set_response_bufsize(r, r->response_buffer_need) == -1) WHY("Cannot render HTTP response, out of memory"); else if (!_render_response(r)) FATAL("Re-render of HTTP response overflowed buffer"); @@ -1656,7 +1866,6 @@ static size_t http_request_drain(struct http_request *r) static void http_request_start_response(struct http_request *r) { assert(r->phase == RECEIVE); - assert(r->response.result_code != 0); if (r->response.content || r->response.content_generator) { assert(r->response.header.content_type != NULL); assert(r->response.header.content_type[0]); @@ -1674,6 +1883,13 @@ static void http_request_start_response(struct http_request *r) http_request_drain(r); if (r->phase != RECEIVE) return; + // Ensure conformance to HTTP standards. + if (r->response.result_code == 401 && r->response.header.www_authenticate.scheme == NOAUTH) { + WHY("HTTP 401 response missing WWW-Authenticate header, sending 500 Server Error instead"); + r->response.result_code = 500; + r->response.content = NULL; + r->response.content_generator = NULL; + } // If the response cannot be rendered, then render a 500 Server Error instead. If that fails, // then just close the connection. http_request_render_response(r); @@ -1681,6 +1897,7 @@ static void http_request_start_response(struct http_request *r) WARN("Cannot render HTTP response, sending 500 Server Error instead"); r->response.result_code = 500; r->response.content = NULL; + r->response.content_generator = NULL; http_request_render_response(r); if (r->response_buffer == NULL) { WHY("Cannot render HTTP 500 Server Error response, closing connection"); @@ -1688,6 +1905,7 @@ static void http_request_start_response(struct http_request *r) return; } } + r->response_buffer_need = 0; r->response_sent = 0; if (r->debug_flag && *r->debug_flag) DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_buffer_length)); @@ -1696,6 +1914,19 @@ static void http_request_start_response(struct http_request *r) watch(&r->alarm); } +void http_request_pause_response(struct http_request *r, time_ms_t until) +{ + if (r->debug_flag && *r->debug_flag) + DEBUGF("Pausing response for %.3f sec", (double)(until - gettime_ms()) / 1000.0); + assert(r->phase == TRANSMIT); + r->phase = PAUSE; + r->alarm.alarm = until; + r->alarm.deadline = until + r->idle_timeout; + unwatch(&r->alarm); + unschedule(&r->alarm); + schedule(&r->alarm); +} + /* Start sending a static (pre-computed) response back to the client. The response's Content-Type * is set by the 'mime_type' parameter (in the standard format "type/subtype"). The response's * content is set from the 'body' and 'bytes' parameters, which need not point to persistent data, @@ -1706,8 +1937,6 @@ static void http_request_start_response(struct http_request *r) void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes) { assert(r->phase == RECEIVE); - assert(result >= 100); - assert(result < 300); assert(mime_type != NULL); assert(mime_type[0]); r->response.result_code = result; @@ -1722,8 +1951,6 @@ void http_request_response_static(struct http_request *r, int result, const char void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR generator) { assert(r->phase == RECEIVE); - assert(result >= 100); - assert(result < 300); assert(mime_type != NULL); assert(mime_type[0]); r->response.result_code = result; @@ -1744,8 +1971,6 @@ void http_request_response_generated(struct http_request *r, int result, const c void http_request_simple_response(struct http_request *r, uint16_t result, const char *body) { assert(r->phase == RECEIVE); - assert(result >= 200); - assert(result < 600); strbuf h = NULL; if (body) { size_t html_len = strlen(body) + 40; @@ -1755,8 +1980,10 @@ void http_request_simple_response(struct http_request *r, uint16_t result, const r->response.result_code = result; r->response.header.content_type = "text/html"; r->response.header.content_range_start = 0; - r->response.header.resource_length = r->response.header.content_length = h ? strbuf_len(h) : 0; - r->response.content = h ? strbuf_str(h) : NULL; + if (h) { + r->response.header.resource_length = r->response.header.content_length = strbuf_len(h); + r->response.content = strbuf_str(h); + } r->response.content_generator = NULL; http_request_start_response(r); } diff --git a/http_server.h b/http_server.h index a647c30e..baab6e75 100644 --- a/http_server.h +++ b/http_server.h @@ -56,13 +56,35 @@ http_size_t http_range_bytes(const struct http_range *range, unsigned nranges); #define CONTENT_LENGTH_UNKNOWN UINT64_MAX +struct mime_content_type { + char type[64]; + char subtype[64]; + char multipart_boundary[71]; + char charset[31]; +}; + + +struct http_client_authorization { + enum http_authorization_scheme { NOAUTH = 0, BASIC } scheme; + union { + struct http_client_credentials_basic { + const char *user; + const char *password; + } basic; + } credentials; +}; + +struct http_www_authenticate { + enum http_authorization_scheme scheme; + const char *realm; +}; + struct http_request_headers { http_size_t content_length; - const char *content_type; - const char *content_subtype; - const char *boundary; + struct mime_content_type content_type; unsigned short content_range_count; struct http_range content_ranges[5]; + struct http_client_authorization authorization; }; struct http_response_headers { @@ -71,15 +93,21 @@ struct http_response_headers { http_size_t resource_length; // size of entire resource const char *content_type; // "type/subtype" const char *boundary; + struct http_www_authenticate www_authenticate; }; -typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *); +struct http_content_generator_result { + size_t generated; + size_t need; +}; + +typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, size_t, struct http_content_generator_result *); struct http_response { uint16_t result_code; struct http_response_headers header; const char *content; - HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content + HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content }; #define MIME_FILENAME_MAXLEN 127 @@ -94,11 +122,16 @@ struct mime_content_disposition { time_t read_date; }; +struct mime_part_headers { + http_size_t content_length; + struct mime_content_type content_type; + struct mime_content_disposition content_disposition; +}; + struct http_mime_handler { void (*handle_mime_preamble)(struct http_request *, const char *, size_t); void (*handle_mime_part_start)(struct http_request *); - void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *); - void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t); + void (*handle_mime_part_header)(struct http_request *, const struct mime_part_headers *); void (*handle_mime_body)(struct http_request *, const char *, size_t); void (*handle_mime_part_end)(struct http_request *); void (*handle_mime_epilogue)(struct http_request *, const char *, size_t); @@ -110,46 +143,67 @@ void http_request_init(struct http_request *r, int sockfd); void http_request_free_response_buffer(struct http_request *r); int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz); void http_request_finalise(struct http_request *r); +void http_request_pause_response(struct http_request *r, time_ms_t until); void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); -void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR); +void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *); void http_request_simple_response(struct http_request *r, uint16_t result, const char *body); typedef int (*HTTP_REQUEST_PARSER)(struct http_request *); struct http_request { struct sched_ent alarm; // MUST BE FIRST ELEMENT - enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase; - bool_t *debug_flag; - bool_t *disable_tx_flag; - time_ms_t initiate_time; // time connection was initiated - time_ms_t idle_timeout; // disconnect if no bytes received for this long - struct sockaddr_in client_sockaddr_in; - HTTP_REQUEST_PARSER parser; // current parser function - HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed - HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed - HTTP_REQUEST_PARSER handle_content_end; // called after all content is received - enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state; - struct http_mime_handler form_data; // called to parse multipart/form-data body + // The following control the lifetime of this struct. + enum http_request_phase { RECEIVE, TRANSMIT, PAUSE, DONE } phase; void (*finalise)(struct http_request *); void (*free)(void*); + // These can be set up to point to config flags, to allow debug to be + // enabled indpendently for different instances HTTP server instances + // that use this code. + bool_t *debug_flag; + bool_t *disable_tx_flag; + // The following are used for parsing the HTTP request. + time_ms_t initiate_time; // time connection was initiated + time_ms_t idle_timeout; // disconnect if no bytes received for this long + struct sockaddr_in client_sockaddr_in; // caller may supply this + // The parsed HTTP request is accumulated into the following fields. const char *verb; // points to nul terminated static string, "GET", "PUT", etc. const char *path; // points into buffer; nul terminated uint8_t version_major; // m from from HTTP/m.n uint8_t version_minor; // n from HTTP/m.n struct http_request_headers request_header; + // Parsing is done by setting 'parser' to point to a series of parsing + // functions as the parsing state progresses. + HTTP_REQUEST_PARSER parser; // current parser function + // The caller may set these up, and they are invoked by the parser as request + // parsing reaches different stages. + HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed + HTTP_REQUEST_PARSER handle_headers; // called after all HTTP headers are parsed + HTTP_REQUEST_PARSER handle_content_end; // called after all content is received + // The following are used for managing the buffer during RECEIVE phase. const char *received; // start of received data in buffer[] const char *end; // end of received data in buffer[] const char *parsed; // start of unparsed data in buffer[] const char *cursor; // for parsing http_size_t request_content_remaining; + // The following are used for parsing a multipart body. + enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state; + struct http_mime_handler form_data; + struct mime_part_headers part_header; + http_size_t part_body_length; + // The following are used for constructing the response that will be sent in + // TRANSMIT phase. struct http_response response; + // The following are used during TRANSMIT phase to control buffering and + // sending. http_size_t response_length; // total response bytes (header + content) http_size_t response_sent; // for counting up to response_length char *response_buffer; + size_t response_buffer_need; size_t response_buffer_size; size_t response_buffer_length; size_t response_buffer_sent; void (*response_free_buffer)(void*); + // This buffer is used during RECEIVE and TRANSMIT phase. char buffer[8 * 1024]; }; diff --git a/keyring.c b/keyring.c index f9c60a6f..3182e6bb 100644 --- a/keyring.c +++ b/keyring.c @@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "overlay_address.h" #include "crypto.h" #include "overlay_packet.h" +#include "keyring.h" +#include "dataformats.h" static void keyring_free_keypair(keypair *kp); static void keyring_free_context(keyring_context *c); @@ -223,7 +225,8 @@ void keyring_free(keyring_file *k) /* Wipe everything, just to be sure. */ bzero(k,sizeof(keyring_file)); - + free(k); + return; } @@ -272,6 +275,7 @@ static void keyring_free_context(keyring_context *c) } if (c->KeyRingSalt) { bzero(c->KeyRingSalt,c->KeyRingSaltLen); + free(c->KeyRingSalt); c->KeyRingSalt = NULL; c->KeyRingSaltLen = 0; } @@ -427,6 +431,7 @@ static const char *keytype_str(unsigned ktype, const char *unknown) case KEYTYPE_CRYPTOSIGN: return "CRYPTOSIGN"; case KEYTYPE_RHIZOME: return "RHIZOME"; case KEYTYPE_DID: return "DID"; + case KEYTYPE_PUBLIC_TAG: return "PUBLIC_TAG"; default: return unknown; } } @@ -437,7 +442,7 @@ struct keytype { size_t packed_size; void (*creator)(keypair *); int (*packer)(const keypair *, struct rotbuf *); - int (*unpacker)(keypair *, struct rotbuf *); + int (*unpacker)(keypair *, struct rotbuf *, int); void (*dumper)(const keypair *, XPRINTF, int); int (*loader)(keypair *, const char *); }; @@ -483,6 +488,12 @@ static int pack_private_only(const keypair *kp, struct rotbuf *rb) return 0; } +static int pack_public_only(const keypair *kp, struct rotbuf *rb) +{ + rotbuf_putbuf(rb, kp->public_key, kp->public_key_len); + return 0; +} + static int pack_private_public(const keypair *kp, struct rotbuf *rb) { rotbuf_putbuf(rb, kp->private_key, kp->private_key_len); @@ -617,20 +628,36 @@ static int load_unknown(keypair *kp, const char *text) return 0; } -static int unpack_private_public(keypair *kp, struct rotbuf *rb) +static int unpack_private_public(keypair *kp, struct rotbuf *rb, int key_length) { rotbuf_getbuf(rb, kp->private_key, kp->private_key_len); rotbuf_getbuf(rb, kp->public_key, kp->public_key_len); return 0; } -static int unpack_private_only(keypair *kp, struct rotbuf *rb) +static int unpack_private_only(keypair *kp, struct rotbuf *rb, int key_length) { + if (!kp->private_key){ + kp->private_key_len = key_length; + if ((kp->private_key = emalloc(kp->private_key_len))==NULL) + return -1; + } rotbuf_getbuf(rb, kp->private_key, kp->private_key_len); return 0; } -static int unpack_cryptobox(keypair *kp, struct rotbuf *rb) +static int unpack_public_only(keypair *kp, struct rotbuf *rb, int key_length) +{ + if (!kp->public_key){ + kp->public_key_len = key_length; + if ((kp->public_key = emalloc(kp->public_key_len))==NULL) + return -1; + } + rotbuf_getbuf(rb, kp->public_key, kp->public_key_len); + return 0; +} + +static int unpack_cryptobox(keypair *kp, struct rotbuf *rb, int key_length) { rotbuf_getbuf(rb, kp->private_key, kp->private_key_len); if (!rb->wrap) @@ -646,9 +673,9 @@ static int pack_did_name(const keypair *kp, struct rotbuf *rb) return pack_private_public(kp, rb); } -static int unpack_did_name(keypair *kp, struct rotbuf *rb) +static int unpack_did_name(keypair *kp, struct rotbuf *rb, int key_length) { - if (unpack_private_public(kp, rb) == -1) + if (unpack_private_public(kp, rb, key_length) == -1) return -1; // Fail if name is not nul terminated. return strnchr((const char *)kp->public_key, kp->public_key_len, '\0') == NULL ? -1 : 0; @@ -761,6 +788,16 @@ const struct keytype keytypes[] = { .unpacker = unpack_did_name, .dumper = dump_did_name, .loader = load_did_name + }, + [KEYTYPE_PUBLIC_TAG] = { + .private_key_size = 0, + .public_key_size = 0, // size is derived from the stored key length + .packed_size = 0, + .creator = NULL, // not included in a newly created identity + .packer = pack_public_only, + .unpacker = unpack_public_only, + .dumper = dump_private_public, + .loader = load_unknown } // ADD MORE KEY TYPES HERE }; @@ -835,13 +872,16 @@ static int keyring_pack_identity(const keyring_identity *id, unsigned char packe unsigned ktype = id->keypairs[kp]->type; const char *kts = keytype_str(ktype, "unknown"); int (*packer)(const keypair *, struct rotbuf *) = NULL; - size_t keypair_len; + size_t keypair_len=0; const struct keytype *kt = &keytypes[ktype]; if (ktype == 0x00) FATALF("ktype=0 in keypair kp=%u", kp); if (ktype < NELS(keytypes)) { packer = kt->packer; keypair_len = kt->packed_size; + if (keypair_len==0){ + keypair_len = id->keypairs[kp]->private_key_len + id->keypairs[kp]->public_key_len; + } } else { packer = pack_private_only; keypair_len = id->keypairs[kp]->private_key_len; @@ -914,16 +954,24 @@ static int cmp_keypair(const keypair *a, const keypair *b) { int c = a->type < b->type ? -1 : a->type > b->type ? 1 : 0; if (c == 0 && a->public_key_len) { - assert(a->public_key_len == b->public_key_len); assert(a->public_key != NULL); assert(b->public_key != NULL); - c = memcmp(a->public_key, b->public_key, a->public_key_len); + int len=a->public_key_len; + if (len>b->public_key_len) + len=b->public_key_len; + c = memcmp(a->public_key, b->public_key, len); + if (c==0 && a->public_key_len!=b->public_key_len) + c = a->public_key_len - b->public_key_len; } if (c == 0 && a->private_key_len) { - assert(a->private_key_len == b->private_key_len); assert(a->private_key != NULL); assert(b->private_key != NULL); - c = memcmp(a->private_key, b->private_key, a->private_key_len); + int len=a->private_key_len; + if (len>b->private_key_len) + len=b->private_key_len; + c = memcmp(a->private_key, b->private_key, len); + if (c==0 && a->private_key_len!=b->private_key_len) + c = a->private_key_len - b->private_key_len; } return c; } @@ -1007,7 +1055,7 @@ static keyring_identity *keyring_unpack_identity(unsigned char *slot, const char if (ktype < NELS(keytypes) && kt->unpacker) { if (config.debug.keyring) DEBUGF("unpack key type = 0x%02x(%s) at offset %u", ktype, keytype_str(ktype, "unknown"), (int)rotbuf_position(&rbo)); - if (kt->unpacker(kp, &rbuf) != 0) { + if (kt->unpacker(kp, &rbuf, keypair_len) != 0) { // If there is an error, it is probably an empty slot. if (config.debug.keyring) DEBUGF("key type 0x%02x does not unpack", ktype); @@ -1413,10 +1461,14 @@ int keyring_set_did(keyring_identity *id, const char *did, const char *name) } /* Store DID unpacked for ease of searching */ - int len=strlen(did); if (len>31) len=31; + int len=strlen(did); + if (len>31) + len=31; bcopy(did,&id->keypairs[i]->private_key[0],len); bzero(&id->keypairs[i]->private_key[len],32-len); - len=strlen(name); if (len>63) len=63; + len=strlen(name); + if (len>63) + len=63; bcopy(name,&id->keypairs[i]->public_key[0],len); bzero(&id->keypairs[i]->public_key[len],64-len); @@ -1427,23 +1479,121 @@ int keyring_set_did(keyring_identity *id, const char *did, const char *name) return 0; } -int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp,char *did) +int keyring_find_did(const keyring_file *k, int *cn, int *in, int *kp, const char *did) { - for (; keyring_sanitise_position(k,cn,in,kp) == 0; ++*kp) { - if (k->contexts[*cn]->identities[*in]->keypairs[*kp]->type==KEYTYPE_DID) { - /* Compare DIDs */ - if ((!did[0]) - ||(did[0]=='*'&&did[1]==0) - ||(!strcasecmp(did,(char *)k->contexts[*cn]->identities[*in] - ->keypairs[*kp]->private_key)) - ) { - return 1; // match - } + for(;keyring_next_keytype(k,cn,in,kp,KEYTYPE_DID);++(*kp)) { + /* Compare DIDs */ + if ((!did[0]) + ||(did[0]=='*'&&did[1]==0) + ||(!strcasecmp(did,(char *)k->contexts[*cn]->identities[*in] + ->keypairs[*kp]->private_key)) + ) { + return 1; // match } } return 0; } +int keyring_unpack_tag(const unsigned char *packed, size_t packed_len, const char **name, const unsigned char **value, size_t *length) +{ + size_t i; + for (i=0;ikeypair_count;i++){ + const char *tag_name; + const unsigned char *tag_value; + size_t tag_length; + if (id->keypairs[i]->type==KEYTYPE_PUBLIC_TAG && + keyring_unpack_tag(id->keypairs[i]->public_key, id->keypairs[i]->public_key_len, + &tag_name, &tag_value, &tag_length)==0 && + strcmp(tag_name, name)==0) { + if (config.debug.keyring) + DEBUG("Found existing public tag"); + break; + } + } + + if (i >= PKR_MAX_KEYPAIRS) + return WHY("Too many key pairs"); + + /* allocate if needed */ + if (i >= id->keypair_count) { + if (config.debug.keyring) + DEBUGF("Creating new public tag @%d", i); + if ((id->keypairs[i] = keyring_alloc_keypair(KEYTYPE_PUBLIC_TAG, 0)) == NULL) + return -1; + ++id->keypair_count; + } + + if (id->keypairs[i]->public_key) + free(id->keypairs[i]->public_key); + + if (keyring_pack_tag(NULL, &id->keypairs[i]->public_key_len, name, value, length)) + return -1; + id->keypairs[i]->public_key = emalloc(id->keypairs[i]->public_key_len); + if (!id->keypairs[i]->public_key) + return -1; + if (keyring_pack_tag(id->keypairs[i]->public_key, &id->keypairs[i]->public_key_len, name, value, length)) + return -1; + + if (config.debug.keyring) + dump("New tag", id->keypairs[i]->public_key, id->keypairs[i]->public_key_len); + return 0; +} + +int keyring_find_public_tag(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char **value, size_t *length) +{ + for(;keyring_next_keytype(k,cn,in,kp,KEYTYPE_PUBLIC_TAG);++(*kp)) { + keypair *keypair=k->contexts[*cn]->identities[*in]->keypairs[*kp]; + const char *tag_name; + if (!keyring_unpack_tag(keypair->public_key, keypair->public_key_len, &tag_name, value, length) && + strcmp(name, tag_name)==0){ + return 1; + } + } + if (value) + *value=NULL; + return 0; +} + +int keyring_find_public_tag_value(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char *value, size_t length) +{ + const unsigned char *stored_value; + size_t stored_length; + for(;keyring_find_public_tag(k, cn, in, kp, name, &stored_value, &stored_length);++(*kp)) { + if (stored_length == length && memcmp(value, stored_value, length)==0) + return 1; + } + return 0; +} + int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype) { int kp; @@ -1468,24 +1618,27 @@ int keyring_next_identity(const keyring_file *k, int *cn, int *in, int *kp) int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp) { - if (!k) return 1; + if (!k) + return 1; /* Sanity check passed in position */ - if ((*cn)>=k->context_count) return 1; - if ((*in)>=k->contexts[*cn]->identity_count) - { - (*in)=0; (*cn)++; - if ((*cn)>=k->context_count) return 1; + while(1){ + if ((*cn)>=k->context_count) + return 1; + + if ((*in)>=k->contexts[*cn]->identity_count){ + (*in)=(*kp)=0; + (*cn)++; + continue; } - if ((*kp)>=k->contexts[*cn]->identities[*in]->keypair_count) - { - *kp=0; (*in)++; - if ((*in)>=k->contexts[*cn]->identity_count) - { - (*in)=0; (*cn)++; - if ((*cn)>=k->context_count) return 1; - } + + if ((*kp)>=k->contexts[*cn]->identities[*in]->keypair_count){ + *kp=0; + (*in)++; + continue; } - return 0; + + return 0; + } } unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public_out) @@ -1493,32 +1646,28 @@ unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsi IN(); int cn=0,in=0,kp=0; - if (!keyring_find_sid(k,&cn,&in,&kp,sidp)) { + if (!keyring_find_sid(k,&cn,&in,&kp,sidp)) RETURNNULL(WHYNULL("Could not find SID in keyring, so can't find SAS")); + + kp = keyring_identity_find_keytype(k, cn, in, KEYTYPE_CRYPTOSIGN); + if (kp==-1) + RETURNNULL(WHYNULL("Identity lacks SAS")); + + unsigned char *sas_private= + k->contexts[cn]->identities[in]->keypairs[kp]->private_key; + unsigned char *sas_public= + k->contexts[cn]->identities[in]->keypairs[kp]->public_key; + if (!rhizome_verify_bundle_privatekey(sas_private,sas_public)){ + /* SAS key is invalid (perhaps because it was a pre 0.90 format one), + so replace it */ + WARN("SAS key is invalid -- regenerating."); + crypto_sign_edwards25519sha512batch_keypair(sas_public, sas_private); + keyring_commit(k); } - - for(kp=0;kpcontexts[cn]->identities[in]->keypair_count;kp++) - if (k->contexts[cn]->identities[in]->keypairs[kp]->type==KEYTYPE_CRYPTOSIGN) - { - unsigned char *sas_private= - k->contexts[cn]->identities[in]->keypairs[kp]->private_key; - unsigned char *sas_public= - k->contexts[cn]->identities[in]->keypairs[kp]->public_key; - if (rhizome_verify_bundle_privatekey(NULL,sas_private,sas_public)) - { - /* SAS key is invalid (perhaps because it was a pre 0.90 format one), - so replace it */ - WARN("SAS key is invalid -- regenerating."); - crypto_sign_edwards25519sha512batch_keypair(sas_public, sas_private); - keyring_commit(k); - } - if (config.debug.keyring) - DEBUGF("Found SAS entry for %s*", alloca_tohex(sidp->binary, 7)); - if (sas_public_out) *sas_public_out=sas_public; - RETURN(sas_private); - } - - RETURNNULL(WHYNULL("Identity lacks SAS")); + if (config.debug.keyring) + DEBUGF("Found SAS entry for %s*", alloca_tohex(sidp->binary, 7)); + if (sas_public_out) *sas_public_out=sas_public; + RETURN(sas_private); OUT(); } @@ -1616,7 +1765,7 @@ static int keyring_respond_sas(keyring_file *k, overlay_mdp_frame *req) alloca_tohex_sid_t(req->out.src.sid), req->out.src.port, alloca_tohex_sid_t(req->out.dst.sid), req->out.dst.port ); - return overlay_mdp_dispatch(req,0,NULL,0); + return overlay_mdp_dispatch(req, NULL); } // someone else is claiming to be me on this network @@ -1642,7 +1791,7 @@ int keyring_send_unlock(struct subscriber *subscriber) if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len)) return -1; mdp.out.payload_length=len; - return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0); + return overlay_mdp_dispatch(&mdp, NULL); } static int keyring_send_challenge(struct subscriber *source, struct subscriber *dest) @@ -1667,7 +1816,7 @@ static int keyring_send_challenge(struct subscriber *source, struct subscriber * bcopy(source->identity->challenge, &mdp.out.payload[1], sizeof(source->identity->challenge)); mdp.out.payload_length+=sizeof(source->identity->challenge); - return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0); + return overlay_mdp_dispatch(&mdp, NULL); } static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_frame *req) @@ -1691,7 +1840,7 @@ static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_ if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len)) return -1; mdp.out.payload_length=len; - return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0); + return overlay_mdp_dispatch(&mdp, NULL); } static int keyring_process_challenge(keyring_file *k, struct subscriber *subscriber, overlay_mdp_frame *req) @@ -1777,7 +1926,7 @@ int keyring_send_sas_request(struct subscriber *subscriber){ mdp.out.payload_length=1; mdp.out.payload[0]=KEYTYPE_CRYPTOSIGN; - if (overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0)) + if (overlay_mdp_dispatch(&mdp, NULL)) return WHY("Failed to send SAS resolution request"); if (config.debug.keyring) DEBUGF("Dispatched SAS resolution request"); @@ -1788,10 +1937,10 @@ int keyring_send_sas_request(struct subscriber *subscriber){ int keyring_find_sid(const keyring_file *k, int *cn, int *in, int *kp, const sid_t *sidp) { - for (; keyring_sanitise_position(k, cn, in, kp) == 0; ++*kp) - if (k->contexts[*cn]->identities[*in]->keypairs[*kp]->type == KEYTYPE_CRYPTOBOX - && memcmp(sidp->binary, k->contexts[*cn]->identities[*in]->keypairs[*kp]->public_key, SID_SIZE) == 0) + for(; keyring_next_keytype(k,cn,in,kp,KEYTYPE_CRYPTOBOX); ++(*kp)) { + if (memcmp(sidp->binary, k->contexts[*cn]->identities[*in]->keypairs[*kp]->public_key, SID_SIZE) == 0) return 1; + } return 0; } diff --git a/keyring.h b/keyring.h new file mode 100644 index 00000000..85492447 --- /dev/null +++ b/keyring.h @@ -0,0 +1,109 @@ +#ifndef __SERVALD_KEYRING_H +#define __SERVALD_KEYRING_H + +typedef struct keypair { + int type; + unsigned char *private_key; + size_t private_key_len; + unsigned char *public_key; + size_t public_key_len; +} keypair; + +/* Contains just the list of private:public key pairs and types, + the pin used to extract them, and the slot in the keyring file + (so that it can be replaced/rewritten as required). */ +#define PKR_MAX_KEYPAIRS 64 +#define PKR_SALT_BYTES 32 +#define PKR_MAC_BYTES 64 +typedef struct keyring_identity { + char *PKRPin; + struct subscriber *subscriber; + time_ms_t challenge_expires; + unsigned char challenge[24]; + unsigned int slot; + unsigned int keypair_count; + keypair *keypairs[PKR_MAX_KEYPAIRS]; +} keyring_identity; + +/* 64K identities, can easily be increased should the need arise, + but keep it low-ish for now so that the 64K pointers don't eat too + much ram on a small device. Should probably think about having + small and large device settings for some of these things */ +#define KEYRING_MAX_IDENTITIES 65536 +typedef struct keyring_context { + char *KeyRingPin; + unsigned char *KeyRingSalt; + int KeyRingSaltLen; + unsigned int identity_count; + keyring_identity *identities[KEYRING_MAX_IDENTITIES]; +} keyring_context; + +#define KEYRING_PAGE_SIZE 4096LL +#define KEYRING_BAM_BYTES 2048LL +#define KEYRING_BAM_BITS (KEYRING_BAM_BYTES<<3) +#define KEYRING_SLAB_SIZE (KEYRING_PAGE_SIZE*KEYRING_BAM_BITS) +typedef struct keyring_bam { + off_t file_offset; + unsigned char bitmap[KEYRING_BAM_BYTES]; + struct keyring_bam *next; +} keyring_bam; + +#define KEYRING_MAX_CONTEXTS 256 +typedef struct keyring_file { + int context_count; + keyring_bam *bam; + keyring_context *contexts[KEYRING_MAX_CONTEXTS]; + FILE *file; + off_t file_size; +} keyring_file; + +void keyring_free(keyring_file *k); +void keyring_release_identity(keyring_file *k, int cn, int id); +#define KEYTYPE_CRYPTOBOX 0x01 // must be lowest +#define KEYTYPE_CRYPTOSIGN 0x02 +#define KEYTYPE_RHIZOME 0x03 +/* DIDs aren't really keys, but the keyring is a real handy place to keep them, + and keep them private if people so desire */ +#define KEYTYPE_DID 0x04 + +/* Arbitrary name / value pairs */ +#define KEYTYPE_PUBLIC_TAG 0x05 + +/* handle to keyring file for use in running instance */ +extern keyring_file *keyring; + +/* Public calls to keyring management */ +keyring_file *keyring_open(const char *path, int writeable); +keyring_file *keyring_open_instance(); +keyring_file *keyring_open_instance_cli(const struct cli_parsed *parsed); +int keyring_enter_pin(keyring_file *k, const char *pin); +int keyring_set_did(keyring_identity *id, const char *did, const char *name); +int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp); +int keyring_next_keytype(const keyring_file *k, int *cn, int *in, int *kp, int keytype); +int keyring_next_identity(const keyring_file *k,int *cn,int *in,int *kp); +int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype); +int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp, const char *did); +int keyring_find_sid(const keyring_file *k,int *cn,int *in,int *kp, const sid_t *sidp); +unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public); +int keyring_send_sas_request(struct subscriber *subscriber); + +int keyring_commit(keyring_file *k); +keyring_identity *keyring_create_identity(keyring_file *k,keyring_context *c, const char *pin); +int keyring_seed(keyring_file *k); +void keyring_identity_extract(const keyring_identity *id, const sid_t **sidp, const char **didp, const char **namep); +int keyring_load(keyring_file *k, const char *keyring_pin, unsigned entry_pinc, const char **entry_pinv, FILE *input); +int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret); + +unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp); + +int keyring_mapping_request(keyring_file *k, struct overlay_frame *frame, overlay_mdp_frame *req); +int keyring_send_unlock(struct subscriber *subscriber); +void keyring_release_subscriber(keyring_file *k, const sid_t *sid); + +int keyring_set_public_tag(keyring_identity *id, const char *name, const unsigned char *value, size_t length); +int keyring_find_public_tag(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char **value, size_t *length); +int keyring_find_public_tag_value(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char *value, size_t length); +int keyring_unpack_tag(const unsigned char *packed, size_t packed_len, const char **name, const unsigned char **value, size_t *length); +int keyring_pack_tag(unsigned char *packed, size_t *packed_len, const char *name, const unsigned char *value, size_t length); + +#endif // __SERVALD_KEYRING_H diff --git a/lsif.c b/lsif.c index 52e80c91..10065455 100644 --- a/lsif.c +++ b/lsif.c @@ -46,7 +46,6 @@ #ifndef SIOCGIFCONF #include #endif -#include #include #if __MACH__ #include diff --git a/mavlink.c b/mavlink.c deleted file mode 100644 index 3150789e..00000000 --- a/mavlink.c +++ /dev/null @@ -1,407 +0,0 @@ -// -*- Mode: C; c-basic-offset: 2; -*- -// -// Copyright (c) 2012 Andrew Tridgell, All Rights Reserved -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// -// o Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// o Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in -// the documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -// OF THE POSSIBILITY OF SUCH DAMAGE. -// - -/* -Portions Copyright (C) 2013 Paul Gardner-Stephen - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation; either version 2 -of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -#include "serval.h" -#include "conf.h" -#include "overlay_buffer.h" -#include "golay.h" - -#define MAVLINK_MSG_ID_RADIO 166 -#define MAVLINK_MSG_ID_DATASTREAM 67 -int MAVLINK_MESSAGE_CRCS[]={72, 39, 190, 92, 191, 217, 104, 119, 0, 219, 60, 186, 10, 0, 0, 0, 0, 0, 0, 0, 89, 159, 162, 121, 0, 149, 222, 110, 179, 136, 66, 126, 185, 147, 112, 252, 162, 215, 229, 128, 9, 106, 101, 213, 4, 229, 21, 214, 215, 14, 206, 50, 157, 126, 108, 213, 95, 5, 127, 0, 0, 0, 57, 126, 130, 119, 193, 191, 236, 158, 143, 0, 0, 104, 123, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 29, 208, 188, 118, 242, 19, 97, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 224, 60, 106, 7}; - -// use '3D' for 3DRadio -#define RADIO_SOURCE_SYSTEM '3' -#define RADIO_SOURCE_COMPONENT 'D' - -uint16_t mavlink_crc(unsigned char *buf,int length) -{ - uint16_t sum = 0xFFFF; - uint8_t i, stoplen; - - stoplen = length + 6; - - // MAVLink 1.0 has an extra CRC seed - buf[length+6] = MAVLINK_MESSAGE_CRCS[buf[5]]; - stoplen++; - - i = 1; - while (i>8) ^ (tmp<<8) ^ (tmp<<3) ^ (tmp>>4); - i++; - } - buf[length+6]=sum&0xff; - buf[length+7]=sum>>8; - - return sum; -} - - -/* - we use a hand-crafted MAVLink packet based on the following - message definition - - - Status generated by radio - local signal strength - remote signal strength - percentage free space in transmit buffer - background noise level - remote background noise level - receive errors - count of error corrected packets - -*/ -struct mavlink_RADIO_v09 { - uint8_t rssi; - uint8_t remrssi; - uint8_t txbuf; - uint8_t noise; - uint8_t remnoise; - uint16_t rxerrors; - uint16_t fixed; -}; -struct mavlink_RADIO_v10 { - uint16_t rxerrors; - uint16_t fixed; - uint8_t rssi; - uint8_t remrssi; - uint8_t txbuf; - uint8_t noise; - uint8_t remnoise; -}; - -/* - Each mavlink frame consists of 0xfe followed by a standard 6 byte header. - Normally the payload plus a 2-byte CRC follows. - We are replacing the CRC check with a Reed-Solomon code to correct as well - as detect upto 16 bytes with errors, in return for a 32-byte overhead. - - The nature of the particular library we are using is that the overhead is - basically fixed, but we can shorten the data section. - - Note that the mavlink headers are not protected against errors. This is a - limitation of the radio firmware at present. One day we will re-write the - radio firmware so that we can send and receive raw radio frames, and get - rid of the mavlink framing altogether, and just send R-S protected payloads. - - Not ideal, but will be fine for now. -*/ - -#include "fec-3.0.1/fixed.h" -void encode_rs_8(data_t *data, data_t *parity,int pad); -int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad); - -int mavlink_encode_packet(struct overlay_interface *interface) -{ - int count = ob_remaining(interface->tx_packet); - int startP = !ob_position(interface->tx_packet); - int endP = 1; - if (count+6+32 > 255){ - count = 255-6-32; - endP = 0; - } - interface->txbuffer[0]=0xfe; // mavlink v1.0 frame - /* payload len, excluding 6 byte header and 2 byte CRC. - But we use a 4-byte CRC, so need to add two to count to make packet lengths - be as expected. - Note that this construction will result in CRC errors by non-servald - programmes, which is probably more helpful than otherwise. - */ - // we need 32 bytes for the parity, but this field assumes - // that there is a 2 byte CRC, so we can save two bytes - int len = count+32 - 2; - interface->txbuffer[1]=len; - interface->txbuffer[2]=(len & 0xF); - interface->txbuffer[3]=0; - golay_encode(&interface->txbuffer[1]); - - interface->txbuffer[4]=(interface->mavlink_seq++) & 0x3f; - if (startP) interface->txbuffer[4]|=0x40; - if (endP) interface->txbuffer[4]|=0x80; - interface->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM; - - ob_get_bytes(interface->tx_packet, &interface->txbuffer[6], count); - - encode_rs_8(&interface->txbuffer[4], &interface->txbuffer[6+count], 223 - (count+2)); - interface->tx_bytes_pending=len + 8; - if (endP){ - ob_free(interface->tx_packet); - interface->tx_packet=NULL; - overlay_queue_schedule_next(gettime_ms()); - } - return 0; -} - -int mavlink_heartbeat(unsigned char *frame,int *outlen) -{ - int count=9; - bzero(frame, count+8); - - frame[0]=0xfe; // mavlink v1.0 frame - // Must be 9 to indicate heartbeat - frame[1]=count; // payload len, excluding 6 byte header and 2 byte CRC - frame[2]=(count & 0xF); // packet sequence - frame[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC) - golay_encode(&frame[1]); - frame[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE) - // Must be zero to indicate heartbeat - frame[5]=0; // message ID type of this frame: DATA_STREAM - - // extra magic number to detect remote heartbeat requests - frame[14]=0x55; - frame[15]=0x05; - golay_encode(&frame[14]); - - *outlen=count+8; - - return 0; -} - -static int parse_heartbeat(struct overlay_interface *interface, const unsigned char *payload) -{ - if (payload[0]==0xFE - && payload[1]==9 - && payload[3]==RADIO_SOURCE_SYSTEM - && payload[4]==RADIO_SOURCE_COMPONENT - && payload[5]==MAVLINK_MSG_ID_RADIO){ - - // we can assume that radio status packets arrive without corruption - interface->radio_rssi=(1.0*payload[10]-payload[13])/1.9; - interface->remote_rssi=(1.0*payload[11] - payload[14])/1.9; - int free_space = payload[12]; - int free_bytes = (free_space * 1280) / 100 - 30; - interface->remaining_space = free_bytes; - if (free_bytes>0) - interface->next_tx_allowed = gettime_ms(); - if (free_bytes>720) - interface->next_heartbeat=gettime_ms()+1000; - if (config.debug.packetradio) { - INFOF("Link budget = %+ddB, remote link budget = %+ddB, buffer space = %d%% (approx %d)", - interface->radio_rssi, - interface->remote_rssi, - free_space, free_bytes); - } - return 1; - } - return 0; -} - -static int mavlink_parse(struct overlay_interface *interface, struct slip_decode_state *state, - int packet_length, unsigned char *payload, int *backtrack) -{ - *backtrack=0; - if (packet_length==9){ - // make sure we've heard the start and end of a remote heartbeat request - int errs=0; - int tail = golay_decode(&errs, &payload[14]); - if (tail == 0x555){ - return 1; - } - return 0; - } - - int data_bytes = packet_length - (32 - 2); - // preserve the last 16 bytes of data - unsigned char old_footer[32]; - unsigned char *payload_footer=&payload[packet_length+8-sizeof(old_footer)]; - bcopy(payload_footer, old_footer, sizeof(old_footer)); - - int pad=223 - (data_bytes + 2); - int errors=decode_rs_8(&payload[4], NULL, 0, pad); - if (errors==-1){ - if (config.debug.mavlink) - DEBUGF("Reed-Solomon error correction failed"); - return 0; - } - *backtrack=errors; - - int seq=payload[4]&0x3f; - - if (config.debug.mavlink){ - DEBUGF("Received RS protected message, len: %d, errors: %d, seq: %d, flags:%s%s", - data_bytes, - errors, - seq, - payload[4]&0x40?" start":"", - payload[4]&0x80?" end":""); - } - - if (seq != ((state->mavlink_seq+1)&0x3f)){ - // reject partial packet if we missed a sequence number - if (config.debug.mavlink) - DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->mavlink_seq, seq); - state->packet_length=sizeof(state->dst)+1; - } - - if (payload[4]&0x40){ - // start a new packet - state->packet_length=0; - } - - state->mavlink_seq=payload[4]&0x3f; - if (state->packet_length + data_bytes > sizeof(state->dst)){ - if (config.debug.mavlink) - DEBUG("Fragmented packet is too long or a previous piece was missed - discarding"); - state->packet_length=sizeof(state->dst)+1; - return 1; - } - - bcopy(&payload[6], &state->dst[state->packet_length], data_bytes); - state->packet_length+=data_bytes; - - if (payload[4]&0x80) { - if (config.debug.mavlink) - DEBUGF("PDU Complete (length=%d)",state->packet_length); - state->dst_offset=0; - - packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL, 0); - state->packet_length=sizeof(state->dst)+1; - } - return 1; -} - -static int decode_length(struct slip_decode_state *state, unsigned char *p) -{ - // look for a valid golay encoded length - int errs=0; - int length = golay_decode(&errs, p); - if (length<0 || ((length >>8) & 0xF) != (length&0xF)) - return -1; - length=length&0xFF; - if (length!=9 && (length<31 || length+8>255)) - return -1; - - if (config.debug.mavlink && (errs || state->mavlink_payload_length!=*p)) - DEBUGF("Decoded length %d to %d with %d errs", *p, length, errs); - - state->mavlink_payload_length=length; - return 0; -} - -int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state, uint8_t c) -{ - if (state->mavlink_payload_start + state->mavlink_payload_offset >= sizeof(state->mavlink_payload)){ - // drop one byte if we run out of space - if (config.debug.mavlink) - DEBUGF("Dropped %02x, buffer full", state->mavlink_payload[0]); - bcopy(state->mavlink_payload+1, state->mavlink_payload, sizeof(state->mavlink_payload) -1); - state->mavlink_payload_start--; - } - - unsigned char *p = &state->mavlink_payload[state->mavlink_payload_start]; - p[state->mavlink_payload_offset++]=c; - - while(1){ - // look for packet length headers - p = &state->mavlink_payload[state->mavlink_payload_start]; - while(state->mavlink_payload_length==0 && state->mavlink_payload_offset>=6){ - if (p[0]==0xFE - && p[1]==9 - && p[3]==RADIO_SOURCE_SYSTEM - && p[4]==RADIO_SOURCE_COMPONENT - && p[5]==MAVLINK_MSG_ID_RADIO){ - //looks like a valid heartbeat response header, read the rest and process it - state->mavlink_payload_length=9; - break; - } - - if (decode_length(state, &p[1])==0) - break; - - state->mavlink_payload_start++; - state->mavlink_payload_offset--; - p++; - } - - // wait for a whole packet - if (!state->mavlink_payload_length || state->mavlink_payload_offset < state->mavlink_payload_length+8) - return 0; - - if (parse_heartbeat(interface, p)){ - // cut the bytes of the heartbeat out of the buffer - state->mavlink_payload_offset -= state->mavlink_payload_length+8; - if (state->mavlink_payload_offset){ - // shuffle bytes backwards - bcopy(&p[state->mavlink_payload_length+8], p, state->mavlink_payload_offset); - } - // restart parsing for a valid header from the beginning of out buffer - state->mavlink_payload_offset+=state->mavlink_payload_start; - state->mavlink_payload_start=0; - state->mavlink_payload_length=0; - continue; - } - - // is this a well formed packet? - int backtrack=0; - if (mavlink_parse(interface, state, state->mavlink_payload_length, p, &backtrack)==1){ - // Since we know we've synced with the remote party, - // and there's nothing we can do about any earlier data - // throw away everything before the end of this packet - if (state->mavlink_payload_start && config.debug.mavlink) - dump("Skipped", state->mavlink_payload, state->mavlink_payload_start); - - // If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet, - // but we may need to examine the last few bytes to find the start of the next packet. - state->mavlink_payload_offset -= state->mavlink_payload_length+8-backtrack; - if (state->mavlink_payload_offset){ - // shuffle all remaining bytes back to the start of the buffer - bcopy(&state->mavlink_payload[state->mavlink_payload_start + state->mavlink_payload_length+8-backtrack], - state->mavlink_payload, state->mavlink_payload_offset); - } - state->mavlink_payload_start=0; - }else{ - // ignore the first byte for now and start looking for another packet header - // we may find a heartbeat in the middle that we need to cut out first - state->mavlink_payload_start++; - state->mavlink_payload_offset--; - } - state->mavlink_payload_length=0; - }; -} diff --git a/mdp_client.c b/mdp_client.c index 02930bef..0c4b6975 100644 --- a/mdp_client.c +++ b/mdp_client.c @@ -27,6 +27,7 @@ #include "overlay_address.h" #include "overlay_packet.h" #include "mdp_client.h" +#include "socket.h" int mdp_socket(void) { @@ -34,49 +35,70 @@ int mdp_socket(void) return overlay_mdp_client_socket(); } -int mdp_close(int socket) +static void mdp_unlink(int mdp_sock) { - // use the same process for closing sockets, though this will need to change once bind is implemented - return overlay_mdp_client_close(socket); + // get the socket name and unlink it from the filesystem if not abstract + struct socket_address addr; + addr.addrlen = sizeof addr.store; + if (getsockname(mdp_sock, &addr.addr, &addr.addrlen)) + WHYF_perror("getsockname(%d)", mdp_sock); + else if (addr.addr.sa_family==AF_UNIX + && addr.addrlen > sizeof addr.local.sun_family + && addr.addrlen <= sizeof addr.local && addr.local.sun_path[0] != '\0') { + if (unlink(addr.local.sun_path) == -1) + WARNF_perror("unlink(%s)", alloca_str_toprint(addr.local.sun_path)); + } + close(mdp_sock); } -int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len) +int mdp_close(int socket) { - struct sockaddr_un addr; - socklen_t addrlen; - if (make_local_sockaddr(&addr, &addrlen, "mdp.2.socket") == -1) + // tell the daemon to drop all bindings + struct mdp_header header={ + .flags = MDP_FLAG_CLOSE, + .local.port = 0, + }; + + mdp_send(socket, &header, NULL, 0); + + // remove socket + mdp_unlink(socket); + return 0; +} + +int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len) +{ + struct socket_address addr; + if (make_local_sockaddr(&addr, "mdp.2.socket") == -1) return -1; - struct iovec iov[]={ - { - .iov_base = (void *)header, - .iov_len = sizeof(struct mdp_header) - }, - { - .iov_base = (void *)payload, - .iov_len = len + struct fragmented_data data={ + .fragment_count=2, + .iov={ + { + .iov_base = (void*)header, + .iov_len = sizeof(struct mdp_header) + }, + { + .iov_base = (void*)payload, + .iov_len = len + } } }; - struct msghdr hdr={ - .msg_name=&addr, - .msg_namelen=addrlen, - .msg_iov=iov, - .msg_iovlen=2, - }; - return sendmsg(socket, &hdr, 0); + return send_message(socket, &addr, &data); } -ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len) +ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len) { /* Construct name of socket to receive from. */ - struct sockaddr_un mdp_addr; - socklen_t mdp_addrlen; - if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.2.socket") == -1) - return -1; + errno=0; + struct socket_address mdp_addr; + if (make_local_sockaddr(&mdp_addr, "mdp.2.socket") == -1) + return WHY("Failed to build socket address"); - struct sockaddr_un addr; + struct socket_address addr; struct iovec iov[]={ { .iov_base = (void *)header, @@ -89,25 +111,26 @@ ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, }; struct msghdr hdr={ - .msg_name=&addr, - .msg_namelen=sizeof(struct sockaddr_un), + .msg_name=&addr.addr, + .msg_namelen=sizeof(addr.store), .msg_iov=iov, .msg_iovlen=2, }; ssize_t len = recvmsg(socket, &hdr, 0); if (lenpacketTypeAndFlags=MDP_ERROR; mdp->error.error=1; snprintf(mdp->error.message,128,"Error sending frame to MDP server."); - return WHY_perror("sendto(f)"); + return -1; } else { if (!(flags&MDP_AWAITREPLY)) { return 0; @@ -177,16 +203,15 @@ int overlay_mdp_client_socket(void) { /* Create local per-client socket to MDP server (connection is always local) */ int mdp_sockfd; - struct sockaddr_un addr; - socklen_t addrlen; + struct socket_address addr; uint32_t random_value; if (urandombytes((unsigned char *)&random_value, sizeof random_value) == -1) return WHY("urandombytes() failed"); - if (make_local_sockaddr(&addr, &addrlen, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1) + if (make_local_sockaddr(&addr, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1) return -1; if ((mdp_sockfd = esocket(AF_UNIX, SOCK_DGRAM, 0)) == -1) return -1; - if (socket_bind(mdp_sockfd, (struct sockaddr *)&addr, addrlen) == -1) { + if (socket_bind(mdp_sockfd, &addr.addr, addr.addrlen) == -1) { close(mdp_sockfd); return -1; } @@ -200,16 +225,8 @@ int overlay_mdp_client_close(int mdp_sockfd) overlay_mdp_frame mdp; mdp.packetTypeAndFlags = MDP_GOODBYE; overlay_mdp_send(mdp_sockfd, &mdp, 0, 0); - // get the socket name and unlink it from the filesystem if not abstract - struct sockaddr_un addr; - socklen_t addrlen = sizeof addr; - if (getsockname(mdp_sockfd, (struct sockaddr *)&addr, &addrlen)) - WHYF_perror("getsockname(%d)", mdp_sockfd); - else if (addrlen > sizeof addr.sun_family && addrlen <= sizeof addr && addr.sun_path[0] != '\0') { - if (unlink(addr.sun_path) == -1) - WARNF_perror("unlink(%s)", alloca_str_toprint(addr.sun_path)); - } - close(mdp_sockfd); + + mdp_unlink(mdp_sockfd); return 0; } @@ -232,36 +249,40 @@ int overlay_mdp_client_poll(int mdp_sockfd, time_ms_t timeout_ms) int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, int *ttl) { /* Construct name of socket to receive from. */ - struct sockaddr_un mdp_addr; - socklen_t mdp_addrlen; - if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.socket") == -1) + struct socket_address mdp_addr; + if (make_local_sockaddr(&mdp_addr, "mdp.socket") == -1) return -1; /* Check if reply available */ - struct sockaddr_un recvaddr; - socklen_t recvaddrlen = sizeof recvaddr; + struct socket_address recvaddr; + recvaddr.addrlen = sizeof recvaddr.store; ssize_t len; mdp->packetTypeAndFlags = 0; set_nonblock(mdp_sockfd); - len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, (struct sockaddr *)&recvaddr, &recvaddrlen); + len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, &recvaddr); + if (len == -1) + WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))", + mdp_sockfd, mdp, sizeof(overlay_mdp_frame), *ttl, + &recvaddr, alloca_socket_address(&recvaddr) + ); set_block(mdp_sockfd); if (len <= 0) return -1; // no packet received // If the received address overflowed the buffer, then it cannot have come from the server, whose // address must always fit within a struct sockaddr_un. - if (recvaddrlen > sizeof recvaddr) + if (recvaddr.addrlen > sizeof recvaddr.store) return WHY("reply did not come from server: address overrun"); // Compare the address of the sender with the address of our server, to ensure they are the same. // If the comparison fails, then try using realpath(3) on the sender address and compare again. - if ( cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0 - && ( recvaddr.sun_family != AF_UNIX - || real_sockaddr(&recvaddr, recvaddrlen, &recvaddr, &recvaddrlen) <= 0 - || cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0 + if ( cmp_sockaddr(&recvaddr, &mdp_addr) != 0 + && ( recvaddr.local.sun_family != AF_UNIX + || real_sockaddr(&recvaddr, &recvaddr) <= 0 + || cmp_sockaddr(&recvaddr, &mdp_addr) != 0 ) ) - return WHYF("reply did not come from server: %s", alloca_sockaddr(&recvaddr, recvaddrlen)); + return WHYF("reply did not come from server: %s", alloca_socket_address(&recvaddr)); // silently drop incoming packets for the wrong port number if (port>0 && port != mdp->in.dst.port){ diff --git a/mdp_client.h b/mdp_client.h index af8f2de0..47fbd33c 100644 --- a/mdp_client.h +++ b/mdp_client.h @@ -31,8 +31,9 @@ struct mdp_sockaddr { #define MDP_FLAG_NO_CRYPT (1<<0) #define MDP_FLAG_NO_SIGN (1<<1) -#define MDP_FLAG_BIND_ALL (1<<2) -#define MDP_FLAG_OK (1<<3) +#define MDP_FLAG_BIND (1<<2) + +#define MDP_FLAG_CLOSE (1<<3) #define MDP_FLAG_ERROR (1<<4) struct mdp_header { @@ -43,19 +44,36 @@ struct mdp_header { uint8_t ttl; }; +#define BIND_PRIMARY SID_ANY +#define BIND_ALL SID_BROADCAST + #define TYPE_SID 1 #define TYPE_PIN 2 #define ACTION_LOCK 1 #define ACTION_UNLOCK 2 +/* Port numbers for commands sent to the local daemon*/ + +#define MDP_LISTEN 0 +/* lock and unlock identities from the local keyring + * Requests start with an mdp_identity_request structure followed by a list of pins or SIDs +*/ +#define MDP_IDENTITY 1 + +/* Search unlocked identities from the running daemon + * If the request is empty, all identities will be returned + * if the request contains a packed tag / value, identities with a matching tag will be returned + * if the value passed in is zero length, all identities with that tag and any value will be returned +*/ +#define MDP_SEARCH_IDS 2 + +// an identity request is sent to port MDP_IDENTITY, sid ANY struct mdp_identity_request{ uint8_t action; uint8_t type; - // followed by a list of SID's or NULL terminated entry pins for the remainder of the payload + // the request is followed by a list of SID's or NULL terminated entry pins for the remainder of the payload }; -#define MDP_IDENTITY 1 - #pragma pack(pop) struct overlay_route_record{ @@ -69,13 +87,15 @@ struct overlay_mdp_scan{ struct in_addr addr; }; -/* V2 interface */ +/* low level V2 mdp interface */ int mdp_socket(void); int mdp_close(int socket); -int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len); -ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len); +int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len); +ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len); int mdp_poll(int socket, time_ms_t timeout_ms); + + /* Client-side MDP function */ int overlay_mdp_client_socket(void); int overlay_mdp_client_close(int mdp_sockfd); diff --git a/mem.c b/mem.c index 569226a8..17ddcb9f 100644 --- a/mem.c +++ b/mem.c @@ -51,10 +51,8 @@ void *_emalloc_zero(struct __sourceloc __whence, size_t bytes) char *_strn_edup(struct __sourceloc __whence, const char *str, size_t len) { char *new = _emalloc(__whence, len + 1); - if (new) { - strncpy(new, str, len); - new[len] = '\0'; - } + if (new) + strncpy(new, str, len)[len] = '\0'; return new; } diff --git a/meshms.c b/meshms.c index 1f51ae26..3c3d18ca 100644 --- a/meshms.c +++ b/meshms.c @@ -1,9 +1,12 @@ +#include #include "serval.h" #include "rhizome.h" #include "log.h" #include "conf.h" #include "crypto.h" #include "strlcpy.h" +#include "keyring.h" +#include "dataformats.h" #define MESHMS_BLOCK_TYPE_ACK 0x01 #define MESHMS_BLOCK_TYPE_MESSAGE 0x02 @@ -82,15 +85,24 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m) return -1; // always consider the content encrypted, we don't need to rely on the manifest itself. - m->payloadEncryption = 1; + rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED); + assert(m->haveSecret); if (m->haveSecret == NEW_BUNDLE_ID) { - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); - if (rhizome_fill_manifest(m, NULL, NULL, NULL) == -1) + rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE); + if (rhizome_fill_manifest(m, NULL, my_sidp) == -1) return WHY("Invalid manifest"); + if (config.debug.meshms) { + char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1]; + rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES); + // The 'meshms' automated test depends on this message; do not alter. + DEBUGF("MESHMS CONVERSATION BUNDLE bid=%s secret=%s", + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), + secret + ); + } } else { - const char *service = rhizome_manifest_get(m, "service", NULL, 0); - if (strcmp(service, RHIZOME_SERVICE_FILE) != 0) - return WHYF("Invalid manifest, service=%s but should be %s", service, RHIZOME_SERVICE_MESHMS2); + if (strcmp(m->service, RHIZOME_SERVICE_FILE) != 0) + return WHYF("Invalid manifest, service=%s but should be %s", m->service, RHIZOME_SERVICE_MESHMS2); } return 0; } @@ -185,33 +197,40 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si return 0; } -static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid){ +static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid) +{ struct conversations *conv=NULL; if (meshms_conversations_list(my_sid, their_sid, &conv)) return NULL; if (!conv){ conv = emalloc_zero(sizeof(struct conversations)); - bcopy(their_sid->binary, conv->them.binary, sizeof(sid_t)); + conv->them = *their_sid; } return conv; } -static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m){ - m->journalTail = 0; - const char *my_sidhex = alloca_tohex_sid_t(*my_sid); - const char *their_sidhex = alloca_tohex_sid_t(conv->them); - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_MESHMS2); - rhizome_manifest_set(m, "sender", my_sidhex); - rhizome_manifest_set(m, "recipient", their_sidhex); - rhizome_manifest_set_ll(m, "tail", m->journalTail); - if (rhizome_fill_manifest(m, NULL, my_sid, NULL)) +static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m) +{ + if (config.debug.meshms) + DEBUGF("Creating ply for my_sid=%s them=%s", + alloca_tohex_sid_t(conv->them), + alloca_tohex_sid_t(*my_sid)); + rhizome_manifest_set_service(m, RHIZOME_SERVICE_MESHMS2); + rhizome_manifest_set_sender(m, my_sid); + rhizome_manifest_set_recipient(m, &conv->them); + rhizome_manifest_set_filesize(m, 0); + rhizome_manifest_set_tail(m, 0); + if (rhizome_fill_manifest(m, NULL, my_sid)) return -1; + assert(m->haveSecret); + assert(m->payloadEncryption == PAYLOAD_ENCRYPTED); conv->my_ply.bundle_id = m->cryptoSignPublic; conv->found_my_ply = 1; return 0; } -static int append_footer(unsigned char *buffer, char type, int payload_len){ +static int append_footer(unsigned char *buffer, char type, int payload_len) +{ payload_len = (payload_len << 4) | (type&0xF); write_uint16(buffer, payload_len); return 2; @@ -223,16 +242,18 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid)); if (rhizome_retrieve_manifest(bid, m)) return -1; - int ret = rhizome_open_decrypt_read(m, NULL, &ply->read); + int ret = rhizome_open_decrypt_read(m, &ply->read); if (ret == 1) WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); if (ret != 0) return ret; - ply->read.offset = ply->read.length = m->fileLength; + assert(m->filesize != RHIZOME_SIZE_UNSET); + ply->read.offset = ply->read.length = m->filesize; return 0; } -static int ply_read_close(struct ply_read *ply){ +static int ply_read_close(struct ply_read *ply) +{ if (ply->buffer){ free(ply->buffer); ply->buffer=NULL; @@ -244,17 +265,22 @@ static int ply_read_close(struct ply_read *ply){ // read the next record from the ply (backwards) // returns 1 on EOF, -1 on failure -static int ply_read_next(struct ply_read *ply){ - ply->record_end_offset=ply->read.offset; +static int ply_read_next(struct ply_read *ply) +{ + ply->record_end_offset = ply->read.offset; unsigned char footer[2]; - ply->read.offset-=sizeof(footer); - if (ply->read.offset<=0){ + if (ply->read.offset <= sizeof footer) { if (config.debug.meshms) DEBUGF("EOF"); return 1; } - if (rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof(footer)) < sizeof(footer)) - return -1; + ply->read.offset -= sizeof footer; + ssize_t read; + read = rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer); + if (read == -1) + return WHYF("rhizome_read_buffered() failed"); + if ((size_t) read != sizeof footer) + return WHYF("Expected %zu bytes read, got %zu", (size_t) sizeof footer, (size_t) read); // (rhizome_read automatically advances the offset by the number of bytes read) ply->record_length=read_uint16(footer); ply->type = ply->record_length & 0xF; @@ -264,7 +290,7 @@ static int ply_read_next(struct ply_read *ply){ DEBUGF("Found record %d, length %d @%"PRId64, ply->type, ply->record_length, ply->record_end_offset); // need to allow for advancing the tail and cutting a message in half. - if (ply->record_length + sizeof(footer) > ply->read.offset){ + if (ply->record_length + sizeof footer > ply->read.offset){ if (config.debug.meshms) DEBUGF("EOF"); return 1; @@ -281,9 +307,11 @@ static int ply_read_next(struct ply_read *ply){ ply->buffer = b; } - int read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length); - if (read!=ply->record_length) - return WHYF("Expected %d bytes read, got %d", ply->record_length, read); + read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length); + if (read == -1) + return WHYF("rhizome_read_buffered() failed"); + if ((size_t) read != ply->record_length) + return WHYF("Expected %u bytes read, got %zu", ply->record_length, (size_t) read); ply->read.offset = record_start; return 0; @@ -298,7 +326,8 @@ static int ply_find_next(struct ply_read *ply, char type){ } } -static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, unsigned char *buffer, int len){ +static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, unsigned char *buffer, int len) +{ int ret=-1; rhizome_manifest *mout = NULL; rhizome_manifest *m = rhizome_new_manifest(); @@ -308,16 +337,17 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, if (conv->found_my_ply){ if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m)) goto end; - // set the author of the manifest as we should already know that - m->author = *my_sid; - if (rhizome_find_bundle_author(m)) + rhizome_authenticate_author(m); + if (!m->haveSecret || m->authorship != AUTHOR_AUTHENTIC) goto end; }else{ if (create_ply(my_sid, conv, m)) goto end; } + assert(m->haveSecret); + assert(m->authorship == AUTHOR_AUTHENTIC); - if (rhizome_append_journal_buffer(m, NULL, 0, buffer, len)) + if (rhizome_append_journal_buffer(m, 0, buffer, len)) goto end; if (rhizome_manifest_finalise(m, &mout, 1)) @@ -476,21 +506,24 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct rhizome_read_buffer buff; bzero(&buff, sizeof(buff)); - int ret = rhizome_open_decrypt_read(m, NULL, &read); + int ret = rhizome_open_decrypt_read(m, &read); if (ret == -1) goto end; unsigned char version=0xFF; - ret=rhizome_read_buffered(&read, &buff, &version, 1); - if (version!=1){ - WARN("Expected version 1"); + ssize_t r = rhizome_read_buffered(&read, &buff, &version, 1); + ret = -1; + if (r == -1) + goto end; + if (version != 1) { + WARNF("Expected version 1 (got 0x%02x)", version); goto end; } - while (1){ + while (1) { sid_t sid; - ret=rhizome_read_buffered(&read, &buff, sid.binary, sizeof(sid)); - if (rettheir_last_message); - if (ret<0) + int bytes = r; + int ofs = 0; + int unpacked = unpack_uint(details, bytes, &ptr->their_last_message); + if (unpacked == -1) break; - ofs+=ret; - - ret=unpack_uint(details+ofs,bytes-ofs, &ptr->read_offset); - if (ret<0) + ofs += unpacked; + unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->read_offset); + if (unpacked == -1) break; - ofs+=ret; - - ret=unpack_uint(details+ofs,bytes-ofs, &ptr->their_size); - if (ret<0) + ofs += unpacked; + unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->their_size); + if (unpacked == -1) break; - ofs+=ret; + ofs += unpacked; read.offset += ofs - bytes; } + ret = 0; end: rhizome_read_close(&read); - return 0; + return ret; } static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv) @@ -583,10 +615,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations * goto end; // then write it - m->version++; - rhizome_manifest_set_ll(m,"version",m->version); - m->fileLength = (size_t) len + 1; - rhizome_manifest_set_ll(m,"filesize",m->fileLength); + rhizome_manifest_set_version(m, m->version + 1); + rhizome_manifest_set_filesize(m, (size_t)len + 1); if (rhizome_write_open_manifest(&write, m) == -1) goto end; @@ -597,8 +627,7 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations * goto end; if (rhizome_finish_write(&write)) goto end; - m->filehash = write.id; - rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); + rhizome_manifest_set_filehash(m, &write.id); if (rhizome_manifest_finalise(m, &mout, 1)) goto end; @@ -612,7 +641,8 @@ end: } // read information about existing conversations from a rhizome payload -static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv){ +static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv) +{ int ret=-1; rhizome_manifest *m = rhizome_new_manifest(); if (!m) @@ -677,13 +707,16 @@ int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context return -1; if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + if (rhizome_opendb() == -1){ + keyring_free(keyring); return -1; + } struct conversations *conv=NULL; - if (meshms_conversations_list(&sid, NULL, &conv)) + if (meshms_conversations_list(&sid, NULL, &conv)){ + keyring_free(keyring); return -1; - + } const char *names[]={ "_id","recipient","read", "last_message", "read_offset" }; @@ -693,10 +726,12 @@ int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context cli_row_count(context, rows); free_conversations(conv); + keyring_free(keyring); return 0; } -int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context *context){ +int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context *context) +{ const char *my_sidhex, *their_sidhex, *message; if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1 || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1 @@ -707,16 +742,21 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context return -1; if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + if (rhizome_opendb() == -1){ + keyring_free(keyring); return -1; + } sid_t my_sid, their_sid; - fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary)); - fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary)); - struct conversations *conv=find_or_create_conv(&my_sid, &their_sid); - if (!conv) + if (str_to_sid_t(&my_sid, my_sidhex) == -1) + return WHY("invalid sender SID"); + if (str_to_sid_t(&their_sid, their_sidhex) == -1) + return WHY("invalid recipient SID"); + struct conversations *conv = find_or_create_conv(&my_sid, &their_sid); + if (!conv) { + keyring_free(keyring); return -1; - + } // construct a message payload int message_len = strlen(message)+1; @@ -727,10 +767,12 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context int ret = append_meshms_buffer(&my_sid, conv, buffer, message_len); free_conversations(conv); + keyring_free(keyring); return ret; } -int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context){ +int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context) +{ const char *my_sidhex, *their_sidhex; if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1 || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1) @@ -740,17 +782,24 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context return -1; if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + if (rhizome_opendb() == -1){ + keyring_free(keyring); return -1; - + } sid_t my_sid, their_sid; - fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary)); - fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary)); - + if (str_to_sid_t(&my_sid, my_sidhex) == -1){ + keyring_free(keyring); + return WHY("invalid sender SID"); + } + if (str_to_sid_t(&their_sid, their_sidhex) == -1){ + keyring_free(keyring); + return WHY("invalid recipient SID"); + } struct conversations *conv=find_or_create_conv(&my_sid, &their_sid); - if (!conv) + if (!conv){ + keyring_free(keyring); return -1; - + } int ret=-1; const char *names[]={ @@ -880,6 +929,7 @@ end: ply_read_close(&read_theirs); } free_conversations(conv); + keyring_free(keyring); return ret; } @@ -915,7 +965,8 @@ static int mark_read(struct conversations *conv, const sid_t *their_sid, const c return ret; } -int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context){ +int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context) +{ const char *my_sidhex, *their_sidhex, *offset_str; if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1 || cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, NULL) == -1 @@ -926,9 +977,10 @@ int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *co return -1; if (!(keyring = keyring_open_instance_cli(parsed))) return -1; - if (rhizome_opendb() == -1) + if (rhizome_opendb() == -1){ + keyring_free(keyring); return -1; - + } sid_t my_sid, their_sid; fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary)); if (their_sidhex) @@ -966,5 +1018,6 @@ end: if (m) rhizome_manifest_free(m); free_conversations(conv); + keyring_free(keyring); return ret; } diff --git a/monitor-client.c b/monitor-client.c index a404a2f1..35ef276e 100644 --- a/monitor-client.c +++ b/monitor-client.c @@ -36,15 +36,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #endif #include #include +#include #include "constants.h" #include "conf.h" #include "log.h" #include "str.h" #include "strbuf_helpers.h" - +#include "socket.h" #include "monitor-client.h" -#include #define STATE_INIT 0 #define STATE_DATA 1 @@ -72,12 +72,11 @@ int monitor_client_open(struct monitor_state **res) int fd; if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) return WHYF_perror("socket(AF_UNIX, SOCK_STREAM, 0)"); - struct sockaddr_un addr; - socklen_t addrlen; - if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1) + struct socket_address addr; + if (make_local_sockaddr(&addr, "monitor.socket") == -1) return -1; - INFOF("Attempting to connect to %s", alloca_sockaddr(&addr, addrlen)); - if (socket_connect(fd, (struct sockaddr*)&addr, addrlen) == -1) { + INFOF("Attempting to connect to %s", alloca_socket_address(&addr)); + if (socket_connect(fd, &addr.addr, addr.addrlen) == -1) { close(fd); return -1; } diff --git a/monitor.c b/monitor.c index b3715a6e..a27e4168 100644 --- a/monitor.c +++ b/monitor.c @@ -32,6 +32,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "strbuf_helpers.h" #include "overlay_address.h" #include "monitor-client.h" +#include "socket.h" +#include "dataformats.h" #ifdef HAVE_UCRED_H #include @@ -80,11 +82,10 @@ int monitor_setup_sockets() int sock = -1; if ((sock = esocket(AF_UNIX, SOCK_STREAM, 0)) == -1) goto error; - struct sockaddr_un addr; - socklen_t addrlen; - if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1) + struct socket_address addr; + if (make_local_sockaddr(&addr, "monitor.socket") == -1) goto error; - if (socket_bind(sock, (struct sockaddr*)&addr, addrlen) == -1) + if (socket_bind(sock, &addr.addr, addr.addrlen) == -1) goto error; if (socket_listen(sock, MAX_MONITOR_SOCKETS) == -1) goto error; @@ -97,7 +98,7 @@ int monitor_setup_sockets() named_socket.poll.fd=sock; named_socket.poll.events=POLLIN; watch(&named_socket); - INFOF("Monitor socket: fd=%d %s", sock, alloca_sockaddr(&addr, addrlen)); + INFOF("Monitor socket: fd=%d %s", sock, alloca_socket_address(&addr)); return 0; error: diff --git a/net.c b/net.c index 6025757a..73c744ca 100644 --- a/net.c +++ b/net.c @@ -22,12 +22,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include -#include #include #include "serval.h" #include "conf.h" #include "net.h" +#include "socket.h" #include "str.h" #include "strbuf_helpers.h" @@ -139,31 +139,25 @@ ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence return _write_all_nonblock(fd, str, strlen(str), __whence); } -ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl, - struct sockaddr *recvaddr, socklen_t *recvaddrlen) +ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl, struct socket_address *recvaddr) { struct msghdr msg; struct iovec iov[1]; - + struct cmsghdr cmsgcmsg[16]; iov[0].iov_base=buffer; iov[0].iov_len=bufferlen; bzero(&msg,sizeof(msg)); - msg.msg_name = recvaddr; - msg.msg_namelen = *recvaddrlen; + msg.msg_name = &recvaddr->store; + msg.msg_namelen = recvaddr->addrlen; msg.msg_iov = &iov[0]; msg.msg_iovlen = 1; - // setting the following makes the data end up in the wrong place - // msg.msg_iov->iov_base=iov_buffer; - // msg.msg_iov->iov_len=sizeof(iov_buffer); - - struct cmsghdr cmsgcmsg[16]; - msg.msg_control = &cmsgcmsg[0]; - msg.msg_controllen = sizeof(struct cmsghdr)*16; + msg.msg_control = cmsgcmsg; + msg.msg_controllen = sizeof cmsgcmsg; msg.msg_flags = 0; ssize_t len = recvmsg(sock,&msg,0); if (len == -1 && errno != EAGAIN && errno != EWOULDBLOCK) - return WHY_perror("recvmsg"); + return WHYF_perror("recvmsg(%d,%p,0)", sock, &msg); #if 0 if (config.debug.packetrx) { @@ -193,7 +187,7 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl, } } } - *recvaddrlen=msg.msg_namelen; + recvaddr->addrlen = msg.msg_namelen; return len; } diff --git a/net.h b/net.h index ab0cf6f7..6ef3e1c0 100644 --- a/net.h +++ b/net.h @@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include // for size_t, ssize_t #include // for struct sockaddr, socklen_t +#ifdef HAVE_NETINET_IN_H #include // for struct in_addr +#endif #include // for in_addr_t #include "log.h" // for __WHENCE__ and struct __sourceloc @@ -50,6 +52,8 @@ ssize_t _write_all_nonblock(int fd, const void *buf, size_t len, struct __source ssize_t _writev_all(int fd, const struct iovec *iov, int iovcnt, struct __sourceloc __whence); ssize_t _write_str(int fd, const char *str, struct __sourceloc __whence); ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence); -ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct sockaddr *recvaddr, socklen_t *recvaddrlen); + +struct socket_address; +ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct socket_address *); #endif // __SERVALD_NET_H diff --git a/os.h b/os.h index e5da065b..f098518e 100644 --- a/os.h +++ b/os.h @@ -54,6 +54,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * } */ typedef int64_t time_ms_t; +#define PRItime_ms_t PRId64 time_ms_t gettime_ms(); time_ms_t sleep_ms(time_ms_t milliseconds); @@ -65,7 +66,7 @@ __SERVALDNA_OS_INLINE void bzero(void *buf, size_t len) { #endif #ifndef HAVE_BCOPY -__SERVALDNA_OS_INLINE void bcopy(void *src, void *dst, size_t len) { +__SERVALDNA_OS_INLINE void bcopy(const void *src, void *dst, size_t len) { memcpy(dst, src, len); } #endif diff --git a/overlay.c b/overlay.c index d16b0ef9..ca4b043b 100644 --- a/overlay.c +++ b/overlay.c @@ -72,6 +72,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "conf.h" #include "rhizome.h" #include "strbuf.h" +#include "keyring.h" int overlayMode=0; diff --git a/overlay_address.c b/overlay_address.c index cc095d15..399e147f 100644 --- a/overlay_address.c +++ b/overlay_address.c @@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. since for things like number resolution we are happy to send repeat requests. */ +#include #include "serval.h" #include "conf.h" #include "str.h" @@ -46,7 +47,7 @@ static struct broadcast bpilist[MAX_BPIS]; // each slot either points to another tree node or a struct subscriber. struct tree_node{ // bit flags for the type of object each element points to - int is_tree; + uint16_t is_tree; union{ struct tree_node *tree_nodes[16]; @@ -66,58 +67,106 @@ static unsigned char get_nibble(const unsigned char *sidp, int pos) return byte&0xF; } -// find a subscriber struct from a whole or abbreviated subscriber id -struct subscriber *find_subscriber(const unsigned char *sidp, int len, int create) +static void free_subscriber(struct subscriber *subscriber) { + if (subscriber->link_state || subscriber->destination) + FATAL("Can't free a subscriber that is being used in routing"); + if (subscriber->sync_state) + FATAL("Can't free a subscriber that is being used by rhizome"); + if (subscriber->identity) + FATAL("Can't free a subscriber that is unlocked in the keyring"); + free(subscriber); +} + +static void free_children(struct tree_node *parent) +{ + int i; + for (i=0;i<16;i++){ + if (parent->is_tree & (1<tree_nodes[i]); + free(parent->tree_nodes[i]); + parent->tree_nodes[i]=NULL; + }else if(parent->subscribers[i]){ + free_subscriber(parent->subscribers[i]); + parent->subscribers[i]=NULL; + } + } + parent->is_tree=0; +} + +void free_subscribers() +{ + // don't attempt to free anything if we're running as a server + // who knows where subscriber ptr's may have leaked to. + if (serverMode) + FATAL("Freeing subscribers from a running daemon is not supported"); + free_children(&root); +} + +// find a subscriber struct from a whole or abbreviated subscriber id +struct subscriber *_find_subscriber(struct __sourceloc __whence, const unsigned char *sidp, int len, int create) +{ + IN(); + if (config.debug.subscriber) + DEBUGF("find_subscriber(sid=%s, create=%d)", alloca_tohex(sidp, len), create); struct tree_node *ptr = &root; int pos=0; if (len!=SID_SIZE) create =0; - - do{ + struct subscriber *ret = NULL; + do { unsigned char nibble = get_nibble(sidp, pos++); - if (ptr->is_tree & (1<tree_nodes[nibble]; - }else if(!ptr->subscribers[nibble]){ // subscriber is not yet known - - if (create){ - struct subscriber *ret=(struct subscriber *)malloc(sizeof(struct subscriber)); - memset(ret,0,sizeof(struct subscriber)); - ptr->subscribers[nibble]=ret; + if (create && (ret = (struct subscriber *) emalloc_zero(sizeof(struct subscriber)))) { + ptr->subscribers[nibble] = ret; ret->sid = *(const sid_t *)sidp; - ret->abbreviate_len=pos; + ret->abbreviate_len = pos; + if (config.debug.subscriber) + DEBUGF("set node[%.*s].subscribers[%c]=%p (sid=%s, abbrev_len=%d)", + pos - 1, alloca_tohex(sidp, len), hexdigit_upper[nibble], + ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len + ); } - return ptr->subscribers[nibble]; - + goto done; }else{ // there's a subscriber in this slot, does it match the rest of the sid we've been given? - struct subscriber *ret = ptr->subscribers[nibble]; + ret = ptr->subscribers[nibble]; if (memcmp(ret->sid.binary, sidp, len) == 0) - return ret; - + goto done; // if we need to insert this subscriber, we have to make a new tree node first - if (!create) - return NULL; - + if (!create) { + ret = NULL; + goto done; + } // create a new tree node and move the existing subscriber into it - struct tree_node *new=(struct tree_node *)malloc(sizeof(struct tree_node)); - memset(new,0,sizeof(struct tree_node)); - ptr->tree_nodes[nibble]=new; + struct tree_node *new = (struct tree_node *) emalloc_zero(sizeof(struct tree_node)); + if (new == NULL) { + ret = NULL; + goto done; + } + if (config.debug.subscriber) + DEBUGF("create node[%.*s]", pos, alloca_tohex(sidp, len)); + ptr->tree_nodes[nibble] = new; ptr->is_tree |= (1<sid.binary, pos); - ptr->subscribers[nibble]=ret; - ret->abbreviate_len=pos+1; + ptr = new; + nibble = get_nibble(ret->sid.binary, pos); + ptr->subscribers[nibble] = ret; + ret->abbreviate_len = pos + 1; + if (config.debug.subscriber) + DEBUGF("set node[%.*s].subscribers[%c]=%p(sid=%s, abbrev_len=%d)", + pos, alloca_tohex(sidp, len), hexdigit_upper[nibble], + ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len + ); // then go around the loop again to compare the next nibble against the sid until we find an empty slot. } - }while(pos < len*2); - - // abbreviation is not unique - return NULL; + } while(pos < len*2); +done: + if (config.debug.subscriber) + DEBUGF("find_subscriber() return %p", ret); + RETURN(ret); } /* @@ -198,35 +247,28 @@ int overlay_broadcast_drop_check(struct broadcast *addr) } } -int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast) +void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast) { - return ob_append_bytes(b, broadcast->id, BROADCAST_LEN); + ob_append_bytes(b, broadcast->id, BROADCAST_LEN); } // append an appropriate abbreviation into the address -int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber) +void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber) { - if (!subscriber) - return WHY("No address supplied"); - - if(context - && subscriber == context->point_to_point_device){ - if (ob_append_byte(b, OA_CODE_P2P_YOU)) - return -1; - }else if(context + assert(subscriber != NULL); + if (context && subscriber == context->point_to_point_device) + ob_append_byte(b, OA_CODE_P2P_YOU); + else if(context && !subscriber->send_full && subscriber == my_subscriber && context->point_to_point_device - && (context->encoding_header==0 || !context->interface->local_echo)){ - if (ob_append_byte(b, OA_CODE_P2P_ME)) - return -1; - }else if (context && subscriber==context->sender){ - if (ob_append_byte(b, OA_CODE_SELF)) - return -1; - }else if(context && subscriber==context->previous){ - if (ob_append_byte(b, OA_CODE_PREVIOUS)) - return -1; - }else{ + && (context->encoding_header==0 || !context->interface->local_echo)) + ob_append_byte(b, OA_CODE_P2P_ME); + else if (context && subscriber==context->sender) + ob_append_byte(b, OA_CODE_SELF); + else if (context && subscriber==context->previous) + ob_append_byte(b, OA_CODE_PREVIOUS); + else { int len=SID_SIZE; if (subscriber->send_full){ subscriber->send_full=0; @@ -237,17 +279,15 @@ int overlay_address_append(struct decode_context *context, struct overlay_buffer if (len>SID_SIZE) len=SID_SIZE; } - if (ob_append_byte(b, len)) - return -1; - if (ob_append_bytes(b, subscriber->sid.binary, len)) - return -1; + ob_append_byte(b, len); + ob_append_bytes(b, subscriber->sid.binary, len); } if (context) context->previous = subscriber; - return 0; } -static int add_explain_response(struct subscriber *subscriber, void *context){ +static int add_explain_response(struct subscriber *subscriber, void *context) +{ struct decode_context *response = context; // only explain a SID once every half second. time_ms_t now = gettime_ms(); @@ -256,8 +296,13 @@ static int add_explain_response(struct subscriber *subscriber, void *context){ subscriber->last_explained = now; if (!response->please_explain){ - response->please_explain = calloc(sizeof(struct overlay_frame),1); - response->please_explain->payload=ob_new(); + if ((response->please_explain = emalloc_zero(sizeof(struct overlay_frame))) == NULL) + return 1; // stop walking + if ((response->please_explain->payload = ob_new()) == NULL) { + free(response->please_explain); + response->please_explain = NULL; + return 1; // stop walking + } ob_limitsize(response->please_explain->payload, 1024); } @@ -265,6 +310,8 @@ static int add_explain_response(struct subscriber *subscriber, void *context){ // the header of this packet must include our full sid. if (subscriber->reachable==REACHABLE_SELF){ if (subscriber==my_subscriber){ + if (config.debug.subscriber) + DEBUGF("Explaining SELF sid=%s", alloca_tohex_sid_t(subscriber->sid)); response->please_explain->source_full=1; return 0; } @@ -272,18 +319,22 @@ static int add_explain_response(struct subscriber *subscriber, void *context){ } // add the whole subscriber id to the payload, stop if we run out of space - DEBUGF("Adding full sid by way of explanation %s", alloca_tohex_sid_t(subscriber->sid)); - if (ob_append_byte(response->please_explain->payload, SID_SIZE)) + if (config.debug.subscriber) + DEBUGF("Explaining sid=%s", alloca_tohex_sid_t(subscriber->sid)); + ob_checkpoint(response->please_explain->payload); + ob_append_byte(response->please_explain->payload, SID_SIZE); + ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE); + if (ob_overrun(response->please_explain->payload)) { + ob_rewind(response->please_explain->payload); return 1; - if (ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE)) - return 1; - + } // let the routing engine know that we had to explain this sid, we probably need to re-send routing info link_explained(subscriber); return 0; } -static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber){ +static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber) +{ if (len<=0 || len>SID_SIZE){ return WHYF("Invalid abbreviation length %d", len); } @@ -309,7 +360,8 @@ static int find_subscr_buffer(struct decode_context *context, struct overlay_buf // add the abbreviation you told me about if (!context->please_explain){ context->please_explain = calloc(sizeof(struct overlay_frame),1); - context->please_explain->payload=ob_new(); + if ((context->please_explain->payload = ob_new()) == NULL) + return -1; ob_limitsize(context->please_explain->payload, MDP_MTU); } @@ -360,7 +412,8 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer // add the abbreviation you told me about if (!context->please_explain){ context->please_explain = calloc(sizeof(struct overlay_frame),1); - context->please_explain->payload=ob_new(); + if ((context->please_explain->payload = ob_new()) == NULL) + return -1; ob_limitsize(context->please_explain->payload, MDP_MTU); } @@ -394,11 +447,13 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer } // once we've finished parsing a packet, complete and send a please explain if required. -int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination){ +int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination) +{ IN(); struct overlay_frame *frame=context->please_explain; - if (!frame) + if (frame == NULL) RETURN(0); + assert(frame->payload != NULL); frame->type = OF_TYPE_PLEASEEXPLAIN; if (source) @@ -430,7 +485,7 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc } frame->queue=OQ_MESH_MANAGEMENT; - if (!overlay_payload_enqueue(frame)) + if (overlay_payload_enqueue(frame) != -1) RETURN(0); op_free(frame); RETURN(-1); @@ -438,7 +493,8 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc } // process an incoming request for explanation of subscriber abbreviations -int process_explain(struct overlay_frame *frame){ +int process_explain(struct overlay_frame *frame) +{ struct overlay_buffer *b=frame->payload; struct decode_context context; @@ -463,14 +519,17 @@ int process_explain(struct overlay_frame *frame){ if (len==SID_SIZE){ // This message is also used to inform people of previously unknown subscribers // make sure we know this one + INFOF("Storing explain response for %s", alloca_tohex(sid, len)); find_subscriber(sid,len,1); }else{ // reply to the sender with all subscribers that match this abbreviation - INFOF("Sending responses for %s", alloca_tohex(sid, len)); + INFOF("Sending explain responses for %s", alloca_tohex(sid, len)); walk_tree(&root, 0, sid, len, sid, len, add_explain_response, &context); } } - - send_please_explain(&context, frame->destination, frame->source); + if (context.please_explain) + send_please_explain(&context, frame->destination, frame->source); + else if (config.debug.subscriber) + DEBUG("No explain responses"); return 0; } diff --git a/overlay_address.h b/overlay_address.h index 7de199b7..1de7801b 100644 --- a/overlay_address.h +++ b/overlay_address.h @@ -57,9 +57,6 @@ struct subscriber{ int max_packet_version; - // overlay routing information - struct overlay_node *node; - // link state routing information struct link_state *link_state; @@ -85,7 +82,7 @@ struct subscriber{ unsigned char sas_valid; // private keys for local identities - keyring_identity *identity; + struct keyring_identity *identity; }; struct broadcast{ @@ -113,7 +110,9 @@ struct decode_context{ extern struct subscriber *my_subscriber; extern struct subscriber *directory_service; -struct subscriber *find_subscriber(const unsigned char *sid, int len, int create); +struct subscriber *_find_subscriber(struct __sourceloc, const unsigned char *sid, int len, int create); +#define find_subscriber(sid, len, create) _find_subscriber(__WHENCE__, sid, len, create) + void enum_subscribers(struct subscriber *start, int(*callback)(struct subscriber *, void *), void *context); int set_reachable(struct subscriber *subscriber, struct network_destination *destination, struct subscriber *next_hop); int load_subscriber_address(struct subscriber *subscriber); @@ -122,11 +121,13 @@ int process_explain(struct overlay_frame *frame); int overlay_broadcast_drop_check(struct broadcast *addr); int overlay_broadcast_generate_address(struct broadcast *addr); -int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast); -int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber); +void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast); +void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber); int overlay_broadcast_parse(struct overlay_buffer *b, struct broadcast *broadcast); int overlay_address_parse(struct decode_context *context, struct overlay_buffer *b, struct subscriber **subscriber); int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination); +void free_subscribers(); + #endif diff --git a/overlay_buffer.c b/overlay_buffer.c index f066838f..78f25218 100644 --- a/overlay_buffer.c +++ b/overlay_buffer.c @@ -17,6 +17,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include "serval.h" #include "conf.h" #include "mem.h" @@ -30,57 +31,64 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. In either case, functions that don't take an offset use and advance the position. */ - - -struct overlay_buffer *ob_new(void) +struct overlay_buffer *_ob_new(struct __sourceloc __whence) { - struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1); - if (!ret) return NULL; - + struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer)); + if (config.debug.overlaybuffer) + DEBUGF("ob_new() return %p", ret); + if (ret == NULL) + return NULL; ob_unlimitsize(ret); - return ret; } // index an existing static buffer. // and allow other callers to use the ob_ convenience methods for reading and writing up to size bytes. -struct overlay_buffer *ob_static(unsigned char *bytes, int size){ - struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1); - if (!ret) return NULL; +struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size) +{ + struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer)); + if (config.debug.overlaybuffer) + DEBUGF("ob_static(bytes=%p, size=%d) return %p", bytes, size, ret); + if (ret == NULL) + return NULL; ret->bytes = bytes; ret->allocSize = size; ret->allocated = NULL; ob_unlimitsize(ret); - return ret; } // create a new overlay buffer from an existing piece of another buffer. // Both buffers will point to the same memory region. // It is up to the caller to ensure this buffer is not used after the parent buffer is freed. -struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length){ +struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length) +{ if (offset+length > b->allocSize) { WHY("Buffer isn't long enough to slice"); - return NULL; + return NULL; } - - struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1); - if (!ret) + struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer)); + if (config.debug.overlaybuffer) + DEBUGF("ob_slice(b=%p, offset=%d, length=%d) return %p", b, offset, length, ret); + if (ret == NULL) return NULL; ret->bytes = b->bytes+offset; ret->allocSize = length; ret->allocated = NULL; ob_unlimitsize(ret); - return ret; } -struct overlay_buffer *ob_dup(struct overlay_buffer *b){ - struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1); +struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b) +{ + struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer)); + if (config.debug.overlaybuffer) + DEBUGF("ob_dup(b=%p) return %p", b, ret); + if (ret == NULL) + return NULL; ret->sizeLimit = b->sizeLimit; ret->position = b->position; ret->checkpointLength = b->checkpointLength; - if (b->bytes && b->allocSize){ // duplicate any bytes that might be relevant int byteCount = b->sizeLimit; @@ -88,97 +96,109 @@ struct overlay_buffer *ob_dup(struct overlay_buffer *b){ byteCount = b->position; if (byteCount > b->allocSize) byteCount = b->allocSize; - - ob_append_bytes(ret, b->bytes, byteCount); + if (byteCount) + ob_append_bytes(ret, b->bytes, byteCount); } return ret; } -int ob_free(struct overlay_buffer *b) +void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b) { - if (!b) return WHY("Asked to free NULL"); - if (b->bytes && b->allocated) free(b->allocated); - // we're about to free this anyway, why are we clearing it? - b->bytes=NULL; - b->allocated=NULL; - b->allocSize=0; - b->sizeLimit=0; + assert(b != NULL); + if (config.debug.overlaybuffer) + DEBUGF("ob_free(b=%p)", b); + if (b->allocated) + free(b->allocated); free(b); +} + +int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b) +{ + assert(b != NULL); + b->checkpointLength = b->position; + if (config.debug.overlaybuffer) + DEBUGF("ob_checkpoint(b=%p) checkpointLength=%d", b, b->checkpointLength); return 0; } -int ob_checkpoint(struct overlay_buffer *b) +int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b) { - if (!b) return WHY("Asked to checkpoint NULL"); - b->checkpointLength=b->position; + assert(b != NULL); + b->position = b->checkpointLength; + if (config.debug.overlaybuffer) + DEBUGF("ob_rewind(b=%p) position=%d", b, b->position); return 0; } -int ob_rewind(struct overlay_buffer *b) +void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b, int bytes) { - if (!b) return WHY("Asked to rewind NULL"); - b->position=b->checkpointLength; - return 0; + assert(b != NULL); + assert(bytes >= 0); + assert(b->position >= 0); + assert(b->position <= bytes); + assert(b->checkpointLength <= bytes); + if (b->bytes && b->allocated == NULL) + assert(bytes <= b->allocSize); + b->sizeLimit = bytes; + if (config.debug.overlaybuffer) + DEBUGF("ob_limitsize(b=%p, bytes=%d) sizeLimit=%d", b, bytes, b->sizeLimit); } -int ob_limitsize(struct overlay_buffer *b,int bytes) +void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b) { - if (!b) return WHY("Asked to limit size of NULL"); - if (b->position>bytes) return WHY("Length of data in buffer already exceeds size limit"); - if (b->checkpointLength>bytes) return WHY("Checkpointed length of data in buffer already exceeds size limit"); - if (b->bytes && (!b->allocated) && bytes > b->allocSize) return WHY("Size limit exceeds buffer size"); - if (bytes<0) return WHY("Can't limit buffer to a negative size"); - b->sizeLimit=bytes; - return 0; + assert(b != NULL); + b->sizeLimit = -1; + if (config.debug.overlaybuffer) + DEBUGF("ob_unlimitsize(b=%p) sizeLimit=%d", b, b->sizeLimit); } -int ob_unlimitsize(struct overlay_buffer *b) +void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b) { - if (!b) return WHY("b is NULL"); - b->sizeLimit=-1; - return 0; + if (config.debug.overlaybuffer) + DEBUGF("ob_flip(b=%p) checkpointLength=0 position=0", b); + b->checkpointLength = 0; + ob_limitsize(b, b->position); + b->position = 0; } -int ob_flip(struct overlay_buffer *b) -{ - b->checkpointLength=0; - if (ob_limitsize(b, b->position)) - return -1; - b->position=0; - return 0; -} - -int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int bytes) +/* Return 1 if space is available, 0 if not. + */ +ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, size_t bytes) { + assert(b != NULL); + if (config.debug.overlaybuffer) + DEBUGF("ob_makespace(b=%p, bytes=%zd) b->bytes=%p b->position=%d b->allocSize=%d", + b, bytes, b->bytes, b->position, b->allocSize); + assert(b->position >= 0); + if (b->sizeLimit != -1) + assert(b->sizeLimit >= 0); + assert(b->allocSize >= 0); + if (b->position) + assert(b->bytes != NULL); if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit) { if (config.debug.packetformats) - DEBUGF("asked for space to %u, beyond size limit of %u", b->position + bytes, b->sizeLimit); - return -1; - } - - // already enough space? - if (b->position + bytes <= b->allocSize) + DEBUGF("ob_makespace(): asked for space to %zu, beyond size limit of %u", b->position + bytes, b->sizeLimit); return 0; - - if (b->bytes && !b->allocated) - return WHY("Can't resize a static buffer"); - - if (0) - DEBUGF("ob_makespace(%p,%d)\n b->bytes=%p,b->position=%d,b->allocSize=%d\n", - b,bytes,b->bytes,b->position,b->allocSize); - + } + if (b->position + bytes <= b->allocSize) + return 1; + // Don't realloc a static buffer. + if (b->bytes && b->allocated == NULL) { + if (config.debug.packetformats) + DEBUGF("ob_makespace(): asked for space to %zu, beyond static buffer size of %u", b->position + bytes, b->allocSize); + return 0; + } int newSize=b->position+bytes; if (newSize<64) newSize=64; if (newSize&63) newSize+=64-(newSize&63); - if (newSize>1024) { - if (newSize&1023) newSize+=1024-(newSize&1023); - } - if (newSize>65536) { - if (newSize&65535) newSize+=65536-(newSize&65535); - } - if (0) DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize); + if (newSize>1024 && (newSize&1023)) + newSize+=1024-(newSize&1023); + if (newSize>65536 && (newSize&65535)) + newSize+=65536-(newSize&65535); + if (config.debug.overlaybuffer) + DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize); /* XXX OSX realloc() seems to be able to corrupt things if the heap is not happy when calling realloc(), making debugging memory corruption much harder. - So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */ + So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */ /* unsigned char *r=realloc(b->bytes,newSize); if (!r) return WHY("realloc() failed"); @@ -196,94 +216,130 @@ int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int byte sleep_ms(36000000); } } - unsigned char *new=malloc(newSize+4096); - if (!new) return WHY("realloc() failed"); + unsigned char *new = emalloc(newSize+4096); { int i; for(i=0;i<4096;i++) new[newSize+i]=0xbd; } #else - unsigned char *new=malloc(newSize); + unsigned char *new = emalloc(newSize); #endif + if (!new) + return 0; bcopy(b->bytes,new,b->position); - if (b->allocated) free(b->allocated); + if (b->allocated) { + assert(b->allocated == b->bytes); + free(b->allocated); + } b->bytes=new; b->allocated=new; b->allocSize=newSize; - return 0; + return 1; } - - /* Functions that append data and increase the size of the buffer if possible / required */ -int _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b,unsigned char byte) +void _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b, unsigned char byte) { - if (_ob_makespace(__whence, b,1)) return WHY("ob_makespace() failed"); - b->bytes[b->position++] = byte; - return 0; + const int bytes = 1; + if (ob_makespace(b, bytes)) { + b->bytes[b->position] = byte; + if (config.debug.overlaybuffer) + DEBUGF("ob_append_byte(b=%p, byte=0x%02x) %p[%d]=%02x position=%d", b, byte, b->bytes, b->position, byte, b->position + bytes); + } else { + if (config.debug.overlaybuffer) + DEBUGF("ob_append_byte(b=%p, byte=0x%02x) OVERRUN position=%d", b, byte, b->position + bytes); + } + b->position += bytes; } -unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b,int count) +unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b, int count) { - if (_ob_makespace(__whence, b,count)) { - WHY("ob_makespace() failed"); - return NULL; - } - - unsigned char *r=&b->bytes[b->position]; - b->position+=count; + assert(count > 0); + unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL; + b->position += count; + if (config.debug.overlaybuffer) + DEBUGF("ob_append_space(b=%p, count=%d) position=%d return %p", b, count, b->position, r); return r; } -int _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count) +void _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count) { - if (_ob_makespace(__whence, b,count)) return WHY("ob_makespace() failed"); - - bcopy(bytes,&b->bytes[b->position],count); - b->position+=count; - return 0; + assert(count > 0); + unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL; + if (r) { + bcopy(bytes, r, count); + if (config.debug.overlaybuffer) + DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) position=%d return %p", b, bytes, count, b->position + count, r); + } else { + if (config.debug.overlaybuffer) + DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) OVERRUN position=%d return NULL", b, bytes, count, b->position + count); + } + if (config.debug.overlaybuffer) + dump("ob_append_bytes", bytes, count); + b->position += count; } -int _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s){ - return _ob_append_bytes(__whence, b, s->bytes, s->position); +void _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s) +{ + ob_append_bytes(b, s->bytes, s->position); } -int _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v) +void _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v) { - if (_ob_makespace(__whence, b, 2)) return WHY("ob_makespace() failed"); - b->bytes[b->position] = (v >> 8) & 0xFF; - b->bytes[b->position+1] = v & 0xFF; - b->position+=2; - return 0; + const int bytes = 2; + if (ob_makespace(b, bytes)) { + b->bytes[b->position] = (v >> 8) & 0xFF; + b->bytes[b->position+1] = v & 0xFF; + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui16(b=%p, v=%u) %p[%d]=%s position=%d", b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes); + } else { + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui16(b=%p, v=%u) OVERRUN position=%d", b, v, b->position + bytes); + } + b->position += bytes; } -int _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v) +void _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v) { - if (_ob_makespace(__whence, b, 4)) return WHY("ob_makespace() failed"); - b->bytes[b->position] = (v >> 24) & 0xFF; - b->bytes[b->position+1] = (v >> 16) & 0xFF; - b->bytes[b->position+2] = (v >> 8) & 0xFF; - b->bytes[b->position+3] = v & 0xFF; - b->position+=4; - return 0; + const int bytes = 4; + if (ob_makespace(b, bytes)) { + b->bytes[b->position] = (v >> 24) & 0xFF; + b->bytes[b->position+1] = (v >> 16) & 0xFF; + b->bytes[b->position+2] = (v >> 8) & 0xFF; + b->bytes[b->position+3] = v & 0xFF; + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") %p[%d]=%s position=%d", + b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes); + } else { + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") OVERRUN position=%d", b, v, b->position + bytes); + } + b->position += bytes; } -int _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v) +void _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v) { - if (_ob_makespace(__whence, b, 8)) return WHY("ob_makespace() failed"); - b->bytes[b->position] = (v >> 56) & 0xFF; - b->bytes[b->position+1] = (v >> 48) & 0xFF; - b->bytes[b->position+2] = (v >> 40) & 0xFF; - b->bytes[b->position+3] = (v >> 32) & 0xFF; - b->bytes[b->position+4] = (v >> 24) & 0xFF; - b->bytes[b->position+5] = (v >> 16) & 0xFF; - b->bytes[b->position+6] = (v >> 8) & 0xFF; - b->bytes[b->position+7] = v & 0xFF; - b->position+=8; - return 0; + const int bytes = 8; + if (ob_makespace(b, bytes)) { + b->bytes[b->position] = (v >> 56) & 0xFF; + b->bytes[b->position+1] = (v >> 48) & 0xFF; + b->bytes[b->position+2] = (v >> 40) & 0xFF; + b->bytes[b->position+3] = (v >> 32) & 0xFF; + b->bytes[b->position+4] = (v >> 24) & 0xFF; + b->bytes[b->position+5] = (v >> 16) & 0xFF; + b->bytes[b->position+6] = (v >> 8) & 0xFF; + b->bytes[b->position+7] = v & 0xFF; + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") %p[%d]=%s position=%d", + b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes); + } else { + if (config.debug.overlaybuffer) + DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") OVERRUN position=%d", b, v, b->position + bytes); + } + b->position += bytes; } int measure_packed_uint(uint64_t v){ @@ -320,36 +376,28 @@ int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v){ return i; } -int _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v) +void _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v) { - do{ - - if (_ob_append_byte(__whence, b, (v&0x7f) | (v>0x7f?0x80:0))) - return -1; - v = v>>7; - - }while(v!=0); - return 0; + do { + ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0)); + v = v >> 7; + } while (v != 0); } -int _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v) +void _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v) { - do{ - - if (ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0))) - return -1; - v = v>>7; - - }while(v!=0); - return 0; + do { + ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0)); + v = v >> 7; + } while (v != 0); } -int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l) +void _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l) { - if (l<0||l>0xffff) return -1; - - b->var_length_offset=b->position; - return _ob_append_ui16(__whence, b,l); + assert(l >= 0); + assert(l <= 0xffff); + b->var_length_offset = b->position; + ob_append_ui16(b, l); } @@ -359,7 +407,8 @@ int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l) // make sure a range of bytes is valid for reading -int test_offset(struct overlay_buffer *b,int start,int length){ +int test_offset(struct overlay_buffer *b,int start,int length) +{ if (!b) return -1; if (start<0) return -1; if (b->sizeLimit>=0 && start+length>b->sizeLimit) return -1; @@ -367,12 +416,17 @@ int test_offset(struct overlay_buffer *b,int start,int length){ return 0; } -int ob_getbyte(struct overlay_buffer *b, int ofs) +// next byte without advancing +int ob_peek(struct overlay_buffer *b) { - if (test_offset(b, ofs, 1)) + if (test_offset(b, b->position, 1)) return -1; - - return b->bytes[ofs]; + return b->bytes[b->position]; +} + +void ob_skip(struct overlay_buffer *b, unsigned n) +{ + b->position += n; } int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len){ @@ -464,46 +518,75 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b) return ret; } -int ob_get(struct overlay_buffer *b){ +int ob_get(struct overlay_buffer *b) +{ if (test_offset(b, b->position, 1)) return -1; - return b->bytes[b->position++]; } -int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v) +void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v) { - if (test_offset(b, offset, 2)) - return -1; - + const int bytes = 2; + assert(b != NULL); + assert(offset >= 0); + if (b->sizeLimit != -1) + assert(offset + bytes <= b->sizeLimit); + assert(offset + bytes <= b->allocSize); b->bytes[offset] = (v >> 8) & 0xFF; b->bytes[offset+1] = v & 0xFF; - return 0; + if (config.debug.overlaybuffer) + DEBUGF("ob_set_ui16(b=%p, offset=%d, v=%u) %p[%d]=%s", b, offset, v, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes)); } -int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte) +void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int offset, unsigned char byte) { - if (test_offset(b, ofs, 1)) - return -1; - b->bytes[ofs] = byte; - return 0; + const int bytes = 1; + assert(b != NULL); + assert(offset >= 0); + if (b->sizeLimit != -1) + assert(offset + bytes <= b->sizeLimit); + assert(offset + bytes <= b->allocSize); + b->bytes[offset] = byte; + if (config.debug.overlaybuffer) + DEBUGF("ob_set(b=%p, offset=%d, byte=0x%02x) %p[%d]=%s", b, offset, byte, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes)); } -int ob_patch_rfs(struct overlay_buffer *b){ - return ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2)); +void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b) +{ + ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2)); } -int ob_position(struct overlay_buffer *b){ +int ob_position(struct overlay_buffer *b) +{ return b->position; } -int ob_limit(struct overlay_buffer *b){ + +int ob_limit(struct overlay_buffer *b) +{ return b->sizeLimit; } -int ob_remaining(struct overlay_buffer *b){ + +int ob_remaining(struct overlay_buffer *b) +{ + assert(b->sizeLimit != -1); return b->sizeLimit - b->position; } -unsigned char *ob_ptr(struct overlay_buffer *b){ + +int _ob_overrun(struct __sourceloc __whence, struct overlay_buffer *b) +{ + assert(b->allocSize >= 0); + if (b->sizeLimit != -1) + assert(b->sizeLimit >= 0); + int ret = b->position > (b->sizeLimit != -1 && b->sizeLimit < b->allocSize ? b->sizeLimit : b->allocSize); + if (config.debug.overlaybuffer) + DEBUGF("ob_overrun(b=%p) return %d", b, ret); + return ret; +} + +unsigned char *ob_ptr(struct overlay_buffer *b) +{ return b->bytes; } diff --git a/overlay_buffer.h b/overlay_buffer.h index 49cc8ca8..cf69a4b9 100644 --- a/overlay_buffer.h +++ b/overlay_buffer.h @@ -17,8 +17,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#ifndef _SERVALD_OVERLAY_BUFFER_H -#define _SERVALD_OVERLAY_BUFFER_H +#ifndef __SERVALD__OVERLAY_BUFFER_H +#define __SERVALD__OVERLAY_BUFFER_H struct overlay_buffer { unsigned char *bytes; @@ -42,31 +42,47 @@ struct overlay_buffer { int var_length_offset; }; -struct overlay_buffer *ob_new(void); -struct overlay_buffer *ob_static(unsigned char *bytes, int size); -struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length); -struct overlay_buffer *ob_dup(struct overlay_buffer *b); -int ob_free(struct overlay_buffer *b); -int ob_checkpoint(struct overlay_buffer *b); -int ob_rewind(struct overlay_buffer *b); -int ob_limitsize(struct overlay_buffer *b,int bytes); -int ob_flip(struct overlay_buffer *b); -int ob_unlimitsize(struct overlay_buffer *b); -int _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b,int bytes); -int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte); +struct overlay_buffer *_ob_new(struct __sourceloc __whence); +struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size); +struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length); +struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b); +void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b); +int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b); +int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b); +void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b,int bytes); +void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b); +void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b); +ssize_t _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b, size_t bytes); +void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int ofs, unsigned char byte); +void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v); +void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b); -int _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte); -int _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count); -int _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s); +void _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte); +void _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count); +void _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s); unsigned char *_ob_append_space(struct __sourceloc whence, struct overlay_buffer *b,int count); -int _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v); -int _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v); -int _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v); -int _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v); -int _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v); -int _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l); +void _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v); +void _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v); +void _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v); +void _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v); +void _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v); +void _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l); +#define ob_new() _ob_new(__WHENCE__) +#define ob_static(bytes, size) _ob_static(__WHENCE__, bytes, size) +#define ob_slice(b, off, len) _ob_slice(__WHENCE__, b, off, len) +#define ob_dup(b) _ob_dup(__WHENCE__, b) +#define ob_free(b) _ob_free(__WHENCE__, b) +#define ob_checkpoint(b) _ob_checkpoint(__WHENCE__, b) +#define ob_rewind(b) _ob_rewind(__WHENCE__, b) +#define ob_limitsize(b, size) _ob_limitsize(__WHENCE__, b, size) +#define ob_flip(b) _ob_flip(__WHENCE__, b) +#define ob_unlimitsize(b) _ob_unlimitsize(__WHENCE__, b) #define ob_makespace(b, bytes) _ob_makespace(__WHENCE__, b, bytes) +#define ob_set(b, off, byte) _ob_set(__WHENCE__, b, off, byte) +#define ob_set_ui16(b, off, v) _ob_set_ui16(__WHENCE__, b, off, v) +#define ob_patch_rfs(b) _ob_patch_rfs(__WHENCE__, b) + #define ob_append_byte(b, byte) _ob_append_byte(__WHENCE__, b, byte) #define ob_append_bytes(b, bytes, count) _ob_append_bytes(__WHENCE__, b, bytes, count) #define ob_append_buffer(b, s) _ob_append_buffer(__WHENCE__, b, s) @@ -78,9 +94,9 @@ int _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l); #define ob_append_packed_ui64(b, v) _ob_append_packed_ui64(__WHENCE__, b, v) #define ob_append_rfs(b, l) _ob_append_rfs(__WHENCE__, b, l) -int ob_patch_rfs(struct overlay_buffer *b); // get one byte, -ve number indicates failure -int ob_getbyte(struct overlay_buffer *b,int ofs); +int ob_peek(struct overlay_buffer *b); +void ob_skip(struct overlay_buffer *b, unsigned n); // get one byte from the current position, -ve number indicates failure int ob_get(struct overlay_buffer *b); int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len); @@ -89,7 +105,6 @@ uint64_t ob_get_ui64(struct overlay_buffer *b); uint32_t ob_get_ui32(struct overlay_buffer *b); uint16_t ob_get_ui16(struct overlay_buffer *b); int ob_dump(struct overlay_buffer *b,char *desc); -int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v); uint32_t ob_get_packed_ui32(struct overlay_buffer *b); uint64_t ob_get_packed_ui64(struct overlay_buffer *b); @@ -98,5 +113,9 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b); int ob_position(struct overlay_buffer *b); int ob_limit(struct overlay_buffer *b); int ob_remaining(struct overlay_buffer *b); +int _ob_overrun(struct __sourceloc, struct overlay_buffer *b); unsigned char* ob_ptr(struct overlay_buffer *b); -#endif + +#define ob_overrun(b) _ob_overrun(__WHENCE__, b) + +#endif //__SERVALD__OVERLAY_BUFFER_H diff --git a/overlay_interface.c b/overlay_interface.c index e67570ff..db34ddc8 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -19,18 +19,20 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include -#include #include #include #include #include #include "serval.h" #include "conf.h" +#include "net.h" +#include "socket.h" #include "strbuf.h" #include "strbuf_helpers.h" #include "overlay_buffer.h" #include "overlay_packet.h" #include "str.h" +#include "radio_link.h" #ifdef HAVE_IFADDRS_H #include @@ -49,7 +51,6 @@ struct profile_total sock_any_stats; static void overlay_interface_poll(struct sched_ent *alarm); static int re_init_socket(int interface_index); -static void write_stream_buffer(overlay_interface *interface); static void overlay_interface_close(overlay_interface *interface){ @@ -59,6 +60,8 @@ overlay_interface_close(overlay_interface *interface){ unschedule(&interface->alarm); unwatch(&interface->alarm); close(interface->alarm.poll.fd); + if (interface->radio_link_state) + radio_link_free(interface); interface->alarm.poll.fd=-1; interface->state=INTERFACE_STATE_DOWN; } @@ -76,8 +79,7 @@ void interface_state_html(struct strbuf *b, struct overlay_interface *interface) switch(interface->type){ case OVERLAY_INTERFACE_PACKETRADIO: strbuf_puts(b, "Type: Packet Radio
"); - strbuf_sprintf(b, "RSSI: %ddB
",interface->radio_rssi); - strbuf_sprintf(b, "Remote RSSI: %ddB
",interface->remote_rssi); + radio_link_state_html(b, interface); break; case OVERLAY_INTERFACE_ETHERNET: strbuf_puts(b, "Type: Ethernet
"); @@ -248,37 +250,39 @@ int overlay_interface_compare(overlay_interface *one, overlay_interface *two) // OSX doesn't recieve broadcast packets on sockets bound to an interface's address // So we have to bind a socket to INADDR_ANY to receive these packets. static void -overlay_interface_read_any(struct sched_ent *alarm){ +overlay_interface_read_any(struct sched_ent *alarm) +{ if (alarm->poll.revents & POLLIN) { int plen=0; int recvttl=1; unsigned char packet[16384]; overlay_interface *interface=NULL; - struct sockaddr src_addr; - socklen_t addrlen = sizeof(src_addr); + struct socket_address recvaddr; + recvaddr.addrlen = sizeof recvaddr.store; /* Read only one UDP packet per call to share resources more fairly, and also enable stats to accurately count packets received */ - plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &src_addr, &addrlen); + plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &recvaddr); if (plen == -1) { - WHY_perror("recvwithttl(c)"); + WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))", + alarm->poll.fd, packet, sizeof packet, recvttl, + &recvaddr, alloca_socket_address(&recvaddr) + ); unwatch(alarm); close(alarm->poll.fd); return; } - struct in_addr src = ((struct sockaddr_in *)&src_addr)->sin_addr; - /* Try to identify the real interface that the packet arrived on */ - interface = overlay_interface_find(src, 0); + interface = overlay_interface_find(recvaddr.inet.sin_addr, 0); /* Drop the packet if we don't find a match */ if (!interface){ if (config.debug.overlayinterfaces) - DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(src)); + DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(recvaddr.inet.sin_addr)); return; } - packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen); + packetOkOverlay(interface, packet, plen, recvttl, &recvaddr); } if (alarm->poll.revents & (POLLHUP | POLLERR)) { INFO("Closing broadcast socket due to error"); @@ -411,8 +415,6 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr set_destination_ref(&interface->destination, NULL); interface->destination = new_destination(interface, ifconfig->encapsulation); - interface->throttle_bytes_per_second = ifconfig->throttle; - interface->throttle_burst_write_size = ifconfig->burst_size; /* Pick a reasonable default MTU. This will ultimately get tuned by the bandwidth and other properties of the interface */ interface->mtu = 1200; @@ -532,10 +534,7 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr switch (ifconfig->socket_type) { case SOCK_STREAM: - interface->slip_decode_state.dst_offset=0; - /* The encapsulation type should be configurable, but for now default to the one that should - be safe on the RFD900 radios, and that also allows us to receive RSSI reports inline */ - interface->slip_decode_state.encapsulator=SLIP_FORMAT_MAVLINK; + radio_link_init(interface); interface->alarm.poll.events=POLLIN|POLLOUT; watch(&interface->alarm); @@ -574,25 +573,27 @@ cleanup: return cleanup_ret; } -static void interface_read_dgram(struct overlay_interface *interface){ +static void interface_read_dgram(struct overlay_interface *interface) +{ int plen=0; unsigned char packet[8096]; - struct sockaddr src_addr; - socklen_t addrlen = sizeof(src_addr); - - + struct socket_address recvaddr; + recvaddr.addrlen = sizeof recvaddr.store; + /* Read only one UDP packet per call to share resources more fairly, and also enable stats to accurately count packets received */ int recvttl=1; - plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &src_addr, &addrlen); + plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &recvaddr); if (plen == -1) { - WHY_perror("recvwithttl(c)"); + WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))", + interface->alarm.poll.fd, packet, sizeof packet, recvttl, + &recvaddr, alloca_socket_address(&recvaddr) + ); overlay_interface_close(interface); return; } - - packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen); + packetOkOverlay(interface, packet, plen, recvttl, &recvaddr); } struct file_packet{ @@ -658,9 +659,6 @@ static void interface_read_file(struct overlay_interface *interface) return; } - if (config.debug.overlayinterfaces) - DEBUGF("Read interface %s (size=%"PRId64") at offset=%d",interface->name, (int64_t)length, interface->recv_offset); - ssize_t nread = read(interface->alarm.poll.fd, &packet, sizeof packet); if (nread == -1){ WHY_perror("read"); @@ -669,14 +667,27 @@ static void interface_read_file(struct overlay_interface *interface) } if (nread == sizeof packet) { + if (config.debug.overlayinterfaces) + DEBUGF("Read from interface %s (filesize=%"PRId64") at offset=%d: src_addr=%s dst_addr=%s pid=%d length=%d", + interface->name, (int64_t)length, interface->recv_offset, + alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr), + alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr), + packet.pid, + packet.payload_length + ); interface->recv_offset += nread; if (should_drop(interface, packet.dst_addr) || (packet.pid == getpid() && !interface->local_echo)){ if (config.debug.packetrx) - DEBUGF("Ignoring packet from %d, addressed to %s:%d", packet.pid, - inet_ntoa(packet.dst_addr.sin_addr), ntohs(packet.dst_addr.sin_port)); + DEBUGF("Ignoring packet from pid=%d src_addr=%s dst_addr=%s", + packet.pid, + alloca_sockaddr_in(&packet.src_addr), + alloca_sockaddr_in(&packet.dst_addr) + ); }else{ - packetOkOverlay(interface, packet.payload, packet.payload_length, -1, - (struct sockaddr*)&packet.src_addr, (socklen_t) sizeof(packet.src_addr)); + struct socket_address srcaddr; + srcaddr.addrlen = sizeof packet.src_addr; + srcaddr.inet = packet.src_addr; + packetOkOverlay(interface, packet.payload, packet.payload_length, -1, &srcaddr); } } } @@ -710,101 +721,15 @@ static void interface_read_stream(struct overlay_interface *interface){ OUT(); return; } - struct slip_decode_state *state=&interface->slip_decode_state; + int i; for (i=0;ithrottle_burst_write_size; - - int total_written=0; - while (interface->tx_bytes_pending>0 || interface->tx_packet || interface->next_heartbeat <= now) { - - if (interface->tx_bytes_pending==0){ - if (interface->next_heartbeat <= now){ - // Queue a hearbeat now - mavlink_heartbeat(interface->txbuffer,&interface->tx_bytes_pending); - if (config.debug.packetradio) - DEBUGF("Sending heartbeat"); - interface->next_heartbeat = now+1000; - }else if(interface->tx_packet && interface->remaining_space >= 256 + 8+9){ - // prepare a new link layer packet in txbuffer - if (mavlink_encode_packet(interface)) - break; - if (interface->remaining_space - interface->tx_bytes_pending < 256 + 8+9) - interface->next_heartbeat = now; - } - } - - if (interface->next_tx_allowed > now) - break; - - int bytes = interface->tx_bytes_pending; - if (interface->throttle_burst_write_size && bytes>bytes_allowed) - bytes=bytes_allowed; - if (bytes<=0) - break; - - int written=write(interface->alarm.poll.fd, interface->txbuffer, bytes); - if (written<=0){ - DEBUGF("Blocking for POLLOUT"); - break; - } - - interface->remaining_space-=written; - interface->tx_bytes_pending-=written; - total_written+=written; - bytes_allowed-=written; - if (interface->tx_bytes_pending){ - bcopy(&interface->txbuffer[written],&interface->txbuffer[0], - interface->tx_bytes_pending); - DEBUGF("Partial write, %d left", interface->tx_bytes_pending); - } - } - - if (total_written>0){ - // Now when are we allowed to send more? - int rate = interface->throttle_bytes_per_second; - if (interface->remaining_space<=0) - rate = 600; - if (rate){ - int delay = total_written*1000/rate; - if (config.debug.throttling) - DEBUGF("Throttling for %dms (%d).", delay, interface->remaining_space); - interface->next_tx_allowed = now + delay; - } - } - - time_ms_t next_write = interface->next_tx_allowed; - if (interface->tx_bytes_pending<=0){ - next_write = interface->next_heartbeat; - } - - if (interface->alarm.alarm==-1 || next_write < interface->alarm.alarm){ - interface->alarm.alarm = next_write; - interface->alarm.deadline = interface->alarm.alarm+10; - } - - if (interface->tx_bytes_pending>0 && next_write <= now){ - // more to write, so set the POLLOUT flag - interface->alarm.poll.events|=POLLOUT; - } else { - // Nothing to write, so clear POLLOUT flag - interface->alarm.poll.events&=~POLLOUT; - } - watch(&interface->alarm); -} - - static void overlay_interface_poll(struct sched_ent *alarm) { struct overlay_interface *interface = (overlay_interface *)alarm; @@ -816,7 +741,7 @@ static void overlay_interface_poll(struct sched_ent *alarm) if (interface->state==INTERFACE_STATE_UP && interface->destination->tick_ms>0 && interface->send_broadcasts - && !interface->tx_packet){ + && !radio_link_is_busy(interface)){ if (now >= interface->destination->last_tx+interface->destination->tick_ms) overlay_send_tick_packet(interface->destination); @@ -827,8 +752,8 @@ static void overlay_interface_poll(struct sched_ent *alarm) switch(interface->socket_type){ case SOCK_STREAM: - write_stream_buffer(interface); - break; + radio_link_tx(interface); + return; case SOCK_DGRAM: break; case SOCK_FILE: @@ -848,14 +773,8 @@ static void overlay_interface_poll(struct sched_ent *alarm) if (alarm->poll.revents & POLLOUT){ switch(interface->socket_type){ case SOCK_STREAM: - write_stream_buffer(interface); - if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) { - if (alarm->alarm < now) - alarm->alarm = now; - unschedule(alarm); - schedule(alarm); - } - break; + radio_link_tx(interface); + return; case SOCK_DGRAM: case SOCK_FILE: //XXX error? fatal? @@ -871,14 +790,9 @@ static void overlay_interface_poll(struct sched_ent *alarm) case SOCK_STREAM: interface_read_stream(interface); // if we read a valid heartbeat packet, we may be able to write more bytes now. - if (interface->state==INTERFACE_STATE_UP && interface->remaining_space>0){ - write_stream_buffer(interface); - if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) { - if (alarm->alarm < now) - alarm->alarm = now; - unschedule(alarm); - schedule(alarm); - } + if (interface->state==INTERFACE_STATE_UP){ + radio_link_tx(interface); + return; } break; case SOCK_FILE: @@ -918,24 +832,7 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o switch(interface->socket_type){ case SOCK_STREAM: - { - if (interface->tx_packet){ - ob_free(buffer); - return WHYF("Cannot send two packets to a stream at the same time"); - } - - // prepare the buffer for reading - ob_flip(buffer); - interface->tx_packet = buffer; - write_stream_buffer(interface); - - if (interface->alarm.alarm!=-1){ - unschedule(&interface->alarm); - schedule(&interface->alarm); - } - - return 0; - } + return radio_link_queue_packet(interface, buffer); case SOCK_FILE: { @@ -962,9 +859,21 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o not support seeking. */ if (errno != ESPIPE) return WHY_perror("lseek"); - DEBUGF("Write to interface %s at unknown offset", interface->name); + DEBUGF("Write to interface %s at offset unknown: src_addr=%s dst_addr=%s pid=%d length=%d", + interface->name, + alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr), + alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr), + packet.pid, + packet.payload_length + ); } else - DEBUGF("Write to interface %s at offset=%"PRId64, interface->name, (int64_t)fsize); + DEBUGF("Write to interface %s at offset=%"PRId64": src_addr=%s dst_addr=%s pid=%d length=%d", + interface->name, (int64_t)fsize, + alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr), + alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr), + packet.pid, + packet.payload_length + ); } ssize_t nwrite = write(interface->alarm.poll.fd, &packet, sizeof(packet)); if (nwrite == -1) @@ -977,13 +886,22 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o case SOCK_DGRAM: { if (config.debug.overlayinterfaces) - DEBUGF("Sending %d byte overlay frame on %s to %s",len,interface->name,inet_ntoa(destination->address.sin_addr)); - int sent=sendto(interface->alarm.poll.fd, - bytes, len, 0, + DEBUGF("Sending %zu byte overlay frame on %s to %s", (size_t)len, interface->name, inet_ntoa(destination->address.sin_addr)); + ssize_t sent = sendto(interface->alarm.poll.fd, + bytes, (size_t)len, 0, (struct sockaddr *)&destination->address, sizeof(destination->address)); ob_free(buffer); - if (sent!= len){ - WHY_perror("sendto(c)"); + if (sent == -1 || (size_t)sent != (size_t)len) { + if (sent == -1) + WHYF_perror("sendto(fd=%d,len=%zu,addr=%s) on interface %s", + interface->alarm.poll.fd, + (size_t)len, + alloca_sockaddr((struct sockaddr *)&destination->address, sizeof destination->address), + interface->name + ); + else + WHYF("sendto() sent %zu bytes of overlay frame (%zu) to interface %s (socket=%d)", + (size_t)sent, (size_t)len, interface->name, interface->alarm.poll.fd); // close the interface if we had any error while sending broadcast packets, // unicast packets should not bring the interface down if (destination == interface->destination) diff --git a/overlay_link.c b/overlay_link.c index 21b42f75..781b456c 100644 --- a/overlay_link.c +++ b/overlay_link.c @@ -1,9 +1,11 @@ +#include #include "serval.h" #include "conf.h" #include "str.h" #include "overlay_address.h" #include "overlay_buffer.h" #include "overlay_packet.h" +#include "keyring.h" #define MIN_BURST_LENGTH 5000 @@ -221,14 +223,14 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest frame->ttl=1; frame->queue=queue; frame->destinations[frame->destination_count++].destination=add_destination_ref(destination); - frame->payload = ob_new(); - frame->source_full = 1; - // TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch... - - if (overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE)){ + if ((frame->payload = ob_new()) == NULL) { op_free(frame); return -1; } + frame->source_full = 1; + // TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch... + + overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE); // not worried about byte order here as we are the only node that should be parsing the contents. unsigned char *dst=ob_append_space(frame->payload, sizeof(struct probe_contents)); if (!dst){ @@ -253,25 +255,21 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest } // append the address of a unicast link into a packet buffer -static int overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff) +static void overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff) { - if (subscriber->destination - && subscriber->destination->unicast - && subscriber->destination->address.sin_family==AF_INET){ - if (overlay_address_append(NULL, buff, subscriber)) - return -1; - if (ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr)) - return -1; - if (ob_append_ui16(buff, subscriber->destination->address.sin_port)) - return -1; - ob_checkpoint(buff); + if ( subscriber->destination + && subscriber->destination->unicast + && subscriber->destination->address.sin_family==AF_INET + ) { + overlay_address_append(NULL, buff, subscriber); + ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr); + ob_append_ui16(buff, subscriber->destination->address.sin_port); if (config.debug.overlayrouting) DEBUGF("Added STUN info for %s", alloca_tohex_sid_t(subscriber->sid)); }else{ if (config.debug.overlayrouting) DEBUGF("Unable to give address of %s, %d", alloca_tohex_sid_t(subscriber->sid),subscriber->reachable); } - return 0; } int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp) @@ -295,29 +293,27 @@ int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp) struct overlay_buffer *replypayload = ob_static(reply.out.payload, sizeof(reply.out.payload)); ob_checkpoint(replypayload); - while(ob_remaining(payload)>0){ + while (ob_remaining(payload) > 0) { struct subscriber *subscriber=NULL; - if (overlay_address_parse(NULL, payload, &subscriber)) break; - if (!subscriber){ if (config.debug.overlayrouting) DEBUGF("Unknown subscriber"); continue; } - - if (overlay_append_unicast_address(subscriber, replypayload)) + overlay_append_unicast_address(subscriber, replypayload); + if (ob_overrun(payload)) break; + ob_checkpoint(replypayload); } - ob_rewind(replypayload); reply.out.payload_length=ob_position(replypayload); if (reply.out.payload_length){ if (config.debug.overlayrouting) DEBUGF("Sending reply"); - overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0); + overlay_mdp_dispatch(&reply, NULL); } ob_free(replypayload); ob_free(payload); @@ -387,11 +383,12 @@ int overlay_send_stun_request(struct subscriber *server, struct subscriber *requ struct overlay_buffer *payload = ob_static(mdp.out.payload, sizeof(mdp.out.payload)); overlay_address_append(NULL, payload, request); - mdp.out.payload_length=ob_position(payload); - if (config.debug.overlayrouting) - DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid)); - overlay_mdp_dispatch(&mdp,0 /* system generated */, - NULL,0); + if (!ob_overrun(payload)) { + mdp.out.payload_length=ob_position(payload); + if (config.debug.overlayrouting) + DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid)); + overlay_mdp_dispatch(&mdp, NULL); + } ob_free(payload); return 0; } diff --git a/overlay_mdp.c b/overlay_mdp.c index 393a09cf..e6089cb4 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -29,9 +29,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "overlay_packet.h" #include "mdp_client.h" #include "crypto.h" +#include "keyring.h" +#include "socket.h" static void overlay_mdp_poll(struct sched_ent *alarm); static void mdp_poll2(struct sched_ent *alarm); +static int overlay_mdp_releasebindings(struct socket_address *client); static struct profile_total mdp_stats = { .name="overlay_mdp_poll" }; static struct sched_ent mdp_sock = { @@ -48,6 +51,8 @@ static struct sched_ent mdp_sock2 = { }; static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame *mdp, time_ms_t now); +static int mdp_send2(struct socket_address *client, struct mdp_header *header, + const uint8_t *payload, size_t payload_len); /* Delete all UNIX socket files in instance directory. */ static void overlay_mdp_clean_socket_files() @@ -76,23 +81,22 @@ static void overlay_mdp_clean_socket_files() static int mdp_bind_socket(const char *name) { - struct sockaddr_un addr; - socklen_t addrlen; + struct socket_address addr; int sock; - if (make_local_sockaddr(&addr, &addrlen, "%s", name) == -1) + if (make_local_sockaddr(&addr, "%s", name) == -1) return -1; - if ((sock = esocket(AF_UNIX, SOCK_DGRAM, 0)) == -1) + if ((sock = esocket(addr.addr.sa_family, SOCK_DGRAM, 0)) == -1) return -1; if (socket_set_reuseaddr(sock, 1) == -1) WARN("Could not set socket to reuse addresses"); - if (socket_bind(sock, (struct sockaddr *)&addr, addrlen) == -1) { + if (socket_bind(sock, &addr.addr, addr.addrlen) == -1) { close(sock); return -1; } socket_set_rcvbufsize(sock, 64 * 1024); - INFOF("Socket %s: fd=%d %s", name, sock, alloca_sockaddr(&addr, addrlen)); + INFOF("Socket %s: fd=%d %s", name, sock, alloca_socket_address(&addr)); return sock; } @@ -121,17 +125,49 @@ int overlay_mdp_setup_sockets() struct mdp_binding{ struct subscriber *subscriber; mdp_port_t port; - char socket_name[MDP_MAX_SOCKET_NAME_LEN]; - int name_len; + int version; + int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len); + struct socket_address client; time_ms_t binding_time; }; struct mdp_binding mdp_bindings[MDP_MAX_BINDINGS]; int mdp_bindings_initialised=0; +mdp_port_t next_port_binding=256; -int overlay_mdp_reply_error(int sock, - struct sockaddr_un *recvaddr, socklen_t recvaddrlen, - int error_number,char *message) +static int compare_client(struct socket_address *one, struct socket_address *two) +{ + if (one->addrlen==two->addrlen + && memcmp(&one->addr, &two->addr, two->addrlen)==0) + return 1; + return 0; +} + +static int overlay_mdp_reply(int sock, struct socket_address *client, + overlay_mdp_frame *mdpreply) +{ + if (!client) return WHY("No reply address"); + + ssize_t replylen = overlay_mdp_relevant_bytes(mdpreply); + if (replylen<0) return WHY("Invalid MDP frame (could not compute length)"); + + ssize_t r=sendto(sock,(char *)mdpreply,replylen,0, &client->addr, client->addrlen); + if (r == -1){ + WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_socket_address(client)); + if (errno == ENOENT){ + /* far-end of socket has died, so drop binding */ + INFOF("Closing dead MDP client '%s'", alloca_socket_address(client)); + overlay_mdp_releasebindings(client); + } + return -1; + } + if (r != replylen) + return WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)r, (size_t)replylen, alloca_socket_address(client)); + return 0; +} + +static int overlay_mdp_reply_error(int sock, struct socket_address *client, + int error_number, char *message) { overlay_mdp_frame mdpreply; @@ -147,50 +183,29 @@ int overlay_mdp_reply_error(int sock, } mdpreply.error.message[127]=0; - return overlay_mdp_reply(sock,recvaddr,recvaddrlen,&mdpreply); + return overlay_mdp_reply(sock, client, &mdpreply); } -int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen, - overlay_mdp_frame *mdpreply) -{ - if (!recvaddr) return WHY("No reply address"); - - ssize_t replylen = overlay_mdp_relevant_bytes(mdpreply); - if (replylen<0) return WHY("Invalid MDP frame (could not compute length)"); - - errno=0; - int r=sendto(sock,(char *)mdpreply,replylen,0, - (struct sockaddr *)recvaddr,recvaddrlen); - if (rsun_path,recvaddrlen)) - mdp_bindings[i].port=0; + if (compare_client(&mdp_bindings[i].client, client)) + mdp_bindings[i].port=0; return 0; } -int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, mdp_port_t port, - int flags, struct sockaddr_un *recvaddr, socklen_t recvaddrlen) +static int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, mdp_port_t port, + int flags, struct socket_address *client) { if (config.debug.mdprequests) DEBUGF("Bind request %s:%"PRImdp_port_t, subscriber ? alloca_tohex_sid_t(subscriber->sid) : "NULL", port); @@ -213,8 +228,7 @@ int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, md for(i=0;isun_path,recvaddrlen)) { + if (compare_client(&mdp_bindings[i].client, client)) { // this client already owns this port binding? INFO("Identical binding exists"); return 0; @@ -250,13 +264,56 @@ int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, md /* Okay, record binding and report success */ mdp_bindings[free].port=port; mdp_bindings[free].subscriber=subscriber; - - mdp_bindings[free].name_len = recvaddrlen - sizeof recvaddr->sun_family; - memcpy(mdp_bindings[free].socket_name, recvaddr->sun_path, mdp_bindings[free].name_len); + mdp_bindings[free].version=0; + mdp_bindings[free].client.addrlen = client->addrlen; + memcpy(&mdp_bindings[free].client.addr, &client->addr, client->addrlen); mdp_bindings[free].binding_time=gettime_ms(); return 0; } +int mdp_bind_internal(struct subscriber *subscriber, mdp_port_t port, + int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len)) +{ + + int i; + struct mdp_binding *free_slot=NULL; + + for(i=0;isubscriber=subscriber; + free_slot->port=port; + free_slot->version=1; + free_slot->internal=internal; + free_slot->binding_time=gettime_ms(); + return 0; +} + +int mdp_unbind_internal(struct subscriber *subscriber, mdp_port_t port, + int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len)) +{ + int i; + for(i=0;iin.src.port = port; - - int len=ob_remaining(buff); - + int len = ob_remaining(buff); if (len<0) return WHY("MDP payload is too short"); mdp->in.payload_length=len; @@ -282,7 +337,6 @@ int overlay_mdp_decrypt(struct overlay_frame *f, overlay_mdp_frame *mdp) /* Indicate MDP message type */ mdp->packetTypeAndFlags=MDP_TX; - switch(f->modifiers&(OF_CRYPTO_CIPHERED|OF_CRYPTO_SIGNED)) { case 0: /* nothing to do, b already points to the plain text */ @@ -316,10 +370,8 @@ int overlay_mdp_decrypt(struct overlay_frame *f, overlay_mdp_frame *mdp) if (!k) RETURN(WHY("I don't have the private key required to decrypt that")); - if (0){ - dump("frame",&f->payload->bytes[f->payload->position], - ob_remaining(f->payload)); - } + if (0) + dump("frame",&f->payload->bytes[f->payload->position], ob_remaining(f->payload)); unsigned char *nonce=ob_get_bytes_ptr(f->payload, nb); if (!nonce) @@ -405,68 +457,86 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame int i; int match=-1; - switch(mdp->packetTypeAndFlags&MDP_TYPE_MASK) { - case MDP_TX: - /* Regular MDP frame addressed to us. Look for matching port binding, - and if available, push to client. Else do nothing, or if we feel nice - send back a connection refused type message? Silence is probably the - more prudent path. - */ + /* Regular MDP frame addressed to us. Look for matching port binding, + and if available, push to client. Else do nothing, or if we feel nice + send back a connection refused type message? Silence is probably the + more prudent path. + */ - if (config.debug.mdprequests) - DEBUGF("Received packet with listener (MDP ports: src=%s*:%"PRImdp_port_t", dst=%"PRImdp_port_t")", - alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14), - mdp->out.src.port, mdp->out.dst.port); + if (config.debug.mdprequests) + DEBUGF("Received packet with listener (MDP ports: src=%s*:%"PRImdp_port_t", dst=%"PRImdp_port_t")", + alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14), + mdp->out.src.port, mdp->out.dst.port); - // TODO pass in dest subscriber as an argument, we should know it by now - struct subscriber *destination = NULL; - if (frame) - destination = frame->destination; - else if (!is_sid_t_broadcast(mdp->out.dst.sid)){ - destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1); + // TODO pass in dest subscriber as an argument, we should know it by now + struct subscriber *destination = NULL; + if (frame) + destination = frame->destination; + else if (!is_sid_t_broadcast(mdp->out.dst.sid)){ + destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1); + } + + for(i=0;iout.dst.port) + continue; + + if ((!destination) || mdp_bindings[i].subscriber == destination){ + /* exact match, so stop searching */ + match=i; + break; + }else if (!mdp_bindings[i].subscriber){ + /* If we find an "ANY" binding, remember it. But we will prefer an exact match if we find one */ + match=i; + } } - - for(i=0;iout.dst.port) - continue; - - if ((!destination) || mdp_bindings[i].subscriber == destination){ - /* exact match, so stop searching */ - match=i; - break; - }else if (!mdp_bindings[i].subscriber){ - /* If we find an "ANY" binding, remember it. But we will prefer an exact match if we find one */ - match=i; + + if (match>-1) { + switch(mdp_bindings[match].version){ + case 0: + { + ssize_t len = overlay_mdp_relevant_bytes(mdp); + if (len < 0) + RETURN(WHY("unsupported MDP packet type")); + struct socket_address *client = &mdp_bindings[match].client; + ssize_t r = sendto(mdp_sock.poll.fd,mdp,len,0, &client->addr, client->addrlen); + if (r == -1){ + WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_socket_address(client)); + if (errno == ENOENT){ + /* far-end of socket has died, so drop binding */ + INFOF("Closing dead MDP client '%s'", alloca_socket_address(client)); + overlay_mdp_releasebindings(client); + } + RETURN(-1); + } + if (r != len) + RETURN(WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)r, (size_t)len, alloca_socket_address(client))); + RETURN(0); + } + case 1: + { + struct mdp_header header; + header.local.sid=mdp->out.dst.sid; + header.local.port=mdp->out.dst.port; + header.remote.sid=mdp->out.src.sid; + header.remote.port=mdp->out.src.port; + header.qos=mdp->out.queue; + header.ttl=mdp->out.ttl; + header.flags=0; + if (mdp->packetTypeAndFlags & MDP_NOCRYPT) + header.flags|=MDP_FLAG_NO_CRYPT; + if (mdp->packetTypeAndFlags & MDP_NOSIGN) + header.flags|=MDP_FLAG_NO_SIGN; + + if (mdp_bindings[match].internal) + RETURN(mdp_bindings[match].internal(&header, mdp->out.payload, mdp->out.payload_length)); + + RETURN(mdp_send2(&mdp_bindings[match].client, &header, mdp->out.payload, mdp->out.payload_length)); } - } - - if (match>-1) { - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - bcopy(mdp_bindings[match].socket_name, addr.sun_path, mdp_bindings[match].name_len); - ssize_t len = overlay_mdp_relevant_bytes(mdp); - if (len < 0) - RETURN(WHY("unsupported MDP packet type")); - socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len; - int r = sendto(mdp_sock.poll.fd,mdp,len,0,(struct sockaddr*)&addr, addrlen); - if (r == len) - RETURN(0); - if (r == -1 && errno == ENOENT) { - /* far-end of socket has died, so drop binding */ - INFOF("Closing dead MDP client '%s'",mdp_bindings[match].socket_name); - overlay_mdp_releasebindings(&addr,mdp_bindings[match].name_len); - } - WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_sockaddr(&addr, addrlen)); - RETURN(WHY("Failed to pass received MDP frame to client")); - } else { - /* No socket is bound, ignore the packet ... except for magic sockets */ - RETURN(overlay_mdp_try_interal_services(frame, mdp)); } - break; - default: - RETURN(WHYF("We should only see MDP_TX frames here (MDP message type = 0x%x)", - mdp->packetTypeAndFlags)); + } else { + /* No socket is bound, ignore the packet ... except for magic sockets */ + RETURN(overlay_mdp_try_internal_services(frame, mdp)); } RETURN(0); @@ -475,6 +545,13 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name) { + if (config.debug.mdprequests) + DEBUGF("MDP_PORT_DNALOOKUP resolved_sid=%s uri=%s did=%s name=%s", + alloca_tohex_sid_t(*resolved_sidp), + alloca_str_toprint(uri), + alloca_str_toprint(did), + alloca_str_toprint(name) + ); overlay_mdp_frame mdpreply; bzero(&mdpreply, sizeof mdpreply); mdpreply.packetTypeAndFlags = MDP_TX; // outgoing MDP message @@ -490,14 +567,14 @@ int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolv return WHY("MDP payload overrun"); mdpreply.out.payload_length = strbuf_len(b) + 1; /* deliver reply */ - return overlay_mdp_dispatch(&mdpreply, 0 /* system generated */, NULL, 0); + return overlay_mdp_dispatch(&mdpreply, NULL); } -int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, int userGeneratedFrameP, - struct sockaddr_un *recvaddr, socklen_t recvaddrlen) +static int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, + struct socket_address *client) { /* System generated frames can send anything they want */ - if (!userGeneratedFrameP) + if (!client) return 0; /* Check if the address is in the list of bound addresses, @@ -509,43 +586,35 @@ int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, in continue; if ((!mdp_bindings[i].subscriber) || mdp_bindings[i].subscriber == subscriber) { /* Binding matches, now make sure the sockets match */ - if ( mdp_bindings[i].name_len == recvaddrlen - sizeof(sa_family_t) - && memcmp(mdp_bindings[i].socket_name, recvaddr->sun_path, recvaddrlen - sizeof(sa_family_t)) == 0 - ) { + if (compare_client(&mdp_bindings[i].client, client)) { /* Everything matches, so this unix socket and MDP address combination is valid */ return 0; } } } - return WHYF("No such binding: recvaddr=%p %s addr=%s port=%"PRImdp_port_t" -- possible spoofing attack", - recvaddr, - recvaddr ? alloca_toprint(-1, recvaddr->sun_path, recvaddrlen - sizeof(sa_family_t)) : "", + return WHYF("No matching binding: addr=%s port=%"PRImdp_port_t" -- possible spoofing attack", alloca_tohex_sid_t(subscriber->sid), port ); } -int overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port) +void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port) { mdp_port_t port = dst_port << 1; if (dst_port == src_port) port |= 1; - if (ob_append_packed_ui32(plaintext, port)) - return -1; - if (dst_port != src_port){ - if (ob_append_packed_ui32(plaintext, src_port)) - return -1; - } - return 0; + ob_append_packed_ui32(plaintext, port); + if (dst_port != src_port) + ob_append_packed_ui32(plaintext, src_port); } static struct overlay_buffer * encrypt_payload( struct subscriber *source, struct subscriber *dest, - const unsigned char *buffer, int cipher_len){ - - int nm=crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES; + const unsigned char *buffer, + int cipher_len) +{ int zb=crypto_box_curve25519xsalsa20poly1305_ZEROBYTES; int nb=crypto_box_curve25519xsalsa20poly1305_NONCEBYTES; int cz=crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES; @@ -561,13 +630,15 @@ static struct overlay_buffer * encrypt_payload( cipher_len+=zb; struct overlay_buffer *ret = ob_new(); + if (ret == NULL) + return NULL; unsigned char *nonce = ob_append_space(ret, nb+cipher_len); - unsigned char *cipher_text = nonce + nb; if (!nonce){ ob_free(ret); return NULL; } + unsigned char *cipher_text = nonce + nb; if (generate_nonce(nonce,nb)){ ob_free(ret); @@ -588,47 +659,105 @@ static struct overlay_buffer * encrypt_payload( } /* Actually authcrypt the payload */ - if (crypto_box_curve25519xsalsa20poly1305_afternm - (cipher_text,plain,cipher_len,nonce,k)){ + if (crypto_box_curve25519xsalsa20poly1305_afternm(cipher_text, plain,cipher_len, nonce, k)) { ob_free(ret); WHY("crypto_box_afternm() failed"); return NULL; } - if (0) { +#if 0 + if (config.debug.crypto) { DEBUG("authcrypted mdp frame"); - dump("nm",k,nm); + dump("nm",k,crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES); dump("plain text",plain,sizeof(plain)); dump("nonce",nonce,nb); dump("cipher text",cipher_text,cipher_len); } +#endif - /* now shuffle down to get rid of the temporary space that crypto_box - uses. + /* now shuffle down to get rid of the temporary space that crypto_box uses. TODO extend overlay buffer so we don't need this. */ bcopy(&cipher_text[cz],&cipher_text[0],cipher_len-cz); ret->position-=cz; - if (0){ +#if 0 + if (config.debug.crypto) dump("frame", &ret->bytes[0], ret->position); - } +#endif return ret; } // encrypt or sign the plaintext, then queue the frame for transmission. -int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plaintext){ +static int overlay_send_frame( + struct subscriber *source, mdp_port_t src_port, + struct subscriber *destination, mdp_port_t dst_port, + const uint8_t *payload, size_t payload_len, + uint8_t ttl, uint8_t qos, uint8_t modifiers) +{ + if (destination && destination->reachable == REACHABLE_SELF) + return 0; - if (!frame->source) - frame->source = my_subscriber; + if (ttl == 0) + ttl = PAYLOAD_TTL_DEFAULT; + else if (ttl > PAYLOAD_TTL_MAX) + return WHYF("Invalid TTL"); + + if (qos == 0) + qos = OQ_ORDINARY; + if (!source) + return WHYF("No source specified"); + + /* Prepare the overlay frame for dispatch */ + struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame)); + if (!frame) + return -1; + + frame->source = source; + frame->destination = destination; + frame->ttl = ttl; + frame->queue = qos; + frame->type = OF_TYPE_DATA; + frame->modifiers = modifiers; + + // copy the plain text message into a new buffer, with the wire encoded port numbers + struct overlay_buffer *plaintext=ob_new(); + if (!plaintext){ + op_free(frame); + return -1; + } + + overlay_mdp_encode_ports(plaintext, dst_port, src_port); + if (payload && payload_len){ + ob_append_bytes(plaintext, payload, payload_len); + } + + if (ob_overrun(plaintext)) { + if (config.debug.mdprequests) + DEBUGF("Frame overrun: position=%d allocSize=%d sizeLimit=%d", + plaintext->position, plaintext->allocSize, plaintext->sizeLimit); + op_free(frame); + ob_free(plaintext); + return -1; + } + if (config.debug.mdprequests) { + DEBUGF("Send frame %u bytes", ob_position(plaintext)); + dump("Frame plaintext", ob_ptr(plaintext), ob_position(plaintext)); + } + /* Work out the disposition of the frame-> For now we are only worried about the crypto matters, and not compression that may be applied before encryption (since applying it after is useless as ciphered text should have maximum entropy). */ - switch(frame->modifiers) { - default: + switch(modifiers) { case OF_CRYPTO_SIGNED|OF_CRYPTO_CIPHERED: + if (!frame->destination){ + ob_free(plaintext); + op_free(frame); + return WHY("Cannot encrypt to broadcast destinations"); + } + /* crypted and signed (using CryptoBox authcryption primitive) */ frame->payload = encrypt_payload(frame->source, frame->destination, ob_ptr(plaintext), ob_position(plaintext)); if (!frame->payload){ @@ -636,22 +765,36 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain op_free(frame); return -1; } +#if 0 + if (config.debug.crypto) + dump("Frame signed ciphertext", ob_ptr(frame->payload), ob_position(frame->payload)); +#endif break; case OF_CRYPTO_SIGNED: // Lets just append some space into the existing payload buffer for the signature, without copying it. frame->payload = plaintext; - ob_makespace(frame->payload,SIGNATURE_BYTES); - if (crypto_sign_message(frame->source, ob_ptr(frame->payload), frame->payload->allocSize, &frame->payload->position)){ + if ( !ob_makespace(frame->payload, SIGNATURE_BYTES) + || crypto_sign_message(frame->source, ob_ptr(frame->payload), frame->payload->allocSize, &frame->payload->position) == -1 + ) { op_free(frame); return -1; } +#if 0 + if (config.debug.crypto) + dump("Frame signed plaintext", ob_ptr(frame->payload), ob_position(frame->payload)); +#endif break; case 0: /* clear text and no signature */ frame->payload = plaintext; break; + + default: + ob_free(plaintext); + op_free(frame); + return WHY("Invalid encrypt / sign combination"); } if (!frame->destination && frame->ttl>1) @@ -671,10 +814,19 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain This is for use by the SERVER. Clients should use overlay_mdp_send() */ -int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP, - struct sockaddr_un *recvaddr, socklen_t recvaddrlen) +int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client) { IN(); + unsigned __d = 0; + if (config.debug.mdprequests) { + __d = fd_depth(); + DEBUGF("[%u] src=%s*:%"PRImdp_port_t", dst=%s*:%"PRImdp_port_t", recv=%s", + __d, + alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14), mdp->out.src.port, + alloca_tohex_sid_t_trunc(mdp->out.dst.sid, 14), mdp->out.dst.port, + client ? alloca_socket_address(client) : "NULL" + ); + } if (mdp->out.payload_length > sizeof(mdp->out.payload)) FATAL("Payload length is past the end of the buffer"); @@ -701,48 +853,47 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP, } } - if (overlay_mdp_check_binding(source, mdp->out.src.port, userGeneratedFrameP, recvaddr, recvaddrlen)){ + if (overlay_mdp_check_binding(source, mdp->out.src.port, client)){ RETURN(overlay_mdp_reply_error (mdp_sock.poll.fd, - (struct sockaddr_un *)recvaddr, - recvaddrlen,8, + client,8, "Source address is invalid (you must bind to a source address before" " you can send packets")); } /* Work out if destination is broadcast or not */ if (is_sid_t_broadcast(mdp->out.dst.sid)){ + if (config.debug.mdprequests) + DEBUGF("[%u] Broadcast packet", __d); /* broadcast packets cannot be encrypted, so complain if MDP_NOCRYPT flag is not set. Also, MDP_NOSIGN must also be applied, until NaCl cryptobox keys can be used for signing. */ if (!(mdp->packetTypeAndFlags&MDP_NOCRYPT)) RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd, - recvaddr,recvaddrlen,5, + client,5, "Broadcast packets cannot be encrypted ")); }else{ destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1); // should we reply with an error if the destination is not currently routable? } - if (mdp->out.ttl == 0) - mdp->out.ttl = PAYLOAD_TTL_DEFAULT; - else if (mdp->out.ttl > PAYLOAD_TTL_MAX) { - RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd, recvaddr,recvaddrlen,9, "TTL out of range")); + if (mdp->out.ttl > PAYLOAD_TTL_MAX) { + RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd, client, 9, "TTL out of range")); } - if (mdp->out.queue == 0) - mdp->out.queue = OQ_ORDINARY; - + if (config.debug.mdprequests) + DEBUGF("[%u] destination->sid=%s", __d, destination ? alloca_tohex_sid_t(destination->sid) : "NULL"); if (!destination || destination->reachable == REACHABLE_SELF){ /* Packet is addressed to us / broadcast, we should process it first. */ overlay_saw_mdp_frame(NULL,mdp,gettime_ms()); if (destination) { - /* Is local, and is not broadcast, so shouldn't get sent out - on the wire. */ + /* Is local, and is not broadcast, so shouldn't get sent out on the wire. */ + if (config.debug.mdprequests) + DEBUGF("[%u] Local packet, not transmitting", __d); RETURN(0); } } - + int modifiers=0; switch(mdp->packetTypeAndFlags&(MDP_NOCRYPT|MDP_NOSIGN)) { @@ -768,37 +919,12 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP, RETURN(WHY("Not implemented")); }; - // copy the plain text message into a new buffer, with the wire encoded port numbers - struct overlay_buffer *plaintext=ob_new(); - if (!plaintext) - RETURN(-1); - - if (overlay_mdp_encode_ports(plaintext, mdp->out.dst.port, mdp->out.src.port)){ - ob_free(plaintext); - RETURN (-1); - } - if (ob_append_bytes(plaintext, mdp->out.payload, mdp->out.payload_length)){ - ob_free(plaintext); - RETURN(-1); - } - - /* Prepare the overlay frame for dispatch */ - struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame)); - if (!frame){ - ob_free(plaintext); - RETURN(-1); - } - - frame->source = source; - frame->destination = destination; - frame->ttl = mdp->out.ttl; - frame->queue = mdp->out.queue; - frame->type=OF_TYPE_DATA; - frame->prev=NULL; - frame->next=NULL; - frame->modifiers=modifiers; - - RETURN(overlay_send_frame(frame, plaintext)); + RETURN(overlay_send_frame( + source, mdp->out.src.port, + destination, mdp->out.dst.port, + mdp->out.payload, mdp->out.payload_length, + mdp->out.ttl, mdp->out.queue, modifiers + )); OUT(); } @@ -826,7 +952,8 @@ static int search_subscribers(struct subscriber *subscriber, void *context){ return 0; } -int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist *response){ +int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist *response) +{ if (config.debug.mdprequests) DEBUGF("MDP_GETADDRS first_sid=%u mode=%d", request->first_sid, request->mode); @@ -851,8 +978,7 @@ int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist } struct routing_state{ - struct sockaddr_un *recvaddr_un; - socklen_t recvaddrlen; + struct socket_address *client; int fd; }; @@ -876,7 +1002,7 @@ static int routing_table(struct subscriber *subscriber, void *context) strcpy(r->interface_name, subscriber->destination->interface->name); else r->interface_name[0]=0; - overlay_mdp_reply(mdp_sock.poll.fd, state->recvaddr_un, state->recvaddrlen, &reply); + overlay_mdp_reply(mdp_sock.poll.fd, state->client, &reply); return 0; } @@ -925,12 +1051,7 @@ static void overlay_mdp_scan(struct sched_ent *alarm) } } -struct mdp_client{ - struct sockaddr_un *addr; - socklen_t addrlen; -}; - -static int mdp_reply2(const struct mdp_client *client, const struct mdp_header *header, +static int mdp_reply2(const struct socket_address *client, const struct mdp_header *header, int flags, const unsigned char *payload, int payload_len) { struct mdp_header response_header; @@ -949,22 +1070,22 @@ static int mdp_reply2(const struct mdp_client *client, const struct mdp_header * }; struct msghdr hdr={ - .msg_name=client->addr, + .msg_name=(void *)&client->addr, .msg_namelen=client->addrlen, .msg_iov=iov, .msg_iovlen=2, }; if (config.debug.mdprequests) - DEBUGF("Replying to %s with code %d", alloca_sockaddr(client->addr, client->addrlen), flags); + DEBUGF("Replying to %s with flags %d", alloca_socket_address(client), flags); return sendmsg(mdp_sock2.poll.fd, &hdr, 0); } #define mdp_reply_error(A,B) mdp_reply2(A,B,MDP_FLAG_ERROR,NULL,0) -#define mdp_reply_ok(A,B) mdp_reply2(A,B,MDP_FLAG_OK,NULL,0) +#define mdp_reply_ok(A,B) mdp_reply2(A,B,MDP_FLAG_CLOSE,NULL,0) -static int mdp_process_identity_request(struct mdp_client *client, struct mdp_header *header, - const unsigned char *payload, int payload_len) +static int mdp_process_identity_request(struct socket_address *client, struct mdp_header *header, + const uint8_t *payload, size_t payload_len) { if (payload_lenpoll.revents & POLLIN) { - unsigned char buffer[1600]; - struct sockaddr_storage addr; - struct mdp_client client={ - .addr = (struct sockaddr_un *)&addr, - .addrlen = sizeof(addr) - }; - int ttl=-1; + int cn=0, in=0, kp=0; + const char *tag=NULL; + const unsigned char *value=NULL; + size_t value_len=0; + + if (payload_len){ + if (keyring_unpack_tag(payload, payload_len, &tag, &value, &value_len)){ + mdp_reply_error(client, header); + return -1; + } + } + + while(1){ + if (value_len){ + if (config.debug.mdprequests) + DEBUGF("Looking for next %s tag & value", tag); + if (!keyring_find_public_tag_value(keyring, &cn, &in, &kp, tag, value, value_len)) + break; + }else if(tag){ + if (config.debug.mdprequests) + DEBUGF("Looking for next %s tag", tag); + if (!keyring_find_public_tag(keyring, &cn, &in, &kp, tag, NULL, NULL)) + break; + }else{ + if (config.debug.mdprequests) + DEBUGF("Looking for next identity"); + if (!keyring_next_identity(keyring, &cn, &in, &kp)) + break; + } + keyring_identity *id = keyring->contexts[cn]->identities[in]; + unsigned char reply_payload[1200]; + size_t ofs=0; - ssize_t len = recvwithttl(alarm->poll.fd, buffer, sizeof(buffer), &ttl, (struct sockaddr *)&addr, &client.addrlen); - - if (len<=sizeof(struct mdp_header)){ - WHYF("Expected length %d, got %d from %s", (int)sizeof(struct mdp_header), (int)len, alloca_sockaddr(client.addr, client.addrlen)); + bcopy(id->subscriber->sid.binary, &reply_payload[ofs], sizeof(id->subscriber->sid)); + ofs+=sizeof(id->subscriber->sid); + + // TODO return other details of this identity + + mdp_reply2(client, header, 0, reply_payload, ofs); + kp++; + } + mdp_reply_ok(client, header); + return 0; +} + +static void mdp_process_packet(struct socket_address *client, struct mdp_header *header, + const uint8_t *payload, size_t payload_len) +{ + if ((header->flags & MDP_FLAG_CLOSE) && header->local.port==0){ + int i; + for(i=0;isid):"All", + mdp_bindings[i].port, + alloca_socket_address(client)); + mdp_bindings[i].port=0; + } + } + // should we expect clients to wait for this? + // mdp_reply_ok(client, header); + return; + } + + // find the source subscriber + struct subscriber *source=NULL; + + if (is_sid_t_broadcast(header->local.sid)){ + // leave source NULL to indicate listening on all local SID's + // note that attempting anything else will fail + }else if (is_sid_t_any(header->local.sid)){ + // leaving the sid blank indicates that we should use our main identity + source = my_subscriber; + header->local.sid = source->sid; + }else{ + // find the matching sid from our keyring + source = find_subscriber(header->local.sid.binary, sizeof(header->local.sid), 0); + if (!source || source->reachable != REACHABLE_SELF){ + mdp_reply_error(client, header); + WHY("Subscriber is not local"); + } + } + + struct mdp_binding *binding=NULL, *free_slot=NULL; + + // assign the next available port number + if (header->local.port==0 && header->flags & MDP_FLAG_BIND){ + if (next_port_binding > 32*1024) + next_port_binding=256; + else + next_port_binding++; + header->local.port=next_port_binding; + } + + // find matching binding + { + int i; + for(i=0;ilocal.port + && mdp_bindings[i].subscriber == source){ + + binding = &mdp_bindings[i]; + break; + } + } + } + + if (header->flags & MDP_FLAG_BIND){ + if (binding){ + mdp_reply_error(client, header); + WHYF("Port %d already bound", header->local.port); return; } - struct mdp_header *header = (struct mdp_header *)buffer; - - unsigned char *payload = &buffer[sizeof(struct mdp_header)]; - int payload_len = len - sizeof(struct mdp_header); - - if (is_sid_t_any(header->remote.sid)){ - // process local commands - switch(header->remote.port){ - case MDP_IDENTITY: - if (config.debug.mdprequests) - DEBUGF("Processing MDP_IDENTITY from %s", alloca_sockaddr(client.addr, client.addrlen)); - mdp_process_identity_request(&client, header, payload, payload_len); - break; - default: - mdp_reply_error(&client, header); - WHY("Unknown port number"); - break; - } - }else{ - // TODO transmit packet - mdp_reply_error(&client, header); - WHY("Transmitting packets is not yet supported"); + if (!free_slot){ + mdp_reply_error(client, header); + WHY("Max supported bindings reached"); + return; } + + if (config.debug.mdprequests) + DEBUGF("Bind MDP %s:%d to %s", + alloca_tohex_sid_t(header->local.sid), + header->local.port, + alloca_socket_address(client)); + + // claim binding + binding = free_slot; + binding->port = header->local.port; + binding->subscriber = source; + bcopy(&client->addr, &binding->client.addr, client->addrlen); + binding->client.addrlen = client->addrlen; + binding->binding_time=gettime_ms(); + binding->version=1; + + // tell the client what we actually bound (with flags & MDP_FLAG_BIND still set) + mdp_reply2(client, header, MDP_FLAG_BIND, NULL, 0); + } + + if (is_sid_t_any(header->remote.sid)){ + // process local commands + switch(header->remote.port){ + case MDP_LISTEN: + // double check that this binding belongs to this connection + if (!binding + || binding->internal + || !compare_client(&binding->client, client)) + mdp_reply_error(client, header); + break; + case MDP_IDENTITY: + if (config.debug.mdprequests) + DEBUGF("Processing MDP_IDENTITY from %s", alloca_socket_address(client)); + mdp_process_identity_request(client, header, payload, payload_len); + break; + // seach unlocked identities + case MDP_SEARCH_IDS: + if (config.debug.mdprequests) + DEBUGF("Processing MDP_SEARCH_IDS from %s", alloca_socket_address(client)); + mdp_search_identities(client, header, payload, payload_len); + break; + default: + mdp_reply_error(client, header); + WHYF("Unknown command port %d", header->remote.port); + break; + } + + }else{ + // double check that this binding belongs to this connection + if (!binding + || binding->internal + || !source + || header->local.port == 0 + || !compare_client(&binding->client, client)){ + mdp_reply_error(client, header); + WHY("No matching binding found"); + return; + } + + struct subscriber *destination=NULL; + if (!is_sid_t_broadcast(header->remote.sid)) + destination = find_subscriber(header->remote.sid.binary, SID_SIZE, 1); + + int modifiers=0; + if ((header->flags & MDP_FLAG_NO_CRYPT) == 0) + modifiers|=OF_CRYPTO_CIPHERED; + if ((header->flags & MDP_FLAG_NO_SIGN) == 0) + modifiers|=OF_CRYPTO_SIGNED; + + if (!destination || destination->reachable==REACHABLE_SELF){ + // TODO deprecate this mdp struct, deal with local delivery in send_frame + overlay_mdp_frame mdp; + mdp.out.src.sid = header->local.sid; + mdp.out.src.port = header->local.port; + mdp.out.dst.sid = header->remote.sid; + mdp.out.dst.port = header->remote.port; + bcopy(payload, mdp.out.payload, payload_len); + mdp.out.payload_length = payload_len; + mdp.out.ttl = header->ttl; + mdp.out.queue = header->qos; + mdp.packetTypeAndFlags=MDP_TX; + if (header->flags&MDP_FLAG_NO_CRYPT) + mdp.packetTypeAndFlags |= MDP_NOCRYPT; + if (header->flags&MDP_FLAG_NO_SIGN) + mdp.packetTypeAndFlags |= MDP_NOSIGN; + + if (config.debug.mdprequests) + DEBUGF("Attempting to process mdp packet locally"); + overlay_saw_mdp_frame(NULL, &mdp, gettime_ms()); + } + + if (config.debug.mdprequests) + DEBUGF("Attempting to queue mdp packet"); + + // construct, encrypt, sign and queue the packet + if (overlay_send_frame( + source, header->local.port, + destination, header->remote.port, + payload, payload_len, + header->ttl, header->qos, modifiers + )){ + mdp_reply_error(client, header); + return; + } + } + + // remove binding + if (binding + && !binding->internal + && header->flags & MDP_FLAG_CLOSE + && compare_client(&binding->client, client)){ + if (config.debug.mdprequests) + DEBUGF("Unbind MDP %s:%d from %s", + binding->subscriber?alloca_tohex_sid_t(binding->subscriber->sid):"All", + binding->port, + alloca_socket_address(client)); + binding->port=0; + binding=NULL; + } +} + +static int mdp_send2(struct socket_address *client, struct mdp_header *header, + const uint8_t *payload, size_t payload_len) +{ + struct iovec iov[]={ + { + .iov_base = (void *)header, + .iov_len = sizeof(struct mdp_header) + }, + { + .iov_base = (void *)payload, + .iov_len = payload_len + } + }; + + struct msghdr hdr={ + .msg_name=&client->addr, + .msg_namelen=client->addrlen, + .msg_iov=iov, + .msg_iovlen=2, + }; + + if (sendmsg(mdp_sock2.poll.fd, &hdr, 0)<0) + return WHY_perror("sendmsg"); + return 0; +} + +static void mdp_poll2(struct sched_ent *alarm) +{ + if (alarm->poll.revents & POLLIN) { + uint8_t payload[1200]; + struct mdp_header header; + struct socket_address client; + client.addrlen=sizeof(client.addr); + + struct iovec iov[]={ + { + .iov_base = (void *)&header, + .iov_len = sizeof(struct mdp_header) + }, + { + .iov_base = (void *)payload, + .iov_len = sizeof(payload) + } + }; + + struct msghdr hdr={ + .msg_name=&client.addr, + .msg_namelen=sizeof(client.store), + .msg_iov=iov, + .msg_iovlen=2, + }; + + ssize_t len = recvmsg(alarm->poll.fd, &hdr, 0); + if (len<0){ + WHY_perror("recvmsg"); + return; + } + if (lenpoll.revents & POLLIN) { unsigned char buffer[16384]; int ttl; - unsigned char recvaddrbuffer[1024]; - struct sockaddr *recvaddr=(struct sockaddr *)&recvaddrbuffer[0]; - socklen_t recvaddrlen=sizeof(recvaddrbuffer); - struct sockaddr_un *recvaddr_un=NULL; + struct socket_address client; + client.addrlen = sizeof client.store; ttl=-1; - bzero((void *)recvaddrbuffer,sizeof(recvaddrbuffer)); - ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, recvaddr, &recvaddrlen); - recvaddr_un=(struct sockaddr_un *)recvaddr; + ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, &client); + if (len == -1) + WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))", + alarm->poll.fd, buffer, sizeof buffer, ttl, + &client, alloca_socket_address(&client) + ); if (len > 0) { - if (recvaddrlen <= sizeof(sa_family_t)) - WHYF("got recvaddrlen=%d too short -- ignoring frame len=%zu", (int)recvaddrlen, (size_t)len); + if (client.addrlen <= sizeof(sa_family_t)) + WHYF("got client.addrlen=%d too short -- ignoring frame len=%zu", (int)client.addrlen, (size_t)len); else { /* Look at overlay_mdp_frame we have received */ overlay_mdp_frame *mdp=(overlay_mdp_frame *)&buffer[0]; @@ -1107,17 +1502,16 @@ static void overlay_mdp_poll(struct sched_ent *alarm) switch (mdp_type) { case MDP_GOODBYE: if (config.debug.mdprequests) - DEBUGF("MDP_GOODBYE from %s", alloca_sockaddr(recvaddr, recvaddrlen)); - overlay_mdp_releasebindings(recvaddr_un,recvaddrlen); + DEBUGF("MDP_GOODBYE from %s", alloca_socket_address(&client)); + overlay_mdp_releasebindings(&client); return; case MDP_ROUTING_TABLE: if (config.debug.mdprequests) - DEBUGF("MDP_ROUTING_TABLE from %s", alloca_sockaddr(recvaddr, recvaddrlen)); + DEBUGF("MDP_ROUTING_TABLE from %s", alloca_socket_address(&client)); { struct routing_state state={ - .recvaddr_un=recvaddr_un, - .recvaddrlen=recvaddrlen, + .client = &client, }; enum_subscribers(NULL, routing_table, &state); @@ -1127,7 +1521,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm) case MDP_GETADDRS: if (config.debug.mdprequests) - DEBUGF("MDP_GETADDRS from %s", alloca_sockaddr(recvaddr, recvaddrlen)); + DEBUGF("MDP_GETADDRS from %s", alloca_socket_address(&client)); { overlay_mdp_frame mdpreply; bzero(&mdpreply, sizeof(overlay_mdp_frame)); @@ -1135,7 +1529,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm) if (!overlay_mdp_address_list(&mdp->addrlist, &mdpreply.addrlist)) /* Send back to caller */ overlay_mdp_reply(alarm->poll.fd, - (struct sockaddr_un *)recvaddr,recvaddrlen, + &client, &mdpreply); return; @@ -1144,18 +1538,18 @@ static void overlay_mdp_poll(struct sched_ent *alarm) case MDP_TX: /* Send payload (and don't treat it as system privileged) */ if (config.debug.mdprequests) - DEBUGF("MDP_TX from %s", alloca_sockaddr(recvaddr, recvaddrlen)); + DEBUGF("MDP_TX from %s", alloca_socket_address(&client)); // Dont allow mdp clients to send very high priority payloads if (mdp->out.queue<=OQ_MESH_MANAGEMENT) mdp->out.queue=OQ_ORDINARY; - overlay_mdp_dispatch(mdp,1,(struct sockaddr_un*)recvaddr,recvaddrlen); + overlay_mdp_dispatch(mdp, &client); return; break; case MDP_BIND: /* Bind to port */ if (config.debug.mdprequests) - DEBUGF("MDP_BIND from %s", alloca_sockaddr(recvaddr, recvaddrlen)); + DEBUGF("MDP_BIND from %s", alloca_socket_address(&client)); { struct subscriber *subscriber=NULL; /* Make sure source address is either all zeros (listen on all), or a valid @@ -1166,24 +1560,24 @@ static void overlay_mdp_poll(struct sched_ent *alarm) if ((!subscriber) || subscriber->reachable != REACHABLE_SELF){ WHYF("Invalid bind request for sid=%s", alloca_tohex_sid_t(mdp->bind.sid)); /* Source address is invalid */ - overlay_mdp_reply_error(alarm->poll.fd, recvaddr_un, recvaddrlen, 7, + overlay_mdp_reply_error(alarm->poll.fd, &client, 7, "Bind address is not valid (must be a local MDP address, or all zeroes)."); return; } } if (overlay_mdp_process_bind_request(alarm->poll.fd, subscriber, mdp->bind.port, - mdp->packetTypeAndFlags, recvaddr_un, recvaddrlen)) - overlay_mdp_reply_error(alarm->poll.fd,recvaddr_un,recvaddrlen,3, "Port already in use"); + mdp->packetTypeAndFlags, &client)) + overlay_mdp_reply_error(alarm->poll.fd, &client, 3, "Port already in use"); else - overlay_mdp_reply_ok(alarm->poll.fd,recvaddr_un,recvaddrlen,"Port bound"); + overlay_mdp_reply_ok(alarm->poll.fd, &client, "Port bound"); return; } break; case MDP_SCAN: if (config.debug.mdprequests) - DEBUGF("MDP_SCAN from %s", alloca_sockaddr(recvaddr, recvaddrlen)); + DEBUGF("MDP_SCAN from %s", alloca_socket_address(&client)); { struct overlay_mdp_scan *scan = (struct overlay_mdp_scan *)&mdp->raw; time_ms_t start=gettime_ms(); @@ -1214,7 +1608,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm) }else{ struct overlay_interface *interface = overlay_interface_find(scan->addr, 1); if (!interface){ - overlay_mdp_reply_error(alarm->poll.fd,recvaddr_un,recvaddrlen, 1, "Unable to find matching interface"); + overlay_mdp_reply_error(alarm->poll.fd, &client, 1, "Unable to find matching interface"); return; } int i = interface - overlay_interfaces; @@ -1229,22 +1623,14 @@ static void overlay_mdp_poll(struct sched_ent *alarm) } } - overlay_mdp_reply_ok(alarm->poll.fd,recvaddr_un,recvaddrlen,"Scan initiated"); + overlay_mdp_reply_ok(alarm->poll.fd, &client, "Scan initiated"); } break; default: /* Client is not allowed to send any other frame type */ - WARNF("Unsupported MDP frame type [%d] from %s", mdp_type, alloca_sockaddr(recvaddr, recvaddrlen)); - mdp->packetTypeAndFlags=MDP_ERROR; - mdp->error.error=2; - snprintf(mdp->error.message,128,"Illegal request type. Clients may use only MDP_TX or MDP_BIND."); - int len=4+4+strlen(mdp->error.message)+1; - errno=0; - /* We ignore the result of the following, because it is just sending an - error message back to the client. If this fails, where would we report - the error to? My point exactly. */ - sendto(alarm->poll.fd,mdp,len,0,(struct sockaddr *)recvaddr,recvaddrlen); + WARNF("Unsupported MDP frame type [%d] from %s", mdp_type, alloca_socket_address(&client)); + overlay_mdp_reply_error(alarm->poll.fd, &client, 2, "Illegal request type. Clients may use only MDP_TX or MDP_BIND."); } } } diff --git a/overlay_mdp_services.c b/overlay_mdp_services.c index b3d42fc7..2abc30d4 100644 --- a/overlay_mdp_services.c +++ b/overlay_mdp_services.c @@ -28,6 +28,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "rhizome.h" #include "crypto.h" #include "log.h" +#include "keyring.h" +#include "dataformats.h" int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength) { @@ -98,7 +100,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, ui reply.out.payload[0]='T'; // send packet - if (overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0)) + if (overlay_mdp_dispatch(&reply, NULL)) break; } @@ -137,11 +139,11 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp) unsigned char *bidprefix=&mdp->out.payload[1]; uint64_t version=read_uint64(&mdp->out.payload[1+16]); uint64_t offset=read_uint64(&mdp->out.payload[1+16+8]); - int count=mdp->out.payload_length-(1+16+8+8); + size_t count = mdp->out.payload_length-(1+16+8+8); unsigned char *bytes=&mdp->out.payload[1+16+8+8]; if (config.debug.rhizome_mdp_rx) - DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%d", + DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%zu", bidprefix[0],bidprefix[1],bidprefix[2],bidprefix[3],offset,count); /* Now see if there is a slot that matches. If so, then @@ -151,7 +153,7 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp) a slot to capture this files as it is being requested by someone else. */ - rhizome_received_content(bidprefix,version,offset,count,bytes,type); + rhizome_received_content(bidprefix,version,offset, count, bytes, type); RETURN(0); } @@ -251,8 +253,7 @@ int overlay_mdp_service_echo(overlay_mdp_frame *mdp) mdp->packetTypeAndFlags&=~(MDP_NOCRYPT|MDP_NOSIGN); /* queue frame for delivery */ - overlay_mdp_dispatch(mdp,0 /* system generated */, - NULL,0); + overlay_mdp_dispatch(mdp, NULL); mdp->packetTypeAndFlags=preserved; /* and switch addresses back around in case the caller was planning on @@ -333,34 +334,33 @@ static int overlay_mdp_service_trace(overlay_mdp_frame *mdp){ ob_unlimitsize(b); // always write a full sid into the payload my_subscriber->send_full=1; - if (overlay_address_append(&context, b, my_subscriber)){ + overlay_address_append(&context, b, my_subscriber); + if (ob_overrun(b)) { ret = WHYF("Unable to append my address to the trace"); goto end; } - mdp->out.payload_length = ob_position(b); mdp->out.src.sid = my_subscriber->sid; mdp->out.dst.sid = next->sid; - - ret = overlay_mdp_dispatch(mdp, 0, NULL, 0); + ret = overlay_mdp_dispatch(mdp, NULL); end: ob_free(b); RETURN(ret); } -static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp) +static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, const uint8_t *payload, size_t len) { int offset=0; - while (offsetout.payload_length) { + while (offsetout.payload[offset]; + const unsigned char *bar = &payload[offset]; if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){ rhizome_advertise_manifest(frame->source, m); // pre-emptively send the payload if it will fit in a single packet - if (m->fileLength > 0 && m->fileLength <= 1024) - rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->fileLength); + if (m->filesize > 0 && m->filesize <= 1024) + rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->filesize); } rhizome_manifest_free(m); offset+=RHIZOME_BAR_BYTES; @@ -368,7 +368,7 @@ static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, ov return 0; } -int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp) +int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp) { IN(); switch(mdp->out.dst.port) { @@ -383,7 +383,7 @@ int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_fr case MDP_PORT_STUN: RETURN(overlay_mdp_service_stun(mdp)); case MDP_PORT_RHIZOME_REQUEST: RETURN(overlay_mdp_service_rhizomerequest(frame, mdp)); case MDP_PORT_RHIZOME_RESPONSE: RETURN(overlay_mdp_service_rhizomeresponse(mdp)); - case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp)); + case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp->out.payload, mdp->out.payload_length)); case MDP_PORT_RHIZOME_SYNC: RETURN(overlay_mdp_service_rhizome_sync(frame, mdp)); } diff --git a/overlay_olsr.c b/overlay_olsr.c index 92422124..01bb48a7 100644 --- a/overlay_olsr.c +++ b/overlay_olsr.c @@ -252,6 +252,8 @@ int olsr_send(struct overlay_frame *frame){ struct decode_context context; bzero(&context, sizeof context); struct overlay_buffer *b=ob_new(); + if (b == NULL) + return 0; // build olsr specific frame header ob_append_byte(b, PACKET_FORMAT_NUMBER); @@ -259,7 +261,6 @@ int olsr_send(struct overlay_frame *frame){ // address the packet as transmitted by me overlay_address_append(&context, b, my_subscriber); - overlay_address_append(&context, b, frame->source); overlay_broadcast_append(b, &frame->broadcast_id); ob_append_byte(b, frame->modifiers); diff --git a/overlay_packetformats.c b/overlay_packetformats.c index c61a53a8..b9261e05 100644 --- a/overlay_packetformats.c +++ b/overlay_packetformats.c @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "conf.h" +#include "socket.h" #include "str.h" #include "strbuf.h" #include "overlay_buffer.h" @@ -42,20 +43,16 @@ int overlay_packet_init_header(int packet_version, int encapsulation, if (encapsulation !=ENCAP_OVERLAY && encapsulation !=ENCAP_SINGLE) return WHY("Invalid packet encapsulation"); - if (ob_append_byte(buff, packet_version)) - return -1; - if (ob_append_byte(buff, encapsulation)) - return -1; + ob_append_byte(buff, packet_version); + ob_append_byte(buff, encapsulation); - if (context->interface->point_to_point - && context->interface->other_device - && packet_version>=1) + if ( context->interface->point_to_point + && context->interface->other_device + && packet_version>=1 + ) context->point_to_point_device = context->interface->other_device; - context->encoding_header=1; - - if (overlay_address_append(context, buff, my_subscriber)) - return -1; + overlay_address_append(context, buff, my_subscriber); context->encoding_header=0; context->sender = my_subscriber; @@ -324,7 +321,7 @@ int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface } int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len, - int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen) + int recvttl, struct socket_address *recvaddr) { IN(); /* @@ -389,8 +386,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s } } - if (recvaddr&&recvaddr->sa_family!=AF_INET) - RETURN(WHYF("Unexpected protocol family %d",recvaddr->sa_family)); + if (recvaddr && recvaddr->addr.sa_family != AF_INET) + RETURN(WHYF("Unexpected protocol family %d", recvaddr->addr.sa_family)); struct overlay_frame f; struct decode_context context; @@ -403,11 +400,11 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s f.interface = interface; if (recvaddr) - f.recvaddr = *((struct sockaddr_in *)recvaddr); + f.recvaddr = recvaddr->inet; else bzero(&f.recvaddr, sizeof f.recvaddr); - int ret=parseEnvelopeHeader(&context, interface, (struct sockaddr_in *)recvaddr, b); + int ret=parseEnvelopeHeader(&context, interface, recvaddr ? &recvaddr->inet : NULL, b); if (ret){ ob_free(b); RETURN(ret); diff --git a/overlay_payload.c b/overlay_payload.c index ec88c117..05f43215 100644 --- a/overlay_payload.c +++ b/overlay_payload.c @@ -51,34 +51,28 @@ static int overlay_frame_build_header(int packet_version, struct decode_context if (type!=OF_TYPE_DATA) flags |= PAYLOAD_FLAG_LEGACY_TYPE; - if (ob_append_byte(buff, flags)) return -1; + ob_append_byte(buff, flags); - if (!(flags & PAYLOAD_FLAG_SENDER_SAME)){ - if (overlay_address_append(context, buff, source)) return -1; - } + if (!(flags & PAYLOAD_FLAG_SENDER_SAME)) + overlay_address_append(context, buff, source); if (flags & PAYLOAD_FLAG_TO_BROADCAST){ - if (!(flags & PAYLOAD_FLAG_ONE_HOP)){ - if (overlay_broadcast_append(buff, broadcast)) return -1; - } - }else{ - if (overlay_address_append(context, buff, destination)) return -1; - if (!(flags & PAYLOAD_FLAG_ONE_HOP)){ - if (overlay_address_append(context, buff, next_hop)) return -1; - } + if (!(flags & PAYLOAD_FLAG_ONE_HOP)) + overlay_broadcast_append(buff, broadcast); + } else { + overlay_address_append(context, buff, destination); + if (!(flags & PAYLOAD_FLAG_ONE_HOP)) + overlay_address_append(context, buff, next_hop); } - if (!(flags & PAYLOAD_FLAG_ONE_HOP)){ - if (ob_append_byte(buff, ttl | ((queue&3)<<5))) return -1; - } + if (!(flags & PAYLOAD_FLAG_ONE_HOP)) + ob_append_byte(buff, ttl | ((queue&3)<<5)); - if (flags & PAYLOAD_FLAG_LEGACY_TYPE){ - if (ob_append_byte(buff, type)) return -1; - } + if (flags & PAYLOAD_FLAG_LEGACY_TYPE) + ob_append_byte(buff, type); if (packet_version >= 1) - if (ob_append_byte(buff, sequence)) - return -1; + ob_append_byte(buff, sequence); return 0; } @@ -112,20 +106,17 @@ int overlay_frame_append_payload(struct decode_context *context, int encapsulati p->queue, p->type, p->modifiers, will_retransmit, p->ttl, p->mdp_sequence&0xFF, broadcast, p->next_hop, - p->destination, p->source)) + p->destination, p->source) == -1) goto cleanup; - if (encapsulation == ENCAP_OVERLAY){ - if (ob_append_ui16(b, ob_position(p->payload))) - goto cleanup; - } + if (encapsulation == ENCAP_OVERLAY) + ob_append_ui16(b, ob_position(p->payload)); - if (ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload))) { - WHYF("could not append payload of %u bytes", ob_position(p->payload)); - goto cleanup; - } - - return 0; + if (ob_position(p->payload)) + ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload)); + + if (!ob_overrun(b)) + return 0; cleanup: ob_rewind(b); @@ -150,13 +141,18 @@ struct overlay_frame *op_dup(struct overlay_frame *in) if (!in) return NULL; /* clone the frame */ - struct overlay_frame *out=malloc(sizeof(struct overlay_frame)); - if (!out) { WHY("malloc() failed"); return NULL; } + struct overlay_frame *out = emalloc(sizeof(struct overlay_frame)); + if (out == NULL) + return NULL; /* copy main data structure */ bcopy(in,out,sizeof(struct overlay_frame)); - if (in->payload) - out->payload=ob_dup(in->payload); + if (in->payload) { + if ((out->payload = ob_dup(in->payload)) == NULL) { + free(out); + return NULL; + } + } return out; } diff --git a/overlay_queue.c b/overlay_queue.c index 88815418..b239b79a 100644 --- a/overlay_queue.c +++ b/overlay_queue.c @@ -17,10 +17,12 @@ */ +#include #include "serval.h" #include "conf.h" #include "overlay_buffer.h" #include "overlay_packet.h" +#include "radio_link.h" #include "str.h" #include "strbuf.h" @@ -150,30 +152,25 @@ int overlay_payload_enqueue(struct overlay_frame *p) Complain if there are too many frames in the queue. */ - if (!p) return WHY("Cannot queue NULL"); - - if (p->queue>=OQ_MAX) - return WHY("Invalid queue specified"); - + assert(p != NULL); + assert(p->queue < OQ_MAX); + assert(p->payload != NULL); overlay_txqueue *queue = &overlay_tx[p->queue]; if (config.debug.packettx) - DEBUGF("Enqueuing packet for %s* (q[%d]length = %d)", + DEBUGF("Enqueuing packet for %s* (q[%d].length = %d)", p->destination?alloca_tohex_sid_t_trunc(p->destination->sid, 14): alloca_tohex(p->broadcast_id.id, BROADCAST_LEN), p->queue, queue->length); - if (p->payload && ob_remaining(p->payload)<0){ - // HACK, maybe should be done in each caller - // set the size of the payload based on the position written - ob_limitsize(p->payload,ob_position(p->payload)); - } + if (ob_overrun(p->payload)) + return WHY("Packet content overrun -- not queueing"); + if (ob_position(p->payload) >= MDP_MTU) + FATAL("Queued packet is too big"); + if (queue->length>=queue->maxLength) return WHYF("Queue #%d congested (size = %d)",p->queue,queue->maxLength); - if (ob_position(p->payload)>=MDP_MTU) - FATAL("Queued packet is too big"); - // it should be safe to try sending all packets with an mdp sequence if (p->packet_version<=0) p->packet_version=1; @@ -226,11 +223,13 @@ int overlay_payload_enqueue(struct overlay_frame *p) return 0; } -static void +static int overlay_init_packet(struct outgoing_packet *packet, int packet_version, - struct network_destination *destination){ + struct network_destination *destination) +{ packet->context.interface = destination->interface; - packet->buffer=ob_new(); + if ((packet->buffer = ob_new()) == NULL) + return -1; packet->packet_version = packet_version; packet->context.packet_version = packet_version; packet->destination = add_destination_ref(destination); @@ -238,20 +237,24 @@ overlay_init_packet(struct outgoing_packet *packet, int packet_version, packet->seq=-1; else packet->seq = destination->sequence_number = (destination->sequence_number + 1) & 0xFFFF; - ob_limitsize(packet->buffer, destination->interface->mtu); - - int i=destination->interface - overlay_interfaces; - overlay_packet_init_header(packet_version, destination->encapsulation, - &packet->context, packet->buffer, - destination->unicast, - i, packet->seq); + int i = destination->interface - overlay_interfaces; + if (overlay_packet_init_header(packet_version, destination->encapsulation, + &packet->context, packet->buffer, + destination->unicast, + i, packet->seq) == -1 + ) { + ob_free(packet->buffer); + packet->buffer = NULL; + return -1; + } packet->header_length = ob_position(packet->buffer); if (config.debug.overlayframes) DEBUGF("Creating %d packet for interface %s, seq %d, %s", packet_version, destination->interface->name, destination->sequence_number, destination->unicast?"unicast":"broadcast"); + return 0; } int overlay_queue_schedule_next(time_ms_t next_allowed_packet){ @@ -288,7 +291,7 @@ overlay_calc_queue_time(overlay_txqueue *queue, struct overlay_frame *frame){ int i; for(i=0;idestination_count;i++) { - if (frame->destinations[i].destination->interface->tx_packet) + if (radio_link_is_busy(frame->destinations[i].destination->interface)) continue; time_ms_t next_packet = limit_next_allowed(&frame->destinations[i].destination->transfer_limit); if (frame->destinations[i].transmit_time){ @@ -331,8 +334,9 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim while(frame){ if (frame->enqueued_at + queue->latencyTarget < now){ if (config.debug.overlayframes) - DEBUGF("Dropping frame type %x for %s due to expiry timeout", - frame->type, frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All"); + DEBUGF("Dropping frame type %x (length %d) for %s due to expiry timeout", + frame->type, frame->payload->checkpointLength, + frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All"); frame = overlay_queue_remove(queue, frame); continue; } @@ -400,8 +404,7 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim } }else{ // skip this interface if the stream tx buffer has data - if (dest->interface->socket_type==SOCK_STREAM - && dest->interface->tx_packet) + if (radio_link_is_busy(dest->interface)) continue; // can we send a packet on this interface now? @@ -411,11 +414,11 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim // send a packet to this destination if (frame->source_full) my_subscriber->send_full=1; - - overlay_init_packet(packet, frame->packet_version, dest); - destination_index=i; - frame->destinations[i].sent_sequence = dest->sequence_number; - break; + if (overlay_init_packet(packet, frame->packet_version, dest) != -1) { + destination_index=i; + frame->destinations[i].sent_sequence = dest->sequence_number; + break; + } } } } @@ -532,12 +535,12 @@ static void overlay_send_packet(struct sched_ent *alarm){ overlay_fill_send_packet(&packet, gettime_ms()); } -int overlay_send_tick_packet(struct network_destination *destination){ +int overlay_send_tick_packet(struct network_destination *destination) +{ struct outgoing_packet packet; bzero(&packet, sizeof(struct outgoing_packet)); - overlay_init_packet(&packet, 0, destination); - - overlay_fill_send_packet(&packet, gettime_ms()); + if (overlay_init_packet(&packet, 0, destination) != -1) + overlay_fill_send_packet(&packet, gettime_ms()); return 0; } diff --git a/performance_timing.c b/performance_timing.c index 97944226..f7d60796 100644 --- a/performance_timing.c +++ b/performance_timing.c @@ -149,17 +149,8 @@ int fd_showstats() stats = stats->_next; } - // Show periodic rhizome transfer information, but only - // if there are some active rhizome transfers. - if (rhizome_active_fetch_count()!=0) - INFOF("Rhizome transfer progress: %d,%d,%d,%d,%d,%d (remaining %d)", - rhizome_active_fetch_bytes_received(0), - rhizome_active_fetch_bytes_received(1), - rhizome_active_fetch_bytes_received(2), - rhizome_active_fetch_bytes_received(3), - rhizome_active_fetch_bytes_received(4), - rhizome_active_fetch_bytes_received(5), - rhizome_fetch_queue_bytes()); + // Show periodic rhizome transfer information + rhizome_fetch_log_short_status(); // Report any functions that take too much time if (!config.debug.timing) @@ -168,8 +159,8 @@ int fd_showstats() while(stats!=NULL){ /* If a function spends more than 1 second in any notionally 3 second period, then dob on it */ - if (stats->total_time>1000 - &&strcmp(stats->name,"Idle (in poll)")) + if ((stats->total_time>1000 || stats->calls > 10000) + && strcmp(stats->name,"Idle (in poll)")) fd_showstat(&total,stats); stats = stats->_next; } @@ -208,6 +199,15 @@ void dump_stack(int log_level) } } +unsigned fd_depth() +{ + unsigned depth = 0; + struct call_stats *call; + for (call = current_call; call; call = call->prev) + ++depth; + return depth; +} + int fd_func_enter(struct __sourceloc __whence, struct call_stats *this_call) { if (config.debug.profiling) diff --git a/radio_link.c b/radio_link.c new file mode 100644 index 00000000..2b2f9224 --- /dev/null +++ b/radio_link.c @@ -0,0 +1,546 @@ +// -*- Mode: C; c-basic-offset: 2; -*- +// +// Copyright (c) 2012 Andrew Tridgell, All Rights Reserved +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// o Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// o Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +// OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* +Portions Copyright (C) 2013 Paul Gardner-Stephen + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "serval.h" +#include "conf.h" +#include "overlay_buffer.h" +#include "golay.h" +#include "radio_link.h" + +#define MAVLINK_MSG_ID_RADIO 166 +#define MAVLINK_MSG_ID_DATASTREAM 67 + +// use '3D' for 3DRadio +#define RADIO_SOURCE_SYSTEM '3' +#define RADIO_SOURCE_COMPONENT 'D' + +/* + we use a hand-crafted MAVLink packet based on the following + message definition + +struct mavlink_RADIO_v10 { + uint16_t rxerrors; // receive errors + uint16_t fixed; // count of error corrected packets + uint8_t rssi; // local signal strength + uint8_t remrssi; // remote signal strength + uint8_t txbuf; // percentage free space in transmit buffer + uint8_t noise; // background noise level + uint8_t remnoise; // remote background noise level +}; + +*/ + +#define FEC_LENGTH 32 +#define FEC_MAX_BYTES 223 +#define RADIO_HEADER_LENGTH 6 +#define RADIO_USED_HEADER_LENGTH 4 +#define RADIO_CRC_LENGTH 2 + +#define LINK_PAYLOAD_MTU (LINK_MTU - FEC_LENGTH - RADIO_HEADER_LENGTH - RADIO_CRC_LENGTH) + +struct radio_link_state{ + // next seq for transmission + int tx_seq; + + // small buffer for parsing incoming bytes from the serial interface, + // looking for recoverable link layer packets + // should be large enough to hold at least one packet from the remote end + // plus one heartbeat packet from the local firmware + uint8_t payload[LINK_MTU*3]; + + // decoded length of next link layer packet + // including all header and footer bytes + int payload_length; + // last rx seq for reassembly + int seq; + // offset within payload that we have found a valid looking header + int payload_start; + // offset after payload_start for incoming bytes + int payload_offset; + + // small buffer for assembling mdp payloads. + uint8_t dst[MDP_MTU]; + // length of recovered packet + int packet_length; + + // next firmware heartbeat + time_ms_t next_heartbeat; + + time_ms_t last_packet; + + // parsed rssi + int radio_rssi; + int remote_rssi; + // estimated firmware buffer space + int32_t remaining_space; + + // next serial write + uint64_t next_tx_allowed; + // partially sent packet + struct overlay_buffer *tx_packet; + + // serial write buffer + uint8_t txbuffer[LINK_MTU]; + int tx_bytes; + int tx_pos; +}; + +/* + Each mavlink frame consists of 0xfe followed by a standard 6 byte header. + Normally the payload plus a 2-byte CRC follows. + We are replacing the CRC check with a Reed-Solomon code to correct as well + as detect upto 16 bytes with errors, in return for a 32-byte overhead. + + The nature of the particular library we are using is that the overhead is + basically fixed, but we can shorten the data section. + + Note that the mavlink headers are not protected against errors. This is a + limitation of the radio firmware at present. One day we will re-write the + radio firmware so that we can send and receive raw radio frames, and get + rid of the mavlink framing altogether, and just send R-S protected payloads. + + Not ideal, but will be fine for now. +*/ + +#include "fec-3.0.1/fixed.h" +void encode_rs_8(data_t *data, data_t *parity,int pad); +int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad); + +int radio_link_free(struct overlay_interface *interface) +{ + if (interface->radio_link_state){ + free(interface->radio_link_state); + interface->radio_link_state=NULL; + } + return 0; +} + +int radio_link_init(struct overlay_interface *interface) +{ + interface->radio_link_state = emalloc_zero(sizeof(struct radio_link_state)); + return 0; +} + +void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface) +{ + struct radio_link_state *state = interface->radio_link_state; + strbuf_sprintf(b, "RSSI: %ddB
", state->radio_rssi); + strbuf_sprintf(b, "Remote RSSI: %ddB
", state->remote_rssi); +} + +// write a new link layer packet to interface->txbuffer +// consuming more bytes from the next interface->tx_packet if required +static int radio_link_encode_packet(struct radio_link_state *link_state) +{ + // if we have nothing interesting left to send, don't create a packet at all + if (!link_state->tx_packet) + return 0; + + int count = ob_remaining(link_state->tx_packet); + int startP = (ob_position(link_state->tx_packet) == 0); + int endP = 1; + if (count > LINK_PAYLOAD_MTU){ + count = LINK_PAYLOAD_MTU; + endP = 0; + } + + link_state->txbuffer[0]=0xfe; // mavlink v1.0 magic header + + // we need to add FEC_LENGTH for FEC, but the length field doesn't include the expected headers or CRC + int len = count + FEC_LENGTH - RADIO_CRC_LENGTH; + link_state->txbuffer[1]=len; // mavlink payload length + link_state->txbuffer[2]=(len & 0xF); + link_state->txbuffer[3]=0; + + // add golay encoding so that decoding the actual length is more reliable + golay_encode(&link_state->txbuffer[1]); + + + link_state->txbuffer[4]=(link_state->tx_seq++) & 0x3f; + if (startP) link_state->txbuffer[4]|=0x40; + if (endP) link_state->txbuffer[4]|=0x80; + link_state->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM; + + ob_get_bytes(link_state->tx_packet, &link_state->txbuffer[6], count); + + encode_rs_8(&link_state->txbuffer[4], &link_state->txbuffer[6+count], FEC_MAX_BYTES - (count+2)); + link_state->tx_bytes=len + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH; + if (endP){ + ob_free(link_state->tx_packet); + link_state->tx_packet=NULL; + overlay_queue_schedule_next(gettime_ms()); + } + return 0; +} + +int radio_link_is_busy(struct overlay_interface *interface) +{ + if (interface->radio_link_state && interface->radio_link_state->tx_packet) + return 1; + return 0; +} + +int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer) +{ + struct radio_link_state *link_state = interface->radio_link_state; + + if (link_state->tx_packet){ + ob_free(buffer); + return WHYF("Cannot send two packets to a stream at the same time"); + } + + // prepare the buffer for reading + ob_flip(buffer); + link_state->tx_packet = buffer; + radio_link_tx(interface); + + return 0; +} + +static int build_heartbeat(struct radio_link_state *link_state) +{ + int count=9; + bzero(link_state->txbuffer, count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH); + + link_state->txbuffer[0]=0xfe; // mavlink v1.0 link_state->txbuffer + // Must be 9 to indicate heartbeat + link_state->txbuffer[1]=count; // payload len, excluding 6 byte header and 2 byte CRC + link_state->txbuffer[2]=(count & 0xF); // packet sequence + link_state->txbuffer[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC) + // we're golay encoding the length to improve the probability of skipping it correctly + golay_encode(&link_state->txbuffer[1]); + link_state->txbuffer[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE) + // Must be zero to indicate heartbeat + link_state->txbuffer[5]=0; // message ID type of this link_state->txbuffer: DATA_STREAM + + // extra magic number to help correctly detect remote heartbeat requests + link_state->txbuffer[14]=0x55; + link_state->txbuffer[15]=0x05; + golay_encode(&link_state->txbuffer[14]); + link_state->tx_bytes = count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH; + if (config.debug.radio_link) + DEBUGF("Produced heartbeat"); + + return 0; +} + +// write a new link layer packet to interface->txbuffer +// consuming more bytes from the next interface->tx_packet if required +int radio_link_tx(struct overlay_interface *interface) +{ + struct radio_link_state *link_state = interface->radio_link_state; + + unschedule(&interface->alarm); + interface->alarm.alarm = 0; + time_ms_t next_tick = interface->destination->last_tx+interface->destination->tick_ms; + time_ms_t now = gettime_ms(); + + while(1){ + + if (link_state->tx_bytes){ + if (link_state->next_tx_allowed > now){ + interface->alarm.alarm = link_state->next_tx_allowed; + break; + } + + int written=write(interface->alarm.poll.fd, &link_state->txbuffer[link_state->tx_pos], link_state->tx_bytes); + if (written<=0){ + interface->alarm.poll.events|=POLLOUT; + break; + } + link_state->remaining_space-=written; + link_state->tx_bytes-=written; + if (link_state->tx_bytes) + link_state->tx_pos+=written; + else + link_state->tx_pos=0; + continue; + } + + interface->alarm.poll.events&=~POLLOUT; + + if (link_state->next_heartbeat<=now){ + build_heartbeat(link_state); + link_state->next_heartbeat = now + 1000; + continue; + } + + // out of space? Don't bother to send anything interesting + // until we hear the next heartbeat response + if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE){ + interface->alarm.alarm = link_state->next_heartbeat; + break; + } + + if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE) + link_state->next_heartbeat = now; + + if (!link_state->tx_packet){ + // finished current packet, wait for more. + interface->alarm.alarm = next_tick; + break; + } + + // encode another packet fragment + radio_link_encode_packet(link_state); + link_state->last_packet = now; + } + + watch(&interface->alarm); + if (interface->alarm.alarmalarm.alarm=now; + if (interface->alarm.alarm){ + interface->alarm.deadline = interface->alarm.alarm+100; + schedule(&interface->alarm); + } + + return 0; +} + +static int parse_heartbeat(struct radio_link_state *state, const unsigned char *payload) +{ + if (payload[0]==0xFE + && payload[1]==9 + && payload[3]==RADIO_SOURCE_SYSTEM + && payload[4]==RADIO_SOURCE_COMPONENT + && payload[5]==MAVLINK_MSG_ID_RADIO){ + + // we can assume that radio status packets arrive without corruption + state->radio_rssi=(1.0*payload[10]-payload[13])/1.9; + state->remote_rssi=(1.0*payload[11] - payload[14])/1.9; + int free_space = payload[12]; + int free_bytes = (free_space * 1280) / 100 - 30; + state->remaining_space = free_bytes; + if (free_bytes>0) + state->next_tx_allowed = gettime_ms(); + if (free_bytes>720) + state->next_heartbeat=gettime_ms()+1000; + if (config.debug.packetradio) + INFOF("Link budget = %+ddB, remote link budget = %+ddB, buffer space = %d%% (approx %d)", + state->radio_rssi, + state->remote_rssi, + free_space, free_bytes); + return 1; + } + return 0; +} + +static int radio_link_parse(struct overlay_interface *interface, struct radio_link_state *state, + size_t packet_length, uint8_t *payload, int *backtrack) +{ + *backtrack=0; + if (packet_length==17){ + // if we've heard the start and end of a remote heartbeat request + // we can skip it without checking anything else + int errs=0; + int tail = golay_decode(&errs, &payload[14]); + if (tail == 0x555){ + if (config.debug.radio_link) + DEBUGF("Decoded remote heartbeat request"); + return 1; + } + return 0; + } + + size_t data_bytes = packet_length - (RADIO_USED_HEADER_LENGTH + FEC_LENGTH); + + int errors=decode_rs_8(&payload[4], NULL, 0, FEC_MAX_BYTES - data_bytes); + if (errors==-1){ + if (config.debug.radio_link) + DEBUGF("Reed-Solomon error correction failed"); + return 0; + } + *backtrack=errors; + data_bytes -= 2; + int seq=payload[4]&0x3f; + + if (config.debug.radio_link){ + DEBUGF("Received RS protected message, len: %zd, errors: %d, seq: %d, flags:%s%s", + data_bytes, + errors, + seq, + payload[4]&0x40?" start":"", + payload[4]&0x80?" end":""); + } + + if (seq != ((state->seq+1)&0x3f)){ + // reject partial packet if we missed a sequence number + if (config.debug.radio_link) + DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->seq, seq); + state->packet_length=sizeof(state->dst)+1; + } + + if (payload[4]&0x40){ + // start a new packet + state->packet_length=0; + } + + state->seq=payload[4]&0x3f; + if (state->packet_length + data_bytes > sizeof(state->dst)){ + if (config.debug.radio_link) + DEBUG("Fragmented packet is too long or a previous piece was missed - discarding"); + state->packet_length=sizeof(state->dst)+1; + return 1; + } + + bcopy(&payload[RADIO_HEADER_LENGTH], &state->dst[state->packet_length], data_bytes); + state->packet_length+=data_bytes; + + if (payload[4]&0x80) { + if (config.debug.radio_link) + DEBUGF("PDU Complete (length=%d)",state->packet_length); + + packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL); + state->packet_length=sizeof(state->dst)+1; + } + return 1; +} + +static int decode_length(struct radio_link_state *state, unsigned char *p) +{ + // look for a valid golay encoded length + int errs=0; + int length = golay_decode(&errs, p); + if (length<0 || ((length >>8) & 0xF) != (length&0xF)) + return -1; + length=length&0xFF; + length += RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH; + + if (length!=17 && (length <= FEC_LENGTH || length > LINK_MTU)) + return -1; + + if (config.debug.radio_link && (errs || state->payload_length!=*p)) + DEBUGF("Decoded length %d to %d with %d errs", *p, length, errs); + + state->payload_length=length; + return 0; +} + +// add one byte at a time from the serial link, and attempt to decode packets +int radio_link_decode(struct overlay_interface *interface, uint8_t c) +{ + IN(); + struct radio_link_state *state=interface->radio_link_state; + + if (state->payload_start + state->payload_offset >= sizeof(state->payload)){ + // drop one byte if we run out of space + if (config.debug.radio_link) + DEBUGF("Dropped %02x, buffer full", state->payload[0]); + bcopy(state->payload+1, state->payload, sizeof(state->payload) -1); + state->payload_start--; + } + + unsigned char *p = &state->payload[state->payload_start]; + p[state->payload_offset++]=c; + + while(1){ + // look for packet length headers + p = &state->payload[state->payload_start]; + while(state->payload_length==0 && state->payload_offset>=6){ + if (p[0]==0xFE + && p[1]==9 + && p[3]==RADIO_SOURCE_SYSTEM + && p[4]==RADIO_SOURCE_COMPONENT + && p[5]==MAVLINK_MSG_ID_RADIO){ + //looks like a valid heartbeat response header, read the rest and process it + state->payload_length=17; + break; + } + + if (decode_length(state, &p[1])==0) + break; + + state->payload_start++; + state->payload_offset--; + p++; + } + + // wait for a whole packet + if (!state->payload_length || state->payload_offset < state->payload_length) + RETURN(0); + + if (parse_heartbeat(state, p)){ + // cut the bytes of the heartbeat out of the buffer + state->payload_offset -= state->payload_length; + if (state->payload_offset){ + // shuffle bytes backwards + bcopy(&p[state->payload_length], p, state->payload_offset); + } + // restart parsing for a valid header from the beginning of out buffer + state->payload_offset+=state->payload_start; + state->payload_start=0; + state->payload_length=0; + continue; + } + + // is this a well formed packet? + int backtrack=0; + if (radio_link_parse(interface, state, state->payload_length, p, &backtrack)==1){ + // Since we know we've synced with the remote party, + // and there's nothing we can do about any earlier data + // throw away everything before the end of this packet + if (state->payload_start && config.debug.radio_link) + dump("Skipped", state->payload, state->payload_start); + + // If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet, + // but we may need to examine the last few bytes to find the start of the next packet. + state->payload_offset -= state->payload_length - backtrack; + if (state->payload_offset){ + // shuffle all remaining bytes back to the start of the buffer + bcopy(&state->payload[state->payload_start + state->payload_length - backtrack], + state->payload, state->payload_offset); + } + state->payload_start=0; + }else{ + // ignore the first byte for now and start looking for another packet header + // we may find a heartbeat in the middle that we need to cut out first + state->payload_start++; + state->payload_offset--; + } + state->payload_length=0; + }; + RETURN(0); +} diff --git a/radio_link.h b/radio_link.h new file mode 100644 index 00000000..1dd92ccf --- /dev/null +++ b/radio_link.h @@ -0,0 +1,15 @@ +#ifndef __SERVALD_RADIO_LINK_H +#define __SERVALD_RADIO_LINK_H + +#define HEARTBEAT_SIZE (8+9) +#define LINK_MTU 255 + +int radio_link_free(struct overlay_interface *interface); +int radio_link_init(struct overlay_interface *interface); +int radio_link_decode(struct overlay_interface *interface, uint8_t c); +int radio_link_tx(struct overlay_interface *interface); +void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface); +int radio_link_is_busy(struct overlay_interface *interface); +int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer); + +#endif \ No newline at end of file diff --git a/rhizome.c b/rhizome.c index 172bc23c..69f9fb1b 100644 --- a/rhizome.c +++ b/rhizome.c @@ -17,11 +17,13 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include +#include #include "serval.h" #include "conf.h" #include "str.h" #include "rhizome.h" -#include +#include "dataformats.h" int is_rhizome_enabled() { @@ -129,14 +131,11 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1) return WHY("could not read manifest file"); - if (rhizome_manifest_verify(m)) + if (!rhizome_manifest_validate(m)) + return WHY("manifest is invalid"); + if (!rhizome_manifest_verify(m)) return WHY("could not verify manifest"); - /* Make sure we store signatures */ - // TODO, why do we need this? Why isn't the state correct from rhizome_read_manifest_file? - // This feels like a hack... - m->manifest_bytes=m->manifest_all_bytes; - /* Do we already have this manifest or newer? */ int64_t dbVersion = -1; if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1) @@ -152,115 +151,130 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, return rhizome_add_manifest(m, 1); } -int rhizome_manifest_check_sanity(rhizome_manifest *m_in) +int rhizome_manifest_check_sanity(rhizome_manifest *m) { /* Ensure manifest meets basic sanity checks. */ - const char *service = rhizome_manifest_get(m_in, "service", NULL, 0); - const char *sender = rhizome_manifest_get(m_in, "sender", NULL, 0); - const char *recipient = rhizome_manifest_get(m_in, "recipient", NULL, 0); - - if (service == NULL || !service[0]) - return WHY("Manifest missing 'service' field"); - if (rhizome_manifest_get_ll(m_in, "date") == -1) - return WHY("Manifest missing 'date' field"); - - /* Get manifest version number. */ - m_in->version = rhizome_manifest_get_ll(m_in, "version"); - if (m_in->version==-1) - return WHY("Manifest must have a version number"); - - if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) { - const char *name = rhizome_manifest_get(m_in, "name", NULL, 0); - if (name == NULL) - return WHY("Manifest missing 'name' field"); - } else if (strcasecmp(service, RHIZOME_SERVICE_MESHMS) == 0 - || strcasecmp(service, RHIZOME_SERVICE_MESHMS2) == 0) { - if (sender == NULL || !sender[0]) - return WHY("MeshMS Manifest missing 'sender' field"); - if (!str_is_subscriber_id(sender)) - return WHYF("MeshMS Manifest contains invalid 'sender' field: %s", sender); - if (recipient == NULL || !recipient[0]) - return WHY("MeshMS Manifest missing 'recipient' field"); - if (!str_is_subscriber_id(recipient)) - return WHYF("MeshMS Manifest contains invalid 'recipient' field: %s", recipient); - } else { - return WHY("Invalid service type"); + int ret = 0; + if (m->version == 0) + ret = WHY("Manifest must have a version number"); + if (m->filesize == RHIZOME_SIZE_UNSET) + ret = WHY("Manifest missing 'filesize' field"); + else if (m->filesize && rhizome_filehash_t_is_zero(m->filehash)) + ret = WHY("Manifest 'filehash' field has not been set"); + if (m->service == NULL) + ret = WHY("Manifest missing 'service' field"); + else if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) { + if (m->name == NULL) + ret = WHY("Manifest with service='" RHIZOME_SERVICE_FILE "' missing 'name' field"); + } else if (strcasecmp(m->service, RHIZOME_SERVICE_MESHMS) == 0 + || strcasecmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) { + if (!m->has_sender) + ret = WHYF("Manifest with service='%s' missing 'sender' field", m->service); + if (!m->has_recipient) + ret = WHYF("Manifest with service='%s' missing 'recipient' field", m->service); } - if (config.debug.rhizome) - DEBUGF("sender='%s'", sender ? sender : "(null)"); - - /* passes all sanity checks */ - return 0; + else if (!rhizome_str_is_manifest_service(m->service)) + ret = WHYF("Manifest invalid 'service' field %s", alloca_str_toprint(m->service)); + if (!m->has_date) + ret = WHY("Manifest missing 'date' field"); + return ret; } - -/* - A bundle can either be an ordinary manifest-payload pair, or a group description. - - - Group descriptions are manifests with no payload that have the "isagroup" variable set. They - get stored in the manifests table AND a reference is added to the grouplist table. Any - manifest, including any group manifest, may be a member of zero or one group. This allows a - nested, i.e., multi-level group hierarchy where sub-groups will only typically be discovered - by joining the parent group. -*/ - -int rhizome_manifest_bind_id(rhizome_manifest *m_in) +/* Sets the bundle key "BK" field of a manifest. Returns 1 if the field was set, 0 if not. + * + * This function must not be called unless the bundle secret is known. + * + * @author Andrew Bettison + */ +int rhizome_manifest_add_bundle_key(rhizome_manifest *m) { - if (rhizome_manifest_createid(m_in) == -1) - return -1; - /* The ID is implicit in transit, but we need to store it in the file, so that reimporting - manifests on receiver nodes works easily. We might implement something that strips the id - variable out of the manifest when sending it, or some other scheme to avoid sending all the - extra bytes. */ - if (!is_sid_t_any(m_in->author)) { - /* Set the BK using the provided authorship information. - Serval Security Framework defines BK as being: - BK = privateKey XOR sha512(RS##BID), where BID = cryptoSignPublic, - and RS is the rhizome secret for the specified author. - The nice thing about this specification is that: - privateKey = BK XOR sha512(RS##BID), so the same function can be used - to encrypt and decrypt the BK field. */ - const unsigned char *rs; - int rs_len=0; - unsigned char bkbytes[RHIZOME_BUNDLE_KEY_BYTES]; - - if (rhizome_find_secret(&m_in->author, &rs_len, &rs)) - return WHYF("Failed to obtain RS for %s to calculate BK", alloca_tohex_sid_t(m_in->author)); - if (!rhizome_secret2bk(&m_in->cryptoSignPublic, rs, rs_len, bkbytes, m_in->cryptoSignSecret)) { - char bkhex[RHIZOME_BUNDLE_KEY_STRLEN + 1]; - (void) tohex(bkhex, RHIZOME_BUNDLE_KEY_STRLEN, bkbytes); - if (config.debug.rhizome) DEBUGF("set BK=%s", bkhex); - rhizome_manifest_set(m_in, "BK", bkhex); - } else - return WHY("Failed to set BK"); + IN(); + assert(m->haveSecret); + switch (m->authorship) { + case ANONYMOUS: // there can be no BK field without an author + case AUTHOR_UNKNOWN: // we already know the author is not in the keyring + case AUTHENTICATION_ERROR: // already tried and failed to get Rhizome Secret + break; + case AUTHOR_NOT_CHECKED: + case AUTHOR_LOCAL: + case AUTHOR_AUTHENTIC: + case AUTHOR_IMPOSTOR: { + /* Set the BK using the provided author. Serval Security Framework defines BK as being: + * BK = privateKey XOR sha512(RS##BID) + * where BID = cryptoSignPublic, + * RS is the rhizome secret for the specified author. + * The nice thing about this specification is that: + * privateKey = BK XOR sha512(RS##BID) + * so the same function can be used to encrypt and decrypt the BK field. + */ + const unsigned char *rs; + size_t rs_len = 0; + enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs); + switch (d) { + case FOUND_RHIZOME_SECRET: { + rhizome_bk_t bkey; + if (rhizome_secret2bk(&m->cryptoSignPublic, rs, rs_len, bkey.binary, m->cryptoSignSecret) == 0) { + rhizome_manifest_set_bundle_key(m, &bkey); + m->authorship = AUTHOR_AUTHENTIC; + RETURN(1); + } else + m->authorship = AUTHENTICATION_ERROR; + } + break; + case IDENTITY_NOT_FOUND: + m->authorship = AUTHOR_UNKNOWN; + break; + case IDENTITY_HAS_NO_RHIZOME_SECRET: + m->authorship = AUTHENTICATION_ERROR; + break; + default: + FATALF("find_rhizome_secret() returned unknown code %d", (int)d); + break; + } + } + break; + default: + FATALF("m->authorship = %d", (int)m->authorship); } - return 0; + rhizome_manifest_del_bundle_key(m); + switch (m->authorship) { + case AUTHOR_UNKNOWN: + WHYF("Cannot set BK because author=%s is not in keyring", alloca_tohex_sid_t(m->author)); + break; + case AUTHENTICATION_ERROR: + WHY("Cannot set BK due to error"); + break; + default: + break; + } + RETURN(0); } -int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) +int rhizome_add_manifest(rhizome_manifest *m, int ttl) { if (config.debug.rhizome) - DEBUGF("rhizome_add_manifest(m_in=%p, ttl=%d)",m_in, ttl); + DEBUGF("rhizome_add_manifest(m=%p, ttl=%d)",m, ttl); - if (m_in->finalised==0) + if (m->finalised==0) return WHY("Manifest must be finalised before being stored"); /* Store time to live, clamped to within legal range */ - m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl; + m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl; - if (rhizome_manifest_check_sanity(m_in)) + if (rhizome_manifest_check_sanity(m)) return -1; - if (m_in->fileLength && !rhizome_exists(&m_in->filehash)) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize > 0 && !rhizome_exists(&m->filehash)) return WHY("File has not been imported"); /* If the manifest already has an ID */ - if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic)) + if (rhizome_bid_t_is_zero(m->cryptoSignPublic)) return WHY("Manifest does not have an ID"); /* Discard the new manifest unless it is newer than the most recent known version with the same ID */ int64_t storedversion = -1; - switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m_in->cryptoSignPublic, END)) { + switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END)) { case -1: return WHY("Select failed"); case 0: @@ -268,18 +282,18 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) break; case 1: if (config.debug.rhizome) - DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m_in->version); - if (m_in->version < storedversion) + DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m->version); + if (m->version < storedversion) return WHY("Newer version exists"); - if (m_in->version == storedversion) - return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m_in->cryptoSignPublic), m_in->version); + if (m->version == storedversion) + return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); break; default: return WHY("Select found too many rows!"); } /* Okay, it is written, and can be put directly into the rhizome database now */ - return rhizome_store_bundle(m_in); + return rhizome_store_bundle(m); } /* When voice traffic is being carried, we need to throttle Rhizome down diff --git a/rhizome.h b/rhizome.h index 61532656..5bfac3d7 100644 --- a/rhizome.h +++ b/rhizome.h @@ -21,12 +21,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define __SERVALDNA__RHIZOME_H #include +#include #include "sha2.h" +#include "uuid.h" #include "str.h" #include "strbuf.h" #include "http_server.h" #include "nacl.h" -#include #ifndef __RHIZOME_INLINE # if __GNUC__ && !__GNUC_STDC_INLINE__ @@ -104,6 +105,8 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) { } #define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary) +int cmp_rhizome_bk_t(const rhizome_bk_t *a, const rhizome_bk_t *b); +int str_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex); extern time_ms_t rhizome_voice_timeout; @@ -118,13 +121,10 @@ extern time_ms_t rhizome_voice_timeout; #define RHIZOME_IDLE_TIMEOUT 20000 -#define EXISTING_BUNDLE_ID 1 -#define NEW_BUNDLE_ID 2 - typedef struct rhizome_signature { unsigned char signature[crypto_sign_edwards25519sha512batch_BYTES +crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES+1]; - int signatureLength; + size_t signatureLength; } rhizome_signature; #define RHIZOME_BAR_BYTES 32 @@ -138,85 +138,246 @@ typedef struct rhizome_signature { #define MAX_MANIFEST_VARS 256 #define MAX_MANIFEST_BYTES 8192 -typedef struct rhizome_manifest { - int manifest_record_number; - int manifest_bytes; - int manifest_all_bytes; - unsigned char manifestdata[MAX_MANIFEST_BYTES]; - unsigned char manifesthash[crypto_hash_sha512_BYTES]; - /* CryptoSign key pair for this manifest. - The filename as distributed on Rhizome will be the public key - of this pair, thus ensuring that noone can tamper with a bundle - except the creator. */ +#define RHIZOME_SIZE_UNSET UINT64_MAX + +typedef struct rhizome_manifest +{ + int manifest_record_number; + + /* CryptoSign key pair for this manifest. The public key is the Bundle ID + * (aka Manifest ID). + */ rhizome_bid_t cryptoSignPublic; unsigned char cryptoSignSecret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]; - /* Whether we have the secret for this manifest on hand */ - int haveSecret; - int var_count; - char *vars[MAX_MANIFEST_VARS]; - char *values[MAX_MANIFEST_VARS]; + /* Whether cryptoSignSecret is correct (ie, bundle secret is known) + */ + enum { SECRET_UNKNOWN = 0, EXISTING_BUNDLE_ID, NEW_BUNDLE_ID } haveSecret; - int sig_count; - /* Parties who have signed this manifest (raw byte format) */ + /* Version of the manifest. Typically the number of milliseconds since 1970. + * A value of zero (0) means it has not been set yet. + * TODO: change this to uint64_t. + */ + int64_t version; + + /* Payload is described by the offset of its tail (number of missing bytes + * before the first byte in the payload), its size (number of bytes) and the + * hash of its content. Bundle size = tail + filesize. + */ + uint64_t tail; + uint64_t filesize; + rhizome_filehash_t filehash; + + /* All the manifest fields in original order (the order affects the manifest + * hash which was used to sign the manifest, so the signature can only be + * checked if order is preserved). + * + * TODO: reduce to only unknown fields. + * + * TODO: store all vars and values as NUL terminated strings within + * manifestdata[], not malloc()/free() heap, to reduce memory fragmentation + * and allow manifest struct copying without string lifetime issues. + */ + unsigned short var_count; + const char *vars[MAX_MANIFEST_VARS]; + const char *values[MAX_MANIFEST_VARS]; + + /* Parties who have signed this manifest (binary format, malloc(3)). + * Recognised signature types: + * 0x17 = crypto_sign_edwards25519sha512batch() + */ + unsigned short sig_count; unsigned char *signatories[MAX_MANIFEST_VARS]; - /* - 0x17 = crypto_sign_edwards25519sha512batch() - */ - unsigned char signatureTypes[MAX_MANIFEST_VARS]; + uint8_t signatureTypes[MAX_MANIFEST_VARS]; - // errors only involve the correctness of fields that are mandatory for - // proper operation of the transport and storage layer - int errors; - // a warning indicates that the manifest cannot be perfectly understood by this version of rhizome - // during add, the manifest should not be finalised and imported - // during extract an error should be displayed. - int warnings; - time_ms_t inserttime; - - /* Set non-zero after variables have been packed and - signature blocks appended. - All fields below may not be valid until the manifest has been finalised */ - int finalised; + /* Set to non-zero if a manifest has been parsed that cannot be fully + * understood by this version of Rhizome (probably from a future or a very + * old past version of Rhizome). During add (local injection), the manifest + * should not be imported. During extract (local decode) a warning or error + * message should be logged. Manifests marked as malformed are still + * transported, imported and exported normally, as long as their signature is + * valid. + */ + unsigned short malformed; + + /* Set non-zero after variables have been packed and signature blocks + * appended. All fields below may not be valid until the manifest has been + * finalised. + */ + bool_t finalised; + + /* Whether the manifest contains a signature that corresponds to the manifest + * id (ie public key). + */ + bool_t selfSigned; + + /* If set, unlink(2) the associated file when freeing the manifest. + */ + bool_t dataFileUnlinkOnFree; + + /* Set if the ID field (cryptoSignPublic) contains a bundle ID. + */ + bool_t has_id; + + /* Set if the tail field is valid, ie, the bundle is a journal. + */ + bool_t is_journal; + + /* Set if the date field is valid, ie, the manifest contains a valid "date" + * field. + */ + bool_t has_date; + + /* Set if the bundle_key field is valid, ie, the manifest contains a valid + * "BK" field. + */ + bool_t has_bundle_key; + + /* Set if the sender and recipient fields are valid, ie, the manifest + * contains a valid "sender"/"recipient" field. + */ + bool_t has_sender; + bool_t has_recipient; + + /* Local authorship. Useful for dividing bundle lists between "sent" and + * "inbox" views. + */ + enum rhizome_bundle_authorship { + ANONYMOUS = 0, // 'author' element is not valid + AUTHOR_NOT_CHECKED, // 'author' element is valid but not checked + AUTHENTICATION_ERROR, // author check failed, don't try again + AUTHOR_UNKNOWN, // author is not a local identity + AUTHOR_LOCAL, // author is in keyring (unlocked) but not verified + AUTHOR_IMPOSTOR, // author is a local identity but fails verification + AUTHOR_AUTHENTIC // a local identity is the verified author + } authorship; /* time-to-live in hops of this manifest. */ int ttl; - /* When finalised, we keep the filehash and maximum priority due to any - group membership handy */ - int64_t fileLength; - int64_t journalTail; - rhizome_filehash_t filehash; - int fileHighestPriority; + /* Absolute path of the file associated with the manifest */ - char *dataFileName; - /* If set, unlink(2) the associated file when freeing the manifest */ - int dataFileUnlinkOnFree; - + const char *dataFileName; + /* Whether the paylaod is encrypted or not */ - int payloadEncryption; + enum rhizome_manifest_crypt { + PAYLOAD_CRYPT_UNKNOWN = 0, + PAYLOAD_CLEAR, + PAYLOAD_ENCRYPTED + } payloadEncryption; unsigned char payloadKey[RHIZOME_CRYPT_KEY_BYTES]; unsigned char payloadNonce[crypto_stream_xsalsa20_NONCEBYTES]; - /* Whether the manifest contains a signature that corresponds to the - manifest id (ie public key) */ - int selfSigned; + /* From the "date" field, if present. The number of milliseconds since 1970 + * when the bundle was last modified. + */ + time_ms_t date; - /* Version of the manifest. Typically the number of milliseconds since 1970. */ - int64_t version; - - int group_count; - char *groups[MAX_MANIFEST_VARS]; + /* From the "service" field, which should always be present. + */ + const char *service; - /* Author of the manifest. A reference to a local keyring entry. Manifests - * not authored locally will have the ANY author (all zeros). + /* From the optional "name" field. NULL if there is no "name" field in the + * manifest. + */ + const char *name; + + /* Bundle Key "BK" field from the manifest. + */ + rhizome_bk_t bundle_key; + + /* Sender and recipient fields, if present in the manifest. + */ + sid_t sender; + sid_t recipient; + + /* Local data, not encapsulated in the bundle. The ROWID of the SQLite + * MANIFESTS table row in which this manifest is stored. Zero if the + * manifest has not been stored yet. + */ + uint64_t rowid; + + /* Local data, not encapsulated in the bundle. The system time of the most + * recent INSERT or UPDATE of the manifest into the store. Zero if the manifest + * has not been stored yet. + */ + time_ms_t inserttime; + + /* Local data, not encapsulated in the bundle. The author of the manifest. + * A reference to a local keyring entry. Manifests not authored locally will + * have an ANY author (all zeros). */ sid_t author; + /* Unused. SHOULD BE DELETED. + */ + unsigned group_count; + char *groups[MAX_MANIFEST_VARS]; + + size_t manifest_body_bytes; + size_t manifest_all_bytes; + unsigned char manifestdata[MAX_MANIFEST_BYTES]; + unsigned char manifesthash[crypto_hash_sha512_BYTES]; + } rhizome_manifest; +/* These setter functions (methods) are needed because the relevant attributes + * are stored in two places: in the vars[] array and in a dedicated struct + * element. + * + * TODO: refactor to remove the redundancy, possibly removing these setter + * functions as well. + * + * @author Andrew Bettison + */ +#define rhizome_manifest_set_id(m,v) _rhizome_manifest_set_id(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_version(m,v) _rhizome_manifest_set_version(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_filesize(m,v) _rhizome_manifest_set_filesize(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_filehash(m,v) _rhizome_manifest_set_filehash(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_tail(m,v) _rhizome_manifest_set_tail(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_bundle_key(m,v) _rhizome_manifest_set_bundle_key(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_bundle_key(m) _rhizome_manifest_del_bundle_key(__WHENCE__,(m)) +#define rhizome_manifest_set_service(m,v) _rhizome_manifest_set_service(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_service(m) _rhizome_manifest_del_service(__WHENCE__,(m)) +#define rhizome_manifest_set_name(m,v) _rhizome_manifest_set_name(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_name(m) _rhizome_manifest_del_name(__WHENCE__,(m)) +#define rhizome_manifest_set_date(m,v) _rhizome_manifest_set_date(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_date(m) _rhizome_manifest_del_date(__WHENCE__,(m)) +#define rhizome_manifest_set_sender(m,v) _rhizome_manifest_set_sender(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_sender(m) _rhizome_manifest_del_sender(__WHENCE__,(m)) +#define rhizome_manifest_set_recipient(m,v) _rhizome_manifest_set_recipient(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_recipient(m) _rhizome_manifest_del_recipient(__WHENCE__,(m)) +#define rhizome_manifest_set_crypt(m,v) _rhizome_manifest_set_crypt(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_rowid(m,v) _rhizome_manifest_set_rowid(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_inserttime(m,v) _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v)) +#define rhizome_manifest_set_author(m,v) _rhizome_manifest_set_author(__WHENCE__,(m),(v)) +#define rhizome_manifest_del_author(m) _rhizome_manifest_del_author(__WHENCE__,(m)) + +void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *); +void _rhizome_manifest_set_version(struct __sourceloc, rhizome_manifest *, int64_t); // TODO change to uint64_t +void _rhizome_manifest_set_filesize(struct __sourceloc, rhizome_manifest *, uint64_t); +void _rhizome_manifest_set_filehash(struct __sourceloc, rhizome_manifest *, const rhizome_filehash_t *); +void _rhizome_manifest_set_tail(struct __sourceloc, rhizome_manifest *, uint64_t); +void _rhizome_manifest_set_bundle_key(struct __sourceloc, rhizome_manifest *, const rhizome_bk_t *); +void _rhizome_manifest_del_bundle_key(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_service(struct __sourceloc, rhizome_manifest *, const char *); +void _rhizome_manifest_del_service(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_name(struct __sourceloc, rhizome_manifest *, const char *); +void _rhizome_manifest_del_name(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_date(struct __sourceloc, rhizome_manifest *, time_ms_t); +void _rhizome_manifest_del_date(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_sender(struct __sourceloc, rhizome_manifest *, const sid_t *); +void _rhizome_manifest_del_sender(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *); +void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *); +void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt); +void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t); +void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t); +void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *); +void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *); + /* Supported service identifiers. These go in the 'service' field of every * manifest, and indicate which application must be used to process the bundle * after it is received by Rhizome. @@ -246,6 +407,7 @@ int create_rhizome_datastore_dir(); #define FORM_RHIZOME_IMPORT_PATH(buf,fmt,...) (form_rhizome_import_path((buf), sizeof(buf), (fmt), ##__VA_ARGS__)) extern sqlite3 *rhizome_db; +uuid_t rhizome_db_uuid; int rhizome_opendb(); int rhizome_close_db(); @@ -261,14 +423,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report); int rhizome_manifest_createid(rhizome_manifest *m); int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed); -int rhizome_strn_is_manifest_id(const char *text); -int rhizome_str_is_manifest_id(const char *text); -int rhizome_strn_is_bundle_key(const char *text); -int rhizome_str_is_bundle_key(const char *text); -int rhizome_strn_is_bundle_crypt_key(const char *text); -int rhizome_str_is_bundle_crypt_key(const char *text); -int rhizome_strn_is_file_hash(const char *text); -int rhizome_str_is_file_hash(const char *text); int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call); @@ -285,35 +439,45 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int #define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1) +struct rhizome_manifest_summary { + rhizome_bid_t bid; + int64_t version; + size_t body_len; +}; + +int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ); + int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append); int rhizome_manifest_selfsign(rhizome_manifest *m); int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority); int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp); int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize); +int rhizome_manifest_validate(rhizome_manifest *m); int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out); -char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen); -int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var); -int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value); -int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value); -int rhizome_manifest_del(rhizome_manifest *m, const char *var); -int64_t rhizome_file_size(char *filename); + void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m); #define rhizome_manifest_free(m) _rhizome_manifest_free(__WHENCE__,m) rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence); #define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__) + int rhizome_manifest_pack_variables(rhizome_manifest *m); int rhizome_store_bundle(rhizome_manifest *m); int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp); -int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); -int rhizome_clean_payload(const char *fileidhex); int rhizome_store_file(rhizome_manifest *m,const unsigned char *key); int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath); -int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp, rhizome_bk_t *bsk); + +int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp); + +int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *); +int rhizome_manifest_add_bundle_key(rhizome_manifest *); + +void rhizome_find_bundle_author_and_secret(rhizome_manifest *m); +int rhizome_lookup_author(rhizome_manifest *m); +void rhizome_authenticate_author(rhizome_manifest *m); int rhizome_manifest_verify(rhizome_manifest *m); int rhizome_manifest_check_sanity(rhizome_manifest *m_in); -int rhizome_manifest_bind_id(rhizome_manifest *m_in); int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate); int rhizome_add_manifest(rhizome_manifest *m_in,int ttl); @@ -355,6 +519,7 @@ enum sqlbind_type { TOHEX, // const unsigned char *binary, unsigned bytes TEXT_TOUPPER, // const char *text, TEXT_LEN_TOUPPER, // const char *text, unsigned bytes + UUID_T, // const uuid_t *uuidp NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ... INDEX = 0xfade0000, // INDEX|INT, int index, ... NAMED = 0xdead0000 // NAMED|INT, const char *label, ... @@ -402,7 +567,7 @@ int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, st #define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__) double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); -int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); +int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs); int rhizome_update_file_priority(const char *fileid); int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found); int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar); @@ -410,9 +575,6 @@ int64_t rhizome_bar_version(const unsigned char *bar); uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar); int rhizome_is_bar_interesting(unsigned char *bar); int rhizome_is_manifest_interesting(rhizome_manifest *m); -int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, - const char *sender_sid, const char *recipient_sid, - int limit, int offset, char count_rows); int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m); int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m); int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m); @@ -426,7 +588,12 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp); int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax); int monitor_announce_bundle(rhizome_manifest *m); -int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned char **rs); +enum rhizome_secret_disposition { + FOUND_RHIZOME_SECRET = 0, + IDENTITY_NOT_FOUND, + IDENTITY_HAS_NO_RHIZOME_SECRET, +}; +enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs); int rhizome_bk_xor_stream( const rhizome_bid_t *bidp, const unsigned char *rs, @@ -448,16 +615,40 @@ int rhizome_secret2bk( const unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES] ); unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m); -int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk); -int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk); int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk, const unsigned char *pk,rhizome_signature *out); -int rhizome_verify_bundle_privatekey(rhizome_manifest *m, const unsigned char *sk, - const unsigned char *pk); -int rhizome_find_bundle_author(rhizome_manifest *m); +int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned char *pk); int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int timeout); int rhizome_ignore_manifest_check(unsigned char *bid_prefix, int prefix_len); +/* Rhizome list cursor for iterating over all or a subset of manifests in the store. + */ +struct rhizome_list_cursor { + // Query parameters that narrow the set of listed bundles. + const char *service; + const char *name; + bool_t is_sender_set; + bool_t is_recipient_set; + sid_t sender; + sid_t recipient; + // If set, then the cursor moves in ascending (chronological) order starting + // from the first bundle with rowid > rowid_since. If zero, then the cursor + // moves in descending (reverse chronological) order starting from the most + // recent bundle. + uint64_t rowid_since; + // Set by calling the next() function. + rhizome_manifest *manifest; + // Private state. + sqlite3_stmt *_statement; + uint64_t _rowid_current; + uint64_t _rowid_last; // for re-opening query +}; + +int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *); +int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *); +void rhizome_list_commit(struct rhizome_list_cursor *); +void rhizome_list_release(struct rhizome_list_cursor *); + /* one manifest is required per candidate, plus a few spare. so MAX_RHIZOME_MANIFESTS must be > MAX_CANDIDATES. */ @@ -471,7 +662,7 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng struct rhizome_write_buffer { struct rhizome_write_buffer *_next; - int64_t offset; + uint64_t offset; size_t buffer_size; size_t data_size; unsigned char data[0]; @@ -483,10 +674,10 @@ struct rhizome_write uint64_t temp_id; char id_known; - int64_t tail; - int64_t file_offset; - int64_t written_offset; - int64_t file_length; + uint64_t tail; + uint64_t file_offset; + uint64_t written_offset; + uint64_t file_length; struct rhizome_write_buffer *buffer_list; size_t buffer_size; @@ -521,9 +712,9 @@ struct rhizome_read int64_t blob_rowid; int blob_fd; - int64_t tail; - int64_t offset; - int64_t length; + uint64_t tail; + uint64_t offset; + uint64_t length; }; /* Rhizome-specific HTTP request handling. @@ -532,40 +723,45 @@ typedef struct rhizome_http_request { struct http_request http; // MUST BE FIRST ELEMENT - /* Identify request from others being run. - Monotonic counter feeds it. Only used for debugging when we write - post-.log files for multi-part form requests. */ + /* Identify request from others being run. Monotonic counter feeds it. Only + * used for debugging when we write post-.log files for multi-part form + * requests. + */ unsigned int uuid; - struct rhizome_read read_state; - - /* File currently being written to while decoding POST multipart form */ + /* For receiving a POST multipart form: + */ + // Which part is currently being received enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part; + // Temporary file currently current part is being written to int part_fd; - /* Which parts have been received in POST multipart form */ + // Which parts have already been received bool_t received_manifest; bool_t received_data; - /* Name of data file supplied */ + // Name of data file supplied in part's Content-Disposition header, filename + // parameter (if any) char data_file_name[MIME_FILENAME_MAXLEN + 1]; - /* The source specification data which are used in different ways by different - request types */ - char source[1024]; - int64_t source_index; - int64_t source_count; - int source_record_size; - unsigned int source_flags; - - const char *sql_table; - const char *sql_row; - int64_t rowid; - /* source_index used for offset in blob */ - int64_t blob_end; + union { + /* For responses that send part or all of a payload. + */ + struct rhizome_read read_state; + + /* For responses that list manifests. + */ + struct { + enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase; + uint64_t rowid_highest; + size_t rowcount; + time_ms_t end_time; + struct rhizome_list_cursor cursor; + } list; + } u; } rhizome_http_request; int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, - uint64_t offset,int count,unsigned char *bytes, + uint64_t offset, size_t count,unsigned char *bytes, int type); int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid, int64_t fileLength,int priority); @@ -702,15 +898,14 @@ enum rhizome_start_fetch_result { enum rhizome_start_fetch_result rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip, const sid_t *sidp, const unsigned char *prefix, size_t prefix_length); int rhizome_any_fetch_active(); int rhizome_any_fetch_queued(); -int rhizome_fetch_queue_bytes(); int rhizome_fetch_status_html(struct strbuf *b); int rhizome_fetch_has_queue_space(unsigned char log2_size); struct http_response_parts { uint16_t code; char *reason; - int64_t range_start; - int64_t content_length; + uint64_t range_start; + uint64_t content_length; char *content_start; }; @@ -719,9 +914,9 @@ int unpack_http_response(char *response, struct http_response_parts *parts); /* rhizome storage methods */ int rhizome_exists(const rhizome_filehash_t *hashp); -int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority); +int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority); int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size); -int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size); +int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size); int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m); int rhizome_write_file(struct rhizome_write *write, const char *filename); int rhizome_fail_write(struct rhizome_write *write); @@ -730,21 +925,20 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath); int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length); int rhizome_stat_file(rhizome_manifest *m, const char *filepath); int rhizome_add_file(rhizome_manifest *m, const char *filepath); -int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk); +int rhizome_derive_payload_key(rhizome_manifest *m); -int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size); -int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len); -int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename); +int rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len); +int rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename); int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length); -int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, +int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset, const unsigned char *key, const unsigned char *nonce); int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp); ssize_t rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length); -int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len); +ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len); int rhizome_read_close(struct rhizome_read *read); -int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); -int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); +int rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state); +int rhizome_extract_file(rhizome_manifest *m, const char *filepath); int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length); int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, size_t length); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 4cf1f7d3..77754411 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -24,66 +24,373 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "conf.h" #include "rhizome.h" #include "str.h" +#include "mem.h" +#include "keyring.h" +#include "dataformats.h" +static const char *rhizome_manifest_get(const rhizome_manifest *m, const char *var) +{ + unsigned i; + for (i = 0; i < m->var_count; ++i) + if (strcmp(m->vars[i], var) == 0) + return m->values[i]; + return NULL; +} + +#if 0 +static int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var) +{ + unsigned i; + for (i = 0; i < m->var_count; ++i) + if (strcmp(m->vars[i], var) == 0) { + int64_t val; + return str_to_int64(m->values[i], 10, &val, NULL) ? val : -1; + } + return -1; +} +#endif + +/* @author Andrew Bettison + */ +static int _rhizome_manifest_del(struct __sourceloc __whence, rhizome_manifest *m, const char *var) +{ + if (config.debug.rhizome_manifest) + DEBUGF("DEL manifest[%d].%s", m->manifest_record_number, var); + int ret = 0; + unsigned i; + for (i = 0; i < m->var_count; ++i) + if (strcmp(m->vars[i], var) == 0) { + free((char *) m->vars[i]); + free((char *) m->values[i]); + --m->var_count; + m->finalised = 0; + ret = 1; + break; + } + for (; i < m->var_count; ++i) { + m->vars[i] = m->vars[i + 1]; + m->values[i] = m->values[i + 1]; + } + return ret; +} + +#define rhizome_manifest_set(m,var,value) _rhizome_manifest_set(__WHENCE__, (m), (var), (value)) +#define rhizome_manifest_set_ll(m,var,value) _rhizome_manifest_set_ll(__WHENCE__, (m), (var), (value)) +#define rhizome_manifest_del(m,var) _rhizome_manifest_del(__WHENCE__, (m), (var)) + +static const char *_rhizome_manifest_set(struct __sourceloc __whence, rhizome_manifest *m, const char *var, const char *value) +{ + if (config.debug.rhizome_manifest) + DEBUGF("SET manifest[%d].%s = %s", m->manifest_record_number, var, alloca_str_toprint(value)); + unsigned i; + for(i=0;ivar_count;i++) + if (strcmp(m->vars[i],var) == 0) { + const char *ret = str_edup(value); + if (ret == NULL) + return NULL; + free((char *)m->values[i]); + m->values[i] = ret; + m->finalised = 0; + return ret; + } + if (m->var_count >= NELS(m->vars)) + return WHYNULL("no more manifest vars"); + if ((m->vars[m->var_count] = str_edup(var)) == NULL) + return NULL; + const char *ret = m->values[m->var_count] = str_edup(value); + if (ret == NULL) { + free((char *)m->vars[i]); + m->vars[i] = NULL; + return NULL; + } + m->var_count++; + m->finalised = 0; + return ret; +} + +static const char *_rhizome_manifest_set_ll(struct __sourceloc __whence, rhizome_manifest *m, char *var, int64_t value) +{ + char str[50]; + snprintf(str, sizeof str, "%" PRId64, value); + return rhizome_manifest_set(m, var, str); +} + +void _rhizome_manifest_set_id(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_bid_t *bidp) +{ + const char *v = rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(*bidp)); + assert(v); // TODO: remove known manifest fields from vars[] + if (bidp != &m->cryptoSignPublic && cmp_rhizome_bid_t(&m->cryptoSignPublic, bidp) != 0) { + m->cryptoSignPublic = *bidp; + // The BID just changed, so the secret key and bundle key are no longer valid. + if (m->haveSecret) { + m->haveSecret = SECRET_UNKNOWN; + bzero(m->cryptoSignSecret, sizeof m->cryptoSignSecret); // not strictly necessary but aids debugging + } + if (m->has_bundle_key) { + m->has_bundle_key = 0; + m->bundle_key = RHIZOME_BK_NONE; // not strictly necessary but aids debugging + } + // Any authenticated author is no longer authenticated, but is still known to be in the keyring. + if (m->authorship == AUTHOR_AUTHENTIC) + m->authorship = AUTHOR_LOCAL; + } +} + +void _rhizome_manifest_set_version(struct __sourceloc __whence, rhizome_manifest *m, int64_t version) +{ + const char *v = rhizome_manifest_set_ll(m, "version", version); + assert(v); // TODO: remove known manifest fields from vars[] + m->version = version; +} + +void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifest *m, uint64_t size) +{ + const char *v = rhizome_manifest_set_ll(m, "filesize", size); + assert(v); // TODO: remove known manifest fields from vars[] + m->filesize = size; + if (m->filesize == 0) + rhizome_manifest_set_filehash(m, NULL); +} + +/* Must always set file size before setting the file hash, to avoid assertion failures. + */ +void _rhizome_manifest_set_filehash(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_filehash_t *hash) +{ + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (hash) { + assert(m->filesize > 0); + const char *v = rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(*hash)); + assert(v); // TODO: remove known manifest fields from vars[] + m->filehash = *hash; + } else { + assert(m->filesize == 0); + rhizome_manifest_del(m, "filehash"); + m->filehash = RHIZOME_FILEHASH_NONE; + } +} + +void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m, uint64_t tail) +{ + const char *v = rhizome_manifest_set_ll(m, "tail", tail); + assert(v); // TODO: remove known manifest fields from vars[] + m->tail = tail; + m->is_journal = (tail != RHIZOME_SIZE_UNSET); +} + +void _rhizome_manifest_set_bundle_key(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_bk_t *bkp) +{ + if (bkp) { + const char *v = rhizome_manifest_set(m, "BK", alloca_tohex_rhizome_bk_t(*bkp)); + assert(v); // TODO: remove known manifest fields from vars[] + m->bundle_key = *bkp; + m->has_bundle_key = 1; + } else + _rhizome_manifest_del_bundle_key(__whence, m); +} + +void _rhizome_manifest_del_bundle_key(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->has_bundle_key) { + rhizome_manifest_del(m, "BK"); + m->has_bundle_key = 0; + m->bundle_key = RHIZOME_BK_NONE; // not strictly necessary, but aids debugging + } else + assert(rhizome_manifest_get(m, "BK") == NULL); + // Once there is no BK field, any authenticated authorship is no longer. + if (m->authorship == AUTHOR_AUTHENTIC) + m->authorship = AUTHOR_LOCAL; +} + +void _rhizome_manifest_set_service(struct __sourceloc __whence, rhizome_manifest *m, const char *service) +{ + if (service) { + const char *v = rhizome_manifest_set(m, "service", service); + assert(v); // TODO: remove known manifest fields from vars[] + m->service = v; + } else + _rhizome_manifest_del_service(__whence, m); +} + +void _rhizome_manifest_del_service(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->service) { + m->service = NULL; + rhizome_manifest_del(m, "service"); + } else + assert(rhizome_manifest_get(m, "service") == NULL); +} + +void _rhizome_manifest_set_name(struct __sourceloc __whence, rhizome_manifest *m, const char *name) +{ + if (name) { + const char *v = rhizome_manifest_set(m, "name", name); + assert(v); // TODO: remove known manifest fields from vars[] + m->name = v; + } else { + rhizome_manifest_del(m, "name"); + m->name = NULL; + } +} + +void _rhizome_manifest_del_name(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->name) { + m->name = NULL; + rhizome_manifest_del(m, "name"); + } else + assert(rhizome_manifest_get(m, "name") == NULL); +} + +void _rhizome_manifest_set_date(struct __sourceloc __whence, rhizome_manifest *m, time_ms_t date) +{ + const char *v = rhizome_manifest_set_ll(m, "date", date); + assert(v); // TODO: remove known manifest fields from vars[] + m->date = date; + m->has_date = 1; +} + +void _rhizome_manifest_set_sender(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp) +{ + if (sidp) { + const char *v = rhizome_manifest_set(m, "sender", alloca_tohex_sid_t(*sidp)); + assert(v); // TODO: remove known manifest fields from vars[] + m->sender = *sidp; + m->has_sender = 1; + } else + _rhizome_manifest_del_sender(__whence, m); +} + +void _rhizome_manifest_del_sender(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->has_sender) { + rhizome_manifest_del(m, "sender"); + m->sender = SID_ANY; + m->has_sender = 0; + } else + assert(rhizome_manifest_get(m, "sender") == NULL); +} + +void _rhizome_manifest_set_recipient(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp) +{ + if (sidp) { + const char *v = rhizome_manifest_set(m, "recipient", alloca_tohex_sid_t(*sidp)); + assert(v); // TODO: remove known manifest fields from vars[] + m->recipient = *sidp; + m->has_recipient = 1; + } else + _rhizome_manifest_del_recipient(__whence, m); +} + +void _rhizome_manifest_del_recipient(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->has_recipient) { + rhizome_manifest_del(m, "recipient"); + m->recipient = SID_ANY; + m->has_recipient = 0; + } else + assert(rhizome_manifest_get(m, "recipient") == NULL); +} + +void _rhizome_manifest_set_crypt(struct __sourceloc __whence, rhizome_manifest *m, enum rhizome_manifest_crypt flag) +{ + switch (flag) { + case PAYLOAD_CRYPT_UNKNOWN: + rhizome_manifest_del(m, "crypt"); + break; + case PAYLOAD_CLEAR: { + const char *v = rhizome_manifest_set(m, "crypt", "0"); + assert(v); // TODO: remove known manifest fields from vars[] + break; + } + case PAYLOAD_ENCRYPTED: { + const char *v = rhizome_manifest_set(m, "crypt", "1"); + assert(v); // TODO: remove known manifest fields from vars[] + break; + } + default: abort(); + } + m->payloadEncryption = flag; +} + +void _rhizome_manifest_set_rowid(struct __sourceloc __whence, rhizome_manifest *m, uint64_t rowid) +{ + m->rowid = rowid; +} + +void _rhizome_manifest_set_inserttime(struct __sourceloc __whence, rhizome_manifest *m, time_ms_t time) +{ + m->inserttime = time; +} + +void _rhizome_manifest_set_author(struct __sourceloc __whence, rhizome_manifest *m, const sid_t *sidp) +{ + if (sidp) { + if (m->authorship == ANONYMOUS || cmp_sid_t(&m->author, sidp) != 0) { + if (config.debug.rhizome_manifest) + DEBUGF("SET manifest[%d] author = %s", m->manifest_record_number, alloca_tohex_sid_t(*sidp)); + m->author = *sidp; + m->authorship = AUTHOR_NOT_CHECKED; + } + } else + _rhizome_manifest_del_author(__whence, m); +} + +void _rhizome_manifest_del_author(struct __sourceloc __whence, rhizome_manifest *m) +{ + if (m->authorship != ANONYMOUS) { + if (config.debug.rhizome_manifest) + DEBUGF("DEL manifest[%d] author", m->manifest_record_number); + m->author = SID_ANY; + m->authorship = ANONYMOUS; + } +} + +/* Compute the hash of the manifest's body, including the NUL byte that separates the body from + * the signature block, and verify that a signature is present and is correct. + * + * Returns 1 if the manifest signature is valid, ie, the signature is a self-signature using the + * manifest's own private key. Sets the m->finalised flag to 1. + * + * Returns 0 if there are no signatures or if the signature block does not verify. + * + * Only call this function on manifests for which rhizome_manifest_validate(m) has returned true. + */ int rhizome_manifest_verify(rhizome_manifest *m) { - int end_of_text=0; - - /* find end of manifest body and start of signatures */ - while(m->manifestdata[end_of_text]&&end_of_textmanifest_all_bytes) - end_of_text++; - end_of_text++; /* include null byte in body for verification purposes */ - - /* Calculate hash of the text part of the file, as we need to couple this with - each signature block to */ - crypto_hash_sha512(m->manifesthash,m->manifestdata,end_of_text); - - /* Read signature blocks from file. */ - int ofs=end_of_text; - while(ofsmanifest_all_bytes) { - if (config.debug.rhizome) DEBUGF("ofs=0x%x, m->manifest_bytes=0x%x", ofs,m->manifest_all_bytes); - if (rhizome_manifest_extract_signature(m,&ofs)) break; - } - - if (m->sig_count==0) { - WHYF("Manifest has zero valid signatures"); - m->errors++; - } - - /* Make sure that id variable is correct */ - { - rhizome_bid_t bid; - char *id = rhizome_manifest_get(m,"id",NULL,0); - if (!id) { - WARN("Manifest lacks 'id' field"); - m->errors++; - } else if (str_to_rhizome_bid_t(&bid, id) == -1) { - WARN("Invalid manifest 'id' field"); - m->errors++; - } else if (m->sig_count == 0 || memcmp(m->signatories[0], bid.binary, sizeof bid.binary) != 0) { - if (config.debug.rhizome) { - if (m->sig_count>0) { - DEBUGF("Manifest id variable does not match first signature block (signature key is %s)", - alloca_tohex(m->signatories[0], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES) - ); - } else { - DEBUG("Manifest has no signature blocks, but should have self-signature block"); - } - } - m->errors++; - m->selfSigned=0; - } else { - m->selfSigned=1; - } + assert(m->manifest_body_bytes > 0); + assert(m->manifest_all_bytes > 0); + assert(m->manifest_body_bytes <= m->manifest_all_bytes); + if (m->manifest_body_bytes == m->manifest_all_bytes) + assert(m->manifestdata[m->manifest_body_bytes - 1] == '\0'); + // Hash the body + crypto_hash_sha512(m->manifesthash, m->manifestdata, m->manifest_body_bytes); + // Read signature blocks + unsigned ofs = m->manifest_body_bytes; + while (ofs < m->manifest_all_bytes) { + if (rhizome_manifest_extract_signature(m, &ofs) == -1) + break; } + assert(ofs <= m->manifest_all_bytes); - /* Mark as finalised, as it is all read and intact, - unless of course it has errors, or is lacking a self-signature. */ - if (!m->errors) m->finalised=1; - else WHY("Verified a manifest that has errors, so marking as not finalised"); - - if (m->errors) return WHY("Manifest verification failed"); - else return 0; + // Make sure the first signatory's public key is the bundle ID + assert(m->has_id); + if (m->sig_count == 0) { + if (config.debug.rhizome) + DEBUG("Manifest has no signature blocks, but should have self-signature block"); + m->selfSigned = 0; + return 0; + } + if (memcmp(m->signatories[0], m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary) != 0) { + if (config.debug.rhizome) + DEBUGF("Manifest id does not match first signature block (signature key is %s)", + alloca_tohex(m->signatories[0], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES) + ); + m->selfSigned = 0; + return 0; + } + m->selfSigned = 1; + m->finalised = 1; + return 1; } ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_size) @@ -99,233 +406,396 @@ ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_s return ret; } -int rhizome_manifest_parse(rhizome_manifest *m) +static void rhizome_manifest_clear(rhizome_manifest *m) +{ + while (m->var_count) { + --m->var_count; + free((char *) m->vars[m->var_count]); + free((char *) m->values[m->var_count]); + m->vars[m->var_count] = m->values[m->var_count] = NULL; + } + while (m->sig_count) { + --m->sig_count; + free(m->signatories[m->sig_count]); + m->signatories[m->sig_count] = NULL; + } + m->malformed = 0; + m->has_id = 0; + m->is_journal = 0; + m->filesize = RHIZOME_SIZE_UNSET; + m->tail = RHIZOME_SIZE_UNSET; + m->version = 0; + // TODO initialise more fields +} + +int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ) +{ + const char *const end = buf + len; + int has_bid = 0; + int has_version = 0; + const char *begin = buf; + enum { Label, Value, Error } state = Label; + const char *p; + for (p = buf; state != Error && p < end && *p; ++p) + switch (state) { + case Label: + if (*p == '=') { + if (p == begin) + state = Error; // nil field name + else { + int *has = NULL; + if (p == begin + 2 && strncmp(begin, "id", 2) == 0) + has = &has_bid; + else if (p == begin + 7 && strncmp(begin, "version", 7) == 0) + has = &has_version; + state = Value; + if (has) { + if (*has) + state = Error; // duplicate + else { + *has = 1; + begin = p + 1; + } + } + } + } else if (!(p == begin ? isalpha(*p) : isalnum(*p))) + state = Error; // bad field name + break; + case Value: + if (*p == '\n') { + const char *eol = p[-1] == '\r' ? p - 1 : p; + if (has_bid == 1) { + const char *e; + if (strn_to_rhizome_bid_t(&summ->bid, begin, &e) == 0 && e == eol) + has_bid = 2; + else + state = Error; // invalid "id" field + } else if (has_version == 1) { + const char *e; + if (str_to_uint64(begin, 10, (uint64_t*)&summ->version, &e) && e == eol) + has_version = 2; + else + state = Error; // invalid "version" field + } + if (state == Value) { + state = Label; + begin = p + 1; + } + } + break; + default: + abort(); + } + if (p < end && *p == '\0') + ++p; + summ->body_len = p - buf; + return state == Label && has_bid == 2 && has_version == 2; +} + +/* Parse a Rhizome text manifest from its internal buffer up to and including the terminating NUL + * character which marks the start of the signature block. + * + * Prior to calling, the caller must set up m->manifest_all_bytes to the length of the manifest + * text, including the signature block, and set m->manifestdata[0..m->manifest_all_bytes-1] to + * contain the manifest text and signature block to be parsed. + * + * A "well formed" manifest consists of a series of zero or more lines with the form: + * + * LABEL "=" VALUE [ CR ] LF + * + * where LABEL matches the regular expression [A-Za-z][A-Za-z0-9]* (identifier without underscore) + * VALUE is any value that does not contain NUL, CR or LF (leading and trailing spaces are + * not stripped from VALUE) + * NUL is ASCII 0 + * CR is ASCII 13 + * LF is ASCII 10 + * + * Unpacks all parsed field labels and string values into the m->vars[] and m->values[] arrays, as + * pointers to malloc(3)ed NUL terminated strings, in the order they appear, and sets m->var_count + * to the number of fields unpacked. Sets m->manifest_body_bytes to the number of bytes in the text + * portion up to and including the optional NUL that starts the signature block (if present). + * + * Returns 1 if the manifest is not well formed (syntax violation), any essential field is + * malformed, or if there are any duplicate fields. In this case the m->vars[] and m->values[] + * arrays are not set and the manifest is returned to the state it was in prior to calling. + * + * Returns 0 if the manifest is well formed, if there are no duplicate fields, and if all essential + * fields are valid. Counts invalid non-essential fields and unrecognised fields in m->malformed. + * + * Returns -1 if there is an unrecoverable error (eg, malloc(3) returns NULL, out of memory). + * + * @author Andrew Bettison + */ +static int rhizome_manifest_parse(rhizome_manifest *m) { IN(); - m->manifest_all_bytes=m->manifest_bytes; - m->var_count=0; - m->journalTail=-1; + assert(m->manifest_all_bytes <= sizeof m->manifestdata); + assert(m->manifest_body_bytes == 0); + assert(m->var_count == 0); + assert(!m->malformed); + assert(!m->has_id); + assert(!m->is_journal); + assert(m->filesize == RHIZOME_SIZE_UNSET); + assert(m->tail == RHIZOME_SIZE_UNSET); + assert(m->version == 0); - /* Parse out variables, signature etc */ - int have_service = 0; - int have_id = 0; - int have_version = 0; - int have_date = 0; - int have_filesize = 0; - int have_filehash = 0; - - int ofs = 0; - while (ofs < m->manifest_bytes && m->manifestdata[ofs]) { - char line[1024]; - int limit = ofs + sizeof line - 1; - if (limit > m->manifest_bytes) - limit = m->manifest_bytes; - char *p = line; - while (ofs < limit && !(m->manifestdata[ofs] == '\0' || m->manifestdata[ofs] == '\n' || m->manifestdata[ofs] == '\r')) - *p++ = m->manifestdata[ofs++]; - *p = '\0'; - if (m->manifestdata[ofs] == '\r') - ++ofs; - if (m->manifestdata[ofs] == '\n') - ++ofs; - /* Ignore blank lines */ - if (line[0] == '\0') - continue; - /* Ignore comment lines */ - if (line[0] == '#' || line[0] == '!') - continue; - /* Parse field=value lines */ - size_t linelen = p - line; - p = strchr(line, '='); - if (p == NULL || p == line) { - m->errors++; - WARNF("Malformed manifest line: %s", alloca_toprint(80, line, linelen)); - } else { - *p++ = '\0'; - char *var = line; - char *value = p; - if (rhizome_manifest_get(m, var, NULL, 0)) { - if (config.debug.rejecteddata) - WARNF("Ill formed manifest file, duplicate variable \"%s\"", var); - m->errors++; - } else if (m->var_count >= MAX_MANIFEST_VARS) { - if (config.debug.rejecteddata) - WARN("Ill formed manifest file, too many variables"); - m->errors++; - } else { - m->vars[m->var_count] = strdup(var); - m->values[m->var_count] = strdup(value); - - // if any of these fields are not well formed, the manifest is invalid and cannot be imported - - if (strcasecmp(var, "id") == 0) { - have_id = 1; - if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) == -1) { - if (config.debug.rejecteddata) - WARNF("Invalid manifest id: %s", value); - m->errors++; - } else { - /* Force to upper case to avoid case sensitive comparison problems later. */ - str_toupper_inplace(m->values[m->var_count]); - } - } else if (strcasecmp(var, "filehash") == 0) { - have_filehash = 1; - if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) { - if (config.debug.rejecteddata) - WARNF("Invalid filehash: %s", value); - m->errors++; - } else { - /* Force to upper case to avoid case sensitive comparison problems later. */ - str_toupper_inplace(m->values[m->var_count]); - } - } else if (strcasecmp(var, "filesize") == 0) { - have_filesize = 1; - uint64_t filesize; - if (!str_to_uint64(value, 10, &filesize, NULL)) { - if (config.debug.rejecteddata) - WARNF("Invalid filesize: %s", value); - m->errors++; - } else { - m->fileLength = filesize; - } - } else if (strcasecmp(var, "version") == 0) { - have_version = 1; - uint64_t version; - if (!str_to_uint64(value, 10, &version, NULL)) { - if (config.debug.rejecteddata) - WARNF("Invalid version: %s", value); - m->errors++; - } else { - m->version = version; - } - // since rhizome *MUST* be able to carry future manifest versions - // if any of these fields are not well formed, the manifest can still be imported and exported - // but the bundle should not be added or exported - } else if (strcasecmp(var, "tail") == 0) { - uint64_t tail; - if (!str_to_uint64(value, 10, &tail, NULL)) { - if (config.debug.rejecteddata) - WARNF("Invalid tail: %s", value); - m->warnings++; - } else { - m->journalTail = tail; - } - } else if (strcasecmp(var, "BK") == 0) { - if (!rhizome_str_is_bundle_key(value)) { - if (config.debug.rejecteddata) - WARNF("Invalid BK: %s", value); - m->warnings++; - } else { - /* Force to upper case to avoid case sensitive comparison problems later. */ - str_toupper_inplace(m->values[m->var_count]); - } - } else if (strcasecmp(var, "service") == 0) { - have_service = 1; - if ( strcasecmp(value, RHIZOME_SERVICE_FILE) == 0 - || strcasecmp(value, RHIZOME_SERVICE_MESHMS) == 0 - || strcasecmp(value, RHIZOME_SERVICE_MESHMS2) == 0) { - } else { - if (config.debug.rejecteddata) - WARNF("Unsupported service: %s", value); - m->warnings++; - } - } else if (strcasecmp(var, "date") == 0) { - have_date = 1; - int64_t date; - if (!str_to_int64(value, 10, &date, NULL)) { - if (config.debug.rejecteddata) - WARNF("Invalid date: %s", value); - m->warnings++; - } - // TODO: store date in manifest struct - } else if (strcasecmp(var, "sender") == 0 || strcasecmp(var, "recipient") == 0) { - if (!str_is_subscriber_id(value)) { - if (config.debug.rejecteddata) - WARNF("Invalid %s: %s", var, value); - m->warnings++; - } else { - /* Force to upper case to avoid case sensitive comparison problems later. */ - str_toupper_inplace(m->values[m->var_count]); - } - } else if (strcasecmp(var, "name") == 0) { - if (value[0] == '\0') { - if (config.debug.rejecteddata) - WARN("Empty name"); - m->warnings++; - } - } else if (strcasecmp(var, "crypt") == 0) { - if (!(strcmp(value, "0") == 0 || strcmp(value, "1") == 0)) { - if (config.debug.rejecteddata) - WARNF("Invalid crypt: %s", value); - m->warnings++; - } else { - m->payloadEncryption = atoi(value); - } - } else { - // An unknown field is not an error... older rhizome nodes must carry newer manifests. - } - m->var_count++; - } + unsigned has_invalid_essential = 0; + unsigned has_duplicate = 0; + + const char *const end = (const char *)m->manifestdata + m->manifest_all_bytes; + const char *p; + unsigned line_number = 0; + for (p = (const char *)m->manifestdata; p < end && *p; ++p) { + ++line_number; + if (!isalpha(*p)) { + if (config.debug.rhizome_manifest) + DEBUGF("Invalid manifest field name at line %u: %s", line_number, alloca_toprint(20, p, end - p)); + break; } + const char *const plabel = p++; + while (p < end && isalnum(*p)) + ++p; + if (*p != '=') { + if (config.debug.rhizome_manifest) + DEBUGF("Invalid manifest field name at line %u: %s", line_number, alloca_toprint(-1, plabel, p - plabel + 1)); + break; + } + const char *const pvalue = ++p; + while (p < end && *p && *p != '\n') + ++p; + if (p >= end || *p != '\n') { + if (config.debug.rhizome_manifest) + DEBUGF("Missing manifest newline at line %u: %s", line_number, alloca_toprint(-1, plabel, p - plabel)); + break; + } + const char *const eol = (p > pvalue && p[-1] == '\r') ? p - 1 : p; + if (m->var_count >= NELS(m->vars)) { + if (config.debug.rhizome_manifest) + DEBUGF("Manifest field limit reached at line %u", line_number); + break; + } + assert(pvalue - plabel - 1 > 0); + const char *label = strn_edup(plabel, pvalue - plabel - 1); + const char *value = strn_edup(pvalue, eol - pvalue); + if (label == NULL || value == NULL) { + free((char *)label); + free((char *)value); + RETURN(-1); + } + enum { FIELD_UNKNOWN, FIELD_OK, FIELD_DUPLICATE, FIELD_MALFORMED, FIELD_INVALID } status = FIELD_UNKNOWN; + if (rhizome_manifest_get(m, label)) + status = FIELD_DUPLICATE; + else if (strcasecmp(label, "id") == 0) { + if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) != -1) { + status = FIELD_OK; + m->has_id = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].id = %s", m->manifest_record_number, alloca_tohex_sid_t(m->cryptoSignPublic)); + } else + status = FIELD_INVALID; + } + else if (strcasecmp(label, "version") == 0) { + uint64_t version; + if (str_to_uint64(value, 10, &version, NULL) && version != 0) { + status = FIELD_OK; + m->version = version; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].version = %"PRIu64, m->manifest_record_number, m->version); + } else + status = FIELD_INVALID; + } + else if (strcasecmp(label, "filehash") == 0) { + if (str_to_rhizome_filehash_t(&m->filehash, value) != -1 && !rhizome_filehash_t_is_zero(m->filehash)) { + status = FIELD_OK; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].filehash = %s", m->manifest_record_number, alloca_tohex_rhizome_filehash_t(m->filehash)); + } else + status = FIELD_INVALID; + } + else if (strcasecmp(label, "filesize") == 0) { + uint64_t filesize; + if (str_to_uint64(value, 10, &filesize, NULL) && filesize != RHIZOME_SIZE_UNSET) { + status = FIELD_OK; + m->filesize = filesize; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].filesize = %"PRIu64, m->manifest_record_number, m->filesize); + } else + status = FIELD_INVALID; + } + else if (strcasecmp(label, "tail") == 0) { + uint64_t tail; + if (str_to_uint64(value, 10, &tail, NULL) && tail != RHIZOME_SIZE_UNSET) { + status = FIELD_OK; + m->tail = tail; + m->is_journal = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].tail = %"PRIu64, m->manifest_record_number, m->tail); + } else + status = FIELD_INVALID; + } + // Since rhizome MUST be able to carry future manifest versions, if any of the following fields + // are not well formed, they are simply not unpacked into their respective struct elements and + // treated as an unrecognised field. The m->malformed flag is set so that the application API + // layer can refuse to add (or export?) the bundle. + else if (strcasecmp(label, "BK") == 0) { + if (str_to_rhizome_bk_t(&m->bundle_key, value) != -1) { + status = FIELD_OK; + m->has_bundle_key = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].BK = %s", m->manifest_record_number, alloca_tohex_rhizome_bk_t(m->bundle_key)); + } else + status = FIELD_MALFORMED; + } + else if (strcasecmp(label, "service") == 0) { + if (rhizome_str_is_manifest_service(value)) { + status = FIELD_OK; + m->service = value; // will be free()d when vars[] and values[] are free()d + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].service = %s", m->manifest_record_number, alloca_str_toprint(m->service)); + } else + status = FIELD_MALFORMED; + } + else if (strcasecmp(label, "date") == 0) { + int64_t date; + if (str_to_int64(value, 10, &date, NULL)) { + status = FIELD_OK; + m->date = date; + m->has_date = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].date = %"PRItime_ms_t, m->manifest_record_number, m->date); + } else + status = FIELD_MALFORMED; + } + else if (strcasecmp(label, "sender") == 0) { + if (str_to_sid_t(&m->sender, value) != -1) { + status = FIELD_OK; + m->has_sender = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].sender = %s", m->manifest_record_number, alloca_tohex_sid_t(m->sender)); + } else + status = FIELD_MALFORMED; + } + else if (strcasecmp(label, "recipient") == 0) { + if (str_to_sid_t(&m->recipient, value) != -1) { + status = FIELD_OK; + m->has_recipient = 1; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].recipient = %s", m->manifest_record_number, alloca_tohex_sid_t(m->recipient)); + } else + status = FIELD_MALFORMED; + } + else if (strcasecmp(label, "name") == 0) { + status = FIELD_OK; + m->name = value; // will be free()d when vars[] and values[] are free()d + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].name = %s", m->manifest_record_number, alloca_str_toprint(m->name)); + } + else if (strcasecmp(label, "crypt") == 0) { + if (strcmp(value, "0") == 0 || strcmp(value, "1") == 0) { + status = FIELD_OK; + m->payloadEncryption = (value[0] == '1') ? PAYLOAD_ENCRYPTED : PAYLOAD_CLEAR; + if (config.debug.rhizome_manifest) + DEBUGF("PARSE manifest[%d].crypt = %u", m->manifest_record_number, m->payloadEncryption == PAYLOAD_ENCRYPTED ? 1 : 0); + } else + status = FIELD_MALFORMED; + } + const char *reason = NULL; + switch (status) { + case FIELD_OK: + m->vars[m->var_count] = label; + m->values[m->var_count] = value; + ++m->var_count; + break; + case FIELD_DUPLICATE: + ++has_duplicate; + reason = "duplicate"; + break; + case FIELD_INVALID: + ++has_invalid_essential; + reason = "invalid"; + break; + case FIELD_UNKNOWN: + ++m->malformed; + reason = "unsupported"; + break; + case FIELD_MALFORMED: + ++m->malformed; + reason = "invalid"; + break; + default: + abort(); + } + if (reason) { + if (config.debug.rhizome_manifest) + DEBUGF("SKIP manifest[%d].%s = %s (%s)", m->manifest_record_number, label, alloca_str_toprint(value), reason); + free((char *)label); + free((char *)value); + } + assert(p < end); + assert(*p == '\n'); } - /* The null byte gets included in the check sum */ - if (ofs < m->manifest_bytes) - ++ofs; - - /* Remember where the text ends */ - int end_of_text=ofs; - m->manifest_bytes = end_of_text; - - // verify that all required fields are consistent. - if (!have_id) { - if (config.debug.rejecteddata) - WARNF("Missing manifest id field"); - m->errors++; + if ((p < end && *p) || has_invalid_essential || has_duplicate) { + rhizome_manifest_clear(m); + RETURN(1); } - if (!have_version) { - if (config.debug.rejecteddata) - WARNF("Missing version field"); - m->errors++; + // The null byte is included in the body (and checksum), not the signature block + if (p < end) { + assert(*p == '\0'); + ++p; } - if (!have_filesize) { - if (config.debug.rejecteddata) - WARNF("Missing filesize field"); - m->errors++; - } - if (!have_filehash && m->fileLength != 0) { - if (config.debug.rejecteddata) - WARNF("Missing filehash field"); - m->errors++; - } - if (have_filehash && m->fileLength == 0) { - if (config.debug.rejecteddata) - WARNF("Spurious filehash field"); - m->errors++; - } - - // warn if expected fields are missing - if (!have_service) { - if (config.debug.rejecteddata) - WARNF("Missing service field"); - m->warnings++; - } - if (!have_date) { - if (config.debug.rejecteddata) - WARNF("Missing date field"); - m->warnings++; - } - - // TODO Determine group membership here. - - if (m->errors || m->warnings) { - if (config.debug.rejecteddata) - dump("manifest body",m->manifestdata,m->manifest_bytes); - } - + m->manifest_body_bytes = p - (const char *)m->manifestdata; RETURN(0); OUT(); } +/* Return 1 if all necessary fields are present, 0 if not. Increment m->malformed if any + * unnecessary fields are missing. + */ +int rhizome_manifest_validate(rhizome_manifest *m) +{ + int ret = 1; + if (!m->has_id) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'id' field"); + ret = 0; + } + if (m->version == 0) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'version' field"); + ret = 0; + } + if (m->filesize == RHIZOME_SIZE_UNSET) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'filesize' field"); + ret = 0; + } + if (rhizome_filehash_t_is_zero(m->filehash)) { + if (m->filesize > 0) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'filehash' field"); + ret = 0; + } + } else { + if (m->filesize == 0) { + if (config.debug.rhizome_manifest) + DEBUG("Spurious 'filehash' field"); + ret = 0; + } + } + // warn if expected fields are missing + if (m->service == NULL) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'service' field"); + ++m->malformed; + } + if (!m->has_date) { + if (config.debug.rhizome_manifest) + DEBUG("Missing 'date' field"); + ++m->malformed; + } + return ret; +} + int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferP) { if (!m) @@ -334,15 +804,19 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t return WHY("Buffer too big"); if (bufferP) { - m->manifest_bytes=bufferP; - memcpy(m->manifestdata, filename, m->manifest_bytes); + m->manifest_all_bytes=bufferP; + memcpy(m->manifestdata, filename, m->manifest_all_bytes); } else { ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata); if (bytes == -1) return -1; - m->manifest_bytes = bytes; + m->manifest_all_bytes = (size_t) bytes; + } + switch (rhizome_manifest_parse(m)) { + case 0: return 0; + case -1: return -1; + default: return WHY("Invalid manifest"); } - return rhizome_manifest_parse(m); } int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out) @@ -353,7 +827,7 @@ int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t implementation. */ // TODO encrypted payloads - if (m && m->payloadEncryption) + if (m && m->payloadEncryption == PAYLOAD_ENCRYPTED) return WHY("Encryption of payloads not implemented"); uint64_t filesize = 0; @@ -389,100 +863,6 @@ int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t return 0; } -char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen) -{ - int i,j; - - if (!m) return NULL; - - for(i=0;ivar_count;i++) - if (!strcmp(m->vars[i],var)) { - if (out) { - for(j=0;(jvalues[i][j]; - if (!out[j]) break; - } - } - return m->values[i]; - } - return NULL; -} - -int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var) -{ - if (!m) - return -1; - int i; - for (i = 0; i < m->var_count; ++i) - if (!strcmp(m->vars[i], var)) { - int64_t val; - return str_to_int64(m->values[i], 10, &val, NULL) ? val : -1; - } - return -1; -} - -double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value) -{ - int i; - - if (!m) return default_value; - - for(i=0;ivar_count;i++) - if (!strcmp(m->vars[i],var)) - return strtod(m->values[i],NULL); - return default_value; -} - -/* @author Andrew Bettison - */ -int rhizome_manifest_del(rhizome_manifest *m, const char *var) -{ - int ret = 0; - int i; - for (i = 0; i < m->var_count; ++i) - if (strcmp(m->vars[i], var) == 0) { - free(m->vars[i]); - free(m->values[i]); - --m->var_count; - m->finalised = 0; - ret = 1; - break; - } - for (; i < m->var_count; ++i) { - m->vars[i] = m->vars[i + 1]; - m->values[i] = m->values[i + 1]; - } - return ret; -} - -int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value) -{ - if (!m) - return WHY("m == NULL"); - int i; - for(i=0;ivar_count;i++) - if (!strcmp(m->vars[i],var)) { - free(m->values[i]); - m->values[i]=strdup(value); - m->finalised=0; - return 0; - } - if (m->var_count >= MAX_MANIFEST_VARS) - return WHY("no more manifest vars"); - m->vars[m->var_count]=strdup(var); - m->values[m->var_count]=strdup(value); - m->var_count++; - m->finalised=0; - return 0; -} - -int rhizome_manifest_set_ll(rhizome_manifest *m, char *var, int64_t value) -{ - char str[50]; - snprintf(str, sizeof str, "%" PRId64, value); - return rhizome_manifest_set(m, var, str); -} - rhizome_manifest manifests[MAX_RHIZOME_MANIFESTS]; char manifest_free[MAX_RHIZOME_MANIFESTS]; int manifest_first_free=-1; @@ -492,7 +872,7 @@ struct __sourceloc manifest_free_whence[MAX_RHIZOME_MANIFESTS]; static void _log_manifest_trace(struct __sourceloc __whence, const char *operation) { int count_free = 0; - int i; + unsigned i; for (i = 0; i != MAX_RHIZOME_MANIFESTS; ++i) if (manifest_free[i]) ++count_free; @@ -503,7 +883,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence) { if (manifest_first_free<0) { /* Setup structures */ - int i; + unsigned i; for(i=0;i=MAX_RHIZOME_MANIFESTS) { - int i; + unsigned i; WHYF("%s(): no free manifest records, this probably indicates a memory leak", __FUNCTION__); WHYF(" Slot# | Last allocated by"); for(i=0;ijournalTail = -1; + // Set global defaults for a manifest (which are not zero) + rhizome_manifest_clear(m); return m; } @@ -554,40 +934,27 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence) void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m) { if (!m) return; - int i; int mid=m->manifest_record_number; - if (m!=&manifests[mid]) { - WHYF("%s(): asked to free manifest %p, which claims to be manifest slot #%d (%p), but isn't", - __FUNCTION__, m, mid, &manifests[mid] + if (m!=&manifests[mid]) + FATALF("%s(): asked to free manifest %p, which claims to be manifest slot #%d (%p), but isn't", + __FUNCTION__, m, mid, &manifests[mid] ); - exit(-1); - } - if (manifest_free[mid]) { - WHYF("%s(): asked to free manifest slot #%d (%p), which was already freed at %s:%d:%s()", - __FUNCTION__, mid, m, - manifest_free_whence[mid].file, - manifest_free_whence[mid].line, - manifest_free_whence[mid].function - ); - exit(-1); - } - - /* Free variable and signature blocks. - XXX These should be moved to malloc-free storage eventually */ - for(i=0;ivar_count;i++) - { free(m->vars[i]); free(m->values[i]); - m->vars[i]=NULL; m->values[i]=NULL; } - for(i=0;isig_count;i++) - { free(m->signatories[i]); - m->signatories[i]=NULL; - } + if (manifest_free[mid]) + FATALF("%s(): asked to free manifest slot #%d (%p), which was already freed at %s:%d:%s()", + __FUNCTION__, mid, m, + manifest_free_whence[mid].file, + manifest_free_whence[mid].line, + manifest_free_whence[mid].function + ); + /* Free variable and signature blocks. */ + rhizome_manifest_clear(m); if (m->dataFileName) { if (m->dataFileUnlinkOnFree && unlink(m->dataFileName) == -1) WARNF_perror("unlink(%s)", alloca_str_toprint(m->dataFileName)); - free(m->dataFileName); + free((char *) m->dataFileName); m->dataFileName = NULL; } @@ -600,48 +967,54 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m) return; } -/* Convert variable list to string, complaining if it ends up - too long. - Signatures etc will be added later. */ +/* Convert variable list into manifest text body and compute the hash. Do not sign. + */ int rhizome_manifest_pack_variables(rhizome_manifest *m) { - int i,ofs=0; - - for(i=0;ivar_count;i++) - { - if ((ofs+strlen(m->vars[i])+1+strlen(m->values[i])+1+1)>MAX_MANIFEST_BYTES) - return WHY("Manifest variables too long in total to fit in MAX_MANIFEST_BYTES"); - snprintf((char *)&m->manifestdata[ofs],MAX_MANIFEST_BYTES-ofs,"%s=%s\n", - m->vars[i],m->values[i]); - ofs+=strlen((char *)&m->manifestdata[ofs]); - } - m->manifestdata[ofs++]=0x00; - m->manifest_bytes=ofs; - if (config.debug.rhizome) DEBUG("Repacked variables in manifest."); - m->manifest_all_bytes=ofs; - - /* Recalculate hash */ - crypto_hash_sha512(m->manifesthash,m->manifestdata,m->manifest_bytes); - + assert(m->var_count <= NELS(m->vars)); + strbuf sb = strbuf_local((char*)m->manifestdata, sizeof m->manifestdata); + unsigned i; + for (i = 0; i < m->var_count; ++i) { + strbuf_puts(sb, m->vars[i]); + strbuf_putc(sb, '='); + strbuf_puts(sb, m->values[i]); + strbuf_putc(sb, '\n'); + } + if (strbuf_overrun(sb)) + return WHYF("Manifest overflow: body of %zu bytes exceeds limit of %zu", strbuf_count(sb) + 1, sizeof m->manifestdata); + m->manifest_body_bytes = strbuf_len(sb) + 1; + if (config.debug.rhizome) + DEBUGF("Repacked variables into manifest: %zu bytes", m->manifest_body_bytes); + m->manifest_all_bytes = m->manifest_body_bytes; + m->selfSigned = 0; return 0; } -/* Sign this manifest using our it's own BID secret key. - TODO: need a third-party signing primitive, eg, to support signing with SAS. +/* Sign this manifest using it's own BID secret key. Manifest must not already be signed. + * Manifest body hash must already be computed. */ int rhizome_manifest_selfsign(rhizome_manifest *m) { + assert(m->manifest_body_bytes > 0); + assert(m->manifest_body_bytes <= sizeof m->manifestdata); + assert(m->manifestdata[m->manifest_body_bytes - 1] == '\0'); + assert(m->manifest_body_bytes == m->manifest_all_bytes); // no signature yet if (!m->haveSecret) return WHY("Need private key to sign manifest"); + crypto_hash_sha512(m->manifesthash, m->manifestdata, m->manifest_body_bytes); rhizome_signature sig; if (rhizome_sign_hash(m, &sig) == -1) return WHY("rhizome_sign_hash() failed"); + assert(sig.signatureLength > 0); /* Append signature to end of manifest data */ - if (sig.signatureLength + m->manifest_bytes > MAX_MANIFEST_BYTES) - return WHY("Manifest plus signatures is too long"); - bcopy(&sig.signature[0], &m->manifestdata[m->manifest_bytes], sig.signatureLength); - m->manifest_bytes += sig.signatureLength; - m->manifest_all_bytes = m->manifest_bytes; + if (sig.signatureLength + m->manifest_body_bytes > sizeof m->manifestdata) + return WHYF("Manifest overflow: body %zu + signature %zu bytes exceeds limit of %zu", + m->manifest_body_bytes, + sig.signatureLength, + sizeof m->manifestdata + ); + bcopy(sig.signature, m->manifestdata + m->manifest_body_bytes, sig.signatureLength); + m->manifest_all_bytes = m->manifest_body_bytes + sig.signatureLength; return 0; } @@ -677,20 +1050,9 @@ int rhizome_write_manifest_file(rhizome_manifest *m, const char *path, char appe return ret; } -/* - Adds a group that this bundle should be present in. If we have the means to sign - the bundle as a member of that group, then we create the appropriate signature block. - The group signature blocks, like all signature blocks, will be appended to the - manifest data during the finalisation process. - */ -int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid) -{ - return WHY("Not implemented."); -} - int rhizome_manifest_dump(rhizome_manifest *m, const char *msg) { - int i; + unsigned i; WHYF("Dumping manifest %s:", msg); for(i=0;ivar_count;i++) WHYF("[%s]=[%s]\n", m->vars[i], m->values[i]); @@ -701,119 +1063,152 @@ int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int { IN(); int ret=0; - + + if (m->filesize == RHIZOME_SIZE_UNSET) + RETURN(WHY("Manifest filesize unknown")); + // if a manifest was supplied with an ID, don't bother to check for a duplicate. // we only want to filter out added files with no existing manifest. if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID && rhizome_find_duplicate(m, mout) == 1) RETURN(2); - + *mout=m; - + /* Convert to final form for signing and writing to disk */ if (rhizome_manifest_pack_variables(m)) RETURN(WHY("Could not convert manifest to wire format")); - + /* Sign it */ if (rhizome_manifest_selfsign(m)) RETURN(WHY("Could not sign manifest")); - + /* mark manifest as finalised */ m->finalised=1; - ret=rhizome_add_manifest(m, 255 /* TTL */); - + ret = rhizome_add_manifest(m, 255 /* TTL */); + RETURN(ret); OUT(); } -int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp, rhizome_bk_t *bsk){ +int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp) +{ /* Fill in a few missing manifest fields, to make it easier to use when adding new files: - - the default service is FILE - - use the current time for "date" + - use the current time for "date" and "version" - if service is file, then use the payload file's basename for "name" */ - const char *service = rhizome_manifest_get(m, "service", NULL, 0); - if (service == NULL) + + /* Set version of manifest from current time if not already set. */ + if (m->version == 0) + rhizome_manifest_set_version(m, gettime_ms()); + + /* Set the manifest's author. This must be done before binding to a new ID (below). If no author + * was specified, then the manifest's "sender" field is used, if present. + */ + if (authorSidp) + rhizome_manifest_set_author(m, authorSidp); + else if (m->has_sender) + rhizome_manifest_set_author(m, &m->sender); + + /* Set the bundle ID (public key) and secret key. + */ + if (!m->haveSecret && rhizome_bid_t_is_zero(m->cryptoSignPublic)) { + if (config.debug.rhizome) + DEBUG("creating new bundle"); + if (rhizome_manifest_createid(m) == -1) + return WHY("Could not bind manifest to an ID"); + if (m->authorship != ANONYMOUS) + rhizome_manifest_add_bundle_key(m); // set the BK field + } else { + if (config.debug.rhizome) + DEBUGF("modifying existing bundle bid=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + // Modifying an existing bundle. Try to discover the bundle secret key and the author. + rhizome_authenticate_author(m); + // TODO assert that new version > old version? + } + + if (m->service == NULL) return WHYF("missing 'service'"); if (config.debug.rhizome) - DEBUGF("manifest service=%s", service); - if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) { - rhizome_manifest_set_ll(m, "date", (int64_t) gettime_ms()); - if (config.debug.rhizome) DEBUGF("missing 'date', set default date=%s", rhizome_manifest_get(m, "date", NULL, 0)); + DEBUGF("manifest service=%s", m->service); + + if (!m->has_date) { + rhizome_manifest_set_date(m, (int64_t) gettime_ms()); + if (config.debug.rhizome) + DEBUGF("missing 'date', set default date=%"PRItime_ms_t, m->date); } - - if (strcasecmp(RHIZOME_SERVICE_FILE, service) == 0) { - const char *name = rhizome_manifest_get(m, "name", NULL, 0); - if (name == NULL) { - if (filepath && *filepath){ - name = strrchr(filepath, '/'); - name = name ? name + 1 : filepath; - }else - name=""; - rhizome_manifest_set(m, "name", name); - if (config.debug.rhizome) DEBUGF("missing 'name', set default name=\"%s\"", name); + + if (strcasecmp(RHIZOME_SERVICE_FILE, m->service) == 0) { + if (m->name == NULL) { + if (filepath && *filepath) { + const char *name = strrchr(filepath, '/'); + rhizome_manifest_set_name(m, name ? name + 1 : filepath); + } else + rhizome_manifest_set_name(m, ""); + if (config.debug.rhizome) + DEBUGF("missing 'name', set default name=%s", alloca_str_toprint(m->name)); } else { - if (config.debug.rhizome) DEBUGF("manifest contains name=\"%s\"", name); - } - } - - /* If the author was not specified, then the manifest's "sender" - field is used, if present. */ - if (authorSidp) - m->author = *authorSidp; - else{ - const char *sender = rhizome_manifest_get(m, "sender", NULL, 0); - if (sender){ - if (str_to_sid_t(&m->author, sender) == -1) - return WHYF("invalid sender: %s", sender); + if (config.debug.rhizome) + DEBUGF("manifest contains name=%s", alloca_str_toprint(m->name)); } } - /* set version of manifest, either from version variable, or using current time */ - if (rhizome_manifest_get(m,"version",NULL,0)==NULL) - { - /* No version set, default to the current time */ - m->version = gettime_ms(); - rhizome_manifest_set_ll(m,"version",m->version); + // Anything sent from one person to another should be considered private and encrypted by default. + if ( m->payloadEncryption == PAYLOAD_CRYPT_UNKNOWN + && m->has_sender + && m->has_recipient + && !is_sid_t_broadcast(m->recipient) + ) { + if (config.debug.rhizome) + DEBUGF("Implicitly adding payload encryption due to presense of sender & recipient fields"); + rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED); } - - if (!m->haveSecret){ - const char *id = rhizome_manifest_get(m, "id", NULL, 0); - if (id == NULL) { - if (config.debug.rhizome) DEBUG("creating new bundle"); - if (rhizome_manifest_bind_id(m) == -1) { - return WHY("Could not bind manifest to an ID"); - } - } else { - if (config.debug.rhizome) DEBUGF("modifying existing bundle bid=%s", id); - - // Modifying an existing bundle. Make sure we can find the bundle secret. - if (rhizome_extract_privatekey_required(m, bsk)) - return -1; - - // TODO assert that new version > old version? - } - } - assert(m->haveSecret); - - int crypt = rhizome_manifest_get_ll(m,"crypt"); - if (crypt==-1){ - // no explicit crypt flag, should we encrypt this bundle? - char *sender = rhizome_manifest_get(m, "sender", NULL, 0); - char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0); - - // anything sent from one person to another should be considered private and encrypted by default - if (sender && recipient){ - sid_t s_sender, s_recipient; - if (cf_opt_sid(&s_sender, sender)==CFOK - && cf_opt_sid(&s_recipient, recipient)==CFOK - && !is_sid_t_broadcast(s_recipient)){ - if (config.debug.rhizome) - DEBUGF("Implicitly adding payload encryption due to presense of sender & recipient fields"); - m->payloadEncryption=1; - rhizome_manifest_set_ll(m,"crypt",1LL); - } - } - } - + return 0; } + +/* Work out the authorship status of the bundle without performing any cryptographic checks. + * Sets the 'authorship' element and returns 1 if an author was found, 0 if not. + * + * @author Andrew Bettison + */ +int rhizome_lookup_author(rhizome_manifest *m) +{ + IN(); + int cn, in, kp; + switch (m->authorship) { + case AUTHOR_NOT_CHECKED: + if (config.debug.rhizome) + DEBUGF("manifest[%d] lookup author=%s", m->manifest_record_number, alloca_tohex_sid_t(m->author)); + cn = 0, in = 0, kp = 0; + if (keyring_find_sid(keyring, &cn, &in, &kp, &m->author)) { + if (config.debug.rhizome) + DEBUGF("found author"); + m->authorship = AUTHOR_LOCAL; + RETURN(1); + } + // fall through + case ANONYMOUS: + if (m->has_sender) { + if (config.debug.rhizome) + DEBUGF("manifest[%d] lookup sender=%s", m->manifest_record_number, alloca_tohex_sid_t(m->sender)); + cn = 0, in = 0, kp = 0; + if (keyring_find_sid(keyring, &cn, &in, &kp, &m->sender)) { + if (config.debug.rhizome) + DEBUGF("found sender"); + rhizome_manifest_set_author(m, &m->sender); + m->authorship = AUTHOR_LOCAL; + RETURN(1); + } + } + case AUTHENTICATION_ERROR: + case AUTHOR_UNKNOWN: + case AUTHOR_IMPOSTOR: + RETURN(0); + case AUTHOR_LOCAL: + case AUTHOR_AUTHENTIC: + RETURN(1); + } + FATALF("m->authorship = %d", m->authorship); + RETURN(0); + OUT(); +} diff --git a/rhizome_crypto.c b/rhizome_crypto.c index 3cbb2a32..db3c990e 100644 --- a/rhizome_crypto.c +++ b/rhizome_crypto.c @@ -17,16 +17,19 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include +#include +#include + #include "crypto_sign_edwards25519sha512batch.h" #include "nacl/src/crypto_sign_edwards25519sha512batch_ref/ge.h" - #include "serval.h" #include "conf.h" #include "str.h" #include "rhizome.h" #include "crypto.h" -#include -#include +#include "keyring.h" +#include "dataformats.h" /* Work out the encrypt/decrypt key for the supplied manifest. If the manifest is not encrypted, then return NULL. @@ -40,8 +43,10 @@ int rhizome_manifest_createid(rhizome_manifest *m) { if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic.binary, m->cryptoSignSecret)) return WHY("Failed to create keypair for manifest ID."); - rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + rhizome_manifest_set_id(m, &m->cryptoSignPublic); m->haveSecret = NEW_BUNDLE_ID; + // A new Bundle ID and secret invalidates any existing BK field. + rhizome_manifest_del_bundle_key(m); return 0; } @@ -56,10 +61,14 @@ static int generate_keypair(const char *seed, struct signing_key *key) unsigned char hash[crypto_hash_sha512_BYTES]; crypto_hash_sha512(hash, (unsigned char *)seed, strlen(seed)); - // The first 256 bits of the hash will be used as the private key of the BID. - bcopy(hash, key->Private, sizeof(key->Private)); - if (crypto_sign_compute_public_key(key->Private, key->Public.binary)) + // The first 256 bits (32 bytes) of the hash will be used as the private key of the BID. + bcopy(hash, key->Private, sizeof key->Private); + if (crypto_sign_compute_public_key(key->Private, key->Public.binary) == -1) return WHY("Could not generate public key"); + // The last 32 bytes of the private key should be identical to the public key. This is what + // crypto_sign_edwards25519sha512batch_keypair() returns, and there is code that depends on it. + // TODO: Refactor the Rhizome private/public keypair to eliminate this duplication. + bcopy(key->Public.binary, key->Private + RHIZOME_BUNDLE_KEY_BYTES, sizeof key->Public.binary); return 0; } @@ -70,16 +79,20 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed) struct signing_key key; if (generate_keypair(seed, &key)) return -1; - - int ret=rhizome_retrieve_manifest(&key.Public, m); + int ret = rhizome_retrieve_manifest(&key.Public, m); if (ret == -1) return -1; - - m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID; - m->cryptoSignPublic = key.Public; + if (ret == 1) { + // manifest not retrieved + rhizome_manifest_set_id(m, &key.Public); // zerofills m->cryptoSignSecret + m->haveSecret = NEW_BUNDLE_ID; + } else { + m->haveSecret = EXISTING_BUNDLE_ID; + } bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret); - if (ret == 1) - rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + // Disabled for performance, these asserts should nevertheless always hold. + //assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &key.Public) == 0); + //assert(memcmp(m->cryptoSignPublic.binary, m->cryptoSignSecret + RHIZOME_BUNDLE_KEY_BYTES, sizeof m->cryptoSignPublic.binary) == 0); return ret; } @@ -114,10 +127,12 @@ int rhizome_bk_xor_stream( return 0; } -/* - CryptoSign Secret Keys in cupercop-20120525 onwards have the public key as the - second half of the secret key. The public key is the BID, so this simplifies - the BK<-->SECRET conversion processes. */ +/* CryptoSign Secret Keys in cupercop-20120525 onwards have the public key as the second half of the + * secret key. The public key is the BID, so this simplifies the BK<-->SECRET conversion processes. + * + * Returns 0 if the BK decodes correctly to the bundle secret, 1 if not. Returns -1 if there is an + * error. + */ int rhizome_bk2secret(rhizome_manifest *m, const rhizome_bid_t *bidp, const unsigned char *rs, const size_t rs_len, @@ -128,21 +143,16 @@ int rhizome_bk2secret(rhizome_manifest *m, { IN(); unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES]; - if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES)) + if (rhizome_bk_xor_stream(bidp, rs, rs_len, xor_stream, RHIZOME_BUNDLE_KEY_BYTES)) RETURN(WHY("rhizome_bk_xor_stream() failed")); - - int i; - /* XOR and store secret part of secret key */ - for(i = 0; i != RHIZOME_BUNDLE_KEY_BYTES; i++) + unsigned i; + for (i = 0; i != RHIZOME_BUNDLE_KEY_BYTES; ++i) secret[i] = bkin[i] ^ xor_stream[i]; - /* Copy BID as public-key part of secret key */ - for(;i!=crypto_sign_edwards25519sha512batch_SECRETKEYBYTES;++i) - secret[i] = bidp->binary[i - RHIZOME_BUNDLE_KEY_BYTES]; - bzero(xor_stream, sizeof xor_stream); - - RETURN(rhizome_verify_bundle_privatekey(m, secret, bidp->binary)); + /* Copy BID as public-key part of secret key */ + bcopy(bidp->binary, secret + RHIZOME_BUNDLE_KEY_BYTES, sizeof bidp->binary); + RETURN(rhizome_verify_bundle_privatekey(secret, bidp->binary) ? 0 : 1); OUT(); } @@ -171,257 +181,245 @@ int rhizome_secret2bk( } -/* Given the SID of a bundle's author, search for an identity in the keyring and return its - * Rhizome secret if found. +/* Given a SID, search the keyring for an identity with the same SID and return its Rhizome secret + * if found. * - * Returns -1 if an error occurs. - * Returns 0 if the author's rhizome secret is found; '*rs' is set to point to the secret key in the - * keyring, and '*rs_len' is set to the key length. - * Returns 2 if the author's identity is not in the keyring. - * Returns 3 if the author's identity is in the keyring but has no rhizome secret. + * Returns FOUND_RHIZOME_SECRET if the author's rhizome secret is found; '*rs' is set to point to + * the secret key in the keyring, and '*rs_len' is set to the key length. + * + * Returns IDENTITY_NOT_FOUND if the SID is not in the keyring. + * + * Returns IDENTITY_HAS_NO_RHIZOME_SECRET if the SID is in the keyring but has no Rhizome Secret. * * @author Andrew Bettison */ -int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned char **rs) +enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs) { + IN(); int cn=0, in=0, kp=0; if (!keyring_find_sid(keyring,&cn,&in,&kp, authorSidp)) { if (config.debug.rhizome) DEBUGF("identity sid=%s is not in keyring", alloca_tohex_sid_t(*authorSidp)); - return 2; + RETURN(IDENTITY_NOT_FOUND); } kp = keyring_identity_find_keytype(keyring, cn, in, KEYTYPE_RHIZOME); if (kp == -1) { - if (config.debug.rhizome) - DEBUGF("identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*authorSidp)); - return 3; + WARNF("Identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*authorSidp)); + RETURN(IDENTITY_HAS_NO_RHIZOME_SECRET); } int rslen = keyring->contexts[cn]->identities[in]->keypairs[kp]->private_key_len; - if (rslen < 16 || rslen > 1024) - return WHYF("identity sid=%s has invalid Rhizome Secret: length=%d", alloca_tohex_sid_t(*authorSidp), rslen); + assert(rslen >= 16); + assert(rslen <= 1024); if (rs_len) *rs_len = rslen; if (rs) *rs = keyring->contexts[cn]->identities[in]->keypairs[kp]->private_key; - return 0; + RETURN(FOUND_RHIZOME_SECRET); } -/* Given the SID of a bundle's author and the bundle ID, XOR a bundle key (private or public) with - * RS##BID where RS is the rhizome secret of the bundle's author, and BID is the bundle's public key - * (aka the Bundle ID). - * - * This will convert a manifest BK field into the bundle's private key, or vice versa. - * - * Returns -1 if an error occurs. - * Returns 0 if the author's private key is located and the XOR is performed successfully. - * Returns 2 if the author's identity is not in the keyring (this return code from - * rhizome_find_secret()). - * Returns 3 if the author's identity is in the keyring but has no rhizome secret (this return code - * from rhizome_find_secret()). - * - * Looks up the SID in the keyring, and if it is present and has a valid-looking RS, calls - * rhizome_bk_xor_rs() to perform the XOR. +/* Attempt to authenticate the authorship of the given bundle, and set the 'authorship' element + * accordingly. If the manifest has nk BK field, then no authentication can be performed. * * @author Andrew Bettison */ - -/* See if the manifest has a BK entry, and if so, use it to obtain the private key for the BID. The - * manifest's 'author' field must contain the (binary) SID of the purported author of the bundle, - * which is used to look up the author's rhizome secret in the keyring. - * - * Returns 0 if a valid private key was extracted, with the private key in the manifest - * 'cryptoSignSecret' field and the 'haveSecret' field set to 1. - * - * Returns 1 if the manifest does not have a BK field. - * - * Returns 2 if the author is not found in the keyring (not unlocked?) -- this return code from - * rhizome_bk_xor(). - * - * Returns 3 if the author is found in the keyring but has no rhizome secret -- this return code - * from rhizome_bk_xor(). - * - * Returns 4 if the author is found in the keyring and has a rhizome secret but the private bundle - * key formed using it does not verify. - * - * Returns -1 on error. - * - * @author Andrew Bettison - - */ -int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk) +void rhizome_authenticate_author(rhizome_manifest *m) { IN(); - unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES]; - char *bk = rhizome_manifest_get(m, "BK", NULL, 0); - int result; - - if (bk){ - if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1) - RETURN(WHYF("invalid BK field: %s", bk)); - - if (is_sid_t_any(m->author)) { - result=rhizome_find_bundle_author(m); - }else{ - int rs_len; - const unsigned char *rs; - result = rhizome_find_secret(&m->author, &rs_len, &rs); - if (result==0) - result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret); - } - - if (result == 0 && bsk && !rhizome_is_bk_none(bsk)){ - // If a bundle secret key was supplied that does not match the secret key derived from the - // author, then warn but carry on using the author's. - if (memcmp(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES) != 0) - WARNF("Supplied bundle secret key is invalid -- ignoring"); - } - - }else if(bsk && !rhizome_is_bk_none(bsk)){ - bcopy(m->cryptoSignPublic.binary, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof m->cryptoSignPublic.binary); - bcopy(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES); - if (rhizome_verify_bundle_privatekey(m, m->cryptoSignSecret, m->cryptoSignPublic.binary)) - result=5; - else - result=0; - }else{ - result=1; + if (!m->has_bundle_key) + RETURNVOID; + switch (m->authorship) { + case ANONYMOUS: + rhizome_find_bundle_author_and_secret(m); + break; + case AUTHOR_NOT_CHECKED: + case AUTHOR_LOCAL: { + if (config.debug.rhizome) + DEBUGF("manifest[%d] authenticate author=%s", m->manifest_record_number, alloca_tohex_sid_t(m->author)); + size_t rs_len; + const unsigned char *rs; + enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs); + switch (d) { + case FOUND_RHIZOME_SECRET: + if (config.debug.rhizome) + DEBUGF("author has Rhizome secret"); + switch (rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, m->cryptoSignSecret)) { + case 0: + if (config.debug.rhizome) + DEBUGF("authentic"); + m->authorship = AUTHOR_AUTHENTIC; + if (!m->haveSecret) + m->haveSecret = EXISTING_BUNDLE_ID; + break; + case -1: + if (config.debug.rhizome) + DEBUGF("error"); + m->authorship = AUTHENTICATION_ERROR; + break; + default: + if (config.debug.rhizome) + DEBUGF("impostor"); + m->authorship = AUTHOR_IMPOSTOR; + break; + } + break; + case IDENTITY_NOT_FOUND: + if (config.debug.rhizome) + DEBUGF("author not found"); + m->authorship = AUTHOR_UNKNOWN; + break; + case IDENTITY_HAS_NO_RHIZOME_SECRET: + if (config.debug.rhizome) + DEBUGF("author has no Rhizome secret"); + m->authorship = AUTHENTICATION_ERROR; + break; + default: + FATALF("find_rhizome_secret() returned unknown code %d", (int)d); + break; + } + } + break; + case AUTHENTICATION_ERROR: + case AUTHOR_UNKNOWN: + case AUTHOR_IMPOSTOR: + case AUTHOR_AUTHENTIC: + // work has already been done, don't repeat it + break; + default: + FATALF("m->authorship = %d", (int)m->authorship); + break; } - - if (result == 0){ - m->haveSecret=EXISTING_BUNDLE_ID; - }else{ - memset(m->cryptoSignSecret, 0, sizeof m->cryptoSignSecret); - m->haveSecret=0; - } - - RETURN(result); OUT(); } -/* Same as rhizome_extract_privatekey, except warnings become errors and are logged */ -int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk) +/* If the given bundle secret key corresponds to the bundle's ID (public key) then store it in the + * manifest structure and mark the secret key as known. Return 1 if the secret key was assigned, + * 0 if not. + * + * This function should only be called on a manifest that already has a public key (ID) and does + * not have a known secret key. + * + * @author Andrew Bettison + */ +int rhizome_apply_bundle_secret(rhizome_manifest *m, const rhizome_bk_t *bsk) { - int result = rhizome_extract_privatekey(m, bsk); - switch (result) { - case -1: - case 0: - return result; - case 1: - return WHY("Bundle contains no BK field, and no bundle secret supplied"); - case 2: - return WHY("Author unknown"); - case 3: - return WHY("Author does not have a Rhizome Secret"); - case 4: - return WHY("Author does not have permission to modify manifest"); - case 5: - return WHY("Bundle secret is not valid for this manifest"); - default: - return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result); + IN(); + if (config.debug.rhizome) + DEBUGF("manifest[%d] bsk=%s", m->manifest_record_number, bsk ? alloca_tohex_rhizome_bk_t(*bsk) : "NULL"); + assert(m->haveSecret == SECRET_UNKNOWN); + assert(is_all_matching(m->cryptoSignSecret, sizeof m->cryptoSignSecret, 0)); + assert(!rhizome_bid_t_is_zero(m->cryptoSignPublic)); + assert(bsk != NULL); + assert(!rhizome_is_bk_none(bsk)); + if (rhizome_verify_bundle_privatekey(bsk->binary, m->cryptoSignPublic.binary)) { + if (config.debug.rhizome) + DEBUG("bundle secret verifies ok"); + bcopy(bsk->binary, m->cryptoSignSecret, sizeof bsk->binary); + bcopy(m->cryptoSignPublic.binary, m->cryptoSignSecret + sizeof bsk->binary, sizeof m->cryptoSignPublic.binary); + m->haveSecret = EXISTING_BUNDLE_ID; + RETURN(1); } + RETURN(0); + OUT(); } /* Discover if the given manifest was created (signed) by any unlocked identity currently in the * keyring. * - * Returns 0 if an identity is found with permission to alter the bundle, after setting the manifest - * 'author' field to the SID of the identity and the manifest 'cryptoSignSecret' field to the bundle - * secret key and the 'haveSecret' field to 1. + * This function must only be called if the bundle secret is not known. If it is known, then + * use * - * Returns 1 if no identity in the keyring is the author of this bundle. + * If the authorship is already known (ie, not ANONYMOUS) then returns without changing anything. + * That means this function can be called several times on the same manifest, but will only perform + * any work the first time. * - * Returns 4 if the manifest has no BK field. + * If the manifest has no bundle key (BK) field, then it is anonymous, so leaves 'authorship' + * unchanged and returns. * - * Returns -1 if an error occurs, eg, the manifest contains an invalid BK field. + * If an identity is found in the keyring with permission to alter the bundle, then sets the + * manifest 'authorship' field to AUTHOR_AUTHENTIC, the 'author' field to the SID of the identity, + * the manifest 'cryptoSignSecret' field to the bundle secret key and the 'haveSecret' field to + * EXISTING_BUNDLE_ID. + * + * If no identity is found in the keyring that combines with the bundle key (BK) field to yield + * the bundle's secret key, then leaves the manifest 'authorship' field as ANONYMOUS. + * + * If an error occurs, eg, the keyring contains an invalid Rhizome Secret or a cryptographic + * operation fails, then sets the 'authorship' field to AUTHENTICATION_ERROR and leaves the + * 'author', 'haveSecret' and 'cryptoSignSecret' fields unchanged. * * @author Andrew Bettison */ -int rhizome_find_bundle_author(rhizome_manifest *m) +void rhizome_find_bundle_author_and_secret(rhizome_manifest *m) { IN(); - char *bk = rhizome_manifest_get(m, "BK", NULL, 0); - if (!bk) { - if (config.debug.rhizome) - DEBUGF("missing BK field"); - RETURN(4); - } - unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES]; - if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1) - RETURN(WHYF("invalid BK field: %s", bk)); + if (m->authorship != ANONYMOUS) + RETURNVOID; + assert(is_sid_t_any(m->author)); + if (!m->has_bundle_key) + RETURNVOID; int cn = 0, in = 0, kp = 0; for (; keyring_next_identity(keyring, &cn, &in, &kp); ++kp) { const sid_t *authorSidp = (const sid_t *) keyring->contexts[cn]->identities[in]->keypairs[kp]->public_key; //if (config.debug.rhizome) DEBUGF("try author identity sid=%s", alloca_tohex_sid_t(*authorSidp)); int rkp = keyring_identity_find_keytype(keyring, cn, in, KEYTYPE_RHIZOME); if (rkp != -1) { - int rs_len = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key_len; - if (rs_len < 16 || rs_len > 1024) - RETURN(WHYF("invalid Rhizome Secret: length=%d", rs_len)); + size_t rs_len = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key_len; + if (rs_len < 16 || rs_len > 1024) { + WHYF("invalid Rhizome Secret: length=%zu", rs_len); + m->authorship = AUTHENTICATION_ERROR; + RETURNVOID; + } const unsigned char *rs = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key; - - if (!rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret)) { - m->haveSecret=EXISTING_BUNDLE_ID; - if (cmp_sid_t(&m->author, authorSidp) != 0){ - m->author = *authorSidp; - if (config.debug.rhizome) - DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(m->author)); - // if this bundle is already in the database, update the author. - if (m->inserttime) - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, - "UPDATE MANIFESTS SET author = ? WHERE id = ?;", - SID_T, &m->author, - RHIZOME_BID_T, &m->cryptoSignPublic, - END); - } - - RETURN(0); // bingo + unsigned char *secretp = m->cryptoSignSecret; + if (m->haveSecret) + secretp = alloca(sizeof m->cryptoSignSecret); + if (rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, secretp) == 0) { + if (m->haveSecret) { + if (memcmp(secretp, m->cryptoSignSecret, sizeof m->cryptoSignSecret) != 0) + FATALF("Bundle secret does not match derived secret"); + } else + m->haveSecret = EXISTING_BUNDLE_ID; + if (config.debug.rhizome) + DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(*authorSidp)); + rhizome_manifest_set_author(m, authorSidp); + m->authorship = AUTHOR_AUTHENTIC; + // if this bundle is already in the database, update the author. + if (m->rowid) + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, + "UPDATE MANIFESTS SET author = ? WHERE rowid = ?;", + SID_T, &m->author, + INT64, m->rowid, + END); + RETURNVOID; // bingo } } } + assert(m->authorship == ANONYMOUS); if (config.debug.rhizome) DEBUG("bundle author not found"); - RETURN(1); OUT(); } -/* Verify the validity of the manifest's secret key, ie, is the given manifest's 'cryptoSignSecret' - * field actually the secret key corresponding to the public key in 'cryptoSignPublic'? - * Return 0 if valid, 1 if not. Return -1 if an error occurs. +/* Verify the validity of a given secret manifest key. Return 1 if valid, 0 if not. * * There is no NaCl API to efficiently test this. We use a modified version of * crypto_sign_keypair() to accomplish this task. */ -int rhizome_verify_bundle_privatekey(rhizome_manifest *m, - const unsigned char *sk, - const unsigned char *pkin) +int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned char *pkin) { IN(); - unsigned char pk[32]; - int i; - crypto_sign_compute_public_key(sk,pk); - for (i = 0;i < 32;++i) - if (pkin[i] != pk[i]) { - if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) - m->haveSecret=0; - RETURN(-1); - } - if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) { - if (config.debug.rhizome) - DEBUGF("We have the private key for this bundle."); - m->haveSecret=EXISTING_BUNDLE_ID; - } - RETURN(0); - OUT(); + rhizome_bid_t pk; + if (crypto_sign_compute_public_key(sk, pk.binary) == -1) + RETURN(0); + int ret = bcmp(pkin, pk.binary, sizeof pk.binary) == 0; + RETURN(ret); } -int rhizome_sign_hash(rhizome_manifest *m, - rhizome_signature *out) +int rhizome_sign_hash(rhizome_manifest *m, rhizome_signature *out) { IN(); - if (!m->haveSecret && rhizome_extract_privatekey_required(m, NULL)) - RETURN(-1); - - int ret=rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out); + assert(m->haveSecret); + int ret = rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out); RETURN(ret); OUT(); } @@ -436,7 +434,7 @@ int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk, int mLen = crypto_hash_sha512_BYTES; int r = crypto_sign_edwards25519sha512batch(signatureBuffer, &sigLen, &hash[0], mLen, sk); if (r) - RETURN(WHY("crypto_sign_edwards25519sha512batch() failed.")); + RETURN(WHY("crypto_sign_edwards25519sha512batch() failed")); /* Here we use knowledge of the internal structure of the signature block to remove the hash, since that is implicitly transported, thus reducing the actual signature size down to 64 bytes. @@ -459,7 +457,7 @@ typedef struct manifest_signature_block_cache { #define SIG_CACHE_SIZE 1024 manifest_signature_block_cache sig_cache[SIG_CACHE_SIZE]; -int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char *sig,int sig_len) +int rhizome_manifest_lookup_signature_validity(const unsigned char *hash, const unsigned char *sig, int sig_len) { IN(); unsigned int slot=0; @@ -504,80 +502,54 @@ int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char OUT(); } -int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs) +int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs) { IN(); - if (!m) - RETURN(WHY("NULL pointer passed in as manifest")); - if (config.debug.rhizome) - DEBUGF("m->manifest_all_bytes=%d m->manifest_bytes=%d *ofs=%d", m->manifest_all_bytes, m->manifest_bytes, *ofs); - - if ((*ofs)>=m->manifest_all_bytes) { RETURN(0); } - - int sigType=m->manifestdata[*ofs]; - int len=(sigType&0x3f)*4+4+1; - - /* Each signature type is required to have a different length to detect it. - At present only crypto_sign_edwards25519sha512batch() signatures are - supported. */ - int r; - if (m->sig_countmanifesthash,&m->manifestdata[(*ofs)+1],96); -#ifdef DEPRECATED - unsigned char sigBuf[256]; - unsigned char verifyBuf[256]; - unsigned char publicKey[256]; - bcopy(&m->manifestdata[(*ofs)+1],&sigBuf[0],64); - bcopy(&m->manifesthash[0],&sigBuf[64],crypto_hash_sha512_BYTES); - /* Get public key of signatory */ - bcopy(&m->manifestdata[(*ofs)+1+64],&publicKey[0],crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); - unsigned long long mlen=0; - int r=crypto_sign_edwards25519sha512batch_open(verifyBuf,&mlen,&sigBuf[0],128, publicKey); -#endif - if (r) { - (*ofs)+=len; - m->errors++; - RETURN(WHY("Error in signature block (verification failed).")); - } else { - /* Signature block passes, so add to list of signatures */ - m->signatureTypes[m->sig_count]=len; - m->signatories[m->sig_count] - =malloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); - if(!m->signatories[m->sig_count]) { - (*ofs)+=len; - RETURN(WHY("malloc() failed when reading signature block")); - } - bcopy(&m->manifestdata[(*ofs)+1+64],m->signatories[m->sig_count], - crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); - m->sig_count++; - if (config.debug.rhizome) DEBUG("Signature passed."); - } - break; - default: - (*ofs)+=len; - m->errors++; - RETURN(WHYF("Encountered illegal or malformed signature block (unknown type=0x%02x @ offset 0x%x)",sigType,(*ofs)-len)); - } - else + if (config.debug.rhizome_manifest) + DEBUGF("*ofs=%u m->manifest_all_bytes=%zu", *ofs, m->manifest_all_bytes); + assert((*ofs) < m->manifest_all_bytes); + const unsigned char *sig = m->manifestdata + *ofs; + uint8_t sigType = m->manifestdata[*ofs]; + uint8_t len = (sigType << 2) + 4 + 1; + if (*ofs + len > m->manifest_all_bytes) { + WARNF("Invalid signature at offset %u: type=%#02x gives len=%u that overruns manifest size", + *ofs, sigType, len); + RETURN(1); + } + *ofs += len; + assert (m->sig_count <= NELS(m->signatories)); + if (m->sig_count == NELS(m->signatories)) { + WARN("Too many signature blocks in manifest"); + RETURN(2); + } + switch (sigType) { + case 0x17: // crypto_sign_edwards25519sha512batch() { - (*ofs)+=len; - WHY("Too many signature blocks in manifest."); - m->errors++; + assert(len == 97); + /* Reconstitute signature block */ + int r = rhizome_manifest_lookup_signature_validity(m->manifesthash, sig + 1, 96); + if (r) { + WARN("Signature verification failed"); + RETURN(4); + } + m->signatureTypes[m->sig_count] = len; + if ((m->signatories[m->sig_count] = emalloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)) == NULL) + RETURN(-1); + bcopy(sig + 1 + 64, m->signatories[m->sig_count], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); + m->sig_count++; + if (config.debug.rhizome) + DEBUG("Signature verified"); + RETURN(0); } - - (*ofs)+=len; - RETURN(0); - OUT(); + } + WARNF("Unsupported signature at ofs=%u: type=%#02x", sig - m->manifestdata, sigType); + RETURN(3); } // add value to nonce, with the same result regardless of CPU endian order // allowing for any carry value up to the size of the whole nonce -static void add_nonce(unsigned char *nonce, int64_t value){ +static void add_nonce(unsigned char *nonce, uint64_t value) +{ int i=crypto_stream_xsalsa20_NONCEBYTES -1; while(i>=0 && value>0){ int x = nonce[i]+(value & 0xFF); @@ -590,13 +562,10 @@ static void add_nonce(unsigned char *nonce, int64_t value){ /* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries * for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE) */ -int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, - const unsigned char *key, const unsigned char *nonce){ - - if (stream_offset<0) - return WHY("Invalid stream offset"); - - int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1); +int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset, + const unsigned char *key, const unsigned char *nonce) +{ + uint64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1); size_t offset=0; unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES]; @@ -632,34 +601,24 @@ int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t s return 0; } -int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk) +int rhizome_derive_payload_key(rhizome_manifest *m) { // don't do anything if the manifest isn't flagged as being encrypted - if (!m->payloadEncryption) + if (m->payloadEncryption != PAYLOAD_ENCRYPTED) return 0; - if (m->payloadEncryption!=1) - return WHYF("Unsupported encryption scheme %d", m->payloadEncryption); - - char *sender = rhizome_manifest_get(m, "sender", NULL, 0); - char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0); - - if (sender && recipient){ - sid_t sender_sid, recipient_sid; - if (cf_opt_sid(&sender_sid, sender)!=CFOK) - return WHYF("Unable to parse sender sid"); - if (cf_opt_sid(&recipient_sid, recipient)!=CFOK) - return WHYF("Unable to parse recipient sid"); - + if (m->has_sender && m->has_recipient){ unsigned char *nm_bytes=NULL; int cn=0,in=0,kp=0; - if (!keyring_find_sid(keyring, &cn, &in, &kp, &sender_sid)){ + if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->sender)){ cn=in=kp=0; - if (!keyring_find_sid(keyring, &cn, &in, &kp, &recipient_sid)){ - return WHYF("Neither the sender %s nor the recipient %s appears in our keyring", sender, recipient); + if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->recipient)){ + return WHYF("Neither the sender %s nor the recipient %s appears in our keyring", + alloca_tohex_sid_t(m->sender), + alloca_tohex_sid_t(m->recipient)); } - nm_bytes=keyring_get_nm_bytes(&recipient_sid, &sender_sid); + nm_bytes=keyring_get_nm_bytes(&m->recipient, &m->sender); }else{ - nm_bytes=keyring_get_nm_bytes(&sender_sid, &recipient_sid); + nm_bytes=keyring_get_nm_bytes(&m->sender, &m->recipient); } if (!nm_bytes) @@ -670,10 +629,8 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk) bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES); }else{ - if(!m->haveSecret){ - if (rhizome_extract_privatekey_required(m, bsk)) - return -1; - } + if (!m->haveSecret) + return WHY("Cannot derive payload key because bundle secret is unknown"); unsigned char raw_key[9+crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]="sasquatch"; bcopy(m->cryptoSignSecret, &raw_key[9], crypto_sign_edwards25519sha512batch_SECRETKEYBYTES); @@ -687,9 +644,9 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk) // journal bundles must always have the same nonce, regardless of version. // otherwise, generate nonce from version#bundle id#version; unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary]; - write_uint64(&raw_nonce[0], m->journalTail>=0?0:m->version); + write_uint64(&raw_nonce[0], m->is_journal ? 0 : m->version); bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary); - write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->journalTail>=0?0:m->version); + write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->is_journal ? 0 : m->version); unsigned char hash[crypto_hash_sha512_BYTES]; diff --git a/rhizome_database.c b/rhizome_database.c index cef548ab..d7a269e1 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "strbuf.h" #include "strbuf_helpers.h" #include "str.h" +#include "keyring.h" static char rhizome_thisdatastore_path[256]; @@ -75,7 +76,8 @@ int create_rhizome_datastore_dir() return emkdirs(rhizome_datastore_path(), 0700); } -sqlite3 *rhizome_db=NULL; +sqlite3 *rhizome_db = NULL; +uuid_t rhizome_db_uuid; /* XXX Requires a messy join that might be slow. */ int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp) @@ -106,11 +108,32 @@ int is_debug_rhizome_ads() static int (*sqlite_trace_func)() = is_debug_rhizome; const struct __sourceloc *sqlite_trace_whence = NULL; +static int sqlite_trace_done; +/* This is called by SQLite when executing a statement using sqlite3_step(). Unfortunately, it is + * not called on PRAGMA statements, and possibly others. Hence the use of the profile callback (see + * below). + * + * @author Andrew Bettison + */ static void sqlite_trace_callback(void *context, const char *rendered_sql) { if (sqlite_trace_func()) logMessage(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql); + ++sqlite_trace_done; +} + +/* This is called by SQLite when an executed statement finishes. We use it to log rendered SQL + * statements when the trace callback (above) has not been called, eg, for PRAGMA statements. This + * requires that the 'sqlite_trace_done' static be reset to zero whenever a new prepared statement + * is about to be stepped. + * + * @author Andrew Bettison + */ +static void sqlite_profile_callback(void *context, const char *rendered_sql, sqlite_uint64 nanosec) +{ + if (!sqlite_trace_done) + sqlite_trace_callback(context, rendered_sql); } /* This function allows code like: @@ -141,33 +164,31 @@ void sqlite_log(void *ignored, int result, const char *msg) WARNF("Sqlite: %d %s", result, msg); } -void verify_bundles(){ +void verify_bundles() +{ // assume that only the manifest itself can be trusted // fetch all manifests and reinsert them. sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; // This cursor must be ordered descending as re-inserting the manifests will give them a new higher manifest id. // If we didn't, we'd get stuck in an infinite loop. sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;"); - while(sqlite_step_retry(&retry, statement)==SQLITE_ROW){ + while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { sqlite3_int64 rowid = sqlite3_column_int64(statement, 0); const void *manifest = sqlite3_column_blob(statement, 1); size_t manifest_length = sqlite3_column_bytes(statement, 1); - rhizome_manifest *m=rhizome_new_manifest(); - int ret=0; - ret = rhizome_read_manifest_file(m, manifest, manifest_length); - if (ret==0 && m->errors) - ret=-1; - if (ret==0) - ret=rhizome_manifest_verify(m); - if (ret==0){ - m->finalised=1; - m->manifest_bytes=m->manifest_all_bytes; - // store it again, to ensure it is valid and stored correctly with matching file content. - ret=rhizome_store_bundle(m); + int ret = -1; + if ( rhizome_read_manifest_file(m, manifest, manifest_length) == 0 + && rhizome_manifest_validate(m) + && rhizome_manifest_verify(m) + ) { + assert(m->finalised); + // Store it again, to ensure that MANIFESTS columns are up to date. + ret = rhizome_store_bundle(m); } - if (ret!=0){ - DEBUGF("Removing invalid manifest entry @%lld", rowid); + if (ret) { + if (config.debug.rhizome) + DEBUGF("Removing invalid manifest entry @%lld", rowid); sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END); } rhizome_manifest_free(m); @@ -176,20 +197,25 @@ void verify_bundles(){ } /* - * The MANIFESTS table 'author' column records the cryptographically verified SID of the author - * that has write permission on the bundle, ie, possesses the Rhizome secret key that generated the - * BID, and hence can derive the Bundle Secret from the bundle's BK field: - * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created - * locally by a non-secret identity, so no verification need be performed for one's own - * bundles while they remain in the Rhizome store. - * - When a bundle is imported, the 'author' column is set to NULL to indicate that no - * verification has passed yet. This includes one's own bundles that have been purged from - * the local Rhizome store then recovered from a remote Rhizome node. - * - When a manifest with NULL 'author' is examined closely, ie extracted, not merely - * listed, the keyring is searched for an identity that is the author. If an author is - * found, the MANIFESTS table 'author' column is updated. This allows one to regain the - * ability to overwrite one's own bundles that have been lost but recovered from an exterior - * Rhizome node. + * The MANIFESTS table 'author' column records the cryptographically verified SID of the author that + * has write permission on the bundle, ie, possesses the Rhizome secret key that generated the BID, + * and hence can derive the Bundle Secret from the bundle's BK field: + * + * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created locally + * by a non-secret identity, so no verification need be performed for one's own bundles while they + * remain in the local Rhizome store. + * + * - When a bundle is imported, the 'author' column is set to NULL to indicate that no verification + * has passed yet. This includes one's own bundles that have been purged from the local Rhizome + * store then recovered from a remote Rhizome node. + * + * - When a manifest with NULL 'author' is examined closely, ie extracted, not merely listed, the + * keyring is searched for an identity that is the author. If the identity is found and its + * Rhizome Secret unlocks the Bundle Key (ie, reveals a Bundle Secret that yields the Bundle's ID + * as its public key), the MANIFESTS table 'author' column is updated. This allows one to regain + * the ability to overwrite one's own bundles that have been lost but + * recovered from an exterior Rhizome node. + * * - The above check automates the "own bundle recovery" mechanism at the expense of a CPU-heavy * cryptographic check every time a foreign bundle is examined, but at least listing is fast. * This will not scale as many identities are added to the keyring. It will eventually have to be @@ -201,7 +227,10 @@ void verify_bundles(){ int rhizome_opendb() { - if (rhizome_db) return 0; + if (rhizome_db) { + assert(uuid_is_valid(&rhizome_db_uuid)); + return 0; + } IN(); @@ -226,6 +255,7 @@ int rhizome_opendb() RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db))); } sqlite3_trace(rhizome_db, sqlite_trace_callback, NULL); + sqlite3_profile(rhizome_db, sqlite_profile_callback, NULL); int loglevel = (config.debug.rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT; /* Read Rhizome configuration */ @@ -251,11 +281,9 @@ int rhizome_opendb() ) { RETURN(WHY("Failed to create schema")); } - /* Create indexes if they don't already exist */ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END); - sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END); } if (version<2){ @@ -267,26 +295,52 @@ int rhizome_opendb() verify_bundles(); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END); } - if (version<3){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END); } - if (version<4){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END); } - + if (version<5){ + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END); + sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END); + } + + char buf[UUID_STRLEN + 1]; + int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END); + if (r == -1) + RETURN(-1); + if (r) { + if (!str_to_uuid(buf, &rhizome_db_uuid, NULL)) { + WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf)); + if (uuid_generate_random(&rhizome_db_uuid) == -1) + RETURN(WHY("Cannot generate new UUID for Rhizome database")); + if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", UUID_T, &rhizome_db_uuid, END) == -1) + RETURN(WHY("Failed to update new UUID in Rhizome database")); + if (config.debug.rhizome) + DEBUGF("Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid)); + } + } else if (r == 0) { + if (uuid_generate_random(&rhizome_db_uuid) == -1) + RETURN(WHY("Cannot generate UUID for Rhizome database")); + if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", UUID_T, &rhizome_db_uuid, END) == -1) + RETURN(WHY("Failed to insert UUID into Rhizome database")); + if (config.debug.rhizome) + DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid)); + } + // TODO recreate tables with collate nocase on hex columns - + /* Future schema updates should be performed here. The above schema can be assumed to exist. All changes should attempt to preserve any existing data */ - + // We can't delete a file that is being transferred in another process at this very moment... if (config.rhizome.clean_on_open) rhizome_cleanup(NULL); + INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_db_uuid)); RETURN(0); OUT(); } @@ -437,6 +491,7 @@ sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite while (1) { switch (sqlite3_prepare_v2(rhizome_db, sqltext, -1, &statement, NULL)) { case SQLITE_OK: + sqlite_trace_done = 0; RETURN(statement); case SQLITE_BUSY: case SQLITE_LOCKED: @@ -504,7 +559,7 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state return -1; } #define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \ - if (config.debug.rhizome_bind) \ + if (config.debug.rhizome_sql_bind) \ DEBUGF("%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement)) #define BIND_RETRY(FUNC, ...) \ do { \ @@ -684,10 +739,10 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state if (hashp == NULL) { BIND_NULL(RHIZOME_FILEHASH_T); } else { - char hash_hex[RHIZOME_FILEHASH_STRLEN]; - tohex(hash_hex, sizeof hash_hex, hashp->binary); - BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%zu,SQLITE_TRANSIENT", hash_hex, sizeof hash_hex); - BIND_RETRY(sqlite3_bind_text, hash_hex, sizeof hash_hex, SQLITE_TRANSIENT); + char hash_hex[RHIZOME_FILEHASH_STRLEN + 1]; + tohex(hash_hex, RHIZOME_FILEHASH_STRLEN, hashp->binary); + BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN); + BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT); } } break; @@ -734,6 +789,19 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state } } break; + case UUID_T: { + const uuid_t *uuidp = va_arg(ap, const uuid_t *); + ++argnum; + if (uuidp == NULL) { + BIND_NULL(UUID_T); + } else { + char uuid_str[UUID_STRLEN + 1]; + uuid_to_str(uuidp, uuid_str); + BIND_DEBUG(UUID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", uuid_str, UUID_STRLEN); + BIND_RETRY(sqlite3_bind_text, uuid_str, UUID_STRLEN, SQLITE_TRANSIENT); + } + } + break; #undef BIND_RETRY default: FATALF("at bind arg %u, unsupported bind code typ=0x%08x: %s", argnum, typ, sqlite3_sql(statement)); @@ -1270,49 +1338,32 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori */ int rhizome_store_bundle(rhizome_manifest *m) { - if (!m->finalised) return WHY("Manifest was not finalised"); + if (!m->finalised) + return WHY("Manifest was not finalised"); - if (m->haveSecret) { - /* We used to store the secret in the database, but we don't anymore, as we use - the BK field in the manifest. So nothing to do here. */ - } else { - /* We don't have the secret for this manifest, so only allow updates if - the self-signature is valid */ - if (!m->selfSigned) - return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt."); - } + // If we don't have the secret for this manifest, only store it if its self-signature is valid + if (!m->haveSecret && !m->selfSigned) + return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt."); /* Bind BAR to data field */ unsigned char bar[RHIZOME_BAR_BYTES]; rhizome_manifest_to_bar(m,bar); /* Store the file (but not if it is already in the database) */ - if (m->fileLength > 0 && !rhizome_exists(&m->filehash)) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize > 0 && !rhizome_exists(&m->filehash)) return WHY("File should already be stored by now"); - const char *name = rhizome_manifest_get(m, "name", NULL, 0); - const char *service = rhizome_manifest_get(m, "service", NULL, 0); - - sid_t *sender = NULL; - const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0); - if (sender_field) { - sender = (sid_t *) alloca(sizeof *sender); - if (str_to_sid_t(sender, sender_field) == -1) - return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field)); - } - - sid_t *recipient = NULL; - const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0); - if (recipient_field) { - recipient = (sid_t *) alloca(sizeof *recipient); - if (str_to_sid_t(recipient, recipient_field) == -1) - return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field)); - } - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) return WHY("Failed to begin transaction"); + time_ms_t now = gettime_ms(); + + // The INSERT OR REPLACE statement will delete a row with the same ID (primary key) if it exists, + // so a new autoincremented ROWID will be allocated whether or not the manifest with this ID is + // already in the table. Other code depends on this property: that ROWID is monotonically + // increasing with time and unique. sqlite3_stmt *stmt; if ((stmt = sqlite_prepare_bind(&retry, "INSERT OR REPLACE INTO MANIFESTS(" @@ -1333,18 +1384,19 @@ int rhizome_store_bundle(rhizome_manifest *m) "?,?,?,?,?,?,?,?,?,?,?,?,?" ");", RHIZOME_BID_T, &m->cryptoSignPublic, - STATIC_BLOB, m->manifestdata, m->manifest_bytes, + STATIC_BLOB, m->manifestdata, m->manifest_all_bytes, INT64, m->version, - INT64, (int64_t) gettime_ms(), + INT64, (int64_t) now, STATIC_BLOB, bar, RHIZOME_BAR_BYTES, - INT64, m->fileLength, - RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL, - SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author, - STATIC_TEXT, service, - STATIC_TEXT|NUL, name, - SID_T|NUL, sender, - SID_T|NUL, recipient, - INT64, m->journalTail, + INT64, m->filesize, + RHIZOME_FILEHASH_T|NUL, m->filesize > 0 ? &m->filehash : NULL, + // Only store the author if it is known to be authentic. + SID_T|NUL, m->authorship == AUTHOR_AUTHENTIC ? &m->author : NULL, + STATIC_TEXT, m->service, + STATIC_TEXT|NUL, m->name, + SID_T|NUL, m->has_sender ? &m->sender : NULL, + SID_T|NUL, m->has_recipient ? &m->recipient : NULL, + INT64, m->tail, END ) ) == NULL) @@ -1353,12 +1405,15 @@ int rhizome_store_bundle(rhizome_manifest *m) goto rollback; sqlite3_finalize(stmt); stmt = NULL; + rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db)); + rhizome_manifest_set_inserttime(m, now); // if (serverMode) // rhizome_sync_bundle_inserted(bar); // TODO remove old payload? +#if 0 if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) { int closed=rhizome_manifest_get_ll(m,"closedgroup"); if (closed<1) closed=0; @@ -1380,11 +1435,13 @@ int rhizome_store_bundle(rhizome_manifest *m) sqlite3_finalize(stmt); stmt = NULL; } +#endif +#if 0 if (m->group_count > 0) { if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS (manifestid, groupid) VALUES (?, ?);")) == NULL) goto rollback; - int i; + unsigned i; for (i=0;igroup_count;i++){ if (sqlite_bind(&retry, stmt, RHIZOME_BID_T, &m->cryptoSignPublic, TEXT, m->groups[i]) == -1) goto rollback; @@ -1395,14 +1452,15 @@ int rhizome_store_bundle(rhizome_manifest *m) sqlite3_finalize(stmt); stmt = NULL; } +#endif + if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){ // This message used in tests; do not modify or remove. - const char *service = rhizome_manifest_get(m, "service", NULL, 0); INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRId64, - service ? service : "NULL", + m->service ? m->service : "NULL", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version - ); + ); monitor_announce_bundle(m); if (serverMode) rhizome_sync_announce(); @@ -1416,183 +1474,188 @@ rollback: return -1; } -int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, - const char *sender_sid, const char *recipient_sid, - int limit, int offset, char count_rows) +/* The cursor struct must be zerofilled and the query parameters optionally filled in prior to + * calling this function. + * + * @author Andrew Bettison + */ +int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c) { + if (config.debug.rhizome) + DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64, + c, + alloca_str_toprint(c->service), + alloca_str_toprint(c->name), + c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET", + c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET", + c->rowid_since, + c->_rowid_last + ); IN(); strbuf b = strbuf_alloca(1024); strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1"); - - if (service && *service) - strbuf_sprintf(b, " AND service = ?1"); - if (name && *name) - strbuf_sprintf(b, " AND name like ?2"); - if (sender_sid && *sender_sid) - strbuf_sprintf(b, " AND sender = ?3"); - if (recipient_sid && *recipient_sid) - strbuf_sprintf(b, " AND recipient = ?4"); - - strbuf_sprintf(b, " ORDER BY inserttime DESC"); - - if (offset) - strbuf_sprintf(b, " OFFSET %u", offset); - + if (c->service) + strbuf_puts(b, " AND service = @service"); + if (c->name) + strbuf_puts(b, " AND name like @name"); + if (c->is_sender_set) + strbuf_puts(b, " AND sender = @sender"); + if (c->is_recipient_set) + strbuf_puts(b, " AND recipient = @recipient"); + if (c->rowid_since) { + strbuf_puts(b, " AND rowid > @last ORDER BY rowid ASC"); // oldest first + if (c->_rowid_last < c->rowid_since) + c->_rowid_last = c->rowid_since; + } else { + if (c->_rowid_last) + strbuf_puts(b, " AND rowid < @last"); + strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first + } if (strbuf_overrun(b)) RETURN(WHYF("SQL command too long: %s", strbuf_str(b))); - - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b)); - if (!statement) + c->_statement = sqlite_prepare(retry, strbuf_str(b)); + if (c->_statement == NULL) RETURN(-1); - - int ret = 0; - - if (service && *service) - ret = sqlite3_bind_text(statement, 1, service, -1, SQLITE_STATIC); - if (ret==SQLITE_OK && name && *name) - ret = sqlite3_bind_text(statement, 2, name, -1, SQLITE_STATIC); - if (ret==SQLITE_OK && sender_sid && *sender_sid) - ret = sqlite3_bind_text(statement, 3, sender_sid, -1, SQLITE_STATIC); - if (ret==SQLITE_OK && recipient_sid && *recipient_sid) - ret = sqlite3_bind_text(statement, 4, recipient_sid, -1, SQLITE_STATIC); - - if (ret!=SQLITE_OK){ - ret = WHYF("Failed to bind parameters: %s", sqlite3_errmsg(rhizome_db)); - goto cleanup; - } - - ret=0; - size_t rows = 0; - - const char *names[]={ - "_id", - "service", - "id", - "version", - "date", - ".inserttime", - ".author", - ".fromhere", - "filesize", - "filehash", - "sender", - "recipient", - "name" - }; - cli_columns(context, 13, names); - - while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { - ++rows; - if (limit>0 && rows>limit) - break; - if (!( sqlite3_column_count(statement) == 6 - && sqlite3_column_type(statement, 0) == SQLITE_TEXT - && sqlite3_column_type(statement, 1) == SQLITE_BLOB - && sqlite3_column_type(statement, 2) == SQLITE_INTEGER - && sqlite3_column_type(statement, 3) == SQLITE_INTEGER - && ( sqlite3_column_type(statement, 4) == SQLITE_TEXT - || sqlite3_column_type(statement, 4) == SQLITE_NULL - ) - )) { - ret = WHY("Incorrect statement column"); - break; - } - rhizome_manifest *m = rhizome_new_manifest(); - if (m == NULL) { - ret = WHY("Out of manifests"); - break; - } - const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0); - const char *manifestblob = (char *) sqlite3_column_blob(statement, 1); - size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() - int64_t q_version = sqlite3_column_int64(statement, 2); - int64_t q_inserttime = sqlite3_column_int64(statement, 3); - const char *q_author = (const char *) sqlite3_column_text(statement, 4); - int64_t rowid = sqlite3_column_int64(statement, 5); - - if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) { - WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); - } else { - int64_t blob_version = rhizome_manifest_get_ll(m, "version"); - if (blob_version != q_version) - WARNF("MANIFESTS row id=%s version=%"PRId64" does not match manifest blob.version=%"PRId64, - q_manifestid, q_version, blob_version); - int match = 1; - - const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0); - if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0)) - match = 0; - const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0); - const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0); - if (match && sender_sid[0]) { - if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0)) - match = 0; - } - if (match && recipient_sid[0]) { - if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0)) - match = 0; - } - - if (match) { - const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0); - int64_t blob_date = rhizome_manifest_get_ll(m, "date"); - const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0); - int from_here = 0; - sid_t senderSid; - sid_t recipientSid; - - if (blob_sender) - str_to_sid_t(&senderSid, blob_sender); - if (blob_recipient) - str_to_sid_t(&recipientSid, blob_recipient); - - if (q_author) { - if (config.debug.rhizome) DEBUGF("q_author=%s", alloca_str_toprint(q_author)); - str_to_sid_t(&m->author, q_author); - int cn = 0, in = 0, kp = 0; - from_here = keyring_find_sid(keyring, &cn, &in, &kp, &m->author); - } - if (!from_here && blob_sender) { - if (config.debug.rhizome) DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender)); - int cn = 0, in = 0, kp = 0; - from_here = keyring_find_sid(keyring, &cn, &in, &kp, &senderSid); - } - - cli_put_long(context, rowid, ":"); - cli_put_string(context, blob_service, ":"); - cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":"); - cli_put_long(context, blob_version, ":"); - cli_put_long(context, blob_date, ":"); - cli_put_long(context, q_inserttime, ":"); - cli_put_hexvalue(context, q_author ? m->author.binary : NULL, sizeof m->author.binary, ":"); - cli_put_long(context, from_here, ":"); - cli_put_long(context, m->fileLength, ":"); - - unsigned char filehash[SHA512_DIGEST_LENGTH]; - if (m->fileLength) - fromhex(filehash, blob_filehash, SHA512_DIGEST_LENGTH); - - cli_put_hexvalue(context, m->fileLength?filehash:NULL, SHA512_DIGEST_LENGTH, ":"); - - cli_put_hexvalue(context, blob_sender ? senderSid.binary : NULL, sizeof senderSid.binary, ":"); - cli_put_hexvalue(context, blob_recipient ? recipientSid.binary : NULL, sizeof recipientSid.binary, ":"); - cli_put_string(context, blob_name, "\n"); - } - } - if (m) rhizome_manifest_free(m); - } - - if (ret==0 && count_rows){ - while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) - ++rows; - } - cli_row_count(context, rows); - -cleanup: - sqlite3_finalize(statement); - RETURN(ret); + if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1) + goto failure; + if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1) + goto failure; + if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1) + goto failure; + if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1) + goto failure; + if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1) + goto failure; + c->manifest = NULL; + c->_rowid_current = 0; + RETURN(0); OUT(); +failure: + sqlite3_finalize(c->_statement); + c->_statement = NULL; + RETURN(-1); + OUT(); +} + +/* Guaranteed to return manifests with monotonically descending rowid. The first manifest will have + * the greatest rowid. + * + * Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest' + * field points to the fetched manifest. Returns 0 if there are no more manifests in the list. + * + * @author Andrew Bettison + */ +int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c) +{ + if (config.debug.rhizome) + DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64, + c, + alloca_str_toprint(c->service), + alloca_str_toprint(c->name), + c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET", + c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET", + c->rowid_since, + c->_rowid_last + ); + IN(); + if (c->_statement == NULL && rhizome_list_open(retry, c) == -1) + RETURN(-1); + while (1) { + if (c->manifest) { + rhizome_manifest_free(c->manifest); + c->_rowid_current = 0; + c->manifest = NULL; + } + if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW) + break; + assert(sqlite3_column_count(c->_statement) == 6); + assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT); + assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB); + assert(sqlite3_column_type(c->_statement, 2) == SQLITE_INTEGER); + assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER); + assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL); + assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER); + uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5); + if (c->_rowid_current && (c->rowid_since ? q_rowid >= c->_rowid_current : q_rowid <= c->_rowid_current)) { + WHYF("Query returned rowid=%"PRIu64" out of order (last was %"PRIu64") -- skipped", q_rowid, c->_rowid_current); + continue; + } + c->_rowid_current = q_rowid; + if (q_rowid <= c->rowid_since) { + WHYF("Query returned rowid=%"PRIu64" <= rowid_since=%"PRIu64" -- skipped", q_rowid, c->rowid_since); + continue; + } + const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0); + const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1); + size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob() + int64_t q_version = sqlite3_column_int64(c->_statement, 2); + int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3); + const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4); + sid_t *author = NULL; + if (q_author) { + author = alloca(sizeof *author); + if (str_to_sid_t(author, q_author) == -1) { + WHYF("MANIFESTS row id=%s has invalid author column %s -- skipped", q_manifestid, alloca_str_toprint(q_author)); + continue; + } + } + rhizome_manifest *m = c->manifest = rhizome_new_manifest(); + if (m == NULL) + RETURN(-1); + if ( rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1 + || !rhizome_manifest_validate(m) + ) { + WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); + continue; + } + if (m->version != q_version) { + WHYF("MANIFESTS row id=%s version=%"PRId64" does not match manifest blob version=%"PRId64" -- skipped", + q_manifestid, q_version, m->version); + continue; + } + if (author) + rhizome_manifest_set_author(m, author); + rhizome_manifest_set_rowid(m, q_rowid); + rhizome_manifest_set_inserttime(m, q_inserttime); + if (c->service && !(m->service && strcasecmp(c->service, m->service) == 0)) + continue; + if (c->is_sender_set && !(m->has_sender && cmp_sid_t(&c->sender, &m->sender) == 0)) + continue; + if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0)) + continue; + assert(c->_rowid_current != 0); + // Don't do rhizome_verify_author(m); too CPU expensive for a listing. Save that for when + // the bundle is extracted or exported. + RETURN(1); + } + assert(c->_rowid_current == 0); + RETURN(0); + OUT(); +} + +void rhizome_list_commit(struct rhizome_list_cursor *c) +{ + if (config.debug.rhizome) + DEBUGF("c=%p c->rowid_since=%"PRIu64" c->_rowid_current=%"PRIu64" c->_rowid_last=%"PRIu64, + c, c->rowid_since, c->_rowid_current, c->_rowid_last); + assert(c->_rowid_current != 0); + if (c->_rowid_last == 0 || (c->rowid_since ? c->_rowid_current > c->_rowid_last : c->_rowid_current < c->_rowid_last)) + c->_rowid_last = c->_rowid_current; +} + +void rhizome_list_release(struct rhizome_list_cursor *c) +{ + if (config.debug.rhizome) + DEBUGF("c=%p", c); + if (c->manifest) { + rhizome_manifest_free(c->manifest); + c->_rowid_current = 0; + c->manifest = NULL; + } + if (c->_statement) { + sqlite3_finalize(c->_statement); + c->_statement = NULL; + } } void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount) @@ -1631,56 +1694,36 @@ int rhizome_update_file_priority(const char *fileid) */ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) { - const char *service = rhizome_manifest_get(m, "service", NULL, 0); - if (service == NULL) + if (m->service == NULL) return WHY("Manifest has no service"); - - const char *name = rhizome_manifest_get(m, "name", NULL, 0); - - sid_t *sender = NULL; - const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0); - if (sender_field) { - sender = (sid_t *) alloca(sizeof *sender); - if (str_to_sid_t(sender, sender_field) == -1) - return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field)); - } - - sid_t *recipient = NULL; - const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0); - if (recipient_field) { - recipient = (sid_t *) alloca(sizeof *recipient); - if (str_to_sid_t(recipient, recipient_field) == -1) - return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field)); - } - char sqlcmd[1024]; strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd); strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?"); - if (m->fileLength != 0) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize > 0) strbuf_puts(b, " AND filehash = ?"); - if (name) + if (m->name) strbuf_puts(b, " AND name = ?"); - if (sender) + if (m->has_sender) strbuf_puts(b, " AND sender = ?"); - if (recipient) + if (m->has_recipient) strbuf_puts(b, " AND recipient = ?"); if (strbuf_overrun(b)) return WHYF("SQL command too long: %s", strbuf_str(b)); - int ret = 0; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END); if (!statement) return -1; int field = 2; - if (m->fileLength != 0) + if (m->filesize > 0) sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END); - if (name) - sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END); - if (sender) - sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END); - if (recipient) - sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END); + if (m->name) + sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, m->name, END); + if (m->has_sender) + sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->sender, END); + if (m->has_recipient) + sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->recipient, END); int rows = 0; while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { @@ -1695,24 +1738,28 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) const unsigned char *q_manifestid = sqlite3_column_text(statement, 0); const char *manifestblob = (char *) sqlite3_column_blob(statement, 1); size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() - if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) { + if ( rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1 + || !rhizome_manifest_validate(blob_m) + ) { WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); goto next; } - if (rhizome_manifest_verify(blob_m)) { + if (!rhizome_manifest_verify(blob_m)) { WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid); goto next; } const char *q_author = (const char *) sqlite3_column_text(statement, 2); if (q_author) { - if (config.debug.rhizome) - strbuf_sprintf(b, " .author=%s", q_author); - str_to_sid_t(&blob_m->author, q_author); + sid_t author; + if (str_to_sid_t(&author, q_author) == -1) + WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_manifestid, alloca_str_toprint(q_author)); + else + rhizome_manifest_set_author(blob_m, &author); } // check that we can re-author this manifest - if (rhizome_extract_privatekey(blob_m, NULL)){ + rhizome_authenticate_author(blob_m); + if (m->authorship != AUTHOR_AUTHENTIC) goto next; - } *found = blob_m; if (config.debug.rhizome) DEBUGF("Found duplicate payload, %s", q_manifestid); @@ -1734,15 +1781,20 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement) int64_t q_inserttime = sqlite3_column_int64(statement, 3); const char *q_author = (const char *) sqlite3_column_text(statement, 4); size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() - if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1) - return WHYF("Manifest %s exists but is invalid", q_id); + uint64_t q_rowid = sqlite3_column_int64(statement, 5); + if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1 || !rhizome_manifest_validate(m)) + return WHYF("Manifest bid=%s in database but invalid", q_id); if (q_author) { - if (str_to_sid_t(&m->author, q_author) == -1) - WARNF("manifest id=%s contains invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author)); + sid_t author; + if (str_to_sid_t(&author, q_author) == -1) + WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author)); + else + rhizome_manifest_set_author(m, &author); } if (m->version != q_version) WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version); - m->inserttime = q_inserttime; + rhizome_manifest_set_rowid(m, q_rowid); + rhizome_manifest_set_inserttime(m, q_inserttime); return 0; } @@ -1757,7 +1809,7 @@ int rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite3_stmt *statement = sqlite_prepare_bind(&retry, - "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id = ?", + "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END); if (!statement) @@ -1787,7 +1839,7 @@ int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned pr like[prefix_strlen] = '%'; like[prefix_strlen + 1] = '\0'; sqlite3_stmt *statement = sqlite_prepare_bind(&retry, - "SELECT id, manifest, version, inserttime, author FROM manifests WHERE id like ?", + "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id like ?", TEXT, like, END); if (!statement) @@ -1937,7 +1989,7 @@ static int is_interesting(const char *id_hex, int64_t version) int rhizome_is_bar_interesting(unsigned char *bar) { int64_t version = rhizome_bar_version(bar); - char id_hex[RHIZOME_MANIFEST_ID_STRLEN]; + char id_hex[RHIZOME_BAR_PREFIX_BYTES *2 + 2]; tohex(id_hex, RHIZOME_BAR_PREFIX_BYTES * 2, &bar[RHIZOME_BAR_PREFIX_OFFSET]); strcat(id_hex, "%"); return is_interesting(id_hex, version); @@ -1945,10 +1997,5 @@ int rhizome_is_bar_interesting(unsigned char *bar) int rhizome_is_manifest_interesting(rhizome_manifest *m) { - char id[RHIZOME_MANIFEST_ID_STRLEN + 1]; - if (!rhizome_manifest_get(m, "id", id, sizeof id)) - // dodgy manifest, we don't want to receive it - return WHY("Ignoring bad manifest (no ID field)"); - str_toupper_inplace(id); - return is_interesting(id, m->version); + return is_interesting(alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); } diff --git a/rhizome_direct.c b/rhizome_direct.c index 02d91833..7d982195 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -442,11 +442,12 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int pref if (!manifestblob) goto error; rhizome_manifest *m=rhizome_new_manifest(); - if (rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1) - { - rhizome_manifest_free(m); - goto error; - } + if ( rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1 + || !rhizome_manifest_validate(m) + ) { + rhizome_manifest_free(m); + goto error; + } DEBUGF("Read manifest"); sqlite3_blob_close(blob); diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index 220fbcbd..b84a5430 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -54,7 +54,7 @@ static void rhizome_direct_clear_temporary_files(rhizome_http_request *r) } } -int rhizome_direct_import_end(struct http_request *hr) +static int rhizome_direct_import_end(struct http_request *hr) { rhizome_http_request *r = (rhizome_http_request *) hr; if (!r->received_manifest) { @@ -168,7 +168,7 @@ int rhizome_direct_enquiry_end(struct http_request *hr) return 0; } -int rhizome_direct_addfile_end(struct http_request *hr) +static int rhizome_direct_addfile_end(struct http_request *hr) { rhizome_http_request *r = (rhizome_http_request *) hr; // If given a file without a manifest, we should only accept if it we are configured to do so, and @@ -222,23 +222,22 @@ int rhizome_direct_addfile_end(struct http_request *hr) return 0; } // If manifest template did not specify a service field, then by default it is "file". - if (rhizome_manifest_get(m, "service", NULL, 0) == NULL) - rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE); - sid_t *author = NULL; - if (!is_sid_t_any(config.rhizome.api.addfile.default_author)) - author = &config.rhizome.api.addfile.default_author; - rhizome_bk_t bsk = config.rhizome.api.addfile.bundle_secret_key; - if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)) { + if (!rhizome_is_bk_none(&config.rhizome.api.addfile.bundle_secret_key)) + rhizome_apply_bundle_secret(m, &config.rhizome.api.addfile.bundle_secret_key); + if (m->service == NULL) + rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE); + const sid_t *author = is_sid_t_any(config.rhizome.api.addfile.default_author) ? NULL : &config.rhizome.api.addfile.default_author; + if (rhizome_fill_manifest(m, r->data_file_name, author)) { rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest"); return 0; } - m->payloadEncryption=0; - rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); + rhizome_manifest_set_crypt(m, PAYLOAD_CLEAR); // import file contents // TODO, stream file into database - if (m->fileLength) { + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize > 0) { if (rhizome_add_file(m, payload_path)) { rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); @@ -258,7 +257,7 @@ int rhizome_direct_addfile_end(struct http_request *hr) if (config.debug.rhizome) DEBUGF("Import sans-manifest appeared to succeed"); /* Respond with the manifest that was added. */ - http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes); + http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_all_bytes); /* clean up after ourselves */ if (mout && mout != m) rhizome_manifest_free(mout); @@ -271,14 +270,14 @@ int rhizome_direct_addfile_end(struct http_request *hr) } } -void rhizome_direct_process_mime_start(struct http_request *hr) +static void rhizome_direct_process_mime_start(struct http_request *hr) { rhizome_http_request *r = (rhizome_http_request *) hr; assert(r->current_part == NONE); assert(r->part_fd == -1); } -void rhizome_direct_process_mime_end(struct http_request *hr) +static void rhizome_direct_process_mime_end(struct http_request *hr) { rhizome_http_request *r = (rhizome_http_request *) hr; if (r->part_fd != -1) { @@ -302,19 +301,19 @@ void rhizome_direct_process_mime_end(struct http_request *hr) r->current_part = NONE; } -void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd) +static void rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h) { rhizome_http_request *r = (rhizome_http_request *) hr; - if (strcmp(cd->name, "data") == 0) { + if (strcmp(h->content_disposition.name, "data") == 0) { r->current_part = DATA; - strncpy(r->data_file_name, cd->filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0'; + strncpy(r->data_file_name, h->content_disposition.filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0'; } - else if (strcmp(cd->name, "manifest") == 0) { + else if (strcmp(h->content_disposition.name, "manifest") == 0) { r->current_part = MANIFEST; } else return; char path[512]; - if (form_temporary_file_path(r, path, cd->name) == -1) { + if (form_temporary_file_path(r, path, h->content_disposition.name) == -1) { http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun"); return; } @@ -325,7 +324,7 @@ void rhizome_direct_process_mime_content_disposition(struct http_request *hr, co } } -void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len) +static void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len) { rhizome_http_request *r = (rhizome_http_request *) hr; if (r->part_fd != -1) { @@ -344,7 +343,7 @@ int rhizome_direct_import(rhizome_http_request *r, const char *remainder) } r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; - r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header; r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; r->http.handle_content_end = rhizome_direct_import_end; r->current_part = NONE; @@ -361,7 +360,7 @@ int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder) } r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; - r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header; r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; r->http.handle_content_end = rhizome_direct_enquiry_end; r->current_part = NONE; @@ -394,7 +393,7 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder) } r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start; r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end; - r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition; + r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header; r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body; r->http.handle_content_end = rhizome_direct_addfile_end; r->current_part = NONE; @@ -637,12 +636,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* Get filehash and size from manifest if present */ if (config.debug.rhizome_tx) { DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); - DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); - DEBUGF("file size = %"PRId64, m->fileLength); + DEBUGF("bundle filehash = %s", alloca_tohex_rhizome_filehash_t(m->filehash)); + DEBUGF("file size = %"PRId64, m->filesize); + DEBUGF("version = %"PRId64, m->version); } - int64_t version = rhizome_manifest_get_ll(m, "version"); - if (config.debug.rhizome_tx) - DEBUGF("version = %"PRId64,version); /* We now have everything we need to compose the POST request and send it. */ @@ -661,15 +658,16 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) "\r\n"; /* Work out what the content length should be */ if (config.debug.rhizome_tx) - DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes); - int content_length - =strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */ - +strlen(boundary) - +m->manifest_all_bytes - +strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */ - +strlen(boundary) - +m->fileLength - +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n"); + DEBUGF("manifest_all_bytes=%u, manifest_body_bytes=%u", m->manifest_all_bytes, m->manifest_body_bytes); + assert(m->filesize != RHIZOME_SIZE_UNSET); + size_t content_length = + strlen(template2) - 2 /* minus 2 for the "%s" that gets replaced */ + + strlen(boundary) + + m->manifest_all_bytes + + strlen(template3) - 2 /* minus 2 for the "%s" that gets replaced */ + + strlen(boundary) + + m->filesize + + strlen("\r\n--") + strlen(boundary) + strlen("--\r\n"); int len=snprintf(buffer,8192,template,content_length,boundary); len+=snprintf(&buffer[len],8192-len,template2,boundary); @@ -705,7 +703,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* send file contents */ { rhizome_filehash_t filehash; - if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1) + if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, m->version, &filehash) == -1) goto closeit; struct rhizome_read read; @@ -713,28 +711,26 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) if (rhizome_open_read(&read, &filehash)) goto closeit; - int64_t read_ofs; - for(read_ofs=0;read_ofsfileLength;){ + uint64_t read_ofs; + for(read_ofs=0;read_ofsfilesize;){ unsigned char buffer[4096]; read.offset=read_ofs; - int bytes_read = rhizome_read(&read, buffer, sizeof buffer); - if (bytes_read<0){ + ssize_t bytes_read = rhizome_read(&read, buffer, sizeof buffer); + if (bytes_read == -1) { rhizome_read_close(&read); goto closeit; } - - int write_ofs=0; - while(write_ofs < bytes_read){ - int written = write(sock, buffer + write_ofs, bytes_read - write_ofs); - if (written<0){ + size_t write_ofs = 0; + while (write_ofs < (size_t) bytes_read){ + ssize_t written = write(sock, buffer + write_ofs, (size_t) bytes_read - write_ofs); + if (written == -1){ WHY_perror("write"); rhizome_read_close(&read); goto closeit; } - write_ofs+=written; + write_ofs += (size_t) written; } - - read_ofs+=bytes_read; + read_ofs += (size_t) bytes_read; } rhizome_read_close(&read); } diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 2e2949fc..efd0b51f 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -26,6 +26,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "str.h" #include "strbuf_helpers.h" #include "overlay_address.h" +#include "socket.h" +#include "dataformats.h" /* Represents a queued fetch of a bundle payload, for which the manifest is already known. */ @@ -73,7 +75,7 @@ struct rhizome_fetch_slot { /* HTTP streaming reception of manifests */ char manifest_buffer[1024]; - int manifest_bytes; + unsigned manifest_bytes; /* MDP transport specific elements */ rhizome_bid_t bid; @@ -100,7 +102,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot); */ struct rhizome_fetch_queue { struct rhizome_fetch_slot active; // must be first element in struct - int candidate_queue_size; + unsigned candidate_queue_size; struct rhizome_fetch_candidate *candidate_queue; unsigned char log_size_threshold; // will only queue payloads smaller than this. }; @@ -114,7 +116,6 @@ struct rhizome_fetch_candidate queue3[4]; struct rhizome_fetch_candidate queue4[2]; struct rhizome_fetch_candidate queue5[2]; -#define NELS(a) (sizeof (a) / sizeof *(a)) #define slotno(slot) (int)((struct rhizome_fetch_queue *)(slot) - &rhizome_fetch_queues[0]) /* Static allocation of the queue structures. Must be in order of ascending log_size_threshold. @@ -150,53 +151,69 @@ static const char * fetch_state(int state) } } -int rhizome_active_fetch_count() -{ - int i,active=0; - for(i=0;i=NQUEUES) return -1; if (rhizome_fetch_queues[q].active.state==RHIZOME_FETCH_FREE) return -1; - return (int)rhizome_fetch_queues[q].active.write_state.file_offset; + return rhizome_fetch_queues[q].active.write_state.file_offset; } -int rhizome_fetch_queue_bytes(){ - int i,j,bytes=0; +static uint64_t rhizome_fetch_queue_bytes() +{ + uint64_t bytes = 0; + unsigned i; for(i=0;ifileLength - received; + assert(rhizome_fetch_queues[i].active.manifest->filesize != RHIZOME_SIZE_UNSET); + bytes += rhizome_fetch_queues[i].active.manifest->filesize - rhizome_fetch_queues[i].active.write_state.file_offset; } + unsigned j; for (j=0;jfileLength; + if (rhizome_fetch_queues[i].candidate_queue[j].manifest) { + assert(rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET); + bytes += rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize; + } } } return bytes; } +void rhizome_fetch_log_short_status() +{ + int i,active=0; + for(i=0;icandidate_queue_size;j++){ if (q->candidate_queue[j].manifest) used++; } - strbuf_sprintf(b, "

Slot %d, (%d of %d): ", i, used, q->candidate_queue_size); + strbuf_sprintf(b, "

Slot %u, (%u of %u): ", i, used, q->candidate_queue_size); if (q->active.state!=RHIZOME_FETCH_FREE){ - strbuf_sprintf(b, "%s %"PRId64" of %"PRId64" from %s*", + strbuf_sprintf(b, "%s %"PRIu64" of %"PRIu64" from %s*", fetch_state(q->active.state), q->active.write_state.file_offset, - q->active.manifest->fileLength, + q->active.manifest->filesize, alloca_tohex_sid_t_trunc(q->active.peer_sid, 16)); }else{ strbuf_puts(b, "inactive"); @@ -206,7 +223,8 @@ int rhizome_fetch_status_html(strbuf b) for (j=0; j< q->candidate_queue_size;j++){ if (q->candidate_queue[j].manifest){ candidates++; - candidate_size += q->candidate_queue[j].manifest->fileLength; + assert(q->candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET); + candidate_size += q->candidate_queue[j].manifest->filesize; } } if (candidates) @@ -224,10 +242,9 @@ static struct profile_total fetch_stats = { .name="rhizome_fetch_poll" }; * * @author Andrew Bettison */ -static struct rhizome_fetch_queue *rhizome_find_queue(uint64_t size) +static struct rhizome_fetch_queue *rhizome_find_queue(unsigned char log_size) { int i; - unsigned char log_size = log2ll(size); for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; if (log_size < q->log_size_threshold) @@ -272,9 +289,10 @@ static struct rhizome_fetch_slot *fetch_search_slot(const unsigned char *id, int // find the first matching candidate for this bundle static struct rhizome_fetch_candidate *fetch_search_candidate(const unsigned char *id, int prefix_length) { - int i, j; + unsigned i; for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; + unsigned j; for (j = 0; j < q->candidate_queue_size; j++) { struct rhizome_fetch_candidate *c = &q->candidate_queue[j]; if (!c->manifest) @@ -304,12 +322,12 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng * * @author Andrew Bettison */ -static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, int i) +static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, unsigned i) { struct rhizome_fetch_candidate * const c = &q->candidate_queue[i]; struct rhizome_fetch_candidate * e = &q->candidate_queue[q->candidate_queue_size - 1]; if (config.debug.rhizome_rx) - DEBUGF("insert queue[%d] candidate[%d]", (int)(q - rhizome_fetch_queues), i); + DEBUGF("insert queue[%d] candidate[%u]", (int)(q - rhizome_fetch_queues), i); assert(i >= 0 && i < q->candidate_queue_size); assert(i == 0 || c[-1].manifest); if (e->manifest) // queue is full @@ -330,7 +348,7 @@ static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch * * @author Andrew Bettison */ -static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i) +static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, unsigned i) { assert(i >= 0 && i < q->candidate_queue_size); struct rhizome_fetch_candidate *c = &q->candidate_queue[i]; @@ -348,10 +366,10 @@ static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i) static void candidate_unqueue(struct rhizome_fetch_candidate *c) { - int i, index; + unsigned i; for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; - index = c - q->candidate_queue; + unsigned index = c - q->candidate_queue; if (index>=0 && index < q->candidate_queue_size){ rhizome_fetch_unqueue(q, index); return; @@ -457,10 +475,9 @@ int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int static int rhizome_import_received_bundle(struct rhizome_manifest *m) { m->finalised = 1; - m->manifest_bytes = m->manifest_all_bytes; // store the signatures too if (config.debug.rhizome_rx) { - DEBUGF("manifest len=%d has %d signatories. Associated file = %"PRId64" bytes", - m->manifest_bytes, m->sig_count, m->fileLength); + DEBUGF("manifest len=%u has %u signatories. Associated filesize=%"PRIu64" bytes", + m->manifest_all_bytes, m->sig_count, m->filesize); dump("manifest", m->manifestdata, m->manifest_all_bytes); } return rhizome_add_manifest(m, m->ttl - 1 /* TTL */); @@ -489,24 +506,27 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) strbuf r = strbuf_local(slot->request, sizeof slot->request); strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash)); - if (slot->manifest->journalTail>=0){ + if (slot->manifest->is_journal){ // if we're fetching a journal bundle, work out how many bytes we have of a previous version // and therefore what range of bytes we should ask for slot->previous = rhizome_new_manifest(); if (rhizome_retrieve_manifest(&slot->manifest->cryptoSignPublic, slot->previous)){ rhizome_manifest_free(slot->previous); slot->previous=NULL; - - // check that the new journal is valid and has some overlapping bytes - }else if (slot->previous->journalTail > slot->manifest->journalTail - || slot->previous->fileLength + slot->previous->journalTail < slot->manifest->journalTail){ + // check that the new journal is valid and has some overlapping bytes + }else if ( !slot->previous->is_journal + || slot->previous->tail > slot->manifest->tail + || slot->previous->filesize + slot->previous->tail < slot->manifest->tail + ){ rhizome_manifest_free(slot->previous); slot->previous=NULL; }else{ - assert(slot->previous->fileLength >= slot->manifest->journalTail); - assert(slot->manifest->fileLength > 0); - strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n", - slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1); + assert(slot->previous->filesize >= slot->manifest->tail); + assert(slot->manifest->filesize > 0); + strbuf_sprintf(r, "Range: bytes=%"PRIu64"-%"PRIu64"\r\n", + slot->previous->filesize - slot->manifest->tail, + slot->manifest->filesize - 1 + ); } } @@ -516,7 +536,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) RETURN(WHY("request overrun")); slot->request_len = strbuf_len(r); - if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT)) + if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->filesize, RHIZOME_PRIORITY_DEFAULT)) RETURN(-1); } else { strbuf r = strbuf_local(slot->request, sizeof slot->request); @@ -526,8 +546,8 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) slot->request_len = strbuf_len(r); slot->manifest_bytes=0; - slot->write_state.file_offset=0; - slot->write_state.file_length=-1; + slot->write_state.file_offset = 0; + slot->write_state.file_length = RHIZOME_SIZE_UNSET; } slot->request_ofs = 0; @@ -647,16 +667,16 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct */ if (config.debug.rhizome_rx) - DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRId64" peerip=%s", + DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRIu64" peerip=%s", slotno(slot), alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, - m->fileLength, + m->filesize, alloca_sockaddr(peerip, sizeof(struct sockaddr_in)) ); // If the payload is empty, no need to fetch, so import now. - if (m->fileLength == 0) { + if (m->filesize == 0) { if (config.debug.rhizome_rx) DEBUGF(" manifest fetch not started -- nil payload, so importing instead"); if (rhizome_import_received_bundle(m) == -1) @@ -769,7 +789,7 @@ static void rhizome_start_next_queued_fetch(struct rhizome_fetch_slot *slot) IN(); struct rhizome_fetch_queue *q; for (q = (struct rhizome_fetch_queue *) slot; q >= rhizome_fetch_queues; --q) { - int i = 0; + unsigned i = 0; struct rhizome_fetch_candidate *c; while (i < q->candidate_queue_size && (c = &q->candidate_queue[i])->manifest) { int result = rhizome_fetch(slot, c->manifest, &c->peer_ipandport, &c->peer_sid); @@ -815,17 +835,14 @@ static void rhizome_start_next_queued_fetches(struct sched_ent *alarm) /* Do we have space to add a fetch candidate of this size? */ int rhizome_fetch_has_queue_space(unsigned char log2_size){ - int i; - for (i = 0; i < NQUEUES; ++i) { - struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; - if (log2_size < q->log_size_threshold){ - // is there an empty candidate? - int j=0; - for (j=0;j < q->candidate_queue_size;j++) - if (!q->candidate_queue[j].manifest) - return 1; - return 0; - } + struct rhizome_fetch_queue *q = rhizome_find_queue(log2_size); + if (q){ + // is there an empty candidate? + unsigned j=0; + for (j=0;j < q->candidate_queue_size;j++) + if (!q->candidate_queue[j].manifest) + return 1; + return 0; } return 0; } @@ -859,8 +876,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock int priority=100; /* normal priority */ if (config.debug.rhizome_rx) - DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", - alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, priority); + DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRIu64" priority=%d:", + alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->filesize, priority); if (!rhizome_is_manifest_interesting(m)) { if (config.debug.rhizome_rx) @@ -875,8 +892,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock DEBUGF(" is new (have version %"PRId64")", stored_version); } - if (m->fileLength == 0) { - if (rhizome_manifest_verify(m) != 0) { + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize == 0) { + if (!rhizome_manifest_verify(m)) { WHY("Error verifying manifest when considering for import"); /* Don't waste time looking at this manifest again for a while */ rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); @@ -889,9 +907,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock } // Find the proper queue for the payload. If there is none suitable, it is an error. - struct rhizome_fetch_queue *qi = rhizome_find_queue(m->fileLength); + struct rhizome_fetch_queue *qi = rhizome_find_queue(log2ll(m->filesize)); if (!qi) { - WHYF("No suitable fetch queue for bundle size=%"PRId64, m->fileLength); + WHYF("No suitable fetch queue for bundle size=%"PRIu64, m->filesize); rhizome_manifest_free(m); RETURN(-1); } @@ -900,9 +918,10 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock // may have changed between versions.) If a newer or the same version is already queued, then // ignore this one. Otherwise, unqueue all older candidates. int ci = -1; - int i, j; + unsigned i; for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; + unsigned j; for (j = 0; j < q->candidate_queue_size; ) { struct rhizome_fetch_candidate *c = &q->candidate_queue[j]; if (c->manifest) { @@ -911,7 +930,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock rhizome_manifest_free(m); RETURN(0); } - if (!m->selfSigned && rhizome_manifest_verify(m)) { + if (!m->selfSigned && !rhizome_manifest_verify(m)) { WHY("Error verifying manifest when considering queuing for import"); /* Don't waste time looking at this manifest again for a while */ rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); @@ -937,7 +956,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock RETURN(1); } - if (!m->selfSigned && rhizome_manifest_verify(m)) { + if (!m->selfSigned && !rhizome_manifest_verify(m)) { WHY("Error verifying manifest when considering queuing for import"); /* Don't waste time looking at this manifest again for a while */ rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); @@ -953,18 +972,18 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (config.debug.rhizome_rx) { DEBUG("Rhizome fetch queues:"); - int i, j; + unsigned i, j; for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i]; for (j = 0; j < q->candidate_queue_size; ++j) { struct rhizome_fetch_candidate *c = &q->candidate_queue[j]; if (!c->manifest) break; - DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRId64, i, j, + DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRIu64, i, j, c->manifest, alloca_tohex_rhizome_bid_t(c->manifest->cryptoSignPublic), c->priority, - c->manifest->fileLength + c->manifest->filesize ); } } @@ -1025,17 +1044,19 @@ static void rhizome_fetch_mdp_slot_callback(struct sched_ent *alarm) struct rhizome_fetch_slot *slot=(struct rhizome_fetch_slot*)alarm; time_ms_t now = gettime_ms(); - if (now-slot->last_write_time>slot->mdpIdleTimeout) { + if (now - slot->last_write_time > slot->mdpIdleTimeout) { DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRId64" of %"PRId64" bytes)", now-slot->last_write_time, - slot->write_state.file_offset, slot->write_state.file_length); + slot->write_state.file_offset, + slot->write_state.file_length); rhizome_fetch_close(slot); OUT(); return; } if (config.debug.rhizome_rx) - DEBUGF("Timeout: Resending request for slot=0x%p (%"PRId64" of %"PRId64" received)", - slot,slot->write_state.file_offset, slot->write_state.file_length); + DEBUGF("Timeout: Resending request for slot=0x%p (%"PRIu64" of %"PRIu64" received)", + slot, slot->write_state.file_offset, + slot->write_state.file_length); rhizome_fetch_mdp_requestblocks(slot); OUT(); } @@ -1050,7 +1071,7 @@ static int rhizome_fetch_mdp_touch_timeout(struct rhizome_fetch_slot *slot) // For now, we will just make the timeout 1 second from the time of the last // received block. unschedule(&slot->alarm); - slot->alarm.alarm=gettime_ms()+1000; + slot->alarm.alarm=gettime_ms()+config.rhizome.mdp_stall_timeout; slot->alarm.deadline=slot->alarm.alarm+500; schedule(&slot->alarm); return 0; @@ -1108,7 +1129,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot) slot->write_state.file_offset, slot->bidVersion); - overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0); + overlay_mdp_dispatch(&mdp, NULL); // remember when we sent the request so that we can adjust the inter-request // interval based on how fast the packets arrive. @@ -1133,11 +1154,14 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){ * [ | written | overlap | new content] */ - uint64_t start = slot->manifest->journalTail - slot->previous->journalTail + slot->write_state.file_offset; - uint64_t length = slot->previous->fileLength - slot->manifest->journalTail - slot->write_state.file_offset; + assert(slot->manifest->tail != RHIZOME_SIZE_UNSET); + assert(slot->previous->tail != RHIZOME_SIZE_UNSET); + assert(slot->previous->filesize != RHIZOME_SIZE_UNSET); + uint64_t start = slot->manifest->tail - slot->previous->tail + slot->write_state.file_offset; + uint64_t length = slot->previous->filesize - slot->manifest->tail - slot->write_state.file_offset; // of course there might not be any overlap - if (start < slot->previous->fileLength && length>0){ + if (start>=0 && start < slot->previous->filesize && length>0){ if (config.debug.rhizome) DEBUGF("Copying %"PRId64" bytes from previous journal", length); rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length); @@ -1170,8 +1194,8 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot) } if (config.debug.rhizome_rx) - DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRId64" bytes)", - slot,slot->write_state.file_length); + DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRIu64" bytes)", + slot, slot->write_state.file_length); /* close socket and stop watching it */ if (slot->alarm.poll.fd>=0) { @@ -1205,8 +1229,15 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot) down too much. Much careful thought is required to optimise this transport. */ - slot->mdpIdleTimeout=config.rhizome.idle_timeout; // give up if nothing received for 5 seconds - slot->mdpRXBlockLength=config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size + slot->mdpIdleTimeout = config.rhizome.idle_timeout; // give up if nothing received for 5 seconds + + unsigned char log_size=log2ll(slot->manifest->filesize); + struct rhizome_fetch_queue *q=rhizome_find_queue(log_size); + // increase the timeout based on the queue number + if (q) + slot->mdpIdleTimeout *= 1+(q - rhizome_fetch_queues); + + slot->mdpRXBlockLength = config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size rhizome_fetch_mdp_requestblocks(slot); RETURN(0); @@ -1289,7 +1320,9 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) call schedule queued items. */ rhizome_manifest *m = rhizome_new_manifest(); if (m) { - if (rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1) { + if ( rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1 + || !rhizome_manifest_validate(m) + ) { DEBUGF("Couldn't read manifest"); rhizome_manifest_free(m); } else { @@ -1308,17 +1341,18 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) time_ms_t interval = now - slot->start_time; if (interval <= 0) interval = 1; - DEBUGF("Closing rhizome fetch slot = 0x%p. Received %"PRId64" bytes in %"PRId64"ms (%"PRId64"KB/sec).", + DEBUGF("Closing rhizome fetch slot = 0x%p. Received %"PRIu64" bytes in %"PRIu64"ms (%"PRIu64"KB/sec).", slot, slot->write_state.file_offset, - (int64_t)interval, - (int64_t)((slot->write_state.file_offset) / interval)); + (uint64_t)interval, + slot->write_state.file_offset / (uint64_t)interval + ); } rhizome_fetch_close(slot); RETURN(-1); } -int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, int bytes) +int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, size_t bytes) { IN(); @@ -1327,17 +1361,18 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer // Truncate to known length of file (handy for reading from journal bundles that // might grow while we are reading from them). - if (bytes>(slot->write_state.file_length-(slot->write_state.file_offset))){ - bytes=slot->write_state.file_length-(slot->write_state.file_offset); + if (bytes > slot->write_state.file_length - slot->write_state.file_offset) { + bytes = slot->write_state.file_length - slot->write_state.file_offset; } if (!slot->manifest){ /* We are reading a manifest. Read it into a buffer. */ - int count=bytes; - if (count+slot->manifest_bytes>1024) count=1024-slot->manifest_bytes; + unsigned count = bytes; + if (count + slot->manifest_bytes > 1024) + count = 1024 - slot->manifest_bytes; bcopy(buffer,&slot->manifest_buffer[slot->manifest_bytes],count); slot->manifest_bytes+=count; - slot->write_state.file_offset+=count; + slot->write_state.file_offset += count; } else { /* We are reading a file. Stream it into the database. */ @@ -1358,7 +1393,7 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer int rhizome_received_content(const unsigned char *bidprefix, uint64_t version, uint64_t offset, - int count,unsigned char *bytes,int type) + size_t count, unsigned char *bytes, int type) { IN(); if (!is_rhizome_mdp_enabled()) { @@ -1368,7 +1403,7 @@ int rhizome_received_content(const unsigned char *bidprefix, if (slot && slot->bidVersion == version && slot->state == RHIZOME_FETCH_RXFILEMDP){ if (config.debug.rhizome) - DEBUGF("Rhizome over MDP receiving %d bytes.",count); + DEBUGF("Rhizome over MDP receiving %zu bytes.", count); if (rhizome_random_write(&slot->write_state, offset, bytes, count)){ if (config.debug.rhizome) DEBUGF("Write failed!"); @@ -1398,18 +1433,18 @@ int rhizome_received_content(const unsigned char *bidprefix, rhizome_manifest *m = NULL; struct rhizome_fetch_candidate *c = NULL; - if (slot && slot->bidVersion == version && slot->manifest->fileLength==count - && slot->state!=RHIZOME_FETCH_RXFILEMDP){ + if (slot && slot->bidVersion == version && slot->manifest->filesize == count + && slot->state != RHIZOME_FETCH_RXFILEMDP) { m=slot->manifest; }else{ slot = NULL; c = fetch_search_candidate(bidprefix, 16); - if (c && c->manifest->version==version && c->manifest->fileLength==count) + if (c && c->manifest->version == version && c->manifest->filesize == count) m=c->manifest; } if (m){ - if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){ + if (rhizome_import_buffer(m, bytes, count) >= 0 && !rhizome_import_received_bundle(m)){ INFOF("Completed MDP transfer in one hit for file %s", alloca_tohex_rhizome_filehash_t(m->filehash)); if (c) @@ -1459,8 +1494,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm) } else { if (errno!=EAGAIN) { if (config.debug.rhizome_rx) - DEBUGF("Empty read, closing connection: received %"PRId64" of %"PRId64" bytes", - slot->write_state.file_offset,slot->write_state.file_length); + DEBUGF("Empty read, closing connection: received %"PRIu64" of %"PRIu64" bytes", + slot->write_state.file_offset, + slot->write_state.file_length); rhizome_fetch_switch_to_mdp(slot); } return; @@ -1509,18 +1545,18 @@ void rhizome_fetch_poll(struct sched_ent *alarm) rhizome_fetch_switch_to_mdp(slot); return; } - if (slot->write_state.file_length==-1) - slot->write_state.file_length=parts.content_length; + if (slot->write_state.file_length == RHIZOME_SIZE_UNSET) + slot->write_state.file_length = parts.content_length; else if (parts.content_length + parts.range_start != slot->write_state.file_length) - WARNF("Expected content length %"PRId64", got %"PRId64" + %"PRId64, + WARNF("Expected content length %"PRIu64", got %"PRIu64" + %"PRIu64, slot->write_state.file_length, parts.content_length, parts.range_start); /* We have all we need. The file is already open, so just write out any initial bytes of the body we read. */ slot->state = RHIZOME_FETCH_RXFILE; if (slot->previous && parts.range_start){ - if (parts.range_start != slot->previous->fileLength - slot->manifest->journalTail) - WARNF("Expected Content-Range header to start @%"PRId64, slot->previous->fileLength - slot->manifest->journalTail); + if (parts.range_start != slot->previous->filesize - slot->manifest->tail) + WARNF("Expected Content-Range header to start @%"PRIu64, slot->previous->filesize - slot->manifest->tail); pipe_journal(slot); } diff --git a/rhizome_http.c b/rhizome_http.c index d08180eb..4b5f201f 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -1,6 +1,6 @@ /* -Serval Distributed Numbering Architecture (DNA) -Copyright (C) 2010 Paul Gardner-Stephen +Serval DNA Rhizome HTTP external interface +Copyright (C) 2013 Serval Project, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -21,37 +21,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #ifdef HAVE_SYS_FILIO_H -#include +# include #endif +#include #include #include "serval.h" #include "overlay_address.h" #include "conf.h" #include "str.h" +#include "strbuf_helpers.h" #include "rhizome.h" #include "http_server.h" #define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32 +typedef int HTTP_HANDLER(rhizome_http_request *r, const char *remainder); + struct http_handler{ const char *path; - int (*parser)(rhizome_http_request *r, const char *remainder); + HTTP_HANDLER *parser; }; -static int rhizome_status_page(rhizome_http_request *r, const char *remainder); -static int rhizome_file_page(rhizome_http_request *r, const char *remainder); -static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder); -static int interface_page(rhizome_http_request *r, const char *remainder); -static int neighbour_page(rhizome_http_request *r, const char *remainder); -static int fav_icon_header(rhizome_http_request *r, const char *remainder); -static int root_page(rhizome_http_request *r, const char *remainder); +static HTTP_HANDLER restful_rhizome_bundlelist_json; +static HTTP_HANDLER restful_rhizome_newsince; -extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder); -extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder); -extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder); +static HTTP_HANDLER rhizome_status_page; +static HTTP_HANDLER rhizome_file_page; +static HTTP_HANDLER manifest_by_prefix_page; +static HTTP_HANDLER interface_page; +static HTTP_HANDLER neighbour_page; +static HTTP_HANDLER fav_icon_header; +static HTTP_HANDLER root_page; + +extern HTTP_HANDLER rhizome_direct_import; +extern HTTP_HANDLER rhizome_direct_enquiry; +extern HTTP_HANDLER rhizome_direct_dispatch; struct http_handler paths[]={ + {"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json}, + {"/restful/rhizome/newsince/", restful_rhizome_newsince}, {"/rhizome/status", rhizome_status_page}, {"/rhizome/file/", rhizome_file_page}, {"/rhizome/import", rhizome_direct_import}, @@ -226,7 +235,7 @@ success: static void rhizome_server_finalise_http_request(struct http_request *_r) { rhizome_http_request *r = (rhizome_http_request *) _r; - rhizome_read_close(&r->read_state); + rhizome_read_close(&r->u.read_state); request_count--; } @@ -246,7 +255,7 @@ void rhizome_server_poll(struct sched_ent *alarm) } else { struct sockaddr_in *peerip=NULL; if (addr.sa_family == AF_INET) { - peerip = (struct sockaddr_in *)&addr; + peerip = (struct sockaddr_in *)&addr; // network order INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u", addr_len, peerip->sin_family, peerip->sin_port, ((unsigned char*)&peerip->sin_addr.s_addr)[0], @@ -267,8 +276,8 @@ void rhizome_server_poll(struct sched_ent *alarm) request_count++; request->uuid = rhizome_http_request_uuid_counter++; request->data_file_name[0] = '\0'; - request->read_state.blob_fd = -1; - request->read_state.blob_rowid = -1; + request->u.read_state.blob_fd = -1; + request->u.read_state.blob_rowid = -1; if (peerip) request->http.client_sockaddr_in = *peerip; request->http.handle_headers = rhizome_dispatch; @@ -312,6 +321,248 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_ OUT(); } +static int is_from_loopback(const struct http_request *r) +{ + return r->client_sockaddr_in.sin_family == AF_INET + && ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET; +} + +/* Return 1 if the given authorization credentials are acceptable. + * Return 0 if not. + */ +static int is_authorized(const struct http_client_authorization *auth) +{ + if (auth->scheme != BASIC) + return 0; + unsigned i; + for (i = 0; i != config.rhizome.api.restful.users.ac; ++i) { + if ( strcmp(config.rhizome.api.restful.users.av[i].key, auth->credentials.basic.user) == 0 + && strcmp(config.rhizome.api.restful.users.av[i].value.password, auth->credentials.basic.password) == 0 + ) + return 1; + } + return 0; +} + +static int authorize(struct http_request *r) +{ + if (!is_from_loopback(r)) { + http_request_simple_response(r, 403, NULL); + return 0; + } + if (!is_authorized(&r->request_header.authorization)) { + r->response.header.www_authenticate.scheme = BASIC; + r->response.header.www_authenticate.realm = "Serval Rhizome"; + http_request_simple_response(r, 401, NULL); + return 0; + } + return 1; +} + +#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8)) +#define alloca_list_token(rowid) list_token_to_str(alloca(LIST_TOKEN_STRLEN + 1), (rowid)) + +static char *list_token_to_str(char *buf, uint64_t rowid) +{ + struct iovec iov[2]; + iov[0].iov_base = rhizome_db_uuid.u.binary; + iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; + iov[1].iov_base = &rowid; + iov[1].iov_len = sizeof rowid; + size_t n = base64url_encodev(buf, iov, 2); + assert(n == LIST_TOKEN_STRLEN); + buf[n] = '\0'; + return buf; +} + +static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp) +{ + unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; + if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token) + return 0; + if (cmp_uuid_t(&rhizome_db_uuid, (uuid_t *) &token) != 0) + return 0; + memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); + return 1; +} + +static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content; + +static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder) +{ + if (!is_rhizome_http_enabled()) + return 1; + if (*remainder) + return 1; + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + if (!authorize(&r->http)) + return 0; + r->u.list.phase = LIST_HEADER; + r->u.list.rowcount = 0; + bzero(&r->u.list.cursor, sizeof r->u.list.cursor); + http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content); + return 0; +} + +static int restful_rhizome_newsince(rhizome_http_request *r, const char *remainder) +{ + if (!is_rhizome_http_enabled()) + return 1; + uint64_t rowid; + const char *end = NULL; + if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0) + return 1; + if (r->http.verb != HTTP_VERB_GET) { + http_request_simple_response(&r->http, 405, NULL); + return 0; + } + if (!authorize(&r->http)) + return 0; + r->u.list.phase = LIST_HEADER; + r->u.list.rowcount = 0; + bzero(&r->u.list.cursor, sizeof r->u.list.cursor); + r->u.list.cursor.rowid_since = rowid; + r->u.list.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000; + http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content); + return 0; +} + +static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b) +{ + const char *headers[] = { + ".token", + "_id", + "service", + "id", + "version", + "date", + ".inserttime", + ".author", + ".fromhere", + "filesize", + "filehash", + "sender", + "recipient", + "name" + }; + switch (r->u.list.phase) { + case LIST_HEADER: + strbuf_puts(b, "{\n\"header\":["); + unsigned i; + for (i = 0; i != NELS(headers); ++i) { + if (i) + strbuf_putc(b, ','); + strbuf_json_string(b, headers[i]); + } + strbuf_puts(b, "],\n\"rows\":["); + if (!strbuf_overrun(b)) + r->u.list.phase = LIST_ROWS; + return 1; + case LIST_ROWS: + { + int ret = rhizome_list_next(retry, &r->u.list.cursor); + if (ret == -1) + return -1; + if (ret == 0) { + time_ms_t now; + if (r->u.list.cursor.rowid_since == 0 || (now = gettime_ms()) >= r->u.list.end_time) { + strbuf_puts(b, "\n]\n}\n"); + if (!strbuf_overrun(b)) + r->u.list.phase = LIST_DONE; + return 0; + } + time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms; + if (wake_at > r->u.list.end_time) + wake_at = r->u.list.end_time; + http_request_pause_response(&r->http, wake_at); + return 0; + } + rhizome_manifest *m = r->u.list.cursor.manifest; + assert(m->filesize != RHIZOME_SIZE_UNSET); + rhizome_lookup_author(m); + if (r->u.list.rowcount != 0) + strbuf_putc(b, ','); + strbuf_puts(b, "\n["); + if (m->rowid > r->u.list.rowid_highest) { + strbuf_json_string(b, alloca_list_token(m->rowid)); + r->u.list.rowid_highest = m->rowid; + } else + strbuf_json_null(b); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRIu64, m->rowid); + strbuf_putc(b, ','); + strbuf_json_string(b, m->service); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRId64, m->version); + strbuf_putc(b, ','); + if (m->has_date) + strbuf_sprintf(b, "%"PRItime_ms_t, m->date); + else + strbuf_json_null(b); + strbuf_putc(b, ','); + strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime); + switch (m->authorship) { + case AUTHOR_LOCAL: + case AUTHOR_AUTHENTIC: + strbuf_json_hex(b, m->author.binary, sizeof m->author.binary); + strbuf_puts(b, ",1,"); + break; + default: + strbuf_json_null(b); + strbuf_puts(b, ",1,"); + break; + } + strbuf_sprintf(b, "%"PRIu64, m->filesize); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary); + strbuf_putc(b, ','); + strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary); + strbuf_putc(b, ','); + strbuf_json_string(b, m->name); + strbuf_puts(b, "]"); + if (!strbuf_overrun(b)) { + rhizome_list_commit(&r->u.list.cursor); + ++r->u.list.rowcount; + } + return 1; + } + case LIST_DONE: + return 0; + } + abort(); +} + +static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) +{ + rhizome_http_request *r = (rhizome_http_request *) hr; + assert(bufsz > 0); + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + int ret = rhizome_list_open(&retry, &r->u.list.cursor); + if (ret == -1) + return -1; + strbuf b = strbuf_local((char *)buf, bufsz); + while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) { + if (strbuf_overrun(b)) { + if (config.debug.rhizome) + DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b)); + result->need = strbuf_count(b) + 1 - result->generated; + break; + } + result->generated = strbuf_len(b); + if (ret == 0) + break; + } + rhizome_list_release(&r->u.list.cursor); + return ret; +} + static int neighbour_page(rhizome_http_request *r, const char *remainder) { if (r->http.verb != HTTP_VERB_GET) { @@ -378,30 +629,31 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder) return 0; } -static int rhizome_file_content(struct http_request *hr) +static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) { + // Only read multiples of 4k from disk. + const size_t blocksz = 1 << 12; + // Ask for a large buffer for all future reads. + const size_t preferred_bufsz = 16 * blocksz; + // Reads the next part of the payload into the supplied buffer. rhizome_http_request *r = (rhizome_http_request *) hr; - assert(r->http.response_buffer_sent == 0); - assert(r->http.response_buffer_length == 0); - assert(r->read_state.offset < r->read_state.length); - uint64_t readlen = r->read_state.length - r->read_state.offset; - size_t suggested_size = 64 * 1024; - if (suggested_size > readlen) - suggested_size = readlen; - if (r->http.response_buffer_size < suggested_size) - http_request_set_response_bufsize(&r->http, suggested_size); - if (r->http.response_buffer == NULL) - http_request_set_response_bufsize(&r->http, 1); - if (r->http.response_buffer == NULL) - return -1; - ssize_t len = rhizome_read(&r->read_state, - (unsigned char *)r->http.response_buffer, - r->http.response_buffer_size); - if (len == -1) - return -1; - assert((size_t) len <= r->http.response_buffer_size); - r->http.response_buffer_length += (size_t) len; - return 0; + assert(r->u.read_state.offset < r->u.read_state.length); + uint64_t remain = r->u.read_state.length - r->u.read_state.offset; + size_t readlen = bufsz; + if (remain < bufsz) + readlen = remain; + else + readlen &= ~(blocksz - 1); + if (readlen > 0) { + ssize_t n = rhizome_read(&r->u.read_state, buf, readlen); + if (n == -1) + return -1; + result->generated = (size_t) n; + } + assert(r->u.read_state.offset <= r->u.read_state.length); + remain = r->u.read_state.length - r->u.read_state.offset; + result->need = remain < preferred_bufsz ? remain : preferred_bufsz; + return remain ? 1 : 0; } static int rhizome_file_page(rhizome_http_request *r, const char *remainder) @@ -422,35 +674,35 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder) rhizome_filehash_t filehash; if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) return 1; - bzero(&r->read_state, sizeof r->read_state); - int n = rhizome_open_read(&r->read_state, &filehash); + bzero(&r->u.read_state, sizeof r->u.read_state); + int n = rhizome_open_read(&r->u.read_state, &filehash); if (n == -1) { http_request_simple_response(&r->http, 500, NULL); return 0; } if (n != 0) return 1; - if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) { - rhizome_read_close(&r->read_state); + if (r->u.read_state.length == -1 && rhizome_read(&r->u.read_state, NULL, 0)) { + rhizome_read_close(&r->u.read_state); return 1; } - assert(r->read_state.length != -1); - r->http.response.header.resource_length = r->read_state.length; + assert(r->u.read_state.length != -1); + r->http.response.header.resource_length = r->u.read_state.length; if (r->http.request_header.content_range_count > 0) { assert(r->http.request_header.content_range_count == 1); struct http_range closed; - unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length); + unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length); if (n == 0 || http_range_bytes(&closed, 1) == 0) { http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable return 0; } r->http.response.header.content_range_start = closed.first; r->http.response.header.content_length = closed.last - closed.first + 1; - r->read_state.offset = closed.first; + r->u.read_state.offset = closed.first; } else { r->http.response.header.content_range_start = 0; r->http.response.header.content_length = r->http.response.header.resource_length; - r->read_state.offset = 0; + r->u.read_state.offset = 0; } http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content); return 0; diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c index 0dbf4477..aa3f7429 100644 --- a/rhizome_packetformats.c +++ b/rhizome_packetformats.c @@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include -#include #include /* Android doesn't have log2(), and we don't really need to do floating point @@ -80,17 +79,18 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar) if (!m) { RETURN(WHY("null manifest passed in")); } - int i; - /* Manifest prefix */ + unsigned i; for(i=0;icryptoSignPublic.binary[i]; /* file length */ - bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->fileLength); + assert(m->filesize != RHIZOME_SIZE_UNSET); + bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->filesize); /* Version */ for(i=0;i<7;i++) bar[RHIZOME_BAR_VERSION_OFFSET+6-i]=(m->version>>(8*i))&0xff; - /* geo bounding box */ +#if 0 + /* geo bounding box TODO: replace with bounding circle!!! */ double minLat=rhizome_manifest_get_double(m,"min_lat",-90); if (minLat<-90) minLat=-90; if (minLat>90) minLat=90; double minLong=rhizome_manifest_get_double(m,"min_long",-180); @@ -99,6 +99,12 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar) if (maxLat<-90) maxLat=-90; if (maxLat>90) maxLat=90; double maxLong=rhizome_manifest_get_double(m,"max_long",+180); if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180; +#else + double minLat = -90; + double minLong = -180; + double maxLat = +90; + double maxLong = +180; +#endif unsigned short v; int o=RHIZOME_BAR_GEOBOX_OFFSET; v=(minLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff; @@ -162,11 +168,12 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons DEBUG("Found a BAR that is the wrong size - ignoring"); continue; } - if (ob_append_bytes(e, (unsigned char *)data, blob_bytes)){ + if (ob_remaining(e) < blob_bytes) { // out of room count--; break; } + ob_append_bytes(e, (unsigned char *)data, blob_bytes); *last_rowid=rowid; } if (statement) @@ -178,7 +185,8 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons Always advertise the most recent 3 manifests in the table, cycle through the rest of the table, adding 17 BAR's at a time */ int64_t bundles_available=0; -void overlay_rhizome_advertise(struct sched_ent *alarm){ +void overlay_rhizome_advertise(struct sched_ent *alarm) +{ bundles_available=0; static int64_t bundle_last_rowid=INT64_MAX; @@ -191,7 +199,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){ int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - // DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER + // TODO: DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER // ESPECIALLY FOR PACKET-RADIO goto end; @@ -210,31 +218,28 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){ frame->source = my_subscriber; frame->ttl = 1; frame->queue = OQ_OPPORTUNISTIC; - frame->payload = ob_new(); + if ((frame->payload = ob_new()) == NULL) { + op_free(frame); + goto end; + } ob_limitsize(frame->payload, 800); - ob_append_byte(frame->payload, 2); ob_append_ui16(frame->payload, rhizome_http_server_port); - int64_t rowid=0; int count = append_bars(frame->payload, &retry, "SELECT BAR,ROWID FROM MANIFESTS ORDER BY ROWID DESC LIMIT 3", &rowid); - if (count>=3){ if (bundle_last_rowid>rowid || bundle_last_rowid<=0) bundle_last_rowid=rowid; - count = append_bars(frame->payload, &retry, "SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < ? ORDER BY ROWID DESC LIMIT 17", &bundle_last_rowid); if (count<17) bundle_last_rowid=INT64_MAX; } - - if (overlay_payload_enqueue(frame)) + if (overlay_payload_enqueue(frame) == -1) op_free(frame); - end: sqlite_set_tracefunc(oldfunc); alarm->alarm = gettime_ms()+config.rhizome.advertise.interval; @@ -256,21 +261,20 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){ else frame->ttl = 1; frame->queue = OQ_OPPORTUNISTIC; - frame->payload = ob_new(); - + if ((frame->payload = ob_new()) == NULL) + goto error; ob_limitsize(frame->payload, 800); - - if (ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS)) goto error; - if (ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0)) goto error; - if (ob_append_ui16(frame->payload, m->manifest_all_bytes)) goto error; - if (ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes)) goto error; + ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS); + ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0); + ob_append_ui16(frame->payload, m->manifest_all_bytes); + ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes); ob_append_byte(frame->payload, 0xFF); - if (overlay_payload_enqueue(frame)) goto error; + if (overlay_payload_enqueue(frame) == -1) + goto error; if (config.debug.rhizome_ads) DEBUGF("Advertising manifest %s %"PRId64" to %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid_t(dest->sid):"broadcast"); return 0; - error: op_free(frame); return -1; @@ -290,7 +294,6 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st int ad_frame_type=ob_get(f->payload); struct sockaddr_in httpaddr = context->addr; httpaddr.sin_port = htons(RHIZOME_HTTP_PORT); - size_t manifest_length; rhizome_manifest *m=NULL; int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads); @@ -301,74 +304,66 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st if (ad_frame_type & HAS_MANIFESTS){ /* Extract whole manifests */ - while(f->payload->position < f->payload->sizeLimit) { - if (ob_getbyte(f->payload, f->payload->position)==0xff){ - f->payload->position++; + while (ob_remaining(f->payload) > 0) { + if (ob_peek(f->payload) == 0xff) { + ob_skip(f->payload, 1); break; } - - manifest_length = ob_get_ui16(f->payload); + + size_t manifest_length = ob_get_ui16(f->payload); if (manifest_length==0) continue; unsigned char *data = ob_get_bytes_ptr(f->payload, manifest_length); if (!data) { WHYF("Illegal manifest length field in rhizome advertisement frame %zu vs %d", - manifest_length, f->payload->sizeLimit - f->payload->position); + manifest_length, ob_remaining(f->payload)); break; } - /* Read manifest without verifying signatures (which would waste lots of - energy, everytime we see a manifest that we already have). - In fact, it would be better here to do a really rough and ready parser - to get the id and version fields out, and avoid the memory copies that - otherwise happen. - But we do need to make sure that at least one signature is there. - */ - m = rhizome_new_manifest(); - if (!m) { - WHY("Out of manifests"); + // Briefly inspect the manifest to see if it looks interesting. + struct rhizome_manifest_summary summ; + if (!rhizome_manifest_inspect((char *)data, manifest_length, &summ)) { + if (config.debug.rhizome_ads) + DEBUG("Ignoring manifest that looks malformed"); goto next; } - if (rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1) { - WHY("Error parsing manifest body"); - goto next; - } - - /* trim manifest ID to a prefix for ease of debugging - (that is the only use of this */ - if (config.debug.rhizome_ads){ - int64_t version = rhizome_manifest_get_ll(m, "version"); - DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), version); - } + if (config.debug.rhizome_ads) + DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(summ.bid), summ.version); - /* Crude signature presence test */ - if (m->manifest_bytes >= m->manifest_all_bytes){ - // no signature was found when parsing? - /* ignore the announcement, but don't ignore other people - offering the same manifest */ + // If it looks like there is no signature at all, ignore the announcement but don't brown-list + // the manifest ID, so that we will still process other offers of the same manifest with + // signatures. + if (summ.body_len == manifest_length) { if (config.debug.rhizome_ads) DEBUG("Ignoring manifest announcment with no signature"); goto next; } - if (rhizome_ignore_manifest_check(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary)){ + if (rhizome_ignore_manifest_check(summ.bid.binary, sizeof summ.bid.binary)){ /* Ignoring manifest that has caused us problems recently */ if (config.debug.rhizome_ads) - DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + DEBUGF("Ignoring manifest with errors bid=%s", alloca_tohex_rhizome_bid_t(summ.bid)); goto next; } - if (m->errors > 0){ - if (config.debug.rhizome_ads) - DEBUG("Unverified manifest has errors - so not processing any further."); - /* Don't waste any time on this manifest in future attempts for at least - a minute. */ + // The manifest looks potentially interesting, so now do a full parse and validation. + if ((m = rhizome_new_manifest()) == NULL) + goto next; + if ( rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1 + || !rhizome_manifest_validate(m) + ) { + WARN("Malformed manifest"); + // Don't attend to this manifest for at least a minute rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000); goto next; } - /* Manifest is okay, so see if it is worth storing */ - + assert(m->has_id); + assert(m->version != 0); + assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &summ.bid) == 0); + assert(m->version == summ.version); + assert(m->manifest_body_bytes == summ.body_len); + // are we already fetching this bundle [or later]? rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary); if (mf && mf->version >= m->version) @@ -479,7 +474,7 @@ next: lookup_time = (end_time - start_time); if (mdp.out.payload_length>0) - overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0); + overlay_mdp_dispatch(&mdp, NULL); end: sqlite_set_tracefunc(oldfunc); diff --git a/rhizome_store.c b/rhizome_store.c index 6016ada9..336c5943 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -14,7 +14,7 @@ int rhizome_exists(const rhizome_filehash_t *hashp) return gotfile; } -int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority) +int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority) { write->blob_fd=-1; @@ -136,13 +136,14 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex * */ // encrypt and hash data, data buffers must be passed in file order. -static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, int data_size){ - if (data_size<=0) +static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size) +{ + if (data_size <= 0) return WHY("No content supplied"); /* Make sure we aren't being asked to write more data than we expected */ if (write_state->file_offset + data_size > write_state->file_length) - return WHYF("Too much content supplied, %"PRId64" + %d > %"PRId64, + return WHYF("Too much content supplied, %"PRIu64" + %zu > %"PRIu64, write_state->file_offset, data_size, write_state->file_length); if (write_state->crypt){ @@ -157,7 +158,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer write_state->file_offset+=data_size; if (config.debug.rhizome) - DEBUGF("Processed %"PRId64" of %"PRId64, write_state->file_offset, write_state->file_length); + DEBUGF("Processed %"PRIu64" of %"PRIu64, write_state->file_offset, write_state->file_length); return 0; } @@ -189,6 +190,11 @@ static int write_get_lock(struct rhizome_write *write_state){ // write data to disk static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size) { + if (config.debug.rhizome) { + DEBUGF("write_state->file_length=%"PRIu64" file_offset=%"PRIu64, write_state->file_length, file_offset); + //dump("buffer", buffer, data_size); + } + if (data_size<=0) return 0; @@ -198,7 +204,8 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u if (write_state->blob_fd != -1) { int ofs=0; // keep trying until all of the data is written. - lseek(write_state->blob_fd, file_offset, SEEK_SET); + if (lseek64(write_state->blob_fd, (off64_t) file_offset, SEEK_SET) == -1) + return WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", write_state->blob_fd, file_offset); while(ofs < data_size){ int r=write(write_state->blob_fd, buffer + ofs, data_size - ofs); if (r<0) @@ -225,10 +232,10 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u } } - write_state->written_offset=file_offset + data_size; + write_state->written_offset = file_offset + data_size; if (config.debug.rhizome) - DEBUGF("Wrote %"PRId64" of %"PRId64, file_offset + data_size, write_state->file_length); + DEBUGF("Wrote %"PRIu64" of %"PRIu64, file_offset + data_size, write_state->file_length); return 0; } @@ -254,8 +261,12 @@ static int write_release_lock(struct rhizome_write *write_state){ // Write data buffers in any order, the data will be cached and streamed into the database in file order. // Though there is an upper bound on the amount of cached data -int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size) +int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size) { + if (config.debug.rhizome) { + DEBUGF("write_state->file_length=%"PRIu64" offset=%"PRIu64, write_state->file_length, offset); + //dump("buffer", buffer, data_size); + } if (offset + data_size > write_state->file_length) data_size = write_state->file_length - offset; @@ -269,10 +280,10 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go. // (Not perfect if the range overlaps) uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size; - if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE) + if (new_size >= write_state->file_length || new_size >= RHIZOME_BUFFER_MAXIMUM_SIZE) should_write = 1; } - int64_t last_offset = write_state->written_offset; + uint64_t last_offset = write_state->written_offset; while(1){ @@ -396,12 +407,10 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){ ret = write_get_lock(write); if (ret) goto end; - while(write->file_offset < write->file_length){ - - int size=sizeof(buffer); + while(write->file_offset < write->file_length) { + size_t size = sizeof buffer; if (write->file_offset + size > write->file_length) - size=write->file_length - write->file_offset; - + size = write->file_length - write->file_offset; size_t r = fread(buffer, 1, size, f); if (ferror(f)){ ret = WHY_perror("fread"); @@ -451,7 +460,7 @@ int rhizome_finish_write(struct rhizome_write *write) } if (write->file_offset < write->file_length){ - WHYF("Only processed %"PRId64" bytes, expected %"PRId64, write->file_offset, write->file_length); + WHYF("Only processed %"PRIu64" bytes, expected %"PRIu64, write->file_offset, write->file_length); } int fd = write->blob_fd; @@ -550,14 +559,15 @@ failure: // import a file for an existing bundle with a known file hash int rhizome_import_file(rhizome_manifest *m, const char *filepath) { - if (m->fileLength<=0) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize == 0) return 0; /* Import the file first, checking the hash as we go */ struct rhizome_write write; bzero(&write, sizeof(write)); - int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); + int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT); if (ret!=0) return ret; @@ -578,16 +588,18 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath) // store a whole payload from a single buffer int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length) { - if (m->fileLength<=0) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (m->filesize == 0) return 0; - if (length!=m->fileLength) - return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length); + + if (length != m->filesize) + return WHYF("Expected %"PRIu64" bytes, got %zu", m->filesize, length); /* Import the file first, checking the hash as we go */ struct rhizome_write write; bzero(&write, sizeof(write)); - int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); + int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT); if (ret!=0) return ret; @@ -607,48 +619,39 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t len int rhizome_stat_file(rhizome_manifest *m, const char *filepath) { - int64_t existing = rhizome_manifest_get_ll(m, "filesize"); - - m->fileLength = 0; + uint64_t size = 0; if (filepath[0]) { struct stat stat; - if (lstat(filepath,&stat)) - return WHYF("Could not stat() payload file '%s'",filepath); - m->fileLength = stat.st_size; + if (lstat(filepath, &stat)) + return WHYF_perror("lstat(%s)", alloca_str_toprint(filepath)); + size = stat.st_size; } - // fail if the file is shorter than specified by the manifest - if (existing > m->fileLength) + // Fail if the file is shorter than already specified by the manifest. + if (m->filesize != RHIZOME_SIZE_UNSET && size < m->filesize) return WHY("Manifest length is longer than the file"); - // if the file is longer than specified by the manifest, ignore the end. - if (existing!=-1 && existing < m->fileLength) - m->fileLength = existing; - - rhizome_manifest_set_ll(m, "filesize", m->fileLength); - - if (m->fileLength == 0){ - m->filehash = RHIZOME_FILEHASH_NONE; - rhizome_manifest_del(m, "filehash"); - } + // If the file is longer than already specified by the manifest, ignore the end of the file. + if (m->filesize == RHIZOME_SIZE_UNSET || size > m->filesize) + rhizome_manifest_set_filesize(m, size); return 0; } -static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_write *write) +static int rhizome_write_derive_key(rhizome_manifest *m, struct rhizome_write *write) { - if (!m->payloadEncryption) + if (m->payloadEncryption != PAYLOAD_ENCRYPTED) return 0; // if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go - if (rhizome_derive_key(m, bsk)) + if (rhizome_derive_payload_key(m)) return -1; if (config.debug.rhizome) DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); write->crypt=1; - if (m->journalTail>0) - write->tail = m->journalTail; + if (m->is_journal && m->tail > 0) + write->tail = m->tail; bcopy(m->payloadKey, write->key, sizeof(write->key)); bcopy(m->payloadNonce, write->nonce, sizeof(write->nonce)); @@ -657,10 +660,11 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m) { - if (rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT)) + assert(m->filesize != RHIZOME_SIZE_UNSET); + if (rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT)) return -1; - if (rhizome_write_derive_key(m, NULL, write)) + if (rhizome_write_derive_key(m, write)) return -1; return 0; } @@ -672,18 +676,14 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath) // Stream the file directly into the database, encrypting & hashing as we go. struct rhizome_write write; bzero(&write, sizeof(write)); - if (rhizome_write_open_manifest(&write, m)) goto failure; if (rhizome_write_file(&write, filepath)) goto failure; if (rhizome_finish_write(&write)) goto failure; - - m->filehash = write.id; - rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); + rhizome_manifest_set_filehash(m, &write.id); return 0; - failure: rhizome_fail_write(&write); return -1; @@ -704,7 +704,7 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp " AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1) return -1; if (read->blob_rowid != -1) { - read->length = -1; // discover the length on opening the db BLOB + read->length = RHIZOME_SIZE_UNSET; // discover the length on opening the db BLOB } else { // No row in FILEBLOBS, look for an external blob file. char blob_path[1024]; @@ -716,8 +716,10 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp return 1; // file not available return WHYF_perror("open(%s)", alloca_str_toprint(blob_path)); } - if ((read->length = lseek(read->blob_fd, 0, SEEK_END)) == -1) - return WHYF_perror("lseek(%s,0,SEEK_END)", alloca_str_toprint(blob_path)); + off64_t pos = lseek64(read->blob_fd, 0, SEEK_END); + if (pos == -1) + return WHYF_perror("lseek64(%s,0,SEEK_END)", alloca_str_toprint(blob_path)); + read->length = pos; if (config.debug.externalblobs) DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length); } @@ -731,8 +733,8 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read { IN(); if (read_state->blob_fd != -1) { - if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1) - RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset)); + if (lseek64(read_state->blob_fd, (off64_t) read_state->offset, SEEK_SET) == -1) + RETURN(WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", read_state->blob_fd, read_state->offset)); if (bufsz == 0) RETURN(0); ssize_t rd = read(read_state->blob_fd, buffer, bufsz); @@ -755,7 +757,7 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db))); } assert(blob != NULL); - if (read_state->length == -1) + if (read_state->length == RHIZOME_SIZE_UNSET) read_state->length = sqlite3_blob_bytes(blob); // A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out // the length. @@ -813,6 +815,7 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz } if (read_state->crypt && buffer && bytes_read>0){ + dump("before decrypt", buffer, bytes_read); if(rhizome_crypt_xor_block( buffer, bytes_read, read_state->offset + read_state->tail, @@ -820,62 +823,73 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz RETURN(-1); } } - read_state->offset+=bytes_read; + read_state->offset += bytes_read; + if (config.debug.rhizome) { + DEBUGF("read %zu bytes, read_state->offset=%"PRIu64, bytes_read, read_state->offset); + //dump("buffer", buffer, bytes_read); + } RETURN(bytes_read); OUT(); } /* Read len bytes from read->offset into data, using *buffer to cache any reads */ -int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len) +ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len) { size_t bytes_copied=0; while (len>0){ + //DEBUGF("len=%zu read->length=%"PRIu64" read->offset=%"PRIu64" buffer->offset=%"PRIu64"", len, read->length, read->offset, buffer->offset); // make sure we only attempt to read data that actually exists - if (read->length !=-1 && read->offset + len > read->length) + if (read->length != RHIZOME_SIZE_UNSET && read->offset + len > read->length) len = read->length - read->offset; // if we can supply either the beginning or end of the data from cache, do that first. - uint64_t ofs=read->offset - buffer->offset; - if (ofs<=buffer->len){ - size_t size=len; - if (size > buffer->len - ofs) - size = buffer->len - ofs; - if (size>0){ - // copy into the start of the data buffer - bcopy(buffer->data + ofs, data, size); - data+=size; - len-=size; - read->offset+=size; - bytes_copied+=size; - continue; + if (read->offset >= buffer->offset) { + assert(read->offset - buffer->offset <= SIZE_MAX); + size_t ofs = read->offset - buffer->offset; + if (ofs <= buffer->len){ + size_t size = len; + if (size > buffer->len - ofs) + size = buffer->len - ofs; + if (size > 0){ + // copy into the start of the data buffer + bcopy(buffer->data + ofs, data, size); + data+=size; + len-=size; + read->offset+=size; + bytes_copied+=size; + continue; + } } } - ofs = (read->offset+len) - buffer->offset; - if (ofs>0 && ofs<=buffer->len){ - size_t size=len; - if (size > ofs) - size = ofs; - if (size>0){ - // copy into the end of the data buffer - bcopy(buffer->data + ofs - size, data + len - size, size); - len-=size; - bytes_copied+=size; - continue; + if (read->offset + len > buffer->offset) { + assert(read->offset + len - buffer->offset <= SIZE_MAX); + size_t ofs = read->offset + len - buffer->offset; + if (ofs <= buffer->len){ + size_t size = len; + if (size > ofs) + size = ofs; + if (size>0){ + // copy into the end of the data buffer + bcopy(buffer->data + ofs - size, data + len - size, size); + len-=size; + bytes_copied+=size; + continue; + } } } // ok, so we need to read a new buffer to fulfill the request. // remember the requested read offset so we can put it back - ofs = read->offset; + uint64_t ofs = read->offset; buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1); - ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data)); + ssize_t r = rhizome_read(read, buffer->data, sizeof buffer->data); read->offset = ofs; buffer->len = 0; - if (len == -1) + if (r == -1) return -1; - buffer->len = (size_t) len; + buffer->len = (size_t) r; } return bytes_copied; } @@ -1066,31 +1080,31 @@ static int write_file(struct rhizome_read *read, const char *filepath){ return ret; } -static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){ - read_state->crypt=m->payloadEncryption; +static int read_derive_key(rhizome_manifest *m, struct rhizome_read *read_state) +{ + read_state->crypt = m->payloadEncryption == PAYLOAD_ENCRYPTED; if (read_state->crypt){ // if the manifest specifies encryption, make sure we can generate the payload key and encrypt // the contents as we go - if (rhizome_derive_key(m, bsk)) { + if (rhizome_derive_payload_key(m)) { rhizome_read_close(read_state); return WHY("Unable to decrypt bundle, valid key not found"); } if (config.debug.rhizome) - DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); - - if (m->journalTail>0) - read_state->tail = m->journalTail; + DEBUGF("Decrypting payload contents for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version); + if (m->is_journal && m->tail > 0) + read_state->tail = m->tail; bcopy(m->payloadKey, read_state->key, sizeof(read_state->key)); bcopy(m->payloadNonce, read_state->nonce, sizeof(read_state->nonce)); } return 0; } -int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state) +int rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state) { int ret = rhizome_open_read(read_state, &m->filehash); if (ret == 0) - ret = read_derive_key(m, bsk, read_state); + ret = read_derive_key(m, read_state); return ret; } @@ -1099,11 +1113,11 @@ int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhi * * Returns -1 on error, 0 if extracted successfully, 1 if not found. */ -int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk) +int rhizome_extract_file(rhizome_manifest *m, const char *filepath) { struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - int ret = rhizome_open_decrypt_read(m, bsk, &read_state); + int ret = rhizome_open_decrypt_read(m, &read_state); if (ret == 0) ret = write_file(&read_state, filepath); rhizome_read_close(&read_state); @@ -1168,28 +1182,24 @@ int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t * } // open an existing journal bundle, advance the head pointer, duplicate the existing content and get ready to add more. -int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size) +int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t new_size) { int ret = 0; - if (advance_by > m->fileLength) + assert(m->filesize != RHIZOME_SIZE_UNSET); + assert(m->is_journal); + if (advance_by > m->filesize) return WHY("Cannot advance past the existing content"); - uint64_t old_length = m->fileLength; - uint64_t copy_length = old_length - advance_by; - - m->fileLength = m->fileLength + new_size - advance_by; - rhizome_manifest_set_ll(m, "filesize", m->fileLength); + uint64_t copy_length = m->filesize - advance_by; + rhizome_manifest_set_filesize(m, m->filesize + new_size - advance_by); - if (advance_by>0){ - m->journalTail += advance_by; - rhizome_manifest_set_ll(m,"tail",m->journalTail); - } + if (advance_by > 0) + rhizome_manifest_set_tail(m, m->tail + advance_by); - m->version = m->fileLength; - rhizome_manifest_set_ll(m,"version",m->version); + rhizome_manifest_set_version(m, m->filesize); - ret = rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT); + ret = rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT); if (ret) goto failure; @@ -1200,7 +1210,7 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, goto failure; } - ret = rhizome_write_derive_key(m, bsk, write); + ret = rhizome_write_derive_key(m, write); if (ret) goto failure; @@ -1212,12 +1222,12 @@ failure: return ret; } -int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len) +int rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len) { struct rhizome_write write; bzero(&write, sizeof write); - int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, (uint64_t) len); + int ret = rhizome_write_open_journal(&write, m, advance_by, (uint64_t) len); if (ret) return -1; @@ -1231,8 +1241,7 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64 if (ret) goto failure; - m->filehash = write.id; - rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); + rhizome_manifest_set_filehash(m, &write.id); return 0; failure: @@ -1241,15 +1250,15 @@ failure: return ret; } -int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename) +int rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename) { struct stat stat; if (lstat(filename,&stat)) - return WHYF("Could not stat() payload file '%s'",filename); + return WHYF_perror("stat(%s)", alloca_str_toprint(filename)); struct rhizome_write write; bzero(&write, sizeof write); - int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, stat.st_size); + int ret = rhizome_write_open_journal(&write, m, advance_by, stat.st_size); if (ret) return -1; @@ -1263,8 +1272,7 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t if (ret) goto failure; - m->filehash = write.id; - rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); + rhizome_manifest_set_filehash(m, &write.id); return 0; diff --git a/rhizome_sync.c b/rhizome_sync.c index 415b0944..be5c7fcb 100644 --- a/rhizome_sync.c +++ b/rhizome_sync.c @@ -93,7 +93,7 @@ static void rhizome_sync_request(struct subscriber *subscriber, uint64_t token, mdp.out.payload_length = ob_position(b); if (config.debug.rhizome) DEBUGF("Sending request to %s for BARs from %"PRIu64" %s", alloca_tohex_sid_t(subscriber->sid), token, forwards?"forwards":"backwards"); - overlay_mdp_dispatch(&mdp,0,NULL,0); + overlay_mdp_dispatch(&mdp, NULL); ob_free(b); } @@ -147,7 +147,7 @@ static void rhizome_sync_send_requests(struct subscriber *subscriber, struct rhi break; } if (mdp.out.payload_length!=0) - overlay_mdp_dispatch(&mdp,0,NULL,0); + overlay_mdp_dispatch(&mdp, NULL); // send request for more bars if we have room to cache them if (state->bar_count >= CACHE_BARS) @@ -321,21 +321,16 @@ static void sync_process_bar_list(struct subscriber *subscriber, struct rhizome_ } -static int append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar) +static void append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar) { - if (ob_append_packed_ui64(b, token)) - return -1; - if (bar){ - if (ob_append_bytes(b, bar, RHIZOME_BAR_BYTES)) - return -1; - }else{ + ob_append_packed_ui64(b, token); + if (bar) + ob_append_bytes(b, bar, RHIZOME_BAR_BYTES); + else{ unsigned char *ptr = ob_append_space(b, RHIZOME_BAR_BYTES); - if (!ptr) - return -1; - bzero(ptr, RHIZOME_BAR_BYTES); + if (ptr) + bzero(ptr, RHIZOME_BAR_BYTES); } - ob_checkpoint(b); - return 0; } static uint64_t max_token=0; @@ -400,24 +395,26 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t // make sure we include the exact rowid that was requested, even if we just deleted / replaced the manifest if (count==0 && rowid!=token){ if (token!=HEAD_FLAG){ - if (append_response(b, token, NULL)) + ob_checkpoint(b); + append_response(b, token, NULL); + if (ob_overrun(b)) ob_rewind(b); - else{ + else { count++; last = token; } }else token = rowid; } - - if (append_response(b, rowid, bar)) + ob_checkpoint(b); + append_response(b, rowid, bar); + if (ob_overrun(b)) ob_rewind(b); else { last = rowid; count++; } } - if (count >= max_count && rowid <= max_token) break; } @@ -427,7 +424,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t // send a zero lower bound if we reached the end of our manifest list if (count && count < max_count && !forwards){ - if (append_response(b, 0, NULL)) + ob_checkpoint(b); + append_response(b, 0, NULL); + if (ob_overrun(b)) ob_rewind(b); else { last = 0; @@ -441,7 +440,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t mdp.out.payload_length = ob_position(b); if (config.debug.rhizome_ads) DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last); - overlay_mdp_dispatch(&mdp,0,NULL,0); + overlay_mdp_dispatch(&mdp, NULL); } ob_free(b); OUT(); diff --git a/route_link.c b/route_link.c index 5fa4edec..c836c7c3 100644 --- a/route_link.c +++ b/route_link.c @@ -4,6 +4,7 @@ #include "overlay_packet.h" #include "str.h" #include "conf.h" +#include "keyring.h" #include /* @@ -254,6 +255,7 @@ static struct neighbour *get_neighbour(struct subscriber *subscriber, char creat n->mdp_ack_sequence = -1; // TODO measure min/max rtt n->rtt = 120; + n->next_neighbour_update = gettime_ms() + 10; neighbours = n; if (config.debug.linkstate) DEBUGF("LINK STATE; new neighbour %s", alloca_tohex_sid_t(n->subscriber->sid)); @@ -466,48 +468,27 @@ static int append_link_state(struct overlay_buffer *payload, char flags, flags|=FLAG_HAS_ACK; if (drop_rate!=-1) flags|=FLAG_HAS_DROP_RATE; - int length_pos = ob_position(payload); - if (ob_append_byte(payload, 0)) - return -1; - - if (ob_append_byte(payload, flags)) - return -1; - - if (overlay_address_append(NULL, payload, receiver)) - return -1; - - if (ob_append_byte(payload, version)) - return -1; - + ob_append_byte(payload, 0); + ob_append_byte(payload, flags); + overlay_address_append(NULL, payload, receiver); + ob_append_byte(payload, version); if (transmitter) - if (overlay_address_append(NULL, payload, transmitter)) - return -1; - - if (interface!=-1) - if (ob_append_byte(payload, interface)) - return -1; - - if (ack_sequence!=-1){ - if (ob_append_byte(payload, ack_sequence)) - return -1; - if (ob_append_ui32(payload, ack_mask)) - return -1; + overlay_address_append(NULL, payload, transmitter); + if (interface != -1) + ob_append_byte(payload, interface); + if (ack_sequence != -1){ + ob_append_byte(payload, ack_sequence); + ob_append_ui32(payload, ack_mask); } - - if (drop_rate!=-1) - if (ob_append_byte(payload, drop_rate)) - return -1; - - + if (drop_rate != -1) + ob_append_byte(payload, drop_rate); // TODO insert future fields here - - + if (ob_overrun(payload)) + return -1; // patch the record length int end_pos = ob_position(payload); - if (ob_set(payload, length_pos, end_pos - length_pos)) - return -1; - + ob_set(payload, length_pos, end_pos - length_pos); ob_checkpoint(payload); return 0; } @@ -709,12 +690,15 @@ static int send_legacy_self_announce_ack(struct neighbour *neighbour, struct lin frame->ttl = 6; frame->destination = neighbour->subscriber; frame->source = my_subscriber; - frame->payload = ob_new(); + if ((frame->payload = ob_new()) == NULL) { + op_free(frame); + return -1; + } ob_append_ui32(frame->payload, neighbour->last_update); ob_append_ui32(frame->payload, now); ob_append_byte(frame->payload, link->neighbour_interface); frame->queue=OQ_MESH_MANAGEMENT; - if (overlay_payload_enqueue(frame)){ + if (overlay_payload_enqueue(frame) == -1) { op_free(frame); return -1; } @@ -786,12 +770,16 @@ static int send_neighbour_link(struct neighbour *n) send_legacy_self_announce_ack(n, n->best_link, now); n->last_update = now; } else { - struct overlay_frame *frame=emalloc_zero(sizeof(struct overlay_frame)); + struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame)); frame->type=OF_TYPE_DATA; frame->source=my_subscriber; frame->ttl=1; frame->queue=OQ_MESH_MANAGEMENT; - frame->payload = ob_new(); + if ((frame->payload = ob_new()) == NULL) { + op_free(frame); + RETURN(-1); + } + frame->send_hook = neighbour_link_sent; frame->send_context = n->subscriber; frame->resend=-1; @@ -824,7 +812,7 @@ static int send_neighbour_link(struct neighbour *n) append_link_state(frame->payload, flags, n->subscriber, my_subscriber, n->best_link->neighbour_interface, 1, n->best_link->ack_sequence, n->best_link->ack_mask, -1); - if (overlay_payload_enqueue(frame)) + if (overlay_payload_enqueue(frame) == -1) op_free(frame); n->best_link->ack_counter = ACK_WINDOW; @@ -884,30 +872,31 @@ static void link_send(struct sched_ent *alarm) frame->source=my_subscriber; frame->ttl=1; frame->queue=OQ_MESH_MANAGEMENT; - frame->payload = ob_new(); - ob_limitsize(frame->payload, 400); - - overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE); - ob_checkpoint(frame->payload); - int pos = ob_position(frame->payload); - - enum_subscribers(NULL, append_link, frame->payload); - - ob_rewind(frame->payload); - - if (ob_position(frame->payload) == pos) - op_free(frame); - else if (overlay_payload_enqueue(frame)) - op_free(frame); - - if (neighbours){ - alarm->deadline = alarm->alarm; - schedule(alarm); - }else - alarm->alarm=0; + if ((frame->payload = ob_new()) == NULL) + WHY("Cannot send link details"); + else { + ob_limitsize(frame->payload, 400); + overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE); + ob_checkpoint(frame->payload); + int pos = ob_position(frame->payload); + enum_subscribers(NULL, append_link, frame->payload); + ob_rewind(frame->payload); + if (ob_position(frame->payload) == pos) + op_free(frame); + else if (overlay_payload_enqueue(frame)) + op_free(frame); + if (neighbours){ + alarm->deadline = alarm->alarm; + schedule(alarm); + }else + alarm->alarm=0; + } } -static void update_alarm(time_ms_t limit){ +static void update_alarm(struct __sourceloc __whence, time_ms_t limit) +{ + if (limit == 0) + FATALF("limit == 0"); if (link_send_alarm.alarm>limit || link_send_alarm.alarm==0){ unschedule(&link_send_alarm); link_send_alarm.alarm = limit; @@ -924,9 +913,11 @@ int link_stop_routing(struct subscriber *subscriber) subscriber->identity=NULL; if (subscriber==my_subscriber) my_subscriber=NULL; - struct link_state *state = get_link_state(subscriber); - state->next_update = gettime_ms(); - update_alarm(state->next_update); + if (subscriber->link_state){ + struct link_state *state = get_link_state(subscriber); + state->next_update = gettime_ms(); + update_alarm(__WHENCE__, state->next_update); + } return 0; } @@ -1044,7 +1035,8 @@ int link_state_should_forward_broadcast(struct subscriber *transmitter) } // when we receive a packet from a neighbour with ourselves as the next hop, make sure we send an ack soon(ish) -int link_state_ack_soon(struct subscriber *subscriber){ +int link_state_ack_soon(struct subscriber *subscriber) +{ IN(); struct neighbour *neighbour = get_neighbour(subscriber, 0); if (!neighbour) @@ -1059,8 +1051,8 @@ int link_state_ack_soon(struct subscriber *subscriber){ if (config.debug.ack) DEBUGF("Asking for next ACK Real Soon Now"); } + update_alarm(__WHENCE__, neighbour->next_neighbour_update); } - update_alarm(neighbour->next_neighbour_update); OUT(); return 0; } @@ -1101,7 +1093,8 @@ int link_unicast_ack(struct subscriber *subscriber, struct overlay_interface *in return 0; } -static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast){ +static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast) +{ struct link_out *ret=emalloc_zero(sizeof(struct link_out)); if (ret){ ret->_next=neighbour->out_links; @@ -1110,15 +1103,14 @@ static struct link_out *create_out_link(struct neighbour *neighbour, overlay_int ret->destination = create_unicast_destination(*addr, interface); else ret->destination = add_destination_ref(interface->destination); - if (config.debug.linkstate) DEBUGF("LINK STATE; Create possible %s link_out for neighbour %s on interface %s", unicast?"unicast":"broadcast", alloca_tohex_sid_t(neighbour->subscriber->sid), interface->name); - - ret->timeout = gettime_ms()+ret->destination->tick_ms*3; - update_alarm(gettime_ms()+5); + time_ms_t now = gettime_ms(); + ret->timeout = now + ret->destination->tick_ms * 3; + update_alarm(__WHENCE__, now + 5); } return ret; } @@ -1206,7 +1198,7 @@ int link_received_packet(struct decode_context *context, int sender_seq, char un send_neighbour_link(neighbour); } - update_alarm(neighbour->next_neighbour_update); + update_alarm(__WHENCE__, neighbour->next_neighbour_update); return 0; } @@ -1385,7 +1377,7 @@ int link_receive(struct overlay_frame *frame, overlay_mdp_frame *mdp) if (config.debug.ack) DEBUGF("LINK STATE; neighbour %s missed ack %d, queue another", alloca_tohex_sid_t(sender->sid), neighbour->last_update_seq); neighbour->next_neighbour_update=now+5; - update_alarm(neighbour->next_neighbour_update); + update_alarm(__WHENCE__, neighbour->next_neighbour_update); } } } @@ -1426,8 +1418,8 @@ void link_explained(struct subscriber *subscriber) { time_ms_t now = gettime_ms(); struct link_state *state = get_link_state(subscriber); - state->next_update = now+5; - update_alarm(now+5); + state->next_update = now + 5; + update_alarm(__WHENCE__, now + 5); } void link_interface_down(struct overlay_interface *interface) diff --git a/serval.h b/serval.h index 283aaf04..035df66e 100644 --- a/serval.h +++ b/serval.h @@ -33,42 +33,41 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #ifdef WIN32 -#include "win32/win32.h" +# include "win32/win32.h" #else -#include -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NET_ROUTE_H - #include -#endif -#ifdef HAVE_LINUX_IF_H -#include -#else -#ifdef HAVE_NET_IF_H -#include -#endif -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif -#ifdef HAVE_LINUX_NETLINK_H -#include -#endif -#ifdef HAVE_LINUX_RTNETLINK_H -#include -#endif -#ifdef HAVE_IFADDRS_H -#include -#endif -#ifdef HAVE_SYS_SOCKIO_H -#include -#endif - -#ifdef HAVE_SYS_UCRED_H -#include -#endif -#endif +# include +# ifdef HAVE_SYS_SOCKET_H +# include +# endif +# ifdef HAVE_NET_ROUTE_H +# include +# endif +# ifdef HAVE_LINUX_IF_H +# include +# else +# ifdef HAVE_NET_IF_H +# include +# endif +# endif +# ifdef HAVE_NETINET_IN_H +# include +# endif +# ifdef HAVE_LINUX_NETLINK_H +# include +# endif +# ifdef HAVE_LINUX_RTNETLINK_H +# include +# endif +# ifdef HAVE_IFADDRS_H +# include +# endif +# ifdef HAVE_SYS_SOCKIO_H +# include +# endif +# ifdef HAVE_SYS_UCRED_H +# include +# endif +#endif //!WIN32 #if !defined(FORASTERISK) && !defined(s_addr) #ifdef HAVE_ARPA_INET_H @@ -103,7 +102,6 @@ struct in_addr { #endif #include -#include #include #include "fdqueue.h" @@ -159,17 +157,12 @@ typedef struct sid_binary { #define is_sid_t_any(SID) is_all_matching((SID).binary, sizeof (*(sid_t*)0).binary, 0) #define alloca_tohex_sid_t(sid) alloca_tohex((sid).binary, sizeof (*(sid_t*)0).binary) -#define alloca_tohex_sid_t_trunc(sid,strlen) tohex((char *)alloca((strlen)+2), (strlen), (sid).binary) +#define alloca_tohex_sid_t_trunc(sid,strlen) tohex((char *)alloca((strlen)+1), (strlen), (sid).binary) int cmp_sid_t(const sid_t *a, const sid_t *b); int str_to_sid_t(sid_t *sid, const char *hex); int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp); -int str_is_subscriber_id(const char *sid); -int strn_is_subscriber_id(const char *sid, size_t *lenp); -int str_is_did(const char *did); -int strn_is_did(const char *did, size_t *lenp); - #define alloca_tohex_sas(sas) alloca_tohex((sas), SAS_SIZE) /* @@ -197,28 +190,6 @@ int formf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, con int vformf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list); void serval_setinstancepath(const char *instancepath); -/* Basic socket operations. - */ -int _make_local_sockaddr(struct __sourceloc, struct sockaddr_un *sockname, socklen_t *addrlen, const char *fmt, ...) - __attribute__((format(printf, 4, 5))); -int _esocket(struct __sourceloc, int domain, int type, int protocol); -int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen); -int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen); -int _socket_listen(struct __sourceloc, int sock, int backlog); -int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP); -int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size); - -#define make_local_sockaddr(sockname, addrlenp, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (addrlenp), (fmt), ##__VA_ARGS__) -#define esocket(domain, type, protocol) _esocket(__WHENCE__, (domain), (type), (protocol)) -#define socket_bind(sock, addr, addrlen) _socket_bind(__WHENCE__, (sock), (addr), (addrlen)) -#define socket_connect(sock, addr, addrlen) _socket_connect(__WHENCE__, (sock), (addr), (addrlen)) -#define socket_listen(sock, backlog) _socket_listen(__WHENCE__, (sock), (backlog)) -#define socket_set_reuseaddr(sock, reuseP) _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP)) -#define socket_set_rcvbufsize(sock, buffer_size) _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size)) - -int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen); -int cmp_sockaddr(const struct sockaddr *, socklen_t, const struct sockaddr *, socklen_t); - #define SERVER_CONFIG_RELOAD_INTERVAL_MS 1000 struct cli_parsed; @@ -240,100 +211,9 @@ extern char *instrumentation_file; extern char *batman_socket; extern char *batman_peerfile; - struct subscriber; struct decode_context; -typedef struct keypair { - int type; - unsigned char *private_key; - size_t private_key_len; - unsigned char *public_key; - size_t public_key_len; -} keypair; - -/* Contains just the list of private:public key pairs and types, - the pin used to extract them, and the slot in the keyring file - (so that it can be replaced/rewritten as required). */ -#define PKR_MAX_KEYPAIRS 64 -#define PKR_SALT_BYTES 32 -#define PKR_MAC_BYTES 64 -typedef struct keyring_identity { - char *PKRPin; - struct subscriber *subscriber; - time_ms_t challenge_expires; - unsigned char challenge[24]; - unsigned int slot; - unsigned int keypair_count; - keypair *keypairs[PKR_MAX_KEYPAIRS]; -} keyring_identity; - -/* 64K identities, can easily be increased should the need arise, - but keep it low-ish for now so that the 64K pointers don't eat too - much ram on a small device. Should probably think about having - small and large device settings for some of these things */ -#define KEYRING_MAX_IDENTITIES 65536 -typedef struct keyring_context { - char *KeyRingPin; - unsigned char *KeyRingSalt; - int KeyRingSaltLen; - unsigned int identity_count; - keyring_identity *identities[KEYRING_MAX_IDENTITIES]; -} keyring_context; - -#define KEYRING_PAGE_SIZE 4096LL -#define KEYRING_BAM_BYTES 2048LL -#define KEYRING_BAM_BITS (KEYRING_BAM_BYTES<<3) -#define KEYRING_SLAB_SIZE (KEYRING_PAGE_SIZE*KEYRING_BAM_BITS) -typedef struct keyring_bam { - off_t file_offset; - unsigned char bitmap[KEYRING_BAM_BYTES]; - struct keyring_bam *next; -} keyring_bam; - -#define KEYRING_MAX_CONTEXTS 256 -typedef struct keyring_file { - int context_count; - keyring_bam *bam; - keyring_context *contexts[KEYRING_MAX_CONTEXTS]; - FILE *file; - off_t file_size; -} keyring_file; - -void keyring_free(keyring_file *k); -void keyring_release_identity(keyring_file *k, int cn, int id); -#define KEYTYPE_CRYPTOBOX 0x01 // must be lowest -#define KEYTYPE_CRYPTOSIGN 0x02 -#define KEYTYPE_RHIZOME 0x03 -/* DIDs aren't really keys, but the keyring is a real handy place to keep them, - and keep them private if people so desire */ -#define KEYTYPE_DID 0x04 - -/* handle to keyring file for use in running instance */ -extern keyring_file *keyring; - -/* Public calls to keyring management */ -keyring_file *keyring_open(const char *path, int writeable); -keyring_file *keyring_open_instance(); -keyring_file *keyring_open_instance_cli(const struct cli_parsed *parsed); -int keyring_enter_pin(keyring_file *k, const char *pin); -int keyring_set_did(keyring_identity *id, const char *did, const char *name); -int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp); -int keyring_next_keytype(const keyring_file *k, int *cn, int *in, int *kp, int keytype); -int keyring_next_identity(const keyring_file *k,int *cn,int *in,int *kp); -int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype); -int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp,char *did); -int keyring_find_sid(const keyring_file *k,int *cn,int *in,int *kp, const sid_t *sidp); -unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public); -int keyring_send_sas_request(struct subscriber *subscriber); - -int keyring_commit(keyring_file *k); -keyring_identity *keyring_create_identity(keyring_file *k,keyring_context *c, const char *pin); -int keyring_seed(keyring_file *k); -void keyring_identity_extract(const keyring_identity *id, const sid_t **sidp, const char **didp, const char **namep); -int keyring_load(keyring_file *k, const char *keyring_pin, unsigned entry_pinc, const char **entry_pinv, FILE *input); -int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret); - /* Make sure we have space to put bytes of the packet as we go along */ #define CHECK_PACKET_LEN(B) {if (((*packet_len)+(B))>=packet_maxlen) { return WHY("Packet composition ran out of space."); } } @@ -392,12 +272,6 @@ struct slip_decode_state{ uint32_t crc; int src_offset; int dst_offset; - - int mavlink_payload_length; - int mavlink_seq; - int mavlink_payload_start; - int mavlink_payload_offset; - uint8_t mavlink_payload[1024]; }; struct overlay_interface; @@ -470,21 +344,7 @@ typedef struct overlay_interface { int recv_count; int tx_count; - // stream socket tx state; - struct overlay_buffer *tx_packet; - unsigned char txbuffer[OVERLAY_INTERFACE_RX_BUFFER_SIZE]; - int tx_bytes_pending; - // Throttle TX rate if required (stream interfaces only for now) - uint32_t throttle_bytes_per_second; - uint32_t throttle_burst_write_size; - uint64_t next_tx_allowed; - int32_t remaining_space; - time_ms_t next_heartbeat; - int mavlink_seq; - int radio_rssi; - int remote_rssi; - - struct slip_decode_state slip_decode_state; + struct radio_link_state *radio_link_state; // copy of ifconfig flags uint16_t drop_packets; @@ -544,7 +404,7 @@ void insertTransactionInCache(unsigned char *transaction_id); int overlay_forward_payload(struct overlay_frame *f); int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len, - int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen); + int recvttl, struct socket_address *recvaddr); int parseMdpPacketHeader(struct decode_context *context, struct overlay_frame *frame, struct overlay_buffer *buffer, struct subscriber **nexthop); int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface *interface, @@ -601,18 +461,14 @@ int rhizome_opendb(); int parseCommandLine(struct cli_context *context, const char *argv0, int argc, const char *const *argv); int overlay_mdp_get_fds(struct pollfd *fds,int *fdcount,int fdmax); -int overlay_mdp_reply_error(int sock, - struct sockaddr_un *recvaddr, socklen_t recvaddrlen, - int error_number,char *message); typedef uint32_t mdp_port_t; -#define PRImdp_port_t "08" PRIx32 +#define PRImdp_port_t "#08" PRIx32 typedef struct sockaddr_mdp { sid_t sid; mdp_port_t port; } sockaddr_mdp; -unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp); typedef struct overlay_mdp_data_frame { sockaddr_mdp src; @@ -664,19 +520,20 @@ typedef struct overlay_mdp_frame { }; } overlay_mdp_frame; -int keyring_mapping_request(keyring_file *k, struct overlay_frame *frame, overlay_mdp_frame *req); -int keyring_send_unlock(struct subscriber *subscriber); -void keyring_release_subscriber(keyring_file *k, const sid_t *sid); - /* Server-side MDP functions */ int overlay_mdp_swap_src_dst(overlay_mdp_frame *mdp); -int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen, - overlay_mdp_frame *mdpreply); -int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP, - struct sockaddr_un *recvaddr, socklen_t recvaddrlen); -int overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port); +struct socket_address; +int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client); +void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port); int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name); +struct mdp_header; +int mdp_bind_internal(struct subscriber *subscriber, mdp_port_t port, + int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len)); +int mdp_unbind_internal(struct subscriber *subscriber, mdp_port_t port, + int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len)); + + struct vomp_call_state; void set_codec_flag(int codec, unsigned char *flags); @@ -786,7 +643,7 @@ int overlay_packetradio_tx_packet(struct overlay_frame *frame); void overlay_dummy_poll(struct sched_ent *alarm); void server_config_reload(struct sched_ent *alarm); void server_shutdown_check(struct sched_ent *alarm); -int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp); +int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp); int overlay_send_probe(struct subscriber *peer, struct network_destination *destination, int queue); int overlay_send_stun_request(struct subscriber *server, struct subscriber *request); void fd_periodicstats(struct sched_ent *alarm); @@ -811,13 +668,6 @@ int limit_init(struct limit_state *state, int rate_micro_seconds); int olsr_init_socket(void); int olsr_send(struct overlay_frame *frame); -void write_uint64(unsigned char *o,uint64_t v); -void write_uint16(unsigned char *o,uint16_t v); -void write_uint32(unsigned char *o,uint32_t v); -uint64_t read_uint64(unsigned char *o); -uint32_t read_uint32(unsigned char *o); -uint16_t read_uint16(unsigned char *o); - int pack_uint(unsigned char *buffer, uint64_t v); int measure_packed_uint(uint64_t v); int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v); @@ -828,8 +678,7 @@ int slip_decode(struct slip_decode_state *state); int upper7_decode(struct slip_decode_state *state,unsigned char byte); uint32_t Crc32_ComputeBuf( uint32_t inCrc32, const void *buf, size_t bufLen ); -int rhizome_active_fetch_count(); -int rhizome_active_fetch_bytes_received(int q); +void rhizome_fetch_log_short_status(); extern int64_t bundles_available; extern char crash_handler_clue[1024]; @@ -851,8 +700,4 @@ int link_stop_routing(struct subscriber *subscriber); int generate_nonce(unsigned char *nonce,int bytes); -int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state,uint8_t c); -int mavlink_heartbeat(unsigned char *frame,int *outlen); -int mavlink_encode_packet(struct overlay_interface *interface); - #endif // __SERVALD_SERVALD_H diff --git a/server.c b/server.c index 56323b24..87186455 100644 --- a/server.c +++ b/server.c @@ -21,7 +21,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include -#include #include #include "serval.h" @@ -108,11 +107,9 @@ int server(const struct cli_parsed *parsed) sigemptyset(&sig.sa_mask); // Block the same signals during handler sigaddset(&sig.sa_mask, SIGHUP); sigaddset(&sig.sa_mask, SIGINT); - sigaddset(&sig.sa_mask, SIGQUIT); sig.sa_flags = 0; sigaction(SIGHUP, &sig, NULL); sigaction(SIGINT, &sig, NULL); - sigaction(SIGQUIT, &sig, NULL); /* Record PID to advertise that the server is now running */ char filename[1024]; @@ -233,19 +230,23 @@ int server_check_stopfile() void serverCleanUp() { - /* Try to remove shutdown and PID files and exit */ - server_remove_stopfile(); - char filename[1024]; - if (FORM_SERVAL_INSTANCE_PATH(filename, PIDFILE_NAME)) - unlink(filename); - - if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket")) { - unlink(filename); + if (serverMode){ + rhizome_close_db(); + dna_helper_shutdown(); } - rhizome_close_db(); + char filename[1024]; + if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket")) + unlink(filename); - dna_helper_shutdown(); + if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.2.socket")) + unlink(filename); + + if (FORM_SERVAL_INSTANCE_PATH(filename, "monitor.socket")) + unlink(filename); + + /* Try to remove shutdown and PID files and exit */ + server_remove_stopfile(); } static void signame(char *buf, size_t len, int signal) diff --git a/slip.c b/slip.c index daa37052..134d3e39 100644 --- a/slip.c +++ b/slip.c @@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" #include "conf.h" #include "log.h" +#include "dataformats.h" #define DEBUG_packet_visualise(M,P,N) logServalPacket(LOG_LEVEL_DEBUG, __WHENCE__, (M), (P), (N)) diff --git a/socket.c b/socket.c index e9839d01..e81482d3 100644 --- a/socket.c +++ b/socket.c @@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "conf.h" #include "log.h" #include "strbuf_helpers.h" +#include "socket.h" /* Form the name of an AF_UNIX (local) socket in the instance directory as an absolute path. * Under Linux, this will create a socket name in the abstract namespace. This permits us to use @@ -40,25 +41,28 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * @author Andrew Bettison * @author Daniel O'Connor */ -int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr, socklen_t *addrlen, const char *fmt, ...) +int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *addr, const char *fmt, ...) { bzero(addr, sizeof(*addr)); + addr->local.sun_family = AF_UNIX; va_list ap; va_start(ap, fmt); - int r = vformf_serval_instance_path(__WHENCE__, addr->sun_path, sizeof addr->sun_path, fmt, ap); + int r = vformf_serval_instance_path(__WHENCE__, addr->local.sun_path, sizeof addr->local.sun_path, fmt, ap); va_end(ap); if (!r) return WHY("socket name overflow"); - if (real_sockaddr(addr, sizeof addr->sun_family + strlen(addr->sun_path) + 1, addr, addrlen) == -1) - return -1; + addr->addrlen=sizeof addr->local.sun_family + strlen(addr->local.sun_path) + 1; +// TODO perform real path transformation in making the serval instance path +// if (real_sockaddr(addr, addr) == -1) +// return -1; + #ifdef USE_ABSTRACT_NAMESPACE // For the abstract name we use the absolute path name with the initial '/' replaced by the // leading nul. This ensures that different instances of the Serval daemon have different socket // names. - addr->sun_path[0] = '\0'; // mark as Linux abstract socket - --*addrlen; // do not count trailing nul in abstract socket name + addr->local.sun_path[0] = '\0'; // mark as Linux abstract socket + --addr->addrlen; // do not count trailing nul in abstract socket name #endif // USE_ABSTRACT_NAMESPACE - addr->sun_family = AF_UNIX; return 0; } @@ -74,31 +78,33 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr, * * @author Andrew Bettison */ -int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen) +int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr) { - int src_path_len = src_addrlen - sizeof src_addr->sun_family; - if ( src_addrlen >= sizeof src_addr->sun_family + 1 - && src_addr->sun_family == AF_UNIX - && src_addr->sun_path[0] != '\0' - && src_addr->sun_path[src_path_len - 1] == '\0' + int src_path_len = src_addr->addrlen - sizeof src_addr->local.sun_family; + if ( src_addr->addrlen >= sizeof src_addr->local.sun_family + 1 + && src_addr->local.sun_family == AF_UNIX + && src_addr->local.sun_path[0] != '\0' + && src_addr->local.sun_path[src_path_len - 1] == '\0' ) { char real_path[PATH_MAX]; size_t real_path_len; - if (realpath(src_addr->sun_path, real_path) == NULL) - return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->sun_path)); - else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->sun_path) - return WHYF("sockaddr overrun: realpath(%s) returned %s", alloca_str_toprint(src_addr->sun_path), alloca_str_toprint(real_path)); + if (realpath(src_addr->local.sun_path, real_path) == NULL) + return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->local.sun_path)); + else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->local.sun_path) + return WHYF("sockaddr overrun: realpath(%s) returned %s", + alloca_str_toprint(src_addr->local.sun_path), alloca_str_toprint(real_path)); else if ( real_path_len != src_path_len - || memcmp(real_path, src_addr->sun_path, src_path_len) != 0 + || memcmp(real_path, src_addr->local.sun_path, src_path_len) != 0 ) { - memcpy(dst_addr->sun_path, real_path, real_path_len); - *dst_addrlen = real_path_len + sizeof dst_addr->sun_family; + memcpy(dst_addr->local.sun_path, real_path, real_path_len); + dst_addr->addrlen = real_path_len + sizeof dst_addr->local.sun_family; return 1; } } - if (dst_addr != src_addr) - memcpy(dst_addr, src_addr, src_addrlen); - *dst_addrlen = src_addrlen; + if (dst_addr != src_addr){ + memcpy(&dst_addr->addr, &src_addr->addr, src_addr->addrlen); + dst_addr->addrlen = src_addr->addrlen; + } return 0; } @@ -107,43 +113,43 @@ int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, str * * @author Andrew Bettison */ -int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct sockaddr *addrB, socklen_t addrlenB) +int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB) { // Two zero-length sockaddrs are equal. - if (addrlenA == 0 && addrlenB == 0) + if (addrA->addrlen == 0 && addrB->addrlen == 0) return 0; // If either sockaddr is truncated, then we compare the bytes we have. - if (addrlenA < sizeof addrA->sa_family || addrlenB < sizeof addrB->sa_family) { - int c = memcmp(addrA, addrB, addrlenA < addrlenB ? addrlenA : addrlenB); + if (addrA->addrlen < sizeof addrA->addr.sa_family || addrB->addrlen < sizeof addrB->addr.sa_family) { + int c = memcmp(addrA, addrB, addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen); if (c == 0) - c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0; + c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0; return c; } // Order first by address family. - if (addrA->sa_family < addrB->sa_family) + if (addrA->addr.sa_family < addrB->addr.sa_family) return -1; - if (addrA->sa_family > addrB->sa_family) + if (addrA->addr.sa_family > addrB->addr.sa_family) return 1; // Both addresses are in the same family... - switch (addrA->sa_family) { + switch (addrA->addr.sa_family) { case AF_UNIX: { - unsigned pathlenA = addrlenA - sizeof ((const struct sockaddr_un *)addrA)->sun_family; - unsigned pathlenB = addrlenB - sizeof ((const struct sockaddr_un *)addrB)->sun_family; + unsigned pathlenA = addrA->addrlen - sizeof (addrA->local.sun_family); + unsigned pathlenB = addrB->addrlen - sizeof (addrB->local.sun_family); int c; if ( pathlenA > 1 && pathlenB > 1 - && ((const struct sockaddr_un *)addrA)->sun_path[0] == '\0' - && ((const struct sockaddr_un *)addrB)->sun_path[0] == '\0' + && addrA->local.sun_path[0] == '\0' + && addrB->local.sun_path[0] == '\0' ) { // Both abstract sockets - just compare names, nul bytes are not terminators. - c = memcmp(&((const struct sockaddr_un *)addrA)->sun_path[1], - &((const struct sockaddr_un *)addrB)->sun_path[1], + c = memcmp(&addrA->local.sun_path[1], + &addrB->local.sun_path[1], (pathlenA < pathlenB ? pathlenA : pathlenB) - 1); } else { // Either or both are named local file sockets. If the file names are identical up to the // first nul, then the addresses are equal. This collates abstract socket names, whose first // character is a nul, ahead of all non-empty file socket names. - c = strncmp(((const struct sockaddr_un *)addrA)->sun_path, - ((const struct sockaddr_un *)addrB)->sun_path, + c = strncmp(addrA->local.sun_path, + addrB->local.sun_path, (pathlenA < pathlenB ? pathlenA : pathlenB)); } if (c == 0) @@ -153,9 +159,10 @@ int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct break; } // Fall back to comparing raw data bytes. - int c = memcmp(addrA->sa_data, addrB->sa_data, (addrlenA < addrlenB ? addrlenA : addrlenB) - sizeof addrA->sa_family); + int c = memcmp(addrA->addr.sa_data, addrB->addr.sa_data, + (addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen) - sizeof addrA->addr.sa_family); if (c == 0) - c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0; + c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0; return c; } @@ -225,3 +232,32 @@ int _socket_set_rcvbufsize(struct __sourceloc __whence, int sock, unsigned buffe DEBUGF("setsockopt(%d, SOL_SOCKET, SO_RCVBUF, &%u, %u)", sock, buffer_size, (unsigned)sizeof buffer_size); return 0; } + +ssize_t _send_message(struct __sourceloc __whence, int fd, const struct socket_address *address, const struct fragmented_data *data) +{ + struct msghdr hdr={ + .msg_name=(void *)&address->addr, + .msg_namelen=address->addrlen, + .msg_iov=(struct iovec*)data->iov, + .msg_iovlen=data->fragment_count, + }; + + ssize_t ret = sendmsg(fd, &hdr, 0); + if (ret==-1) + WHYF_perror("sendmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen); + return ret; +} + +ssize_t _recv_message(struct __sourceloc __whence, int fd, struct socket_address *address, struct fragmented_data *data) +{ + struct msghdr hdr={ + .msg_name=(void *)&address->addr, + .msg_namelen=address->addrlen, + .msg_iov=data->iov, + .msg_iovlen=data->fragment_count, + }; + ssize_t ret = recvmsg(fd, &hdr, 0); + if (ret==-1) + WHYF_perror("recvmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen); + return ret; +} diff --git a/socket.h b/socket.h new file mode 100644 index 00000000..5a167f43 --- /dev/null +++ b/socket.h @@ -0,0 +1,57 @@ +#ifndef __SERVALD_SOCKET_H +#define __SERVALD_SOCKET_H + +#ifndef WIN32 +#include +#endif + +struct socket_address{ + socklen_t addrlen; + union{ + struct sockaddr addr; + struct sockaddr_un local; // name "unix" is a predefined macro + struct sockaddr_in inet; + struct sockaddr_storage store; + }; +}; + +/* Basic socket operations. + */ +int _make_local_sockaddr(struct __sourceloc, struct socket_address *addr, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); +int _esocket(struct __sourceloc, int domain, int type, int protocol); +int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen); +int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen); +int _socket_listen(struct __sourceloc, int sock, int backlog); +int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP); +int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size); + +#define make_local_sockaddr(sockname, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (fmt), ##__VA_ARGS__) +#define esocket(domain, type, protocol) _esocket(__WHENCE__, (domain), (type), (protocol)) +#define socket_bind(sock, addr, addrlen) _socket_bind(__WHENCE__, (sock), (addr), (addrlen)) +#define socket_connect(sock, addr, addrlen) _socket_connect(__WHENCE__, (sock), (addr), (addrlen)) +#define socket_listen(sock, backlog) _socket_listen(__WHENCE__, (sock), (backlog)) +#define socket_set_reuseaddr(sock, reuseP) _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP)) +#define socket_set_rcvbufsize(sock, buffer_size) _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size)) + +int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr); +int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB); + +// helper functions for manipulating fragmented packet data +#define MAX_FRAGMENTS 8 +struct fragmented_data{ + int fragment_count; + struct iovec iov[MAX_FRAGMENTS]; +}; + +int prepend_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len); +int append_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len); +size_t copy_fragment(struct fragmented_data *src, uint8_t *dest, size_t length); + +ssize_t _send_message(struct __sourceloc, int fd, const struct socket_address *address, const struct fragmented_data *data); +ssize_t _recv_message(struct __sourceloc, int fd, struct socket_address *address, struct fragmented_data *data); + +#define send_message(fd, address, data) _send_message(__WHENCE__, (fd), (address), (data)) +#define recv_message(fd, address, data) _recv_message(__WHENCE__, (fd), (address), (data)) + +#endif diff --git a/sourcefiles.mk b/sourcefiles.mk index 1b4516ac..86dc3dcd 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -19,7 +19,7 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)log.c \ $(SERVAL_BASE)lsif.c \ $(SERVAL_BASE)main.c \ - $(SERVAL_BASE)mavlink.c \ + $(SERVAL_BASE)radio_link.c \ $(SERVAL_BASE)meshms.c \ $(SERVAL_BASE)mdp_client.c \ $(SERVAL_BASE)os.c \ @@ -69,10 +69,12 @@ SERVAL_SOURCES = \ $(SERVAL_BASE)strbuf.c \ $(SERVAL_BASE)strbuf_helpers.c \ $(SERVAL_BASE)strlcpy.c \ + $(SERVAL_BASE)uuid.c \ $(SERVAL_BASE)vomp.c \ $(SERVAL_BASE)vomp_console.c \ $(SERVAL_BASE)xprintf.c \ $(SERVAL_BASE)fec-3.0.1/ccsds_tables.c \ $(SERVAL_BASE)fec-3.0.1/decode_rs_8.c \ $(SERVAL_BASE)fec-3.0.1/encode_rs_8.c \ - $(SERVAL_BASE)fec-3.0.1/init_rs_char.c + $(SERVAL_BASE)fec-3.0.1/init_rs_char.c \ + $(SERVAL_BASE)context1.c diff --git a/str.c b/str.c index 182a7842..4849f8a0 100644 --- a/str.c +++ b/str.c @@ -17,37 +17,33 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#define __STR_INLINE +#define __SERVAL_DNA_STR_INLINE #include "str.h" #include "strbuf_helpers.h" #include "constants.h" #include #include +#include #include #include #include #include #include -const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; +const char hexdigit_upper[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; +const char hexdigit_lower[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; char *tohex(char *dstHex, size_t dstStrLen, const unsigned char *srcBinary) { char *p; size_t i; for (p = dstHex, i = 0; i < dstStrLen; ++i) - *p++ = (i & 1) ? hexdigit[*srcBinary++ & 0xf] : hexdigit[*srcBinary >> 4]; + *p++ = (i & 1) ? hexdigit_upper[*srcBinary++ & 0xf] : hexdigit_upper[*srcBinary >> 4]; *p = '\0'; return dstHex; } -/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] to nbinary bytes of data. Can be used to - * perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); Returns -1 if a non-hex-digit - * character is encountered, otherwise returns the number of binary bytes produced (= nbinary). - * - * @author Andrew Bettison - */ size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary) { if (strn_fromhex(dstBinary, nbinary, srcHex, NULL) == nbinary) @@ -55,13 +51,6 @@ size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary) return -1; } -/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] followed by a nul '\0' character to nbinary - * bytes of data. Can be used to perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); - * Returns -1 if a non-hex-digit character is encountered or the character immediately following the - * last hex digit is not a nul, otherwise returns zero. - * - * @author Andrew Bettison - */ int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary) { const char *p; @@ -70,21 +59,6 @@ int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary) return -1; } -/* Decode pairs of ASCII hex characters [0-9A-Fa-f] into binary data with an optional upper limit on - * the number of binary bytes produced (destination buffer size). Returns the number of binary - * bytes decoded. If 'afterHex' is not NULL, then sets *afterHex to point to the source character - * immediately following the last hex digit consumed. - * - * Can be used to perform a conversion in-place, eg: - * - * strn_fromhex((unsigned char *)buf, n, (const char *)buf, NULL); - * - * Can also be used to count hex digits without converting, eg: - * - * strn_fromhex(NULL, -1, buf, NULL); - * - * @author Andrew Bettison - */ size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *srcHex, const char **afterHex) { unsigned char *dstorig = dstBinary; @@ -106,6 +80,441 @@ size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *srcHex return dstBinary - dstorig; } +const char base64_symbols[65] = { + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f', + 'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v', + 'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/', + '=' +}; + +const char base64url_symbols[65] = { + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f', + 'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v', + 'w','x','y','z','0','1','2','3','4','5','6','7','8','9','-','_', + '=' +}; + +static size_t _base64_encodev(const char symbols[], char *dstBase64, const struct iovec *const iov, int const iovcnt) +{ + char *dst = dstBase64; + unsigned place = 0; + unsigned char buf = 0; + int iovc = 0; + for (iovc = 0; iovc != iovcnt; ++iovc) { + unsigned char *src = iov[iovc].iov_base; + size_t cnt = iov[iovc].iov_len; + for (; cnt; --cnt, ++src) { + switch (place) { + case 0: + *dst++ = symbols[*src >> 2]; + buf = (*src << 4) & 0x3f; + place = 1; + break; + case 1: + *dst++ = symbols[(*src >> 4) | buf]; + buf = (*src << 2) & 0x3f; + place = 2; + break; + case 2: + *dst++ = symbols[(*src >> 6) | buf]; + *dst++ = symbols[*src & 0x3f]; + place = 0; + break; + } + } + } + if (place) + *dst++ = symbols[buf]; + switch (place) { + case 2: + *dst++ = symbols[64]; + case 1: + *dst++ = symbols[64]; + } + return dst - dstBase64; +} + +size_t base64_encodev(char *dstBase64, const struct iovec *const iov, int const iovcnt) +{ + return _base64_encodev(base64_symbols, dstBase64, iov, iovcnt); +} + +size_t base64url_encodev(char *dstBase64, const struct iovec *const iov, int const iovcnt) +{ + return _base64_encodev(base64url_symbols, dstBase64, iov, iovcnt); +} + +size_t base64_encode(char *const dstBase64, const unsigned char *src, size_t srclen) +{ + struct iovec iov; + iov.iov_base = (void *) src; + iov.iov_len = srclen; + return _base64_encodev(base64_symbols, dstBase64, &iov, 1); +} + +size_t base64url_encode(char *const dstBase64, const unsigned char *src, size_t srclen) +{ + struct iovec iov; + iov.iov_base = (void *) src; + iov.iov_len = srclen; + return _base64_encodev(base64url_symbols, dstBase64, &iov, 1); +} + +char *to_base64_str(char *const dstBase64, const unsigned char *srcBinary, size_t srcBytes) +{ + dstBase64[base64_encode(dstBase64, srcBinary, srcBytes)] = '\0'; + return dstBase64; +} + +char *to_base64url_str(char *const dstBase64, const unsigned char *srcBinary, size_t srcBytes) +{ + dstBase64[base64url_encode(dstBase64, srcBinary, srcBytes)] = '\0'; + return dstBase64; +} + +static size_t _base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen, + const char **afterp, int flags, int (*skip_pred)(char), + int (*isdigit_pred)(char), int (*ispad_pred)(char), uint8_t (*todigit)(char) + ) +{ + uint8_t buf = 0; + size_t digits = 0; + unsigned pads = 0; + size_t bytes = 0; + const char *const srcend = srcBase64 + srclen; + const char *src = srcBase64; + const char *first_pad = NULL; + for (; srclen == 0 || (src < srcend); ++src) { + int isdigit = isdigit_pred(*src); + int ispad = ispad_pred(*src); + if (!isdigit && !ispad && skip_pred && skip_pred(*src)) + continue; + assert(pads <= 2); + if (pads == 2) + break; + int place = digits & 3; + if (pads == 1) { + if (place == 3) + break; + assert(place == 2); + if (ispad) { + ++pads; + continue; // consume trailing space before ending + } + // If only one pad character was present but there should be two, then don't consume the first + // one. + assert(first_pad != NULL); + src = first_pad; + break; + } + assert(pads == 0); + if (ispad && place >= 2) { + first_pad = src; + ++pads; + continue; + } + if (!isdigit) + break; + ++digits; + if (dstBinary && bytes < dstsiz) { + uint8_t d = todigit(*src); + switch (place) { + case 0: + buf = d << 2; + break; + case 1: + dstBinary[bytes++] = buf | (d >> 4); + buf = d << 4; + break; + case 2: + dstBinary[bytes++] = buf | (d >> 2); + buf = d << 6; + break; + case 3: + dstBinary[bytes++] = buf | d; + break; + } + } else if (flags & B64_CONSUME_ALL) { + switch (place) { + case 1: case 2: case 3: ++bytes; + } + } else + break; + } + if (afterp) + *afterp = src; + else if (*src) + return 0; + return bytes; +} + +size_t base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen, + const char **afterp, int flags, int (*skip_pred)(char)) +{ + return _base64_decode(dstBinary, dstsiz, srcBase64, srclen, afterp, flags, skip_pred, is_base64_digit, is_base64_pad, base64_digit); +} + + +size_t base64url_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen, + const char **afterp, int flags, int (*skip_pred)(char)) +{ + return _base64_decode(dstBinary, dstsiz, srcBase64, srclen, afterp, flags, skip_pred, is_base64url_digit, is_base64url_pad, base64url_digit); +} + + +#define _B64 _SERVAL_CTYPE_0_BASE64 +#define _B64U _SERVAL_CTYPE_0_BASE64URL + +uint8_t _serval_ctype_0[UINT8_MAX] = { + ['A'] = _B64 | _B64U | 0, + ['B'] = _B64 | _B64U | 1, + ['C'] = _B64 | _B64U | 2, + ['D'] = _B64 | _B64U | 3, + ['E'] = _B64 | _B64U | 4, + ['F'] = _B64 | _B64U | 5, + ['G'] = _B64 | _B64U | 6, + ['H'] = _B64 | _B64U | 7, + ['I'] = _B64 | _B64U | 8, + ['J'] = _B64 | _B64U | 9, + ['K'] = _B64 | _B64U | 10, + ['L'] = _B64 | _B64U | 11, + ['M'] = _B64 | _B64U | 12, + ['N'] = _B64 | _B64U | 13, + ['O'] = _B64 | _B64U | 14, + ['P'] = _B64 | _B64U | 15, + ['Q'] = _B64 | _B64U | 16, + ['R'] = _B64 | _B64U | 17, + ['S'] = _B64 | _B64U | 18, + ['T'] = _B64 | _B64U | 19, + ['U'] = _B64 | _B64U | 20, + ['V'] = _B64 | _B64U | 21, + ['W'] = _B64 | _B64U | 22, + ['X'] = _B64 | _B64U | 23, + ['Y'] = _B64 | _B64U | 24, + ['Z'] = _B64 | _B64U | 25, + ['a'] = _B64 | _B64U | 26, + ['b'] = _B64 | _B64U | 27, + ['c'] = _B64 | _B64U | 28, + ['d'] = _B64 | _B64U | 29, + ['e'] = _B64 | _B64U | 30, + ['f'] = _B64 | _B64U | 31, + ['g'] = _B64 | _B64U | 32, + ['h'] = _B64 | _B64U | 33, + ['i'] = _B64 | _B64U | 34, + ['j'] = _B64 | _B64U | 35, + ['k'] = _B64 | _B64U | 36, + ['l'] = _B64 | _B64U | 37, + ['m'] = _B64 | _B64U | 38, + ['n'] = _B64 | _B64U | 39, + ['o'] = _B64 | _B64U | 40, + ['p'] = _B64 | _B64U | 41, + ['q'] = _B64 | _B64U | 42, + ['r'] = _B64 | _B64U | 43, + ['s'] = _B64 | _B64U | 44, + ['t'] = _B64 | _B64U | 45, + ['u'] = _B64 | _B64U | 46, + ['v'] = _B64 | _B64U | 47, + ['w'] = _B64 | _B64U | 48, + ['x'] = _B64 | _B64U | 49, + ['y'] = _B64 | _B64U | 50, + ['z'] = _B64 | _B64U | 51, + ['0'] = _B64 | _B64U | 52, + ['1'] = _B64 | _B64U | 53, + ['2'] = _B64 | _B64U | 54, + ['3'] = _B64 | _B64U | 55, + ['4'] = _B64 | _B64U | 56, + ['5'] = _B64 | _B64U | 57, + ['6'] = _B64 | _B64U | 58, + ['7'] = _B64 | _B64U | 59, + ['8'] = _B64 | _B64U | 60, + ['9'] = _B64 | _B64U | 61, + ['+'] = _B64 | 62, + ['/'] = _B64 | 63, + ['-'] = _B64U | 62, + ['_'] = _B64U | 63, +}; + +#define _SEP _SERVAL_CTYPE_1_HTTP_SEPARATOR +#define _URI_SCHEME _SERVAL_CTYPE_1_URI_SCHEME +#define _URI_UNRES _SERVAL_CTYPE_1_URI_UNRESERVED +#define _URI_RES _SERVAL_CTYPE_1_URI_RESERVED + +uint8_t _serval_ctype_1[UINT8_MAX] = { + ['A'] = _URI_SCHEME | _URI_UNRES | 0xA, + ['B'] = _URI_SCHEME | _URI_UNRES | 0xB, + ['C'] = _URI_SCHEME | _URI_UNRES | 0xC, + ['D'] = _URI_SCHEME | _URI_UNRES | 0xD, + ['E'] = _URI_SCHEME | _URI_UNRES | 0xE, + ['F'] = _URI_SCHEME | _URI_UNRES | 0xF, + ['G'] = _URI_SCHEME | _URI_UNRES, + ['H'] = _URI_SCHEME | _URI_UNRES, + ['I'] = _URI_SCHEME | _URI_UNRES, + ['J'] = _URI_SCHEME | _URI_UNRES, + ['K'] = _URI_SCHEME | _URI_UNRES, + ['L'] = _URI_SCHEME | _URI_UNRES, + ['M'] = _URI_SCHEME | _URI_UNRES, + ['N'] = _URI_SCHEME | _URI_UNRES, + ['O'] = _URI_SCHEME | _URI_UNRES, + ['P'] = _URI_SCHEME | _URI_UNRES, + ['Q'] = _URI_SCHEME | _URI_UNRES, + ['R'] = _URI_SCHEME | _URI_UNRES, + ['S'] = _URI_SCHEME | _URI_UNRES, + ['T'] = _URI_SCHEME | _URI_UNRES, + ['U'] = _URI_SCHEME | _URI_UNRES, + ['V'] = _URI_SCHEME | _URI_UNRES, + ['W'] = _URI_SCHEME | _URI_UNRES, + ['X'] = _URI_SCHEME | _URI_UNRES, + ['Y'] = _URI_SCHEME | _URI_UNRES, + ['Z'] = _URI_SCHEME | _URI_UNRES, + ['a'] = _URI_SCHEME | _URI_UNRES | 0xa, + ['b'] = _URI_SCHEME | _URI_UNRES | 0xb, + ['c'] = _URI_SCHEME | _URI_UNRES | 0xc, + ['d'] = _URI_SCHEME | _URI_UNRES | 0xd, + ['e'] = _URI_SCHEME | _URI_UNRES | 0xe, + ['f'] = _URI_SCHEME | _URI_UNRES | 0xf, + ['g'] = _URI_SCHEME | _URI_UNRES, + ['h'] = _URI_SCHEME | _URI_UNRES, + ['i'] = _URI_SCHEME | _URI_UNRES, + ['j'] = _URI_SCHEME | _URI_UNRES, + ['k'] = _URI_SCHEME | _URI_UNRES, + ['l'] = _URI_SCHEME | _URI_UNRES, + ['m'] = _URI_SCHEME | _URI_UNRES, + ['n'] = _URI_SCHEME | _URI_UNRES, + ['o'] = _URI_SCHEME | _URI_UNRES, + ['p'] = _URI_SCHEME | _URI_UNRES, + ['q'] = _URI_SCHEME | _URI_UNRES, + ['r'] = _URI_SCHEME | _URI_UNRES, + ['s'] = _URI_SCHEME | _URI_UNRES, + ['t'] = _URI_SCHEME | _URI_UNRES, + ['u'] = _URI_SCHEME | _URI_UNRES, + ['v'] = _URI_SCHEME | _URI_UNRES, + ['w'] = _URI_SCHEME | _URI_UNRES, + ['x'] = _URI_SCHEME | _URI_UNRES, + ['y'] = _URI_SCHEME | _URI_UNRES, + ['z'] = _URI_SCHEME | _URI_UNRES, + ['0'] = _URI_SCHEME | _URI_UNRES | 0, + ['1'] = _URI_SCHEME | _URI_UNRES | 1, + ['2'] = _URI_SCHEME | _URI_UNRES | 2, + ['3'] = _URI_SCHEME | _URI_UNRES | 3, + ['4'] = _URI_SCHEME | _URI_UNRES | 4, + ['5'] = _URI_SCHEME | _URI_UNRES | 5, + ['6'] = _URI_SCHEME | _URI_UNRES | 6, + ['7'] = _URI_SCHEME | _URI_UNRES | 7, + ['8'] = _URI_SCHEME | _URI_UNRES | 8, + ['9'] = _URI_SCHEME | _URI_UNRES | 9, + ['\t'] = _SEP, + [' '] = _SEP, + ['_'] = _URI_UNRES, + ['='] = _SEP | _URI_RES, + ['<'] = _SEP, + ['>'] = _SEP, + [';'] = _SEP | _URI_RES, + [':'] = _SEP | _URI_RES, + ['\\'] = _SEP, + ['\''] = _URI_RES, + ['"'] = _SEP, + ['/'] = _SEP | _URI_RES, + ['['] = _SEP | _URI_RES, + [']'] = _SEP | _URI_RES, + ['{'] = _SEP, + ['}'] = _SEP, + ['('] = _SEP | _URI_RES, + [')'] = _SEP | _URI_RES, + [','] = _SEP | _URI_RES, + ['.'] = _URI_SCHEME | _URI_UNRES, + ['?'] = _SEP | _URI_RES, + ['!'] = _URI_RES, + ['+'] = _URI_SCHEME | _URI_RES, + ['-'] = _URI_SCHEME | _URI_UNRES, + ['*'] = _URI_RES, + ['$'] = _URI_RES, + ['&'] = _URI_RES, + ['#'] = _URI_RES, + ['@'] = _SEP | _URI_RES, + ['~'] = _URI_UNRES, +}; + +#define _BND _SERVAL_CTYPE_2_MULTIPART_BOUNDARY + +uint8_t _serval_ctype_2[UINT8_MAX] = { + ['A'] = _BND, + ['B'] = _BND, + ['C'] = _BND, + ['D'] = _BND, + ['E'] = _BND, + ['F'] = _BND, + ['G'] = _BND, + ['H'] = _BND, + ['I'] = _BND, + ['J'] = _BND, + ['K'] = _BND, + ['L'] = _BND, + ['M'] = _BND, + ['N'] = _BND, + ['O'] = _BND, + ['P'] = _BND, + ['Q'] = _BND, + ['R'] = _BND, + ['S'] = _BND, + ['T'] = _BND, + ['U'] = _BND, + ['V'] = _BND, + ['W'] = _BND, + ['X'] = _BND, + ['Y'] = _BND, + ['Z'] = _BND, + ['a'] = _BND, + ['b'] = _BND, + ['c'] = _BND, + ['d'] = _BND, + ['e'] = _BND, + ['f'] = _BND, + ['g'] = _BND, + ['h'] = _BND, + ['i'] = _BND, + ['j'] = _BND, + ['k'] = _BND, + ['l'] = _BND, + ['m'] = _BND, + ['n'] = _BND, + ['o'] = _BND, + ['p'] = _BND, + ['q'] = _BND, + ['r'] = _BND, + ['s'] = _BND, + ['t'] = _BND, + ['u'] = _BND, + ['v'] = _BND, + ['w'] = _BND, + ['x'] = _BND, + ['y'] = _BND, + ['z'] = _BND, + ['0'] = _BND, + ['1'] = _BND, + ['2'] = _BND, + ['3'] = _BND, + ['4'] = _BND, + ['5'] = _BND, + ['6'] = _BND, + ['7'] = _BND, + ['8'] = _BND, + ['9'] = _BND, + ['+'] = _BND, + ['/'] = _BND, + ['='] = _BND, + ['-'] = _BND, + ['.'] = _BND, + [':'] = _BND, + ['_'] = _BND, + ['('] = _BND, + [')'] = _BND, + [','] = _BND, + ['?'] = _BND, + [' '] = _BND, +}; + /* Does this whole buffer contain the same value? */ int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value) { diff --git a/str.h b/str.h index 7f9827be..86f81644 100644 --- a/str.h +++ b/str.h @@ -17,8 +17,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#ifndef __STR_H__ -#define __STR_H__ +#ifndef __SERVAL_DNA_STR_H__ +#define __SERVAL_DNA_STR_H__ #include #include @@ -26,20 +26,34 @@ #include #include -#ifndef __STR_INLINE +#ifndef __SERVAL_DNA_STR_INLINE # if __GNUC__ && !__GNUC_STDC_INLINE__ -# define __STR_INLINE extern inline +# define __SERVAL_DNA_STR_INLINE extern inline # else -# define __STR_INLINE inline +# define __SERVAL_DNA_STR_INLINE inline # endif #endif +/* -------------------- Useful functions and macros -------------------- */ + +#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str)) + +int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value); + +char *str_toupper_inplace(char *s); +char *str_tolower_inplace(char *s); + +/* -------------------- Hexadecimal strings -------------------- */ + +extern const char hexdigit_upper[16]; +extern const char hexdigit_lower[16]; + /* Return true iff 'len' bytes starting at 'text' are hex digits, upper or lower case. * Does not check the following byte. * * @author Andrew Bettison */ -__STR_INLINE int is_xsubstring(const char *text, int len) +__SERVAL_DNA_STR_INLINE int is_xsubstring(const char *text, int len) { while (len--) if (!isxdigit(*text++)) @@ -52,7 +66,7 @@ __STR_INLINE int is_xsubstring(const char *text, int len) * * @author Andrew Bettison */ -__STR_INLINE int is_xstring(const char *text, int len) +__SERVAL_DNA_STR_INLINE int is_xstring(const char *text, int len) { while (len--) if (!isxdigit(*text++)) @@ -60,43 +74,205 @@ __STR_INLINE int is_xstring(const char *text, int len) return *text == '\0'; } -extern const char hexdigit[16]; +/* Converts a given binary blob to uppercase ASCII hexadecimal with a NUL terminator on the end. + * 'dstHex' must point to a buffer of at least 'dstStrLen' + 1 bytes. + */ char *tohex(char *dstHex, size_t dstStrlen, const unsigned char *srcBinary); -size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary); -int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary); -size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp); - #define alloca_tohex(buf,bytes) tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf)) -#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str)) +/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] to nbinary bytes of data. Can be used to + * perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); Returns -1 if a non-hex-digit + * character is encountered, otherwise returns the number of binary bytes produced (= nbinary). + * Does not insist that the last hex digit is followed by a NUL or any particular character. + * + * @author Andrew Bettison + */ +size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary); -__STR_INLINE int hexvalue(char c) -{ - switch (c) { - case '0': return 0; - case '1': return 1; - case '2': return 2; - case '3': return 3; - case '4': return 4; - case '5': return 5; - case '6': return 6; - case '7': return 7; - case '8': return 8; - case '9': return 9; - case 'a': case 'A': return 10; - case 'b': case 'B': return 11; - case 'c': case 'C': return 12; - case 'd': case 'D': return 13; - case 'e': case 'E': return 14; - case 'f': case 'F': return 15; - } - return -1; +/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] followed by a NUL '\0' character to nbinary + * bytes of data. Can be used to perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); + * Returns -1 if a non-hex-digit character is encountered or the character immediately following the + * last hex digit is not a NUL, otherwise returns zero. + * + * @author Andrew Bettison + */ +int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary); + +/* Decode pairs of ASCII hex characters [0-9A-Fa-f] into binary data with an optional upper limit on + * the number of binary bytes produced (destination buffer size). Returns the number of binary + * bytes decoded. If 'afterHex' is not NULL, then sets *afterHex to point to the source character + * immediately following the last hex digit consumed. + * + * Can be used to perform a conversion in-place, eg: + * + * strn_fromhex((unsigned char *)buf, n, (const char *)buf, NULL); + * + * Can also be used to count hex digits without converting, eg: + * + * strn_fromhex(NULL, -1, buf, NULL); + * + * The fromhex() and fromhexstr() functions are both implemented using strn_fromhex(). + * + * @author Andrew Bettison + */ +size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp); + +/* -------------------- Base64 encoding and decoding -------------------- */ + +/* Return the number of bytes required to represent 'binaryBytes' bytes of binary data encoded + * into Base64 form. + * + * @author Andrew Bettison + */ +#define BASE64_ENCODED_LEN(binaryBytes) (((size_t)(binaryBytes) + 2) / 3 * 4) + +/* Array of encoding symbols. Entry [64] is the pad character (usually '='). + */ +const char base64_symbols[65]; +const char base64url_symbols[65]; + +/* Encode 'srcBytes' bytes of binary data at 'srcBinary' into Base64 representation at 'dstBase64' + * (or Base64-URL representation at 'dstBase64url'), which must point to at least + * 'BASE64_ENCODED_LEN(srcBytes)' bytes. The encoding is terminated by a "=" or "==" pad to bring + * the total number of encoded bytes up to a multiple of 4. + * + * Returns the total number of encoded bytes writtent at 'dstBase64'. + * + * The base64_encodev() is a multi-buffer gather variant, analagous to readv(2) and writev(2). + * + * @author Andrew Bettison + */ +size_t base64_encode(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes); +size_t base64url_encode(char *dstBase64url, const unsigned char *srcBinary, size_t srcBytes); +struct iovec; +size_t base64_encode(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes); +size_t base64url_encodev(char *dstBase64url, const struct iovec *iov, int iovcnt); + +/* The same as base64_encode() but appends a terminating NUL character to the encoded string, + * so 'dstBase64' must point to at least 'BASE64_ENCODED_LEN(srcBytes) + 1' bytes. + * + * @author Andrew Bettison + */ +char *to_base64_str(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes); +char *to_base64url_str(char *dstBase64url, const unsigned char *srcBinary, size_t srcBytes); + +#define alloca_base64(buf,len) to_base64_str(alloca(BASE64_ENCODED_LEN(len) + 1), (buf), (len)) +#define alloca_base64url(buf,len) to_base64url_str(alloca(BASE64_ENCODED_LEN(len) + 1), (buf), (len)) + +/* Decode the string at 'srcBase64' as ASCII Base64 or Base64-URL (as per RFC-4648), writing up to + * 'dstsiz' decoded binary bytes at 'dstBinary'. Returns the number of decoded binary bytes + * produced. If 'dstsiz' is zero or 'dstBinary' is NULL, no binary bytes are produced and returns + * zero. + * + * If the 'afterp' pointer is not NULL, then sets *afterp to point to the first character in + * 'srcBase64' where decoding stopped for whatever reason. + * + * If 'srclen' is 0, then the string at 'stcBase64' is assumed to be NUL-terminated, and decoding + * runs until the first non-Base64-digit is encountered. If 'srclen' is nonzero, then decoding will + * cease at the first non-Base64-digit or when 'srclen' bytes at 'srcBase64' have been decoded, + * whichever comes first. + * + * If 'skip_pred' is not NULL, then all leading, internal and trailing characters C which are not a + * valid Base64 digit or pad '=' will be skipped if skip_pred(C) returns true. Otherwise, decoding + * ends at C. + * + * If the B64_CONSUME_ALL flag is set, then once the 'dstsiz' limit is reached (or if 'dstBinary' is + * NULL), the Base64 decoding process continues without actually writing decoded bytes, but instead + * counts them and advances through the 'srcBase64' buffer as usual. The return value is then the + * number of binary bytes that would be decoded were all available Base64 decoded from 'srcBase64', + * and *afterp points to the first character beyond the end of the decoded source characters. + * + * @author Andrew Bettison + */ +size_t base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen, + const char **afterp, int flags, int (*skip_pred)(char)); +size_t base64url_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64url, size_t srclen, + const char **afterp, int flags, int (*skip_pred)(char)); + +#define B64_CONSUME_ALL (1 << 0) + +/* -------------------- Character classes -------------------- */ + +#define _SERVAL_CTYPE_0_BASE64_MASK 0x3f +#define _SERVAL_CTYPE_0_BASE64 (1 << 6) +#define _SERVAL_CTYPE_0_BASE64URL (1 << 7) + +#define _SERVAL_CTYPE_1_HEX_MASK 0xf +#define _SERVAL_CTYPE_1_HTTP_SEPARATOR (1 << 4) +#define _SERVAL_CTYPE_1_URI_SCHEME (1 << 5) +#define _SERVAL_CTYPE_1_URI_UNRESERVED (1 << 6) +#define _SERVAL_CTYPE_1_URI_RESERVED (1 << 7) + +#define _SERVAL_CTYPE_2_MULTIPART_BOUNDARY (1 << 0) + +extern uint8_t _serval_ctype_0[UINT8_MAX]; +extern uint8_t _serval_ctype_1[UINT8_MAX]; +extern uint8_t _serval_ctype_2[UINT8_MAX]; + +__SERVAL_DNA_STR_INLINE int is_http_char(char c) { + return isascii(c); } -int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value); +__SERVAL_DNA_STR_INLINE int is_http_ctl(char c) { + return iscntrl(c); +} -char *str_toupper_inplace(char *s); -char *str_tolower_inplace(char *s); +__SERVAL_DNA_STR_INLINE int is_base64_digit(char c) { + return (_serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64) != 0; +} + +__SERVAL_DNA_STR_INLINE int is_base64url_digit(char c) { + return (_serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64URL) != 0; +} + +__SERVAL_DNA_STR_INLINE int is_base64_pad(char c) { + return c == '='; +} + +__SERVAL_DNA_STR_INLINE int is_base64url_pad(char c) { + return c == '='; +} + +__SERVAL_DNA_STR_INLINE uint8_t base64_digit(char c) { + return _serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64_MASK; +} + +__SERVAL_DNA_STR_INLINE uint8_t base64url_digit(char c) { + return _serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64_MASK; +} + +__SERVAL_DNA_STR_INLINE int is_multipart_boundary(char c) { + return (_serval_ctype_2[(unsigned char) c] & _SERVAL_CTYPE_2_MULTIPART_BOUNDARY) != 0; +} + +__SERVAL_DNA_STR_INLINE int is_valid_multipart_boundary_string(const char *s) +{ + if (s[0] == '\0') + return 0; + for (; *s; ++s) + if (!is_multipart_boundary(*s)) + return 0; + return s[-1] != ' '; +} + +__SERVAL_DNA_STR_INLINE int is_http_separator(char c) { + return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_HTTP_SEPARATOR) != 0; +} + +__SERVAL_DNA_STR_INLINE int is_http_token(char c) { + return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c); +} + +/* Convert the given ASCII hex digit character into its radix value, eg, '0' -> + * 0, 'b' -> 11. If the argument is not an ASCII hex digit, returns -1. + * + * @author Andrew Bettison + */ +__SERVAL_DNA_STR_INLINE int hexvalue(char c) { + return isxdigit(c) ? _serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_HEX_MASK : -1; +} + +/* -------------------- Printable string representation -------------------- */ char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]); char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]); @@ -104,10 +280,14 @@ size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2]); size_t toprint_str_len(const char *srcStr, const char quotes[2]); size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp); -#define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), "``") + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), "``") +#define alloca_toprint_quoted(dstlen,buf,len,quotes) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), (quotes)) + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), (quotes)) +#define alloca_toprint(dstlen,buf,len) alloca_toprint_quoted(dstlen,buf,len,"``") + #define alloca_str_toprint_quoted(str, quotes) toprint_str((char *)alloca(toprint_str_len((str), (quotes)) + 1), -1, (str), (quotes)) #define alloca_str_toprint(str) alloca_str_toprint_quoted(str, "``") +/* -------------------- Useful string primitives -------------------- */ + /* Like strchr(3), but only looks for 'c' in the first 'n' characters of 's', stopping at the first * nul char in 's'. * @@ -122,24 +302,24 @@ const char *strnchr(const char *s, size_t n, char c); * @author Andrew Bettison */ -__STR_INLINE ssize_t str_index_dfl(const char *s, char c, ssize_t dfl) +__SERVAL_DNA_STR_INLINE ssize_t str_index_dfl(const char *s, char c, ssize_t dfl) { const char *r = strchr(s, c); return r ? r - s : dfl; } -__STR_INLINE ssize_t str_rindex_dfl(const char *s, char c, ssize_t dfl) +__SERVAL_DNA_STR_INLINE ssize_t str_rindex_dfl(const char *s, char c, ssize_t dfl) { const char *r = strrchr(s, c); return r ? r - s : dfl; } -__STR_INLINE ssize_t str_index(const char *s, char c) +__SERVAL_DNA_STR_INLINE ssize_t str_index(const char *s, char c) { return str_index_dfl(s, c, -1); } -__STR_INLINE ssize_t str_rindex(const char *s, char c) +__SERVAL_DNA_STR_INLINE ssize_t str_rindex(const char *s, char c) { return str_rindex_dfl(s, c, -1); } @@ -267,25 +447,16 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft */ int str_is_uri(const char *uri); -__STR_INLINE int is_uri_char_scheme(char c) -{ - return isalpha(c) || isdigit(c) || c == '+' || c == '-' || c == '.'; +__SERVAL_DNA_STR_INLINE int is_uri_char_scheme(char c) { + return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_SCHEME) != 0; } -__STR_INLINE int is_uri_char_unreserved(char c) -{ - return isalpha(c) || isdigit(c) || c == '-' || c == '.' || c == '_' || c == '~'; +__SERVAL_DNA_STR_INLINE int is_uri_char_unreserved(char c) { + return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_UNRESERVED) != 0; } -__STR_INLINE int is_uri_char_reserved(char c) -{ - switch (c) { - case ':': case '/': case '?': case '#': case '[': case ']': case '@': - case '!': case '$': case '&': case '\'': case '(': case ')': - case '*': case '+': case ',': case ';': case '=': - return 1; - } - return 0; +__SERVAL_DNA_STR_INLINE int is_uri_char_reserved(char c) { + return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_RESERVED) != 0; } /* Return true if the string resembles a URI scheme without the terminating colon. @@ -293,7 +464,7 @@ __STR_INLINE int is_uri_char_reserved(char c) * * @author Andrew Bettison */ -__STR_INLINE int str_is_uri_scheme(const char *scheme) +__SERVAL_DNA_STR_INLINE int str_is_uri_scheme(const char *scheme) { if (!isalpha(*scheme++)) return 0; @@ -359,4 +530,4 @@ int str_uri_authority_port(const char *auth, uint16_t *portp); int parse_argv(char *cmdline, char delim, char **argv, int max_argv); -#endif +#endif // __SERVAL_DNA_STR_H__ diff --git a/strbuf.c b/strbuf.c index df37a0a1..23efa133 100644 --- a/strbuf.c +++ b/strbuf.c @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define __STRBUF_INLINE #include "strbuf.h" +#include "str.h" static inline size_t min(size_t a, size_t b) { return a < b ? a : b; @@ -76,7 +77,6 @@ strbuf strbuf_puts(strbuf sb, const char *text) strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data) { - static char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; char *p = sb->current; sb->current += strlen; if (sb->start) { @@ -84,7 +84,7 @@ strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data) // The following loop could overwrite the '\0' at *sp->end. size_t i; for (i = 0; i < strlen && p < e; ++i) - *p++ = (i & 1) ? hexdigit[*data++ & 0xf] : hexdigit[*data >> 4]; + *p++ = (i & 1) ? hexdigit_upper[*data++ & 0xf] : hexdigit_upper[*data >> 4]; // This will restore the '\0' at *sp->end if it was overwritten. *e = '\0'; } diff --git a/strbuf_helpers.c b/strbuf_helpers.c index 66c6898d..5d0db6ce 100644 --- a/strbuf_helpers.c +++ b/strbuf_helpers.c @@ -36,6 +36,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include "http_server.h" #include "strbuf_helpers.h" +#include "str.h" +#include "socket.h" static inline strbuf _toprint(strbuf sb, char c) { @@ -315,7 +317,7 @@ strbuf strbuf_append_socket_type(strbuf sb, int type) strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr) { - strbuf_sprintf(sb, " %u.%u.%u.%u", + strbuf_sprintf(sb, "%u.%u.%u.%u", ((unsigned char *) &addr->s_addr)[0], ((unsigned char *) &addr->s_addr)[1], ((unsigned char *) &addr->s_addr)[2], @@ -323,13 +325,21 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr) return sb; } +strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr) +{ + assert(addr->sin_family == AF_INET); + strbuf_puts(sb, "AF_INET:"); + strbuf_append_in_addr(sb, &addr->sin_addr); + strbuf_sprintf(sb, ":%u", ntohs(addr->sin_port)); + return sb; +} + strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen) { - strbuf_append_socket_domain(sb, addr->sa_family); switch (addr->sa_family) { case AF_UNIX: { + strbuf_puts(sb, "AF_UNIX:"); size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0; - strbuf_putc(sb, ' '); if (addr->sa_data[0]) { strbuf_toprint_quoted_len(sb, "\"\"", addr->sa_data, len); if (len < 2) @@ -346,24 +356,30 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t break; case AF_INET: { const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr; - strbuf_putc(sb, ' '); - strbuf_append_in_addr(sb, &addr_in->sin_addr); - strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port)); + strbuf_append_sockaddr_in(sb, addr_in); if (addrlen != sizeof(struct sockaddr_in)) strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in)); } break; default: { + strbuf_append_socket_domain(sb, addr->sa_family); size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0; int i; - for (i = 0; i < len; ++i) - strbuf_sprintf(sb, " %02x", addr->sa_data[i]); + for (i = 0; i < len; ++i) { + strbuf_putc(sb, i ? ',' : ':'); + strbuf_sprintf(sb, "%02x", addr->sa_data[i]); + } } break; } return sb; } +strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr) +{ + return strbuf_append_sockaddr(sb, &addr->addr, addr->addrlen); +} + strbuf strbuf_append_strftime(strbuf sb, const char *format, const struct tm *tm) { // First, try calling strftime(3) directly on the buffer in the strbuf, if there is one and it @@ -402,6 +418,69 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt) return sb; } +strbuf strbuf_append_quoted_string(strbuf sb, const char *str) +{ + strbuf_putc(sb, '"'); + for (; *str; ++str) { + if (*str == '"' || *str == '\\') + strbuf_putc(sb, '\\'); + strbuf_putc(sb, *str); + } + strbuf_putc(sb, '"'); + return sb; +} + +strbuf strbuf_json_null(strbuf sb) +{ + strbuf_puts(sb, "null"); + return sb; +} + +strbuf strbuf_json_string(strbuf sb, const char *str) +{ + if (str) { + strbuf_putc(sb, '"'); + for (; *str; ++str) { + if (*str == '"' || *str == '\\') { + strbuf_putc(sb, '\\'); + strbuf_putc(sb, *str); + } + else if (*str == '\b') + strbuf_puts(sb, "\\b"); + else if (*str == '\f') + strbuf_puts(sb, "\\f"); + else if (*str == '\n') + strbuf_puts(sb, "\\n"); + else if (*str == '\r') + strbuf_puts(sb, "\\r"); + else if (*str == '\t') + strbuf_puts(sb, "\\t"); + else if (iscntrl(*str)) + strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str); + else + strbuf_putc(sb, *str); + } + strbuf_putc(sb, '"'); + } else + strbuf_json_null(sb); + return sb; +} + +strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len) +{ + if (buf) { + strbuf_putc(sb, '"'); + size_t i; + for (i = 0; i != len; ++i) { + strbuf_putc(sb, hexdigit_upper[*buf >> 4]); + strbuf_putc(sb, hexdigit_upper[*buf++ & 0xf]); + } + strbuf_putc(sb, '"'); + } else + strbuf_json_null(sb); + return sb; +} + strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels) { unsigned i; @@ -427,31 +506,47 @@ strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, uns return sb; } -strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd) +strbuf strbuf_append_mime_content_type(strbuf sb, const struct mime_content_type *ct) { - strbuf_puts(sb, "type="); - strbuf_toprint_quoted(sb, "``", cd->type); - strbuf_puts(sb, " name="); - strbuf_toprint_quoted(sb, "``", cd->name); - strbuf_puts(sb, " filename="); - strbuf_toprint_quoted(sb, "``", cd->filename); - strbuf_puts(sb, " size="); - strbuf_sprintf(sb, "%"PRIhttp_size_t, cd->size); - struct tm tm; - strbuf_puts(sb, " creation_date="); - if (cd->creation_date) - strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->creation_date, &tm)); - else - strbuf_puts(sb, "0"); - strbuf_puts(sb, " modification_date="); - if (cd->modification_date) - strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->modification_date, &tm)); - else - strbuf_puts(sb, "0"); - strbuf_puts(sb, " read_date="); - if (cd->read_date) - strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->read_date, &tm)); - else - strbuf_puts(sb, "0"); + strbuf_puts(sb, ct->type); + strbuf_putc(sb, '/'); + strbuf_puts(sb, ct->subtype); + if (ct->charset) { + strbuf_puts(sb, "; charset="); + strbuf_append_quoted_string(sb, ct->charset); + } + if (ct->multipart_boundary) { + strbuf_puts(sb, "; boundary="); + strbuf_append_quoted_string(sb, ct->multipart_boundary); + } + return sb; +} + +strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd) +{ + strbuf_puts(sb, cd->type); + if (cd->name) { + strbuf_puts(sb, "; name="); + strbuf_append_quoted_string(sb, cd->name); + } + if (cd->filename) { + strbuf_puts(sb, "; filename="); + strbuf_append_quoted_string(sb, cd->filename); + } + if (cd->size) + strbuf_sprintf(sb, "; size=%"PRIhttp_size_t, cd->size); + struct tm tm; + if (cd->creation_date) { + strbuf_puts(sb, " creation_date="); + strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->creation_date, &tm)); + } + if (cd->modification_date) { + strbuf_puts(sb, " modification_date="); + strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->modification_date, &tm)); + } + if (cd->read_date) { + strbuf_puts(sb, " read_date="); + strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->read_date, &tm)); + } return sb; } diff --git a/strbuf_helpers.h b/strbuf_helpers.h index 289c7ddf..4fee86cb 100644 --- a/strbuf_helpers.h +++ b/strbuf_helpers.h @@ -128,10 +128,21 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr); /* Append a textual description of a struct sockaddr_in. * @author Andrew Bettison */ +struct sockaddr_in; +strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr); +#define alloca_sockaddr_in(addr) strbuf_str(strbuf_append_sockaddr_in(strbuf_alloca(45), (const struct sockaddr_in *)(addr))) + +/* Append a textual description of a struct sockaddr. + * @author Andrew Bettison + */ struct sockaddr; strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen); #define alloca_sockaddr(addr, addrlen) strbuf_str(strbuf_append_sockaddr(strbuf_alloca(200), (const struct sockaddr *)(addr), (addrlen))) +struct socket_address; +strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr); +#define alloca_socket_address(addr) strbuf_str(strbuf_append_socket_address(strbuf_alloca(200), (addr))) + /* Append a strftime(3) string. * @author Andrew Bettison */ @@ -145,6 +156,19 @@ struct iovec; strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt); #define alloca_iovec(iov,cnt) strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt))) +/* Append a string using HTTP quoted-string format: delimited by double quotes (") and + * internal double quotes and backslash escaped by leading backslash. + * @author Andrew Bettison + */ +strbuf strbuf_append_quoted_string(strbuf sb, const char *str); + +/* Append various JSON elements. + * @author Andrew Bettison + */ +strbuf strbuf_json_null(strbuf sb); +strbuf strbuf_json_string(strbuf sb, const char *str); +strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len); + /* Append a representation of a struct http_range[] array. * @author Andrew Bettison */ @@ -152,7 +176,14 @@ struct http_range; strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels); #define alloca_http_ranges(ra) strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra))) -/* Append a representation of a struct mime_content_disposition struct. +/* Append a representation of a struct mime_content_type in HTTP header format. + * @author Andrew Bettison + */ +struct mime_content_type; +strbuf strbuf_append_mime_content_type(strbuf, const struct mime_content_type *); +#define alloca_mime_content_type(ct) strbuf_str(strbuf_append_mime_content_type(strbuf_alloca(500), (ct))) + +/* Append a representation of a struct mime_content_disposition, in HTTP header format. * @author Andrew Bettison */ struct mime_content_disposition; diff --git a/testdefs.sh b/testdefs.sh index 2e083640..def06464 100644 --- a/testdefs.sh +++ b/testdefs.sh @@ -57,14 +57,17 @@ extract_stdout_keyvalue_optional() { esac local _label_re=$(escape_grep_basic "$_label") local _delim_re=$(escape_grep_basic "$_delim") - local _line=$(replayStdout | $GREP "^$_label_re$_delim_re") + local _line=$($GREP "^$_label_re$_delim_re" "$TFWSTDOUT") local _value= local _return=1 if [ -n "$_line" ]; then _value="${_line#*$_delim}" _return=0 fi - [ -n "$_var" ] && eval $_var="\$_value" + if [ -n "$_var" ]; then + eval $_var="\$_value" + eval tfw_log "$_var=\$(shellarg "\${$_var}")" + fi return $_return } diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh index 984cc253..631371cd 100644 --- a/testdefs_rhizome.sh +++ b/testdefs_rhizome.sh @@ -78,6 +78,63 @@ assert_rhizome_list() { rhizome_list_file_count=$(( $(replayStdout | wc -l) - 2 )) } +# Parse the standard output produced by the immediately preceding "rhizome list" +# command into the following shell variables: +# NCOLS the number of columns +# NROWS the number of data rows (not counting headers) +# HEADER[c] the C-th header label, 0 <= C <= NCOLS-1 +#