diff --git a/commandline.c b/commandline.c index ad5d6005..07cde19a 100644 --- a/commandline.c +++ b/commandline.c @@ -1275,10 +1275,11 @@ int app_rhizome_hash_file(const struct cli_parsed *parsed, struct cli_context *c return the hash of the file unencrypted. */ const char *filepath; cli_arg(parsed, "filepath", &filepath, NULL, ""); - char hexhash[RHIZOME_FILEHASH_STRLEN + 1]; - if (rhizome_hash_file(NULL, filepath, hexhash)) + rhizome_filehash_t hash; + uint64_t size; + if (rhizome_hash_file(NULL, filepath, &hash, &size) == -1) return -1; - cli_put_string(context, hexhash, "\n"); + cli_put_string(context, size ? alloca_tohex_rhizome_filehash_t(hash) : "", "\n"); return 0; } @@ -1425,7 +1426,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co cli_put_long(context, mout->fileLength, "\n"); if (mout->fileLength != 0) { cli_field_name(context, "filehash", ":"); - cli_put_string(context, mout->fileHexHash, "\n"); + cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n"); } const char *name = rhizome_manifest_get(mout, "name", NULL, 0); if (name) { @@ -1528,7 +1529,7 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex cli_put_long(context, m->fileLength, "\n"); if (m->fileLength != 0) { cli_field_name(context, "filehash", ":"); - cli_put_string(context, m->fileHexHash, "\n"); + cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n"); } const char *name = rhizome_manifest_get(m, "name", NULL, 0); if (name) { @@ -1588,12 +1589,10 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) { if (!fileid) return WHY("missing argument"); - unsigned char filehash[RHIZOME_FILEHASH_BYTES]; - if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1) - return WHY("Invalid file ID"); - char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1]; - tohex(fileIDUpper, RHIZOME_FILEHASH_STRLEN, filehash); - ret = rhizome_delete_file(fileIDUpper); + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, fileid) == -1) + return WHYF("invalid argument: %s", alloca_str_toprint(fileid)); + ret = rhizome_delete_file(&hash); } else { if (!manifestid) return WHY("missing argument"); @@ -1688,7 +1687,8 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con cli_field_name(context, ".readonly", ":"); cli_put_long(context, m->haveSecret?0:1, "\n"); cli_field_name(context, "filesize", ":"); cli_put_long(context, m->fileLength, "\n"); if (m->fileLength != 0) { - cli_field_name(context, "filehash", ":"); cli_put_string(context, m->fileHexHash, "\n"); + cli_field_name(context, "filehash", ":"); + cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n"); } } @@ -1702,7 +1702,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con }else{ // Save the file without attempting to decrypt int64_t length; - retfile = rhizome_dump_file(m->fileHexHash, filepath, &length); + retfile = rhizome_dump_file(&m->filehash, filepath, &length); } } @@ -1741,18 +1741,21 @@ int app_rhizome_export_file(const struct cli_parsed *parsed, struct cli_context if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1 || cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1) return -1; + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, fileid) == -1) + return WHYF("invalid argument: %s", alloca_str_toprint(fileid)); if (create_serval_instance_dir() == -1) return -1; if (rhizome_opendb() == -1) return -1; - if (!rhizome_exists(fileid)) + if (!rhizome_exists(&hash)) return 1; int64_t length; - int ret = rhizome_dump_file(fileid, filepath, &length); + int ret = rhizome_dump_file(&hash, filepath, &length); if (ret) return ret == -1 ? -1 : 1; cli_field_name(context, "filehash", ":"); - cli_put_string(context, fileid, "\n"); + cli_put_string(context, alloca_tohex_rhizome_filehash_t(hash), "\n"); cli_field_name(context, "filesize", ":"); cli_put_long(context, length, "\n"); return 0; diff --git a/dataformats.c b/dataformats.c index a6830b47..48396ef7 100644 --- a/dataformats.c +++ b/dataformats.c @@ -94,6 +94,28 @@ int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp return 0; } +int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b) +{ + return memcmp(a, b, sizeof a->binary); +} + +int str_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex) +{ + return fromhexstr(hashp->binary, hex, sizeof hashp->binary); +} + +int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const char **endp) +{ + rhizome_filehash_t tmp; + int n = fromhex(tmp.binary, hex, sizeof tmp.binary); + if (n != sizeof tmp.binary) + return -1; + *hashp = tmp; + if (endp) + *endp = hex + sizeof hashp->binary * 2; + return 0; +} + int rhizome_strn_is_manifest_id(const char *id) { return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN); diff --git a/meshms.c b/meshms.c index 4479290c..87a25073 100644 --- a/meshms.c +++ b/meshms.c @@ -561,7 +561,8 @@ static int write_conversation(struct rhizome_write *write, struct conversations return len; } -static int write_known_conversations(rhizome_manifest *m, struct conversations *conv){ +static int write_known_conversations(rhizome_manifest *m, struct conversations *conv) +{ rhizome_manifest *mout=NULL; struct rhizome_write write; @@ -590,8 +591,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations * goto end; if (rhizome_finish_write(&write)) goto end; - strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); - rhizome_manifest_set(m, "filehash", m->fileHexHash); + m->filehash = write.id; + rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); if (rhizome_manifest_finalise(m, &mout, 1)) goto end; diff --git a/rhizome.c b/rhizome.c index 4dc22787..3148d547 100644 --- a/rhizome.c +++ b/rhizome.c @@ -73,7 +73,7 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, filepath ? alloca_str_toprint(filepath) : "NULL"); unsigned char buffer[MAX_MANIFEST_BYTES]; - int buffer_len=0; + size_t buffer_len = 0; // manifest has been appended to the end of the file. if (strcmp(manifest_path, filepath)==0){ @@ -251,10 +251,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl) if (rhizome_manifest_check_sanity(m_in)) return -1; - if (m_in->fileLength){ - if (!rhizome_exists(m_in->fileHexHash)) - return WHY("File has not been imported"); - } + if (m_in->fileLength && !rhizome_exists(&m_in->filehash)) + return WHY("File has not been imported"); /* If the manifest already has an ID */ if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic)) diff --git a/rhizome.h b/rhizome.h index 96b6bd73..bc21f05b 100644 --- a/rhizome.h +++ b/rhizome.h @@ -52,6 +52,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define RHIZOME_HTTP_PORT 4110 #define RHIZOME_HTTP_PORT_MAX 4150 +/* Fundamental data type: Rhizome Bundle ID + * + * @author Andrew Bettison + */ + typedef struct rhizome_bid_binary { unsigned char binary[RHIZOME_MANIFEST_ID_BYTES]; } rhizome_bid_t; @@ -65,6 +70,28 @@ int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b); int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex); int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp); +/* Fundamental data type: Rhizome File Hash + * + * @author Andrew Bettison + */ + +typedef struct rhizome_filehash_binary { + unsigned char binary[RHIZOME_FILEHASH_BYTES]; +} rhizome_filehash_t; + +#define RHIZOME_FILEHASH_NONE ((rhizome_filehash_t){{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}}) +#define rhizome_filehash_t_is_zero(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0) +#define rhizome_filehash_t_is_max(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0xff) +#define alloca_tohex_rhizome_filehash_t(fh) alloca_tohex((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary) +int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b); +int str_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex); +int strn_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, const char **endp); + +/* Fundamental data type: Rhizome Bundle Key + * + * @author Andrew Bettison + */ + typedef struct rhizome_bk_binary { unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES]; } rhizome_bk_t; @@ -77,6 +104,7 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) { #define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary) + extern time_ms_t rhizome_voice_timeout; #define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE @@ -158,7 +186,7 @@ typedef struct rhizome_manifest { group membership handy */ int64_t fileLength; int64_t journalTail; - char fileHexHash[SHA512_DIGEST_STRING_LENGTH]; + rhizome_filehash_t filehash; int fileHighestPriority; /* Absolute path of the file associated with the manifest */ @@ -258,10 +286,10 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append); int rhizome_manifest_selfsign(rhizome_manifest *m); -int rhizome_drop_stored_file(const char *id,int maximum_priority); +int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority); int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp); int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize); -int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out); +int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out); char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen); int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var); int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value); @@ -274,7 +302,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence); #define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__) int rhizome_manifest_pack_variables(rhizome_manifest *m); int rhizome_store_bundle(rhizome_manifest *m); -int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid); +int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp); int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); int rhizome_clean_payload(const char *fileidhex); int rhizome_store_file(rhizome_manifest *m,const unsigned char *key); @@ -308,25 +336,25 @@ int is_debug_rhizome_ads(); enum sqlbind_type { END = 0xbabecafe, - INT = 1, // int value - INT_TOSTR, // int value - UINT_TOSTR, // unsigned value - INT64, // int64_t value - INT64_TOSTR, // int64_t value - UINT64_TOSTR, // uint64_t value - TEXT, // const char *text, - TEXT_LEN, // const char *text, int bytes - STATIC_TEXT, // const char *text, - STATIC_TEXT_LEN, // const char *text, int bytes - STATIC_BLOB, // const void *blob, int bytes - ZEROBLOB, // int bytes - SID_T, // const sid_t *sidp - RHIZOME_BID_T, // const rhizome_bid_t *bidp - FILEHASH_T, // const unsigned char hash_binary[RHIZOME_FILEHASH_BYTES] - TOHEX, // const unsigned char *binary, unsigned bytes - TEXT_TOUPPER, // const char *text, - TEXT_LEN_TOUPPER, // const char *text, unsigned bytes - NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ... + INT = 1, // int value + INT_TOSTR, // int value + UINT_TOSTR, // unsigned value + INT64, // int64_t value + INT64_TOSTR, // int64_t value + UINT64_TOSTR, // uint64_t value + TEXT, // const char *text, + TEXT_LEN, // const char *text, int bytes + STATIC_TEXT, // const char *text, + STATIC_TEXT_LEN, // const char *text, int bytes + STATIC_BLOB, // const void *blob, int bytes + ZEROBLOB, // int bytes + SID_T, // const sid_t *sidp + RHIZOME_BID_T, // const rhizome_bid_t *bidp + RHIZOME_FILEHASH_T, // const rhizome_filehash_t *hashp + TOHEX, // const unsigned char *binary, unsigned bytes + TEXT_TOUPPER, // const char *text, + TEXT_LEN_TOUPPER, // const char *text, unsigned bytes + NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ... INDEX = 0xfade0000, // INDEX|INT, int index, ... NAMED = 0xdead0000 // NAMED|INT, const char *label, ... }; @@ -390,7 +418,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m); int rhizome_delete_bundle(const rhizome_bid_t *bidp); int rhizome_delete_manifest(const rhizome_bid_t *bidp); int rhizome_delete_payload(const rhizome_bid_t *bidp); -int rhizome_delete_file(const char *fileid); +int rhizome_delete_file(const rhizome_filehash_t *hashp); #define RHIZOME_DONTVERIFY 0 #define RHIZOME_VERIFY 1 @@ -439,7 +467,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_length); /* Rhizome file storage api */ -struct rhizome_write_buffer{ +struct rhizome_write_buffer +{ struct rhizome_write_buffer *_next; int64_t offset; int buffer_size; @@ -447,8 +476,9 @@ struct rhizome_write_buffer{ unsigned char data[0]; }; -struct rhizome_write{ - char id[SHA512_DIGEST_STRING_LENGTH+1]; +struct rhizome_write +{ + rhizome_filehash_t id; uint64_t temp_id; char id_known; @@ -475,8 +505,9 @@ struct rhizome_read_buffer{ int len; }; -struct rhizome_read{ - char id[SHA512_DIGEST_STRING_LENGTH+1]; +struct rhizome_read +{ + rhizome_filehash_t id; int crypt; unsigned char key[RHIZOME_CRYPT_KEY_BYTES]; @@ -741,8 +772,8 @@ int unpack_http_response(char *response, struct http_response_parts *parts); /* rhizome storage methods */ -int rhizome_exists(const char *fileHash); -int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority); +int rhizome_exists(const rhizome_filehash_t *hashp); +int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority); int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size); int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size); int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m); @@ -758,23 +789,22 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk); int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size); int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len); int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename); -int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length); +int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length); int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, const unsigned char *key, const unsigned char *nonce); -int rhizome_open_read(struct rhizome_read *read, const char *fileid); +int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp); int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length); int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len); int rhizome_read_close(struct rhizome_read *read); int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); -int rhizome_dump_file(const char *id, const char *filepath, int64_t *length); +int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length); int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length); int rhizome_cache_close(); -int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]); - +int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp); int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp); int rhizome_sync_announce(); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 477c41d6..c942e971 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -169,14 +169,13 @@ int rhizome_manifest_parse(rhizome_manifest *m) } } else if (strcasecmp(var, "filehash") == 0) { have_filehash = 1; - if (!rhizome_str_is_file_hash(value)) { + if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) { if (config.debug.rejecteddata) WARNF("Invalid filehash: %s", value); m->errors++; } else { /* Force to upper case to avoid case sensitive comparison problems later. */ str_toupper_inplace(m->values[m->var_count]); - strcpy(m->fileHexHash, m->values[m->var_count]); } } else if (strcasecmp(var, "filesize") == 0) { have_filesize = 1; @@ -344,7 +343,7 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bu return rhizome_manifest_parse(m); } -int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out) +int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out) { /* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(), @@ -355,35 +354,36 @@ int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out) if (m && m->payloadEncryption) return WHY("Encryption of payloads not implemented"); - size_t filesize = 0; + uint64_t filesize = 0; SHA512_CTX context; SHA512_Init(&context); - if (filename[0]) { - FILE *f = fopen(filename, "r"); - if (!f) { - WHY_perror("fopen"); - return WHYF("Could not open %s to calculate SHA512 hash.", filename); - } - while (!feof(f)) { - unsigned char buffer[8192]; - int r = fread(buffer, 1, 8192, f); + if (path[0]) { + int fd = open(path, O_RDONLY); + if (fd == -1) + return WHYF_perror("open(%s,O_RDONLY)", alloca_str_toprint(path)); + unsigned char buffer[8192]; + ssize_t r; + while ((r = read(fd, buffer, sizeof buffer))) { if (r == -1) { - WHY_perror("fread"); - fclose(f); - return WHYF("Error reading %s to calculate SHA512 hash", filename); + WHYF_perror("read(%s,%u)", alloca_str_toprint(path), sizeof buffer); + close(fd); + return -1; } - if (r > 0) - SHA512_Update(&context, buffer, r); - filesize += r; + SHA512_Update(&context, buffer, (size_t) r); + filesize += (size_t) r; } - fclose(f); + close(fd); } - SHA512_End(&context, (char *)hash_out); - // Empty files (including null filename) have no hash. - if (filesize > 0) - str_toupper_inplace(hash_out); - else - hash_out[0] = '\0'; + // Empty files (including empty path) have no hash. + if (hash_out) { + if (filesize > 0) + SHA512_Final(hash_out->binary, &context); + else + *hash_out = RHIZOME_FILEHASH_NONE; + } + if (size_out) + *size_out = filesize; + SHA512_End(&context, NULL); return 0; } diff --git a/rhizome_database.c b/rhizome_database.c index 3e3dbec0..d73fc360 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -32,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. static char rhizome_thisdatastore_path[256]; static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); -static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid); +static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp); static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); const char *rhizome_datastore_path() @@ -678,14 +678,16 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state } } break; - case FILEHASH_T: { - const char *hash_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_FILEHASH_BYTES); + case RHIZOME_FILEHASH_T: { + const rhizome_filehash_t *hashp = va_arg(ap, const rhizome_filehash_t *); ++argnum; - if (hash_hex == NULL) { - BIND_NULL(FILEHASH_T); + if (hashp == NULL) { + BIND_NULL(RHIZOME_FILEHASH_T); } else { - BIND_DEBUG(FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN); - BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT); + char hash_hex[RHIZOME_FILEHASH_STRLEN]; + tohex(hash_hex, sizeof hash_hex, hashp->binary); + BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, sizeof hash_hex); + BIND_RETRY(sqlite3_bind_text, hash_hex, sizeof hash_hex, SQLITE_TRANSIENT); } } break; @@ -1014,20 +1016,24 @@ int64_t rhizome_database_used_bytes() return db_page_size * (db_page_count - db_free_page_count); } -int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]) +int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp) { IN(); - strbuf hash_sb = strbuf_local(hash, SHA512_DIGEST_STRING_LENGTH); - RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;", - INT64, version, RHIZOME_BID_T, bidp, END)); + strbuf hash_sb = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1); + if ( sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;", + INT64, version, RHIZOME_BID_T, bidp, END) == -1) + RETURN(-1); + if (strbuf_overrun(hash_sb) || str_to_rhizome_filehash_t(hashp, strbuf_str(hash_sb)) == -1) + RETURN(WHYF("malformed file hash for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(*bidp), version)); + RETURN(0); OUT(); } -static int rhizome_delete_external(const char *fileid) +static int rhizome_delete_external(const rhizome_filehash_t *hashp) { // attempt to remove any external blob char blob_path[1024]; - if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid)) + if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(*hashp))) return -1; return unlink(blob_path); } @@ -1039,18 +1045,18 @@ static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry) END); } -int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid) +int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp) { int ret = 0; if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, "DELETE FROM FILES WHERE id = ? and datavalid = 0;", - TEXT_TOUPPER, fileid, END + RHIZOME_FILEHASH_T, hashp, END ) == -1 ) ret = -1; if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, "DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );", - TEXT_TOUPPER, fileid, END + RHIZOME_FILEHASH_T, hashp, END ) == -1 ) ret = -1; @@ -1081,18 +1087,24 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { candidates++; const char *id = (const char *) sqlite3_column_text(statement, 0); - if (rhizome_delete_external(id) == 0 && report) - ++report->deleted_stale_incoming_files; + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, id) == -1) + WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id)); + else if (rhizome_delete_external(&hash) == 0 && report) + ++report->deleted_stale_incoming_files; } sqlite3_finalize(statement); - + statement = sqlite_prepare_bind(&retry, "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", INT64, insert_horizon_no_manifest, END); while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { candidates++; const char *id = (const char *) sqlite3_column_text(statement, 0); - if (rhizome_delete_external(id) == 0 && report) + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, id) == -1) + WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id)); + else if (rhizome_delete_external(&hash) == 0 && report) ++report->deleted_orphan_files; } sqlite3_finalize(statement); @@ -1152,11 +1164,10 @@ int rhizome_make_space(int group_priority, uint64_t bytes) && sqlite_step_retry(&retry, statement) == SQLITE_ROW ) { /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */ - const unsigned char *id; - + const char *id; /* Get values */ if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) - id = sqlite3_column_text(statement, 0); + id = (const char *) sqlite3_column_text(statement, 0); else { WHY("Incorrect type in id column of files table"); break; @@ -1167,10 +1178,16 @@ int rhizome_make_space(int group_priority, uint64_t bytes) WHY("Incorrect type in length column of files table"); break; } - /* Try to drop this file from storage, discarding any references that do not trump the priority - of this request. The query done earlier should ensure this, but it doesn't hurt to be - paranoid, and it also protects against inconsistency in the database. */ - rhizome_drop_stored_file((char *)id, group_priority + 1); + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, id) == -1) + WHYF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id)); + else { + /* Try to drop this file from storage, discarding any references that do not trump the + * priority of this request. The query done earlier should ensure this, but it doesn't hurt + * to be paranoid, and it also protects against inconsistency in the database. + */ + rhizome_drop_stored_file(&hash, group_priority + 1); + } } sqlite3_finalize(statement); @@ -1187,14 +1204,12 @@ int rhizome_make_space(int group_priority, uint64_t bytes) /* Drop the specified file from storage, and any manifests that reference it, provided that none of * those manifests are being retained at a higher priority than the maximum specified here. */ -int rhizome_drop_stored_file(const char *id, int maximum_priority) +int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority) { - if (!rhizome_str_is_file_hash(id)) - return WHYF("invalid file hash id=%s", alloca_str_toprint(id)); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", TEXT_TOUPPER, id, END); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", RHIZOME_FILEHASH_T, hashp, END); if (!statement) - return WHYF("Could not drop stored file id=%s", id); + return WHYF("Could not drop stored file id=%s", alloca_tohex_rhizome_filehash_t(*hashp)); int can_drop = 1; while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { /* Find manifests for this file */ @@ -1214,9 +1229,9 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority) that are lower priority, and thus free up a little space. */ int priority = rhizome_manifest_priority(&retry, &bid); if (priority == -1) - WHYF("Cannot drop fileid=%s due to error, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); + WHYF("Cannot drop fileid=%s due to error, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid)); else if (priority > maximum_priority) { - WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); + WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid)); can_drop = 0; } else { if (config.debug.rhizome) @@ -1228,7 +1243,7 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority) } sqlite3_finalize(statement); if (can_drop) - rhizome_delete_file_retry(&retry, id); + rhizome_delete_file_retry(&retry, hashp); return 0; } @@ -1272,16 +1287,8 @@ int rhizome_store_bundle(rhizome_manifest *m) rhizome_manifest_to_bar(m,bar); /* Store the file (but not if it is already in the database) */ - char filehash[RHIZOME_FILEHASH_STRLEN + 1]; - if (m->fileLength > 0) { - strncpy(filehash, m->fileHexHash, sizeof filehash); - str_toupper_inplace(filehash); - - if (!rhizome_exists(filehash)) - return WHY("File should already be stored by now"); - } else { - filehash[0] = '\0'; - } + if (m->fileLength > 0 && !rhizome_exists(&m->filehash)) + return WHY("File should already be stored by now"); const char *name = rhizome_manifest_get(m, "name", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0); @@ -1331,7 +1338,7 @@ int rhizome_store_bundle(rhizome_manifest *m) INT64, (int64_t) gettime_ms(), STATIC_BLOB, bar, RHIZOME_BAR_BYTES, INT64, m->fileLength, - TEXT_TOUPPER|NUL, filehash, + RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL, SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author, STATIC_TEXT, service, STATIC_TEXT|NUL, name, @@ -1627,15 +1634,28 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) const char *service = rhizome_manifest_get(m, "service", NULL, 0); if (service == NULL) return WHY("Manifest has no service"); - + const char *name = rhizome_manifest_get(m, "name", NULL, 0); - const char *sender = rhizome_manifest_get(m, "sender", NULL, 0); - const char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0); - + + sid_t *sender = NULL; + const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0); + if (sender_field) { + sender = (sid_t *) alloca(sizeof *sender); + if (str_to_sid_t(sender, sender_field) == -1) + return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field)); + } + + sid_t *recipient = NULL; + const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0); + if (recipient_field) { + recipient = (sid_t *) alloca(sizeof *recipient); + if (str_to_sid_t(recipient, recipient_field) == -1) + return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field)); + } + char sqlcmd[1024]; strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd); strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?"); - if (m->fileLength != 0) strbuf_puts(b, " AND filehash = ?"); if (name) @@ -1644,73 +1664,60 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) strbuf_puts(b, " AND sender = ?"); if (recipient) strbuf_puts(b, " AND recipient = ?"); - if (strbuf_overrun(b)) return WHYF("SQL command too long: %s", strbuf_str(b)); - + int ret = 0; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b)); + sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END); if (!statement) return -1; - - int field = 1; - sqlite3_bind_int(statement, field++, m->fileLength); - sqlite3_bind_text(statement, field++, service, -1, SQLITE_STATIC); - + int field = 2; if (m->fileLength != 0) - sqlite3_bind_text(statement, field++, m->fileHexHash, -1, SQLITE_STATIC); + sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END); if (name) - sqlite3_bind_text(statement, field++, name, -1, SQLITE_STATIC); + sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END); if (sender) - sqlite3_bind_text(statement, field++, sender, -1, SQLITE_STATIC); + sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END); if (recipient) - sqlite3_bind_text(statement, field++, recipient, -1, SQLITE_STATIC); - + sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END); + int rows = 0; while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { ++rows; if (config.debug.rhizome) DEBUGF("Row %d", rows); - rhizome_manifest *blob_m = rhizome_new_manifest(); if (blob_m == NULL) { ret = WHY("Out of manifests"); break; } - const unsigned char *q_manifestid = sqlite3_column_text(statement, 0); - const char *manifestblob = (char *) sqlite3_column_blob(statement, 1); size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) { WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); goto next; } - if (rhizome_manifest_verify(blob_m)) { WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid); goto next; } - const char *q_author = (const char *) sqlite3_column_text(statement, 2); if (q_author) { if (config.debug.rhizome) strbuf_sprintf(b, " .author=%s", q_author); str_to_sid_t(&blob_m->author, q_author); } - // check that we can re-author this manifest if (rhizome_extract_privatekey(blob_m, NULL)){ goto next; } - *found = blob_m; if (config.debug.rhizome) DEBUGF("Found duplicate payload, %s", q_manifestid); ret = 1; break; - next: if (blob_m) rhizome_manifest_free(blob_m); @@ -1807,14 +1814,14 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom return sqlite3_changes(rhizome_db) ? 0 : 1; } -static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid) +static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp) { int ret = 0; - rhizome_delete_external(fileid); - sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", TEXT_TOUPPER, fileid, END); + rhizome_delete_external(hashp); + sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END); if (!statement || sqlite_exec_retry(retry, statement) == -1) ret = -1; - statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", TEXT_TOUPPER, fileid, END); + statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END); if (!statement || sqlite_exec_retry(retry, statement) == -1) ret = -1; return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1; @@ -1826,7 +1833,10 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END); if (rows == -1) return -1; - if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1) + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, strbuf_str(fh)) == -1) + return WHYF("invalid field FILES.id=%s", strbuf_str(fh)); + if (rows && rhizome_delete_file_retry(retry, &hash) == -1) return -1; return 0; } @@ -1887,10 +1897,10 @@ int rhizome_delete_payload(const rhizome_bid_t *bidp) * * @author Andrew Bettison */ -int rhizome_delete_file(const char *fileid) +int rhizome_delete_file(const rhizome_filehash_t *hashp) { sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - return rhizome_delete_file_retry(&retry, fileid); + return rhizome_delete_file_retry(&retry, hashp); } static int is_interesting(const char *id_hex, int64_t version) @@ -1901,7 +1911,7 @@ static int is_interesting(const char *id_hex, int64_t version) // do we have this bundle [or later]? sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite3_stmt *statement = sqlite_prepare_bind(&retry, - "SELECT filehash FROM manifests WHERE id like ? and version >= ?", + "SELECT filehash FROM MANIFESTS WHERE id LIKE ? AND version >= ?", TEXT_TOUPPER, id_hex, INT64, version, END); @@ -1910,8 +1920,14 @@ static int is_interesting(const char *id_hex, int64_t version) if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){ const char *q_filehash = (const char *) sqlite3_column_text(statement, 0); ret=0; - if (q_filehash && *q_filehash && !rhizome_exists(q_filehash)) - ret=1; + if (q_filehash && *q_filehash) { + rhizome_filehash_t hash; + if (str_to_rhizome_filehash_t(&hash, q_filehash) == -1) { + WARNF("invalid field MANIFESTS.filehash=%s -- ignored", alloca_str_toprint(q_filehash)); + ret = 1; + } else if (!rhizome_exists(&hash)) + ret = 1; + } } sqlite3_finalize(statement); RETURN(ret); diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index ef3d334e..c44ea7fb 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -892,10 +892,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* Get filehash and size from manifest if present */ DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); - const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0); - DEBUGF("bundle file hash = '%s'",hash); - int64_t filesize = rhizome_manifest_get_ll(m, "filesize"); - DEBUGF("file size = %"PRId64,filesize); + DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash)); + DEBUGF("file size = %"PRId64, m->fileLength); int64_t version = rhizome_manifest_get_ll(m, "version"); DEBUGF("version = %"PRId64,version); @@ -923,7 +921,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) +m->manifest_all_bytes +strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */ +strlen(boundary) - +filesize + +m->fileLength +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n"); /* XXX For some reason the above is four bytes out, so fix that */ @@ -961,17 +959,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) /* send file contents */ { - char filehash[SHA512_DIGEST_STRING_LENGTH]; - if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, filehash) == -1) + rhizome_filehash_t filehash; + if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1) goto closeit; struct rhizome_read read; bzero(&read, sizeof read); - if (rhizome_open_read(&read, filehash)) + if (rhizome_open_read(&read, &filehash)) goto closeit; - int read_ofs; - for(read_ofs=0;read_ofsfileLength;){ unsigned char buffer[4096]; read.offset=read_ofs; int bytes_read = rhizome_read(&read, buffer, sizeof buffer); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 82981f68..b66a9c38 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -487,7 +487,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) slot->manifest->dataFileUnlinkOnFree = 0; strbuf r = strbuf_local(slot->request, sizeof slot->request); - strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", slot->manifest->fileHexHash); + strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash)); if (slot->manifest->journalTail>=0){ // if we're fetching a journal bundle, work out how many bytes we have of a previous version @@ -509,12 +509,12 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot) } strbuf_puts(r, "\r\n"); - + if (strbuf_overrun(r)) RETURN(WHY("request overrun")); slot->request_len = strbuf_len(r); - if (rhizome_open_write(&slot->write_state, slot->manifest->fileHexHash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT)) + if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT)) RETURN(-1); } else { strbuf r = strbuf_local(slot->request, sizeof slot->request); @@ -690,9 +690,9 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct for (i = 0; i < NQUEUES; ++i) { struct rhizome_fetch_slot *as = &rhizome_fetch_queues[i].active; const rhizome_manifest *am = as->manifest; - if (as->state != RHIZOME_FETCH_FREE && strcasecmp(m->fileHexHash, am->fileHexHash) == 0) { + if (as->state != RHIZOME_FETCH_FREE && cmp_rhizome_filehash_t(&m->filehash, &am->filehash) == 0) { if (config.debug.rhizome_rx) - DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, m->fileHexHash); + DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, alloca_tohex_rhizome_filehash_t(m->filehash)); RETURN(SAMEPAYLOAD); } } @@ -707,7 +707,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct DEBUGF(" is new"); // If the payload is already available, no need to fetch, so import now. - if (rhizome_exists(m->fileHexHash)){ + if (rhizome_exists(&m->filehash)){ if (config.debug.rhizome_rx) DEBUGF(" fetch not started - payload already present, so importing instead"); if (rhizome_add_manifest(m, m->ttl-1) == -1) @@ -1138,8 +1138,7 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){ if (start>=0 && start < slot->previous->fileLength && length>0){ if (config.debug.rhizome) DEBUGF("Copying %"PRId64" bytes from previous journal", length); - rhizome_journal_pipe(&slot->write_state, slot->previous->fileHexHash, - start, length); + rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length); } // and we don't need to do this again, so drop the manifest @@ -1274,10 +1273,11 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) } INFOF("Completed http request from %s:%u for file %s", buf, ntohs(slot->peer_ipandport.sin_port), - slot->manifest->fileHexHash); + alloca_tohex_rhizome_filehash_t(slot->manifest->filehash)); } else { INFOF("Completed MDP request from %s for file %s", - alloca_tohex_sid_t(slot->peer_sid), slot->manifest->fileHexHash); + alloca_tohex_sid_t(slot->peer_sid), + alloca_tohex_rhizome_filehash_t(slot->manifest->filehash)); } } else { /* This was to fetch the manifest, so now fetch the file if needed */ @@ -1409,7 +1409,8 @@ int rhizome_received_content(const unsigned char *bidprefix, if (m){ if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){ - INFOF("Completed MDP transfer in one hit for file %s", m->fileHexHash); + INFOF("Completed MDP transfer in one hit for file %s", + alloca_tohex_rhizome_filehash_t(m->filehash)); if (c) candidate_unqueue(c); } diff --git a/rhizome_http.c b/rhizome_http.c index 31e1acfc..3ae5c495 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -449,14 +449,15 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder, con if (!is_rhizome_http_enabled()) return 1; - if (!rhizome_str_is_file_hash(remainder)) + rhizome_filehash_t filehash; + if (str_to_rhizome_filehash_t(&filehash, remainder) == -1) return -1; bzero(&r->read_state, sizeof(r->read_state)); /* Refuse to honour HTTP request if required (used for debugging and testing transition from HTTP to MDP) */ - if (rhizome_open_read(&r->read_state, remainder)) + if (rhizome_open_read(&r->read_state, &filehash)) return 1; if (r->read_state.length==-1){ diff --git a/rhizome_store.c b/rhizome_store.c index 1703bcf8..5e761852 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -5,22 +5,22 @@ #define RHIZOME_BUFFER_MAXIMUM_SIZE (1024*1024) -int rhizome_exists(const char *fileHash) +int rhizome_exists(const rhizome_filehash_t *hashp) { int64_t gotfile = 0; - if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", TEXT_TOUPPER, fileHash, END) != 1) + if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", RHIZOME_FILEHASH_T, hashp, END) != 1) return 0; return gotfile; } -int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority) +int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority) { write->blob_fd=-1; - if (expectedFileHash){ - if (rhizome_exists(expectedFileHash)) + if (expectedHashp){ + if (rhizome_exists(expectedHashp)) return 1; - strlcpy(write->id, expectedFileHash, SHA512_DIGEST_STRING_LENGTH); + write->id = *expectedHashp; write->id_known=1; }else{ write->id_known=0; @@ -429,7 +429,7 @@ int rhizome_fail_write(struct rhizome_write *write) write->buffer_list=n->_next; free(n); } - rhizome_delete_file(write->id); + rhizome_delete_file(&write->id); return 0; } @@ -460,39 +460,39 @@ int rhizome_finish_write(struct rhizome_write *write) if (write_release_lock(write)) goto failure; - char hash_out[SHA512_DIGEST_STRING_LENGTH + 1]; - SHA512_End(&write->sha512_context, hash_out); - str_toupper_inplace(hash_out); + rhizome_filehash_t hash_out; + SHA512_Final(hash_out.binary, &write->sha512_context); + SHA512_End(&write->sha512_context, NULL); if (write->id_known) { - if (strcasecmp(write->id, hash_out) != 0) { - WHYF("expected filehash=%s, got %s", write->id, hash_out); + if (cmp_rhizome_filehash_t(&write->id, &hash_out) != 0) { + WHYF("expected filehash=%s, got %s", alloca_tohex_rhizome_filehash_t(write->id), alloca_tohex_rhizome_filehash_t(hash_out)); goto failure; } } else { - strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH); + write->id = hash_out; } sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - rhizome_remove_file_datainvalid(&retry, write->id); - if (rhizome_exists(write->id)) { + rhizome_remove_file_datainvalid(&retry, &write->id); + if (rhizome_exists(&write->id)) { // we've already got that payload, delete the new copy sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); if (config.debug.rhizome) - DEBUGF("File id='%s' already present, removed id='%"PRId64"'", write->id, write->temp_id); + DEBUGF("File id=%s already present, removed id='%"PRId64"'", alloca_tohex_rhizome_filehash_t(write->id), write->temp_id); } else { if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) goto dbfailure; // delete any half finished records - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", STATIC_TEXT, write->id, END); - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", STATIC_TEXT, write->id, END); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END); + sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END); if (sqlite_exec_void_retry( &retry, "UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?", - STATIC_TEXT, write->id, + RHIZOME_FILEHASH_T, &write->id, INT64, gettime_ms(), UINT64_TOSTR, write->temp_id, END @@ -507,7 +507,7 @@ int rhizome_finish_write(struct rhizome_write *write) WHYF("Failed to generate file path"); goto dbfailure; } - if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, write->id)){ + if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, alloca_tohex_rhizome_filehash_t(write->id))){ WHYF("Failed to generate file path"); goto dbfailure; } @@ -521,7 +521,7 @@ int rhizome_finish_write(struct rhizome_write *write) if (sqlite_exec_void_retry( &retry, "UPDATE FILEBLOBS SET id = ? WHERE rowid = ?", - STATIC_TEXT, write->id, + RHIZOME_FILEHASH_T, &write->id, INT64, write->blob_rowid, END ) == -1 @@ -531,7 +531,7 @@ int rhizome_finish_write(struct rhizome_write *write) if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1) goto dbfailure; if (config.debug.rhizome) - DEBUGF("Stored file %s", write->id); + DEBUGF("Stored file %s", alloca_tohex_rhizome_filehash_t(write->id)); } write->blob_rowid=-1; return 0; @@ -553,7 +553,7 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath) struct rhizome_write write; bzero(&write, sizeof(write)); - int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); + int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); if (ret!=0) return ret; @@ -583,7 +583,7 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length struct rhizome_write write; bzero(&write, sizeof(write)); - int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); + int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); if (ret!=0) return ret; @@ -624,7 +624,7 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath) rhizome_manifest_set_ll(m, "filesize", m->fileLength); if (m->fileLength == 0){ - m->fileHexHash[0] = '\0'; + m->filehash = RHIZOME_FILEHASH_NONE; rhizome_manifest_del(m, "filehash"); } return 0; @@ -676,8 +676,8 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath) if (rhizome_finish_write(&write)) goto failure; - strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); - rhizome_manifest_set(m, "filehash", m->fileHexHash); + m->filehash = write.id; + rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); return 0; failure: @@ -687,26 +687,24 @@ failure: /* Return -1 on error, 0 if file blob found, 1 if not found. */ -int rhizome_open_read(struct rhizome_read *read, const char *fileid) +int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp) { - strncpy(read->id, fileid, sizeof read->id); - read->id[RHIZOME_FILEHASH_STRLEN] = '\0'; - str_toupper_inplace(read->id); + read->id = *hashp; read->blob_rowid = -1; read->blob_fd = -1; - if (sqlite_exec_int64(&read->blob_rowid, + if (sqlite_exec_int64(&read->blob_rowid, "SELECT FILEBLOBS.rowid " "FROM FILEBLOBS, FILES " "WHERE FILEBLOBS.id = FILES.id" " AND FILES.id = ?" - " AND FILES.datavalid != 0", STATIC_TEXT, read->id, END) == -1) + " AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1) return -1; if (read->blob_rowid != -1) { read->length = -1; // discover the length on opening the db BLOB } else { // No row in FILEBLOBS, look for an external blob file. char blob_path[1024]; - if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, read->id)) + if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(read->id))) return -1; read->blob_fd = open(blob_path, O_RDONLY); if (read->blob_fd == -1) { @@ -787,13 +785,13 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf read_state->hash_offset += bytes_read; // if we hash everything and the has doesn't match, we need to delete the payload if (read_state->hash_offset>=read_state->length){ - char hash_out[SHA512_DIGEST_STRING_LENGTH+1]; - SHA512_End(&read_state->sha512_context, hash_out); - - if (strcasecmp(read_state->id, hash_out)){ + rhizome_filehash_t hash_out; + SHA512_Final(hash_out.binary, &read_state->sha512_context); + SHA512_End(&read_state->sha512_context, NULL); + if (cmp_rhizome_filehash_t(&read_state->id, &hash_out) != 0) { // hash failure, mark the payload as invalid read_state->invalid = 1; - RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out)); + RETURN(WHYF("Expected hash=%s, got %s", alloca_tohex_rhizome_filehash_t(read_state->id), alloca_tohex_rhizome_filehash_t(hash_out))); } } } @@ -874,7 +872,7 @@ int rhizome_read_close(struct rhizome_read *read) read->blob_fd = -1; if (read->invalid){ // delete payload! - rhizome_delete_file(read->id); + rhizome_delete_file(&read->id); } return 0; } @@ -980,8 +978,7 @@ int rhizome_cache_count() } // read a block of data, caching meta data for reuse -int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, - uint64_t fileOffset, unsigned char *buffer, int length) +int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length) { // look for a cached entry struct cache_entry **ptr = find_entry_location(&root, bidp, version); @@ -989,15 +986,13 @@ int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t t // if we don't have one yet, create one and open it if (!entry){ - char filehash[SHA512_DIGEST_STRING_LENGTH]; - if (rhizome_database_filehash_from_id(bidp, version, filehash) == -1) + rhizome_filehash_t filehash; + if (rhizome_database_filehash_from_id(bidp, version, &filehash) == -1) return -1; - entry = emalloc_zero(sizeof(struct cache_entry)); - - if (rhizome_open_read(&entry->read_state, filehash)){ + if (rhizome_open_read(&entry->read_state, &filehash)){ free(entry); - return WHYF("Payload %s not found", filehash); + return WHYF("Payload %s not found", alloca_tohex_rhizome_filehash_t(filehash)); } entry->bundle_id = *bidp; entry->version = version; @@ -1073,8 +1068,9 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom return 0; } -int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){ - int ret = rhizome_open_read(read_state, m->fileHexHash); +int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state) +{ + int ret = rhizome_open_read(read_state, &m->filehash); if (ret == 0) ret = read_derive_key(m, bsk, read_state); return ret; @@ -1100,12 +1096,12 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t * * Returns -1 on error, 0 if dumped successfully, 1 if not found. */ -int rhizome_dump_file(const char *id, const char *filepath, int64_t *length) +int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length) { struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - int ret = rhizome_open_read(&read_state, id); + int ret = rhizome_open_read(&read_state, hashp); if (ret == 0) { ret = write_file(&read_state, filepath); @@ -1141,13 +1137,12 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write, return 0; } -int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length) +int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length) { struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - if (rhizome_open_read(&read_state, fileHash)) + if (rhizome_open_read(&read_state, hashp)) return -1; - read_state.offset = start_offset; int ret = rhizome_pipe(&read_state, write, length); rhizome_read_close(&read_state); @@ -1182,7 +1177,7 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, if (copy_length>0){ // note that we don't need to bother decrypting the existing journal payload - ret = rhizome_journal_pipe(write, m->fileHexHash, advance_by, copy_length); + ret = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length); if (ret) goto failure; } @@ -1218,8 +1213,8 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64 if (ret) goto failure; - strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); - rhizome_manifest_set(m, "filehash", m->fileHexHash); + m->filehash = write.id; + rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); return 0; failure: @@ -1250,8 +1245,8 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t if (ret) goto failure; - strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); - rhizome_manifest_set(m, "filehash", m->fileHexHash); + m->filehash = write.id; + rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash)); return 0;