ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Issue #11: Use rhizome_filehash_t everywhere

Browse Source
This commit is contained in:
Andrew Bettison 2013-10-10 18:23:06 +10:30
parent c97bd4a69a
commit 974c7a56a0
11 changed files with 316 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
commandline.c
View File

@ -1275,10 +1275,11 @@ int app_rhizome_hash_file(const struct cli_parsed *parsed, struct cli_context *c
return the hash of the file unencrypted. */ return the hash of the file unencrypted. */
const char *filepath; const char *filepath;
cli_arg(parsed, "filepath", &filepath, NULL, ""); cli_arg(parsed, "filepath", &filepath, NULL, "");
char hexhash[RHIZOME_FILEHASH_STRLEN + 1]; rhizome_filehash_t hash;
if (rhizome_hash_file(NULL, filepath, hexhash)) uint64_t size;
if (rhizome_hash_file(NULL, filepath, &hash, &size) == -1)
return -1; return -1;
cli_put_string(context, hexhash, "\n"); cli_put_string(context, size ? alloca_tohex_rhizome_filehash_t(hash) : "", "\n");
return 0; return 0;
} }
@ -1425,7 +1426,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
cli_put_long(context, mout->fileLength, "\n"); cli_put_long(context, mout->fileLength, "\n");
if (mout->fileLength != 0) { if (mout->fileLength != 0) {
cli_field_name(context, "filehash", ":"); cli_field_name(context, "filehash", ":");
cli_put_string(context, mout->fileHexHash, "\n"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n");
} }
const char *name = rhizome_manifest_get(mout, "name", NULL, 0); const char *name = rhizome_manifest_get(mout, "name", NULL, 0);
if (name) { if (name) {
@ -1528,7 +1529,7 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
cli_put_long(context, m->fileLength, "\n"); cli_put_long(context, m->fileLength, "\n");
if (m->fileLength != 0) { if (m->fileLength != 0) {
cli_field_name(context, "filehash", ":"); cli_field_name(context, "filehash", ":");
cli_put_string(context, m->fileHexHash, "\n"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
} }
const char *name = rhizome_manifest_get(m, "name", NULL, 0); const char *name = rhizome_manifest_get(m, "name", NULL, 0);
if (name) { if (name) {
@ -1588,12 +1589,10 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) { if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
if (!fileid) if (!fileid)
return WHY("missing <fileid> argument"); return WHY("missing <fileid> argument");
unsigned char filehash[RHIZOME_FILEHASH_BYTES]; rhizome_filehash_t hash;
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1) if (str_to_rhizome_filehash_t(&hash, fileid) == -1)
return WHY("Invalid file ID"); return WHYF("invalid <fileid> argument: %s", alloca_str_toprint(fileid));
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1]; ret = rhizome_delete_file(&hash);
tohex(fileIDUpper, RHIZOME_FILEHASH_STRLEN, filehash);
ret = rhizome_delete_file(fileIDUpper);
} else { } else {
if (!manifestid) if (!manifestid)
return WHY("missing <manifestid> argument"); return WHY("missing <manifestid> argument");
@ -1688,7 +1687,8 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
cli_field_name(context, ".readonly", ":"); cli_put_long(context, m->haveSecret?0:1, "\n"); cli_field_name(context, ".readonly", ":"); cli_put_long(context, m->haveSecret?0:1, "\n");
cli_field_name(context, "filesize", ":"); cli_put_long(context, m->fileLength, "\n"); cli_field_name(context, "filesize", ":"); cli_put_long(context, m->fileLength, "\n");
if (m->fileLength != 0) { if (m->fileLength != 0) {
cli_field_name(context, "filehash", ":"); cli_put_string(context, m->fileHexHash, "\n"); cli_field_name(context, "filehash", ":");
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
} }
} }
@ -1702,7 +1702,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
}else{ }else{
// Save the file without attempting to decrypt // Save the file without attempting to decrypt
int64_t length; int64_t length;
retfile = rhizome_dump_file(m->fileHexHash, filepath, &length); retfile = rhizome_dump_file(&m->filehash, filepath, &length);
} }
} }
@ -1741,18 +1741,21 @@ int app_rhizome_export_file(const struct cli_parsed *parsed, struct cli_context
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1 if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1) || cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1; return -1;
rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, fileid) == -1)
return WHYF("invalid <fileid> argument: %s", alloca_str_toprint(fileid));
if (create_serval_instance_dir() == -1) if (create_serval_instance_dir() == -1)
return -1; return -1;
if (rhizome_opendb() == -1) if (rhizome_opendb() == -1)
return -1; return -1;
if (!rhizome_exists(fileid)) if (!rhizome_exists(&hash))
return 1; return 1;
int64_t length; int64_t length;
int ret = rhizome_dump_file(fileid, filepath, &length); int ret = rhizome_dump_file(&hash, filepath, &length);
if (ret) if (ret)
return ret == -1 ? -1 : 1; return ret == -1 ? -1 : 1;
cli_field_name(context, "filehash", ":"); cli_field_name(context, "filehash", ":");
cli_put_string(context, fileid, "\n"); cli_put_string(context, alloca_tohex_rhizome_filehash_t(hash), "\n");
cli_field_name(context, "filesize", ":"); cli_field_name(context, "filesize", ":");
cli_put_long(context, length, "\n"); cli_put_long(context, length, "\n");
return 0; return 0;

22
dataformats.c
View File

@ -94,6 +94,28 @@ int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp
return 0; return 0;
} }
int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b)
{
return memcmp(a, b, sizeof a->binary);
}
int str_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex)
{
return fromhexstr(hashp->binary, hex, sizeof hashp->binary);
}
int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const char **endp)
{
rhizome_filehash_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
*hashp = tmp;
if (endp)
*endp = hex + sizeof hashp->binary * 2;
return 0;
}
int rhizome_strn_is_manifest_id(const char *id) int rhizome_strn_is_manifest_id(const char *id)
{ {
return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN); return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN);

7
meshms.c
View File

@ -561,7 +561,8 @@ static int write_conversation(struct rhizome_write *write, struct conversations
return len; return len;
} }
static int write_known_conversations(rhizome_manifest *m, struct conversations *conv){ static int write_known_conversations(rhizome_manifest *m, struct conversations *conv)
{
rhizome_manifest *mout=NULL; rhizome_manifest *mout=NULL;
struct rhizome_write write; struct rhizome_write write;
@ -590,8 +591,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
goto end; goto end;
if (rhizome_finish_write(&write)) if (rhizome_finish_write(&write))
goto end; goto end;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); m->filehash = write.id;
rhizome_manifest_set(m, "filehash", m->fileHexHash); rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
if (rhizome_manifest_finalise(m, &mout, 1)) if (rhizome_manifest_finalise(m, &mout, 1))
goto end; goto end;

8
rhizome.c
View File

@ -73,7 +73,7 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
filepath ? alloca_str_toprint(filepath) : "NULL"); filepath ? alloca_str_toprint(filepath) : "NULL");
unsigned char buffer[MAX_MANIFEST_BYTES]; unsigned char buffer[MAX_MANIFEST_BYTES];
int buffer_len=0; size_t buffer_len = 0;
// manifest has been appended to the end of the file. // manifest has been appended to the end of the file.
if (strcmp(manifest_path, filepath)==0){ if (strcmp(manifest_path, filepath)==0){
@ -251,10 +251,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
if (rhizome_manifest_check_sanity(m_in)) if (rhizome_manifest_check_sanity(m_in))
return -1; return -1;
if (m_in->fileLength){ if (m_in->fileLength && !rhizome_exists(&m_in->filehash))
if (!rhizome_exists(m_in->fileHexHash)) return WHY("File has not been imported");
return WHY("File has not been imported");
}
/* If the manifest already has an ID */ /* If the manifest already has an ID */
if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic)) if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic))

102
rhizome.h
View File

@ -52,6 +52,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define RHIZOME_HTTP_PORT 4110 #define RHIZOME_HTTP_PORT 4110
#define RHIZOME_HTTP_PORT_MAX 4150 #define RHIZOME_HTTP_PORT_MAX 4150
/* Fundamental data type: Rhizome Bundle ID
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
typedef struct rhizome_bid_binary { typedef struct rhizome_bid_binary {
unsigned char binary[RHIZOME_MANIFEST_ID_BYTES]; unsigned char binary[RHIZOME_MANIFEST_ID_BYTES];
} rhizome_bid_t; } rhizome_bid_t;
@ -65,6 +70,28 @@ int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b);
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex); int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex);
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp); int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp);
/* Fundamental data type: Rhizome File Hash
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
typedef struct rhizome_filehash_binary {
unsigned char binary[RHIZOME_FILEHASH_BYTES];
} rhizome_filehash_t;
#define RHIZOME_FILEHASH_NONE ((rhizome_filehash_t){{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}})
#define rhizome_filehash_t_is_zero(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0)
#define rhizome_filehash_t_is_max(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0xff)
#define alloca_tohex_rhizome_filehash_t(fh) alloca_tohex((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary)
int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b);
int str_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex);
int strn_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, const char **endp);
/* Fundamental data type: Rhizome Bundle Key
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
typedef struct rhizome_bk_binary { typedef struct rhizome_bk_binary {
unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES]; unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES];
} rhizome_bk_t; } rhizome_bk_t;
@ -77,6 +104,7 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) {
#define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary) #define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary)
extern time_ms_t rhizome_voice_timeout; extern time_ms_t rhizome_voice_timeout;
#define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE #define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE
@ -158,7 +186,7 @@ typedef struct rhizome_manifest {
group membership handy */ group membership handy */
int64_t fileLength; int64_t fileLength;
int64_t journalTail; int64_t journalTail;
char fileHexHash[SHA512_DIGEST_STRING_LENGTH]; rhizome_filehash_t filehash;
int fileHighestPriority; int fileHighestPriority;
/* Absolute path of the file associated with the manifest */ /* Absolute path of the file associated with the manifest */
@ -258,10 +286,10 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append); int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m); int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const char *id,int maximum_priority); int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp); int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize); int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize);
int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out); int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen); char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen);
int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var); int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var);
int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value); int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value);
@ -274,7 +302,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence);
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__) #define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
int rhizome_manifest_pack_variables(rhizome_manifest *m); int rhizome_manifest_pack_variables(rhizome_manifest *m);
int rhizome_store_bundle(rhizome_manifest *m); int rhizome_store_bundle(rhizome_manifest *m);
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid); int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
int rhizome_clean_payload(const char *fileidhex); int rhizome_clean_payload(const char *fileidhex);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key); int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
@ -308,25 +336,25 @@ int is_debug_rhizome_ads();
enum sqlbind_type { enum sqlbind_type {
END = 0xbabecafe, END = 0xbabecafe,
INT = 1, // int value INT = 1, // int value
INT_TOSTR, // int value INT_TOSTR, // int value
UINT_TOSTR, // unsigned value UINT_TOSTR, // unsigned value
INT64, // int64_t value INT64, // int64_t value
INT64_TOSTR, // int64_t value INT64_TOSTR, // int64_t value
UINT64_TOSTR, // uint64_t value UINT64_TOSTR, // uint64_t value
TEXT, // const char *text, TEXT, // const char *text,
TEXT_LEN, // const char *text, int bytes TEXT_LEN, // const char *text, int bytes
STATIC_TEXT, // const char *text, STATIC_TEXT, // const char *text,
STATIC_TEXT_LEN, // const char *text, int bytes STATIC_TEXT_LEN, // const char *text, int bytes
STATIC_BLOB, // const void *blob, int bytes STATIC_BLOB, // const void *blob, int bytes
ZEROBLOB, // int bytes ZEROBLOB, // int bytes
SID_T, // const sid_t *sidp SID_T, // const sid_t *sidp
RHIZOME_BID_T, // const rhizome_bid_t *bidp RHIZOME_BID_T, // const rhizome_bid_t *bidp
FILEHASH_T, // const unsigned char hash_binary[RHIZOME_FILEHASH_BYTES] RHIZOME_FILEHASH_T, // const rhizome_filehash_t *hashp
TOHEX, // const unsigned char *binary, unsigned bytes TOHEX, // const unsigned char *binary, unsigned bytes
TEXT_TOUPPER, // const char *text, TEXT_TOUPPER, // const char *text,
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ... NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ...
INDEX = 0xfade0000, // INDEX|INT, int index, ... INDEX = 0xfade0000, // INDEX|INT, int index, ...
NAMED = 0xdead0000 // NAMED|INT, const char *label, ... NAMED = 0xdead0000 // NAMED|INT, const char *label, ...
}; };
@ -390,7 +418,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
int rhizome_delete_bundle(const rhizome_bid_t *bidp); int rhizome_delete_bundle(const rhizome_bid_t *bidp);
int rhizome_delete_manifest(const rhizome_bid_t *bidp); int rhizome_delete_manifest(const rhizome_bid_t *bidp);
int rhizome_delete_payload(const rhizome_bid_t *bidp); int rhizome_delete_payload(const rhizome_bid_t *bidp);
int rhizome_delete_file(const char *fileid); int rhizome_delete_file(const rhizome_filehash_t *hashp);
#define RHIZOME_DONTVERIFY 0 #define RHIZOME_DONTVERIFY 0
#define RHIZOME_VERIFY 1 #define RHIZOME_VERIFY 1
@ -439,7 +467,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_length); rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_length);
/* Rhizome file storage api */ /* Rhizome file storage api */
struct rhizome_write_buffer{ struct rhizome_write_buffer
{
struct rhizome_write_buffer *_next; struct rhizome_write_buffer *_next;
int64_t offset; int64_t offset;
int buffer_size; int buffer_size;
@ -447,8 +476,9 @@ struct rhizome_write_buffer{
unsigned char data[0]; unsigned char data[0];
}; };
struct rhizome_write{ struct rhizome_write
char id[SHA512_DIGEST_STRING_LENGTH+1]; {
rhizome_filehash_t id;
uint64_t temp_id; uint64_t temp_id;
char id_known; char id_known;
@ -475,8 +505,9 @@ struct rhizome_read_buffer{
int len; int len;
}; };
struct rhizome_read{ struct rhizome_read
char id[SHA512_DIGEST_STRING_LENGTH+1]; {
rhizome_filehash_t id;
int crypt; int crypt;
unsigned char key[RHIZOME_CRYPT_KEY_BYTES]; unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
@ -741,8 +772,8 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
/* rhizome storage methods */ /* rhizome storage methods */
int rhizome_exists(const char *fileHash); int rhizome_exists(const rhizome_filehash_t *hashp);
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority); int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size); int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size);
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size); int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size);
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m); int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
@ -758,23 +789,22 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size); int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len); int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len);
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename); int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length); int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce); const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const char *fileid); int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length); int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len); int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
int rhizome_read_close(struct rhizome_read *read); int rhizome_read_close(struct rhizome_read *read);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length); int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length);
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, int length); uint64_t fileOffset, unsigned char *buffer, int length);
int rhizome_cache_close(); int rhizome_cache_close();
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]); int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);
int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp); int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp);
int rhizome_sync_announce(); int rhizome_sync_announce();

52
rhizome_bundle.c
View File

@ -169,14 +169,13 @@ int rhizome_manifest_parse(rhizome_manifest *m)
} }
} else if (strcasecmp(var, "filehash") == 0) { } else if (strcasecmp(var, "filehash") == 0) {
have_filehash = 1; have_filehash = 1;
if (!rhizome_str_is_file_hash(value)) { if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) {
if (config.debug.rejecteddata) if (config.debug.rejecteddata)
WARNF("Invalid filehash: %s", value); WARNF("Invalid filehash: %s", value);
m->errors++; m->errors++;
} else { } else {
/* Force to upper case to avoid case sensitive comparison problems later. */ /* Force to upper case to avoid case sensitive comparison problems later. */
str_toupper_inplace(m->values[m->var_count]); str_toupper_inplace(m->values[m->var_count]);
strcpy(m->fileHexHash, m->values[m->var_count]);
} }
} else if (strcasecmp(var, "filesize") == 0) { } else if (strcasecmp(var, "filesize") == 0) {
have_filesize = 1; have_filesize = 1;
@ -344,7 +343,7 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bu
return rhizome_manifest_parse(m); return rhizome_manifest_parse(m);
} }
int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out) int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out)
{ {
/* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one /* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(), go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(),
@ -355,35 +354,36 @@ int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out)
if (m && m->payloadEncryption) if (m && m->payloadEncryption)
return WHY("Encryption of payloads not implemented"); return WHY("Encryption of payloads not implemented");
size_t filesize = 0; uint64_t filesize = 0;
SHA512_CTX context; SHA512_CTX context;
SHA512_Init(&context); SHA512_Init(&context);
if (filename[0]) { if (path[0]) {
FILE *f = fopen(filename, "r"); int fd = open(path, O_RDONLY);
if (!f) { if (fd == -1)
WHY_perror("fopen"); return WHYF_perror("open(%s,O_RDONLY)", alloca_str_toprint(path));
return WHYF("Could not open %s to calculate SHA512 hash.", filename); unsigned char buffer[8192];
} ssize_t r;
while (!feof(f)) { while ((r = read(fd, buffer, sizeof buffer))) {
unsigned char buffer[8192];
int r = fread(buffer, 1, 8192, f);
if (r == -1) { if (r == -1) {
WHY_perror("fread"); WHYF_perror("read(%s,%u)", alloca_str_toprint(path), sizeof buffer);
fclose(f); close(fd);
return WHYF("Error reading %s to calculate SHA512 hash", filename); return -1;
} }
if (r > 0) SHA512_Update(&context, buffer, (size_t) r);
SHA512_Update(&context, buffer, r); filesize += (size_t) r;
filesize += r;
} }
fclose(f); close(fd);
} }
SHA512_End(&context, (char *)hash_out); // Empty files (including empty path) have no hash.
// Empty files (including null filename) have no hash. if (hash_out) {
if (filesize > 0) if (filesize > 0)
str_toupper_inplace(hash_out); SHA512_Final(hash_out->binary, &context);
else else
hash_out[0] = '\0'; *hash_out = RHIZOME_FILEHASH_NONE;
}
if (size_out)
*size_out = filesize;
SHA512_End(&context, NULL);
return 0; return 0;
} }

180
rhizome_database.c
View File

@ -32,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
static char rhizome_thisdatastore_path[256]; static char rhizome_thisdatastore_path[256];
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid); static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
const char *rhizome_datastore_path() const char *rhizome_datastore_path()
@ -678,14 +678,16 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
} }
} }
break; break;
case FILEHASH_T: { case RHIZOME_FILEHASH_T: {
const char *hash_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_FILEHASH_BYTES); const rhizome_filehash_t *hashp = va_arg(ap, const rhizome_filehash_t *);
++argnum; ++argnum;
if (hash_hex == NULL) { if (hashp == NULL) {
BIND_NULL(FILEHASH_T); BIND_NULL(RHIZOME_FILEHASH_T);
} else { } else {
BIND_DEBUG(FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN); char hash_hex[RHIZOME_FILEHASH_STRLEN];
BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT); tohex(hash_hex, sizeof hash_hex, hashp->binary);
BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, sizeof hash_hex);
BIND_RETRY(sqlite3_bind_text, hash_hex, sizeof hash_hex, SQLITE_TRANSIENT);
} }
} }
break; break;
@ -1014,20 +1016,24 @@ int64_t rhizome_database_used_bytes()
return db_page_size * (db_page_count - db_free_page_count); return db_page_size * (db_page_count - db_free_page_count);
} }
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]) int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp)
{ {
IN(); IN();
strbuf hash_sb = strbuf_local(hash, SHA512_DIGEST_STRING_LENGTH); strbuf hash_sb = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;", if ( sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;",
INT64, version, RHIZOME_BID_T, bidp, END)); INT64, version, RHIZOME_BID_T, bidp, END) == -1)
RETURN(-1);
if (strbuf_overrun(hash_sb) || str_to_rhizome_filehash_t(hashp, strbuf_str(hash_sb)) == -1)
RETURN(WHYF("malformed file hash for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(*bidp), version));
RETURN(0);
OUT(); OUT();
} }
static int rhizome_delete_external(const char *fileid) static int rhizome_delete_external(const rhizome_filehash_t *hashp)
{ {
// attempt to remove any external blob // attempt to remove any external blob
char blob_path[1024]; char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid)) if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(*hashp)))
return -1; return -1;
return unlink(blob_path); return unlink(blob_path);
} }
@ -1039,18 +1045,18 @@ static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
END); END);
} }
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid) int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
{ {
int ret = 0; int ret = 0;
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
"DELETE FROM FILES WHERE id = ? and datavalid = 0;", "DELETE FROM FILES WHERE id = ? and datavalid = 0;",
TEXT_TOUPPER, fileid, END RHIZOME_FILEHASH_T, hashp, END
) == -1 ) == -1
) )
ret = -1; ret = -1;
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry, if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
"DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );", "DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
TEXT_TOUPPER, fileid, END RHIZOME_FILEHASH_T, hashp, END
) == -1 ) == -1
) )
ret = -1; ret = -1;
@ -1081,18 +1087,24 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
candidates++; candidates++;
const char *id = (const char *) sqlite3_column_text(statement, 0); const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_delete_external(id) == 0 && report) rhizome_filehash_t hash;
++report->deleted_stale_incoming_files; if (str_to_rhizome_filehash_t(&hash, id) == -1)
WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
else if (rhizome_delete_external(&hash) == 0 && report)
++report->deleted_stale_incoming_files;
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
statement = sqlite_prepare_bind(&retry, statement = sqlite_prepare_bind(&retry,
"SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
INT64, insert_horizon_no_manifest, END); INT64, insert_horizon_no_manifest, END);
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
candidates++; candidates++;
const char *id = (const char *) sqlite3_column_text(statement, 0); const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_delete_external(id) == 0 && report) rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, id) == -1)
WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
else if (rhizome_delete_external(&hash) == 0 && report)
++report->deleted_orphan_files; ++report->deleted_orphan_files;
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
@ -1152,11 +1164,10 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW && sqlite_step_retry(&retry, statement) == SQLITE_ROW
) { ) {
/* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */ /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
const unsigned char *id; const char *id;
/* Get values */ /* Get values */
if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
id = sqlite3_column_text(statement, 0); id = (const char *) sqlite3_column_text(statement, 0);
else { else {
WHY("Incorrect type in id column of files table"); WHY("Incorrect type in id column of files table");
break; break;
@ -1167,10 +1178,16 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
WHY("Incorrect type in length column of files table"); WHY("Incorrect type in length column of files table");
break; break;
} }
/* Try to drop this file from storage, discarding any references that do not trump the priority rhizome_filehash_t hash;
of this request. The query done earlier should ensure this, but it doesn't hurt to be if (str_to_rhizome_filehash_t(&hash, id) == -1)
paranoid, and it also protects against inconsistency in the database. */ WHYF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
rhizome_drop_stored_file((char *)id, group_priority + 1); else {
/* Try to drop this file from storage, discarding any references that do not trump the
* priority of this request. The query done earlier should ensure this, but it doesn't hurt
* to be paranoid, and it also protects against inconsistency in the database.
*/
rhizome_drop_stored_file(&hash, group_priority + 1);
}
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
@ -1187,14 +1204,12 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
/* Drop the specified file from storage, and any manifests that reference it, provided that none of /* Drop the specified file from storage, and any manifests that reference it, provided that none of
* those manifests are being retained at a higher priority than the maximum specified here. * those manifests are being retained at a higher priority than the maximum specified here.
*/ */
int rhizome_drop_stored_file(const char *id, int maximum_priority) int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority)
{ {
if (!rhizome_str_is_file_hash(id))
return WHYF("invalid file hash id=%s", alloca_str_toprint(id));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", TEXT_TOUPPER, id, END); sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement) if (!statement)
return WHYF("Could not drop stored file id=%s", id); return WHYF("Could not drop stored file id=%s", alloca_tohex_rhizome_filehash_t(*hashp));
int can_drop = 1; int can_drop = 1;
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
/* Find manifests for this file */ /* Find manifests for this file */
@ -1214,9 +1229,9 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority)
that are lower priority, and thus free up a little space. */ that are lower priority, and thus free up a little space. */
int priority = rhizome_manifest_priority(&retry, &bid); int priority = rhizome_manifest_priority(&retry, &bid);
if (priority == -1) if (priority == -1)
WHYF("Cannot drop fileid=%s due to error, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); WHYF("Cannot drop fileid=%s due to error, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
else if (priority > maximum_priority) { else if (priority > maximum_priority) {
WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", id, alloca_tohex_rhizome_bid_t(bid)); WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
can_drop = 0; can_drop = 0;
} else { } else {
if (config.debug.rhizome) if (config.debug.rhizome)
@ -1228,7 +1243,7 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority)
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
if (can_drop) if (can_drop)
rhizome_delete_file_retry(&retry, id); rhizome_delete_file_retry(&retry, hashp);
return 0; return 0;
} }
@ -1272,16 +1287,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
rhizome_manifest_to_bar(m,bar); rhizome_manifest_to_bar(m,bar);
/* Store the file (but not if it is already in the database) */ /* Store the file (but not if it is already in the database) */
char filehash[RHIZOME_FILEHASH_STRLEN + 1]; if (m->fileLength > 0 && !rhizome_exists(&m->filehash))
if (m->fileLength > 0) { return WHY("File should already be stored by now");
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
if (!rhizome_exists(filehash))
return WHY("File should already be stored by now");
} else {
filehash[0] = '\0';
}
const char *name = rhizome_manifest_get(m, "name", NULL, 0); const char *name = rhizome_manifest_get(m, "name", NULL, 0);
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
@ -1331,7 +1338,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
INT64, (int64_t) gettime_ms(), INT64, (int64_t) gettime_ms(),
STATIC_BLOB, bar, RHIZOME_BAR_BYTES, STATIC_BLOB, bar, RHIZOME_BAR_BYTES,
INT64, m->fileLength, INT64, m->fileLength,
TEXT_TOUPPER|NUL, filehash, RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL,
SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author, SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author,
STATIC_TEXT, service, STATIC_TEXT, service,
STATIC_TEXT|NUL, name, STATIC_TEXT|NUL, name,
@ -1627,15 +1634,28 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service == NULL) if (service == NULL)
return WHY("Manifest has no service"); return WHY("Manifest has no service");
const char *name = rhizome_manifest_get(m, "name", NULL, 0); const char *name = rhizome_manifest_get(m, "name", NULL, 0);
const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
const char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0); sid_t *sender = NULL;
const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
if (sender_field) {
sender = (sid_t *) alloca(sizeof *sender);
if (str_to_sid_t(sender, sender_field) == -1)
return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
}
sid_t *recipient = NULL;
const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
if (recipient_field) {
recipient = (sid_t *) alloca(sizeof *recipient);
if (str_to_sid_t(recipient, recipient_field) == -1)
return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
}
char sqlcmd[1024]; char sqlcmd[1024];
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd); strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?"); strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
if (m->fileLength != 0) if (m->fileLength != 0)
strbuf_puts(b, " AND filehash = ?"); strbuf_puts(b, " AND filehash = ?");
if (name) if (name)
@ -1644,73 +1664,60 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
strbuf_puts(b, " AND sender = ?"); strbuf_puts(b, " AND sender = ?");
if (recipient) if (recipient)
strbuf_puts(b, " AND recipient = ?"); strbuf_puts(b, " AND recipient = ?");
if (strbuf_overrun(b)) if (strbuf_overrun(b))
return WHYF("SQL command too long: %s", strbuf_str(b)); return WHYF("SQL command too long: %s", strbuf_str(b));
int ret = 0; int ret = 0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b)); sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END);
if (!statement) if (!statement)
return -1; return -1;
int field = 2;
int field = 1;
sqlite3_bind_int(statement, field++, m->fileLength);
sqlite3_bind_text(statement, field++, service, -1, SQLITE_STATIC);
if (m->fileLength != 0) if (m->fileLength != 0)
sqlite3_bind_text(statement, field++, m->fileHexHash, -1, SQLITE_STATIC); sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END);
if (name) if (name)
sqlite3_bind_text(statement, field++, name, -1, SQLITE_STATIC); sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END);
if (sender) if (sender)
sqlite3_bind_text(statement, field++, sender, -1, SQLITE_STATIC); sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END);
if (recipient) if (recipient)
sqlite3_bind_text(statement, field++, recipient, -1, SQLITE_STATIC); sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END);
int rows = 0; int rows = 0;
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
++rows; ++rows;
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("Row %d", rows); DEBUGF("Row %d", rows);
rhizome_manifest *blob_m = rhizome_new_manifest(); rhizome_manifest *blob_m = rhizome_new_manifest();
if (blob_m == NULL) { if (blob_m == NULL) {
ret = WHY("Out of manifests"); ret = WHY("Out of manifests");
break; break;
} }
const unsigned char *q_manifestid = sqlite3_column_text(statement, 0); const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1); const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) { if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
goto next; goto next;
} }
if (rhizome_manifest_verify(blob_m)) { if (rhizome_manifest_verify(blob_m)) {
WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid); WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
goto next; goto next;
} }
const char *q_author = (const char *) sqlite3_column_text(statement, 2); const char *q_author = (const char *) sqlite3_column_text(statement, 2);
if (q_author) { if (q_author) {
if (config.debug.rhizome) if (config.debug.rhizome)
strbuf_sprintf(b, " .author=%s", q_author); strbuf_sprintf(b, " .author=%s", q_author);
str_to_sid_t(&blob_m->author, q_author); str_to_sid_t(&blob_m->author, q_author);
} }
// check that we can re-author this manifest // check that we can re-author this manifest
if (rhizome_extract_privatekey(blob_m, NULL)){ if (rhizome_extract_privatekey(blob_m, NULL)){
goto next; goto next;
} }
*found = blob_m; *found = blob_m;
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("Found duplicate payload, %s", q_manifestid); DEBUGF("Found duplicate payload, %s", q_manifestid);
ret = 1; ret = 1;
break; break;
next: next:
if (blob_m) if (blob_m)
rhizome_manifest_free(blob_m); rhizome_manifest_free(blob_m);
@ -1807,14 +1814,14 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom
return sqlite3_changes(rhizome_db) ? 0 : 1; return sqlite3_changes(rhizome_db) ? 0 : 1;
} }
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid) static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
{ {
int ret = 0; int ret = 0;
rhizome_delete_external(fileid); rhizome_delete_external(hashp);
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", TEXT_TOUPPER, fileid, END); sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1) if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1; ret = -1;
statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", TEXT_TOUPPER, fileid, END); statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1) if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1; ret = -1;
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1; return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
@ -1826,7 +1833,10 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END); int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
if (rows == -1) if (rows == -1)
return -1; return -1;
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1) rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, strbuf_str(fh)) == -1)
return WHYF("invalid field FILES.id=%s", strbuf_str(fh));
if (rows && rhizome_delete_file_retry(retry, &hash) == -1)
return -1; return -1;
return 0; return 0;
} }
@ -1887,10 +1897,10 @@ int rhizome_delete_payload(const rhizome_bid_t *bidp)
* *
* @author Andrew Bettison <andrew@servalproject.com> * @author Andrew Bettison <andrew@servalproject.com>
*/ */
int rhizome_delete_file(const char *fileid) int rhizome_delete_file(const rhizome_filehash_t *hashp)
{ {
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_retry(&retry, fileid); return rhizome_delete_file_retry(&retry, hashp);
} }
static int is_interesting(const char *id_hex, int64_t version) static int is_interesting(const char *id_hex, int64_t version)
@ -1901,7 +1911,7 @@ static int is_interesting(const char *id_hex, int64_t version)
// do we have this bundle [or later]? // do we have this bundle [or later]?
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT filehash FROM manifests WHERE id like ? and version >= ?", "SELECT filehash FROM MANIFESTS WHERE id LIKE ? AND version >= ?",
TEXT_TOUPPER, id_hex, TEXT_TOUPPER, id_hex,
INT64, version, INT64, version,
END); END);
@ -1910,8 +1920,14 @@ static int is_interesting(const char *id_hex, int64_t version)
if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){ if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){
const char *q_filehash = (const char *) sqlite3_column_text(statement, 0); const char *q_filehash = (const char *) sqlite3_column_text(statement, 0);
ret=0; ret=0;
if (q_filehash && *q_filehash && !rhizome_exists(q_filehash)) if (q_filehash && *q_filehash) {
ret=1; rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, q_filehash) == -1) {
WARNF("invalid field MANIFESTS.filehash=%s -- ignored", alloca_str_toprint(q_filehash));
ret = 1;
} else if (!rhizome_exists(&hash))
ret = 1;
}
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
RETURN(ret); RETURN(ret);

18
rhizome_direct_http.c
View File

@ -892,10 +892,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* Get filehash and size from manifest if present */ /* Get filehash and size from manifest if present */
DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0); DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
DEBUGF("bundle file hash = '%s'",hash); DEBUGF("file size = %"PRId64, m->fileLength);
int64_t filesize = rhizome_manifest_get_ll(m, "filesize");
DEBUGF("file size = %"PRId64,filesize);
int64_t version = rhizome_manifest_get_ll(m, "version"); int64_t version = rhizome_manifest_get_ll(m, "version");
DEBUGF("version = %"PRId64,version); DEBUGF("version = %"PRId64,version);
@ -923,7 +921,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
+m->manifest_all_bytes +m->manifest_all_bytes
+strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */ +strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */
+strlen(boundary) +strlen(boundary)
+filesize +m->fileLength
+strlen("\r\n--")+strlen(boundary)+strlen("--\r\n"); +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n");
/* XXX For some reason the above is four bytes out, so fix that */ /* XXX For some reason the above is four bytes out, so fix that */
@ -961,17 +959,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* send file contents */ /* send file contents */
{ {
char filehash[SHA512_DIGEST_STRING_LENGTH]; rhizome_filehash_t filehash;
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, filehash) == -1) if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1)
goto closeit; goto closeit;
struct rhizome_read read; struct rhizome_read read;
bzero(&read, sizeof read); bzero(&read, sizeof read);
if (rhizome_open_read(&read, filehash)) if (rhizome_open_read(&read, &filehash))
goto closeit; goto closeit;
int read_ofs; int64_t read_ofs;
for(read_ofs=0;read_ofs<filesize;){ for(read_ofs=0;read_ofs<m->fileLength;){
unsigned char buffer[4096]; unsigned char buffer[4096];
read.offset=read_ofs; read.offset=read_ofs;
int bytes_read = rhizome_read(&read, buffer, sizeof buffer); int bytes_read = rhizome_read(&read, buffer, sizeof buffer);

23
rhizome_fetch.c
View File

@ -487,7 +487,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
slot->manifest->dataFileUnlinkOnFree = 0; slot->manifest->dataFileUnlinkOnFree = 0;
strbuf r = strbuf_local(slot->request, sizeof slot->request); strbuf r = strbuf_local(slot->request, sizeof slot->request);
strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", slot->manifest->fileHexHash); strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
if (slot->manifest->journalTail>=0){ if (slot->manifest->journalTail>=0){
// if we're fetching a journal bundle, work out how many bytes we have of a previous version // if we're fetching a journal bundle, work out how many bytes we have of a previous version
@ -509,12 +509,12 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
} }
strbuf_puts(r, "\r\n"); strbuf_puts(r, "\r\n");
if (strbuf_overrun(r)) if (strbuf_overrun(r))
RETURN(WHY("request overrun")); RETURN(WHY("request overrun"));
slot->request_len = strbuf_len(r); slot->request_len = strbuf_len(r);
if (rhizome_open_write(&slot->write_state, slot->manifest->fileHexHash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT)) if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT))
RETURN(-1); RETURN(-1);
} else { } else {
strbuf r = strbuf_local(slot->request, sizeof slot->request); strbuf r = strbuf_local(slot->request, sizeof slot->request);
@ -690,9 +690,9 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
for (i = 0; i < NQUEUES; ++i) { for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_slot *as = &rhizome_fetch_queues[i].active; struct rhizome_fetch_slot *as = &rhizome_fetch_queues[i].active;
const rhizome_manifest *am = as->manifest; const rhizome_manifest *am = as->manifest;
if (as->state != RHIZOME_FETCH_FREE && strcasecmp(m->fileHexHash, am->fileHexHash) == 0) { if (as->state != RHIZOME_FETCH_FREE && cmp_rhizome_filehash_t(&m->filehash, &am->filehash) == 0) {
if (config.debug.rhizome_rx) if (config.debug.rhizome_rx)
DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, m->fileHexHash); DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, alloca_tohex_rhizome_filehash_t(m->filehash));
RETURN(SAMEPAYLOAD); RETURN(SAMEPAYLOAD);
} }
} }
@ -707,7 +707,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
DEBUGF(" is new"); DEBUGF(" is new");
// If the payload is already available, no need to fetch, so import now. // If the payload is already available, no need to fetch, so import now.
if (rhizome_exists(m->fileHexHash)){ if (rhizome_exists(&m->filehash)){
if (config.debug.rhizome_rx) if (config.debug.rhizome_rx)
DEBUGF(" fetch not started - payload already present, so importing instead"); DEBUGF(" fetch not started - payload already present, so importing instead");
if (rhizome_add_manifest(m, m->ttl-1) == -1) if (rhizome_add_manifest(m, m->ttl-1) == -1)
@ -1138,8 +1138,7 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){
if (start>=0 && start < slot->previous->fileLength && length>0){ if (start>=0 && start < slot->previous->fileLength && length>0){
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("Copying %"PRId64" bytes from previous journal", length); DEBUGF("Copying %"PRId64" bytes from previous journal", length);
rhizome_journal_pipe(&slot->write_state, slot->previous->fileHexHash, rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length);
start, length);
} }
// and we don't need to do this again, so drop the manifest // and we don't need to do this again, so drop the manifest
@ -1274,10 +1273,11 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
} }
INFOF("Completed http request from %s:%u for file %s", INFOF("Completed http request from %s:%u for file %s",
buf, ntohs(slot->peer_ipandport.sin_port), buf, ntohs(slot->peer_ipandport.sin_port),
slot->manifest->fileHexHash); alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
} else { } else {
INFOF("Completed MDP request from %s for file %s", INFOF("Completed MDP request from %s for file %s",
alloca_tohex_sid_t(slot->peer_sid), slot->manifest->fileHexHash); alloca_tohex_sid_t(slot->peer_sid),
alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
} }
} else { } else {
/* This was to fetch the manifest, so now fetch the file if needed */ /* This was to fetch the manifest, so now fetch the file if needed */
@ -1409,7 +1409,8 @@ int rhizome_received_content(const unsigned char *bidprefix,
if (m){ if (m){
if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){ if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){
INFOF("Completed MDP transfer in one hit for file %s", m->fileHexHash); INFOF("Completed MDP transfer in one hit for file %s",
alloca_tohex_rhizome_filehash_t(m->filehash));
if (c) if (c)
candidate_unqueue(c); candidate_unqueue(c);
} }

5
rhizome_http.c
View File

@ -449,14 +449,15 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder, con
if (!is_rhizome_http_enabled()) if (!is_rhizome_http_enabled())
return 1; return 1;
if (!rhizome_str_is_file_hash(remainder)) rhizome_filehash_t filehash;
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
return -1; return -1;
bzero(&r->read_state, sizeof(r->read_state)); bzero(&r->read_state, sizeof(r->read_state));
/* Refuse to honour HTTP request if required (used for debugging and /* Refuse to honour HTTP request if required (used for debugging and
testing transition from HTTP to MDP) */ testing transition from HTTP to MDP) */
if (rhizome_open_read(&r->read_state, remainder)) if (rhizome_open_read(&r->read_state, &filehash))
return 1; return 1;
if (r->read_state.length==-1){ if (r->read_state.length==-1){

115
rhizome_store.c
View File

@ -5,22 +5,22 @@
#define RHIZOME_BUFFER_MAXIMUM_SIZE (1024*1024) #define RHIZOME_BUFFER_MAXIMUM_SIZE (1024*1024)
int rhizome_exists(const char *fileHash) int rhizome_exists(const rhizome_filehash_t *hashp)
{ {
int64_t gotfile = 0; int64_t gotfile = 0;
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", TEXT_TOUPPER, fileHash, END) != 1) if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", RHIZOME_FILEHASH_T, hashp, END) != 1)
return 0; return 0;
return gotfile; return gotfile;
} }
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority) int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority)
{ {
write->blob_fd=-1; write->blob_fd=-1;
if (expectedFileHash){ if (expectedHashp){
if (rhizome_exists(expectedFileHash)) if (rhizome_exists(expectedHashp))
return 1; return 1;
strlcpy(write->id, expectedFileHash, SHA512_DIGEST_STRING_LENGTH); write->id = *expectedHashp;
write->id_known=1; write->id_known=1;
}else{ }else{
write->id_known=0; write->id_known=0;
@ -429,7 +429,7 @@ int rhizome_fail_write(struct rhizome_write *write)
write->buffer_list=n->_next; write->buffer_list=n->_next;
free(n); free(n);
} }
rhizome_delete_file(write->id); rhizome_delete_file(&write->id);
return 0; return 0;
} }
@ -460,39 +460,39 @@ int rhizome_finish_write(struct rhizome_write *write)
if (write_release_lock(write)) if (write_release_lock(write))
goto failure; goto failure;
char hash_out[SHA512_DIGEST_STRING_LENGTH + 1]; rhizome_filehash_t hash_out;
SHA512_End(&write->sha512_context, hash_out); SHA512_Final(hash_out.binary, &write->sha512_context);
str_toupper_inplace(hash_out); SHA512_End(&write->sha512_context, NULL);
if (write->id_known) { if (write->id_known) {
if (strcasecmp(write->id, hash_out) != 0) { if (cmp_rhizome_filehash_t(&write->id, &hash_out) != 0) {
WHYF("expected filehash=%s, got %s", write->id, hash_out); WHYF("expected filehash=%s, got %s", alloca_tohex_rhizome_filehash_t(write->id), alloca_tohex_rhizome_filehash_t(hash_out));
goto failure; goto failure;
} }
} else { } else {
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH); write->id = hash_out;
} }
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
rhizome_remove_file_datainvalid(&retry, write->id); rhizome_remove_file_datainvalid(&retry, &write->id);
if (rhizome_exists(write->id)) { if (rhizome_exists(&write->id)) {
// we've already got that payload, delete the new copy // we've already got that payload, delete the new copy
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("File id='%s' already present, removed id='%"PRId64"'", write->id, write->temp_id); DEBUGF("File id=%s already present, removed id='%"PRId64"'", alloca_tohex_rhizome_filehash_t(write->id), write->temp_id);
} else { } else {
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
goto dbfailure; goto dbfailure;
// delete any half finished records // delete any half finished records
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", STATIC_TEXT, write->id, END); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", STATIC_TEXT, write->id, END); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END);
if (sqlite_exec_void_retry( if (sqlite_exec_void_retry(
&retry, &retry,
"UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?", "UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?",
STATIC_TEXT, write->id, RHIZOME_FILEHASH_T, &write->id,
INT64, gettime_ms(), INT64, gettime_ms(),
UINT64_TOSTR, write->temp_id, UINT64_TOSTR, write->temp_id,
END END
@ -507,7 +507,7 @@ int rhizome_finish_write(struct rhizome_write *write)
WHYF("Failed to generate file path"); WHYF("Failed to generate file path");
goto dbfailure; goto dbfailure;
} }
if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, write->id)){ if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, alloca_tohex_rhizome_filehash_t(write->id))){
WHYF("Failed to generate file path"); WHYF("Failed to generate file path");
goto dbfailure; goto dbfailure;
} }
@ -521,7 +521,7 @@ int rhizome_finish_write(struct rhizome_write *write)
if (sqlite_exec_void_retry( if (sqlite_exec_void_retry(
&retry, &retry,
"UPDATE FILEBLOBS SET id = ? WHERE rowid = ?", "UPDATE FILEBLOBS SET id = ? WHERE rowid = ?",
STATIC_TEXT, write->id, RHIZOME_FILEHASH_T, &write->id,
INT64, write->blob_rowid, INT64, write->blob_rowid,
END END
) == -1 ) == -1
@ -531,7 +531,7 @@ int rhizome_finish_write(struct rhizome_write *write)
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1) if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1)
goto dbfailure; goto dbfailure;
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("Stored file %s", write->id); DEBUGF("Stored file %s", alloca_tohex_rhizome_filehash_t(write->id));
} }
write->blob_rowid=-1; write->blob_rowid=-1;
return 0; return 0;
@ -553,7 +553,7 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
struct rhizome_write write; struct rhizome_write write;
bzero(&write, sizeof(write)); bzero(&write, sizeof(write));
int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
if (ret!=0) if (ret!=0)
return ret; return ret;
@ -583,7 +583,7 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length
struct rhizome_write write; struct rhizome_write write;
bzero(&write, sizeof(write)); bzero(&write, sizeof(write));
int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT); int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
if (ret!=0) if (ret!=0)
return ret; return ret;
@ -624,7 +624,7 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
rhizome_manifest_set_ll(m, "filesize", m->fileLength); rhizome_manifest_set_ll(m, "filesize", m->fileLength);
if (m->fileLength == 0){ if (m->fileLength == 0){
m->fileHexHash[0] = '\0'; m->filehash = RHIZOME_FILEHASH_NONE;
rhizome_manifest_del(m, "filehash"); rhizome_manifest_del(m, "filehash");
} }
return 0; return 0;
@ -676,8 +676,8 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
if (rhizome_finish_write(&write)) if (rhizome_finish_write(&write))
goto failure; goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); m->filehash = write.id;
rhizome_manifest_set(m, "filehash", m->fileHexHash); rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
return 0; return 0;
failure: failure:
@ -687,26 +687,24 @@ failure:
/* Return -1 on error, 0 if file blob found, 1 if not found. /* Return -1 on error, 0 if file blob found, 1 if not found.
*/ */
int rhizome_open_read(struct rhizome_read *read, const char *fileid) int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp)
{ {
strncpy(read->id, fileid, sizeof read->id); read->id = *hashp;
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(read->id);
read->blob_rowid = -1; read->blob_rowid = -1;
read->blob_fd = -1; read->blob_fd = -1;
if (sqlite_exec_int64(&read->blob_rowid, if (sqlite_exec_int64(&read->blob_rowid,
"SELECT FILEBLOBS.rowid " "SELECT FILEBLOBS.rowid "
"FROM FILEBLOBS, FILES " "FROM FILEBLOBS, FILES "
"WHERE FILEBLOBS.id = FILES.id" "WHERE FILEBLOBS.id = FILES.id"
" AND FILES.id = ?" " AND FILES.id = ?"
" AND FILES.datavalid != 0", STATIC_TEXT, read->id, END) == -1) " AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1)
return -1; return -1;
if (read->blob_rowid != -1) { if (read->blob_rowid != -1) {
read->length = -1; // discover the length on opening the db BLOB read->length = -1; // discover the length on opening the db BLOB
} else { } else {
// No row in FILEBLOBS, look for an external blob file. // No row in FILEBLOBS, look for an external blob file.
char blob_path[1024]; char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, read->id)) if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(read->id)))
return -1; return -1;
read->blob_fd = open(blob_path, O_RDONLY); read->blob_fd = open(blob_path, O_RDONLY);
if (read->blob_fd == -1) { if (read->blob_fd == -1) {
@ -787,13 +785,13 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
read_state->hash_offset += bytes_read; read_state->hash_offset += bytes_read;
// if we hash everything and the has doesn't match, we need to delete the payload // if we hash everything and the has doesn't match, we need to delete the payload
if (read_state->hash_offset>=read_state->length){ if (read_state->hash_offset>=read_state->length){
char hash_out[SHA512_DIGEST_STRING_LENGTH+1]; rhizome_filehash_t hash_out;
SHA512_End(&read_state->sha512_context, hash_out); SHA512_Final(hash_out.binary, &read_state->sha512_context);
SHA512_End(&read_state->sha512_context, NULL);
if (strcasecmp(read_state->id, hash_out)){ if (cmp_rhizome_filehash_t(&read_state->id, &hash_out) != 0) {
// hash failure, mark the payload as invalid // hash failure, mark the payload as invalid
read_state->invalid = 1; read_state->invalid = 1;
RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out)); RETURN(WHYF("Expected hash=%s, got %s", alloca_tohex_rhizome_filehash_t(read_state->id), alloca_tohex_rhizome_filehash_t(hash_out)));
} }
} }
} }
@ -874,7 +872,7 @@ int rhizome_read_close(struct rhizome_read *read)
read->blob_fd = -1; read->blob_fd = -1;
if (read->invalid){ if (read->invalid){
// delete payload! // delete payload!
rhizome_delete_file(read->id); rhizome_delete_file(&read->id);
} }
return 0; return 0;
} }
@ -980,8 +978,7 @@ int rhizome_cache_count()
} }
// read a block of data, caching meta data for reuse // read a block of data, caching meta data for reuse
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length)
uint64_t fileOffset, unsigned char *buffer, int length)
{ {
// look for a cached entry // look for a cached entry
struct cache_entry **ptr = find_entry_location(&root, bidp, version); struct cache_entry **ptr = find_entry_location(&root, bidp, version);
@ -989,15 +986,13 @@ int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t t
// if we don't have one yet, create one and open it // if we don't have one yet, create one and open it
if (!entry){ if (!entry){
char filehash[SHA512_DIGEST_STRING_LENGTH]; rhizome_filehash_t filehash;
if (rhizome_database_filehash_from_id(bidp, version, filehash) == -1) if (rhizome_database_filehash_from_id(bidp, version, &filehash) == -1)
return -1; return -1;
entry = emalloc_zero(sizeof(struct cache_entry)); entry = emalloc_zero(sizeof(struct cache_entry));
if (rhizome_open_read(&entry->read_state, &filehash)){
if (rhizome_open_read(&entry->read_state, filehash)){
free(entry); free(entry);
return WHYF("Payload %s not found", filehash); return WHYF("Payload %s not found", alloca_tohex_rhizome_filehash_t(filehash));
} }
entry->bundle_id = *bidp; entry->bundle_id = *bidp;
entry->version = version; entry->version = version;
@ -1073,8 +1068,9 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
return 0; return 0;
} }
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){ int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state)
int ret = rhizome_open_read(read_state, m->fileHexHash); {
int ret = rhizome_open_read(read_state, &m->filehash);
if (ret == 0) if (ret == 0)
ret = read_derive_key(m, bsk, read_state); ret = read_derive_key(m, bsk, read_state);
return ret; return ret;
@ -1100,12 +1096,12 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
* *
* Returns -1 on error, 0 if dumped successfully, 1 if not found. * Returns -1 on error, 0 if dumped successfully, 1 if not found.
*/ */
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length) int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length)
{ {
struct rhizome_read read_state; struct rhizome_read read_state;
bzero(&read_state, sizeof read_state); bzero(&read_state, sizeof read_state);
int ret = rhizome_open_read(&read_state, id); int ret = rhizome_open_read(&read_state, hashp);
if (ret == 0) { if (ret == 0) {
ret = write_file(&read_state, filepath); ret = write_file(&read_state, filepath);
@ -1141,13 +1137,12 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write,
return 0; return 0;
} }
int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length) int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length)
{ {
struct rhizome_read read_state; struct rhizome_read read_state;
bzero(&read_state, sizeof read_state); bzero(&read_state, sizeof read_state);
if (rhizome_open_read(&read_state, fileHash)) if (rhizome_open_read(&read_state, hashp))
return -1; return -1;
read_state.offset = start_offset; read_state.offset = start_offset;
int ret = rhizome_pipe(&read_state, write, length); int ret = rhizome_pipe(&read_state, write, length);
rhizome_read_close(&read_state); rhizome_read_close(&read_state);
@ -1182,7 +1177,7 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m,
if (copy_length>0){ if (copy_length>0){
// note that we don't need to bother decrypting the existing journal payload // note that we don't need to bother decrypting the existing journal payload
ret = rhizome_journal_pipe(write, m->fileHexHash, advance_by, copy_length); ret = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);
if (ret) if (ret)
goto failure; goto failure;
} }
@ -1218,8 +1213,8 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
if (ret) if (ret)
goto failure; goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); m->filehash = write.id;
rhizome_manifest_set(m, "filehash", m->fileHexHash); rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
return 0; return 0;
failure: failure:
@ -1250,8 +1245,8 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t
if (ret) if (ret)
goto failure; goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH); m->filehash = write.id;
rhizome_manifest_set(m, "filehash", m->fileHexHash); rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
return 0; return 0;