mirror of
https://github.com/servalproject/serval-dna.git
synced 2024-12-18 20:57:56 +00:00
Issue #11: Use rhizome_filehash_t everywhere
This commit is contained in:
Andrew Bettison
parent
c97bd4a69a
commit
974c7a56a0
@ -1275,10 +1275,11 @@ int app_rhizome_hash_file(const struct cli_parsed *parsed, struct cli_context *c
|
||||
return the hash of the file unencrypted. */
|
||||
const char *filepath;
|
||||
cli_arg(parsed, "filepath", &filepath, NULL, "");
|
||||
char hexhash[RHIZOME_FILEHASH_STRLEN + 1];
|
||||
if (rhizome_hash_file(NULL, filepath, hexhash))
|
||||
rhizome_filehash_t hash;
|
||||
uint64_t size;
|
||||
if (rhizome_hash_file(NULL, filepath, &hash, &size) == -1)
|
||||
return -1;
|
||||
cli_put_string(context, hexhash, "\n");
|
||||
cli_put_string(context, size ? alloca_tohex_rhizome_filehash_t(hash) : "", "\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1425,7 +1426,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
|
||||
cli_put_long(context, mout->fileLength, "\n");
|
||||
if (mout->fileLength != 0) {
|
||||
cli_field_name(context, "filehash", ":");
|
||||
cli_put_string(context, mout->fileHexHash, "\n");
|
||||
cli_put_string(context, alloca_tohex_rhizome_filehash_t(mout->filehash), "\n");
|
||||
}
|
||||
const char *name = rhizome_manifest_get(mout, "name", NULL, 0);
|
||||
if (name) {
|
||||
@ -1528,7 +1529,7 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
|
||||
cli_put_long(context, m->fileLength, "\n");
|
||||
if (m->fileLength != 0) {
|
||||
cli_field_name(context, "filehash", ":");
|
||||
cli_put_string(context, m->fileHexHash, "\n");
|
||||
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
|
||||
}
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
if (name) {
|
||||
@ -1588,12 +1589,10 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont
|
||||
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
|
||||
if (!fileid)
|
||||
return WHY("missing <fileid> argument");
|
||||
unsigned char filehash[RHIZOME_FILEHASH_BYTES];
|
||||
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1)
|
||||
return WHY("Invalid file ID");
|
||||
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1];
|
||||
tohex(fileIDUpper, RHIZOME_FILEHASH_STRLEN, filehash);
|
||||
ret = rhizome_delete_file(fileIDUpper);
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, fileid) == -1)
|
||||
return WHYF("invalid <fileid> argument: %s", alloca_str_toprint(fileid));
|
||||
ret = rhizome_delete_file(&hash);
|
||||
} else {
|
||||
if (!manifestid)
|
||||
return WHY("missing <manifestid> argument");
|
||||
@ -1688,7 +1687,8 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
|
||||
cli_field_name(context, ".readonly", ":"); cli_put_long(context, m->haveSecret?0:1, "\n");
|
||||
cli_field_name(context, "filesize", ":"); cli_put_long(context, m->fileLength, "\n");
|
||||
if (m->fileLength != 0) {
|
||||
cli_field_name(context, "filehash", ":"); cli_put_string(context, m->fileHexHash, "\n");
|
||||
cli_field_name(context, "filehash", ":");
|
||||
cli_put_string(context, alloca_tohex_rhizome_filehash_t(m->filehash), "\n");
|
||||
}
|
||||
}
|
||||
|
||||
@ -1702,7 +1702,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
|
||||
}else{
|
||||
// Save the file without attempting to decrypt
|
||||
int64_t length;
|
||||
retfile = rhizome_dump_file(m->fileHexHash, filepath, &length);
|
||||
retfile = rhizome_dump_file(&m->filehash, filepath, &length);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1741,18 +1741,21 @@ int app_rhizome_export_file(const struct cli_parsed *parsed, struct cli_context
|
||||
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|
||||
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
|
||||
return -1;
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, fileid) == -1)
|
||||
return WHYF("invalid <fileid> argument: %s", alloca_str_toprint(fileid));
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
if (rhizome_opendb() == -1)
|
||||
return -1;
|
||||
if (!rhizome_exists(fileid))
|
||||
if (!rhizome_exists(&hash))
|
||||
return 1;
|
||||
int64_t length;
|
||||
int ret = rhizome_dump_file(fileid, filepath, &length);
|
||||
int ret = rhizome_dump_file(&hash, filepath, &length);
|
||||
if (ret)
|
||||
return ret == -1 ? -1 : 1;
|
||||
cli_field_name(context, "filehash", ":");
|
||||
cli_put_string(context, fileid, "\n");
|
||||
cli_put_string(context, alloca_tohex_rhizome_filehash_t(hash), "\n");
|
||||
cli_field_name(context, "filesize", ":");
|
||||
cli_put_long(context, length, "\n");
|
||||
return 0;
|
||||
|
||||
@ -94,6 +94,28 @@ int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b)
|
||||
{
|
||||
return memcmp(a, b, sizeof a->binary);
|
||||
}
|
||||
|
||||
int str_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex)
|
||||
{
|
||||
return fromhexstr(hashp->binary, hex, sizeof hashp->binary);
|
||||
}
|
||||
|
||||
int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const char **endp)
|
||||
{
|
||||
rhizome_filehash_t tmp;
|
||||
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
|
||||
if (n != sizeof tmp.binary)
|
||||
return -1;
|
||||
*hashp = tmp;
|
||||
if (endp)
|
||||
*endp = hex + sizeof hashp->binary * 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rhizome_strn_is_manifest_id(const char *id)
|
||||
{
|
||||
return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN);
|
||||
|
||||
7
meshms.c
7
meshms.c
@ -561,7 +561,8 @@ static int write_conversation(struct rhizome_write *write, struct conversations
|
||||
return len;
|
||||
}
|
||||
|
||||
static int write_known_conversations(rhizome_manifest *m, struct conversations *conv){
|
||||
static int write_known_conversations(rhizome_manifest *m, struct conversations *conv)
|
||||
{
|
||||
rhizome_manifest *mout=NULL;
|
||||
|
||||
struct rhizome_write write;
|
||||
@ -590,8 +591,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
|
||||
goto end;
|
||||
if (rhizome_finish_write(&write))
|
||||
goto end;
|
||||
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
|
||||
rhizome_manifest_set(m, "filehash", m->fileHexHash);
|
||||
m->filehash = write.id;
|
||||
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
if (rhizome_manifest_finalise(m, &mout, 1))
|
||||
goto end;
|
||||
|
||||
|
||||
@ -73,7 +73,7 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
|
||||
filepath ? alloca_str_toprint(filepath) : "NULL");
|
||||
|
||||
unsigned char buffer[MAX_MANIFEST_BYTES];
|
||||
int buffer_len=0;
|
||||
size_t buffer_len = 0;
|
||||
|
||||
// manifest has been appended to the end of the file.
|
||||
if (strcmp(manifest_path, filepath)==0){
|
||||
@ -251,10 +251,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
|
||||
if (rhizome_manifest_check_sanity(m_in))
|
||||
return -1;
|
||||
|
||||
if (m_in->fileLength){
|
||||
if (!rhizome_exists(m_in->fileHexHash))
|
||||
return WHY("File has not been imported");
|
||||
}
|
||||
if (m_in->fileLength && !rhizome_exists(&m_in->filehash))
|
||||
return WHY("File has not been imported");
|
||||
|
||||
/* If the manifest already has an ID */
|
||||
if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic))
|
||||
|
||||
102
rhizome.h
102
rhizome.h
@ -52,6 +52,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#define RHIZOME_HTTP_PORT 4110
|
||||
#define RHIZOME_HTTP_PORT_MAX 4150
|
||||
|
||||
/* Fundamental data type: Rhizome Bundle ID
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
|
||||
typedef struct rhizome_bid_binary {
|
||||
unsigned char binary[RHIZOME_MANIFEST_ID_BYTES];
|
||||
} rhizome_bid_t;
|
||||
@ -65,6 +70,28 @@ int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b);
|
||||
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex);
|
||||
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp);
|
||||
|
||||
/* Fundamental data type: Rhizome File Hash
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
|
||||
typedef struct rhizome_filehash_binary {
|
||||
unsigned char binary[RHIZOME_FILEHASH_BYTES];
|
||||
} rhizome_filehash_t;
|
||||
|
||||
#define RHIZOME_FILEHASH_NONE ((rhizome_filehash_t){{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}})
|
||||
#define rhizome_filehash_t_is_zero(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0)
|
||||
#define rhizome_filehash_t_is_max(fh) is_all_matching((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary, 0xff)
|
||||
#define alloca_tohex_rhizome_filehash_t(fh) alloca_tohex((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary)
|
||||
int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b);
|
||||
int str_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex);
|
||||
int strn_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, const char **endp);
|
||||
|
||||
/* Fundamental data type: Rhizome Bundle Key
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
|
||||
typedef struct rhizome_bk_binary {
|
||||
unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES];
|
||||
} rhizome_bk_t;
|
||||
@ -77,6 +104,7 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) {
|
||||
|
||||
#define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary)
|
||||
|
||||
|
||||
extern time_ms_t rhizome_voice_timeout;
|
||||
|
||||
#define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE
|
||||
@ -158,7 +186,7 @@ typedef struct rhizome_manifest {
|
||||
group membership handy */
|
||||
int64_t fileLength;
|
||||
int64_t journalTail;
|
||||
char fileHexHash[SHA512_DIGEST_STRING_LENGTH];
|
||||
rhizome_filehash_t filehash;
|
||||
|
||||
int fileHighestPriority;
|
||||
/* Absolute path of the file associated with the manifest */
|
||||
@ -258,10 +286,10 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
|
||||
|
||||
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
|
||||
int rhizome_manifest_selfsign(rhizome_manifest *m);
|
||||
int rhizome_drop_stored_file(const char *id,int maximum_priority);
|
||||
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
|
||||
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
|
||||
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize);
|
||||
int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out);
|
||||
int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
|
||||
char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen);
|
||||
int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var);
|
||||
int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value);
|
||||
@ -274,7 +302,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence);
|
||||
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
|
||||
int rhizome_manifest_pack_variables(rhizome_manifest *m);
|
||||
int rhizome_store_bundle(rhizome_manifest *m);
|
||||
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid);
|
||||
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
|
||||
int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
|
||||
int rhizome_clean_payload(const char *fileidhex);
|
||||
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
|
||||
@ -308,25 +336,25 @@ int is_debug_rhizome_ads();
|
||||
|
||||
enum sqlbind_type {
|
||||
END = 0xbabecafe,
|
||||
INT = 1, // int value
|
||||
INT_TOSTR, // int value
|
||||
UINT_TOSTR, // unsigned value
|
||||
INT64, // int64_t value
|
||||
INT64_TOSTR, // int64_t value
|
||||
UINT64_TOSTR, // uint64_t value
|
||||
TEXT, // const char *text,
|
||||
TEXT_LEN, // const char *text, int bytes
|
||||
STATIC_TEXT, // const char *text,
|
||||
STATIC_TEXT_LEN, // const char *text, int bytes
|
||||
STATIC_BLOB, // const void *blob, int bytes
|
||||
ZEROBLOB, // int bytes
|
||||
SID_T, // const sid_t *sidp
|
||||
RHIZOME_BID_T, // const rhizome_bid_t *bidp
|
||||
FILEHASH_T, // const unsigned char hash_binary[RHIZOME_FILEHASH_BYTES]
|
||||
TOHEX, // const unsigned char *binary, unsigned bytes
|
||||
TEXT_TOUPPER, // const char *text,
|
||||
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
|
||||
NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ...
|
||||
INT = 1, // int value
|
||||
INT_TOSTR, // int value
|
||||
UINT_TOSTR, // unsigned value
|
||||
INT64, // int64_t value
|
||||
INT64_TOSTR, // int64_t value
|
||||
UINT64_TOSTR, // uint64_t value
|
||||
TEXT, // const char *text,
|
||||
TEXT_LEN, // const char *text, int bytes
|
||||
STATIC_TEXT, // const char *text,
|
||||
STATIC_TEXT_LEN, // const char *text, int bytes
|
||||
STATIC_BLOB, // const void *blob, int bytes
|
||||
ZEROBLOB, // int bytes
|
||||
SID_T, // const sid_t *sidp
|
||||
RHIZOME_BID_T, // const rhizome_bid_t *bidp
|
||||
RHIZOME_FILEHASH_T, // const rhizome_filehash_t *hashp
|
||||
TOHEX, // const unsigned char *binary, unsigned bytes
|
||||
TEXT_TOUPPER, // const char *text,
|
||||
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
|
||||
NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ...
|
||||
INDEX = 0xfade0000, // INDEX|INT, int index, ...
|
||||
NAMED = 0xdead0000 // NAMED|INT, const char *label, ...
|
||||
};
|
||||
@ -390,7 +418,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
|
||||
int rhizome_delete_bundle(const rhizome_bid_t *bidp);
|
||||
int rhizome_delete_manifest(const rhizome_bid_t *bidp);
|
||||
int rhizome_delete_payload(const rhizome_bid_t *bidp);
|
||||
int rhizome_delete_file(const char *fileid);
|
||||
int rhizome_delete_file(const rhizome_filehash_t *hashp);
|
||||
|
||||
#define RHIZOME_DONTVERIFY 0
|
||||
#define RHIZOME_VERIFY 1
|
||||
@ -439,7 +467,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
|
||||
rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_length);
|
||||
|
||||
/* Rhizome file storage api */
|
||||
struct rhizome_write_buffer{
|
||||
struct rhizome_write_buffer
|
||||
{
|
||||
struct rhizome_write_buffer *_next;
|
||||
int64_t offset;
|
||||
int buffer_size;
|
||||
@ -447,8 +476,9 @@ struct rhizome_write_buffer{
|
||||
unsigned char data[0];
|
||||
};
|
||||
|
||||
struct rhizome_write{
|
||||
char id[SHA512_DIGEST_STRING_LENGTH+1];
|
||||
struct rhizome_write
|
||||
{
|
||||
rhizome_filehash_t id;
|
||||
uint64_t temp_id;
|
||||
char id_known;
|
||||
|
||||
@ -475,8 +505,9 @@ struct rhizome_read_buffer{
|
||||
int len;
|
||||
};
|
||||
|
||||
struct rhizome_read{
|
||||
char id[SHA512_DIGEST_STRING_LENGTH+1];
|
||||
struct rhizome_read
|
||||
{
|
||||
rhizome_filehash_t id;
|
||||
|
||||
int crypt;
|
||||
unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
|
||||
@ -741,8 +772,8 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
|
||||
|
||||
/* rhizome storage methods */
|
||||
|
||||
int rhizome_exists(const char *fileHash);
|
||||
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority);
|
||||
int rhizome_exists(const rhizome_filehash_t *hashp);
|
||||
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
|
||||
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size);
|
||||
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size);
|
||||
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
|
||||
@ -758,23 +789,22 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
|
||||
int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
|
||||
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len);
|
||||
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
|
||||
int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length);
|
||||
int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
|
||||
|
||||
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
|
||||
const unsigned char *key, const unsigned char *nonce);
|
||||
int rhizome_open_read(struct rhizome_read *read, const char *fileid);
|
||||
int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
|
||||
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
|
||||
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
|
||||
int rhizome_read_close(struct rhizome_read *read);
|
||||
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
|
||||
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
|
||||
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length);
|
||||
int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length);
|
||||
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
|
||||
uint64_t fileOffset, unsigned char *buffer, int length);
|
||||
int rhizome_cache_close();
|
||||
|
||||
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]);
|
||||
|
||||
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);
|
||||
|
||||
int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp);
|
||||
int rhizome_sync_announce();
|
||||
|
||||
@ -169,14 +169,13 @@ int rhizome_manifest_parse(rhizome_manifest *m)
|
||||
}
|
||||
} else if (strcasecmp(var, "filehash") == 0) {
|
||||
have_filehash = 1;
|
||||
if (!rhizome_str_is_file_hash(value)) {
|
||||
if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) {
|
||||
if (config.debug.rejecteddata)
|
||||
WARNF("Invalid filehash: %s", value);
|
||||
m->errors++;
|
||||
} else {
|
||||
/* Force to upper case to avoid case sensitive comparison problems later. */
|
||||
str_toupper_inplace(m->values[m->var_count]);
|
||||
strcpy(m->fileHexHash, m->values[m->var_count]);
|
||||
}
|
||||
} else if (strcasecmp(var, "filesize") == 0) {
|
||||
have_filesize = 1;
|
||||
@ -344,7 +343,7 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bu
|
||||
return rhizome_manifest_parse(m);
|
||||
}
|
||||
|
||||
int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out)
|
||||
int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out)
|
||||
{
|
||||
/* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
|
||||
go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(),
|
||||
@ -355,35 +354,36 @@ int rhizome_hash_file(rhizome_manifest *m, const char *filename, char *hash_out)
|
||||
if (m && m->payloadEncryption)
|
||||
return WHY("Encryption of payloads not implemented");
|
||||
|
||||
size_t filesize = 0;
|
||||
uint64_t filesize = 0;
|
||||
SHA512_CTX context;
|
||||
SHA512_Init(&context);
|
||||
if (filename[0]) {
|
||||
FILE *f = fopen(filename, "r");
|
||||
if (!f) {
|
||||
WHY_perror("fopen");
|
||||
return WHYF("Could not open %s to calculate SHA512 hash.", filename);
|
||||
}
|
||||
while (!feof(f)) {
|
||||
unsigned char buffer[8192];
|
||||
int r = fread(buffer, 1, 8192, f);
|
||||
if (path[0]) {
|
||||
int fd = open(path, O_RDONLY);
|
||||
if (fd == -1)
|
||||
return WHYF_perror("open(%s,O_RDONLY)", alloca_str_toprint(path));
|
||||
unsigned char buffer[8192];
|
||||
ssize_t r;
|
||||
while ((r = read(fd, buffer, sizeof buffer))) {
|
||||
if (r == -1) {
|
||||
WHY_perror("fread");
|
||||
fclose(f);
|
||||
return WHYF("Error reading %s to calculate SHA512 hash", filename);
|
||||
WHYF_perror("read(%s,%u)", alloca_str_toprint(path), sizeof buffer);
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
if (r > 0)
|
||||
SHA512_Update(&context, buffer, r);
|
||||
filesize += r;
|
||||
SHA512_Update(&context, buffer, (size_t) r);
|
||||
filesize += (size_t) r;
|
||||
}
|
||||
fclose(f);
|
||||
close(fd);
|
||||
}
|
||||
SHA512_End(&context, (char *)hash_out);
|
||||
// Empty files (including null filename) have no hash.
|
||||
if (filesize > 0)
|
||||
str_toupper_inplace(hash_out);
|
||||
else
|
||||
hash_out[0] = '\0';
|
||||
// Empty files (including empty path) have no hash.
|
||||
if (hash_out) {
|
||||
if (filesize > 0)
|
||||
SHA512_Final(hash_out->binary, &context);
|
||||
else
|
||||
*hash_out = RHIZOME_FILEHASH_NONE;
|
||||
}
|
||||
if (size_out)
|
||||
*size_out = filesize;
|
||||
SHA512_End(&context, NULL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
static char rhizome_thisdatastore_path[256];
|
||||
|
||||
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
|
||||
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid);
|
||||
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
|
||||
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
|
||||
|
||||
const char *rhizome_datastore_path()
|
||||
@ -678,14 +678,16 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
|
||||
}
|
||||
}
|
||||
break;
|
||||
case FILEHASH_T: {
|
||||
const char *hash_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_FILEHASH_BYTES);
|
||||
case RHIZOME_FILEHASH_T: {
|
||||
const rhizome_filehash_t *hashp = va_arg(ap, const rhizome_filehash_t *);
|
||||
++argnum;
|
||||
if (hash_hex == NULL) {
|
||||
BIND_NULL(FILEHASH_T);
|
||||
if (hashp == NULL) {
|
||||
BIND_NULL(RHIZOME_FILEHASH_T);
|
||||
} else {
|
||||
BIND_DEBUG(FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
|
||||
BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT);
|
||||
char hash_hex[RHIZOME_FILEHASH_STRLEN];
|
||||
tohex(hash_hex, sizeof hash_hex, hashp->binary);
|
||||
BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, sizeof hash_hex);
|
||||
BIND_RETRY(sqlite3_bind_text, hash_hex, sizeof hash_hex, SQLITE_TRANSIENT);
|
||||
}
|
||||
}
|
||||
break;
|
||||
@ -1014,20 +1016,24 @@ int64_t rhizome_database_used_bytes()
|
||||
return db_page_size * (db_page_count - db_free_page_count);
|
||||
}
|
||||
|
||||
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH])
|
||||
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp)
|
||||
{
|
||||
IN();
|
||||
strbuf hash_sb = strbuf_local(hash, SHA512_DIGEST_STRING_LENGTH);
|
||||
RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;",
|
||||
INT64, version, RHIZOME_BID_T, bidp, END));
|
||||
strbuf hash_sb = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
|
||||
if ( sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;",
|
||||
INT64, version, RHIZOME_BID_T, bidp, END) == -1)
|
||||
RETURN(-1);
|
||||
if (strbuf_overrun(hash_sb) || str_to_rhizome_filehash_t(hashp, strbuf_str(hash_sb)) == -1)
|
||||
RETURN(WHYF("malformed file hash for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(*bidp), version));
|
||||
RETURN(0);
|
||||
OUT();
|
||||
}
|
||||
|
||||
static int rhizome_delete_external(const char *fileid)
|
||||
static int rhizome_delete_external(const rhizome_filehash_t *hashp)
|
||||
{
|
||||
// attempt to remove any external blob
|
||||
char blob_path[1024];
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid))
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(*hashp)))
|
||||
return -1;
|
||||
return unlink(blob_path);
|
||||
}
|
||||
@ -1039,18 +1045,18 @@ static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
|
||||
END);
|
||||
}
|
||||
|
||||
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid)
|
||||
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
|
||||
{
|
||||
int ret = 0;
|
||||
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
|
||||
"DELETE FROM FILES WHERE id = ? and datavalid = 0;",
|
||||
TEXT_TOUPPER, fileid, END
|
||||
RHIZOME_FILEHASH_T, hashp, END
|
||||
) == -1
|
||||
)
|
||||
ret = -1;
|
||||
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
|
||||
"DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
|
||||
TEXT_TOUPPER, fileid, END
|
||||
RHIZOME_FILEHASH_T, hashp, END
|
||||
) == -1
|
||||
)
|
||||
ret = -1;
|
||||
@ -1081,18 +1087,24 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
|
||||
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
||||
candidates++;
|
||||
const char *id = (const char *) sqlite3_column_text(statement, 0);
|
||||
if (rhizome_delete_external(id) == 0 && report)
|
||||
++report->deleted_stale_incoming_files;
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, id) == -1)
|
||||
WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
|
||||
else if (rhizome_delete_external(&hash) == 0 && report)
|
||||
++report->deleted_stale_incoming_files;
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
|
||||
|
||||
statement = sqlite_prepare_bind(&retry,
|
||||
"SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
|
||||
INT64, insert_horizon_no_manifest, END);
|
||||
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
||||
candidates++;
|
||||
const char *id = (const char *) sqlite3_column_text(statement, 0);
|
||||
if (rhizome_delete_external(id) == 0 && report)
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, id) == -1)
|
||||
WARNF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
|
||||
else if (rhizome_delete_external(&hash) == 0 && report)
|
||||
++report->deleted_orphan_files;
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
@ -1152,11 +1164,10 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
|
||||
&& sqlite_step_retry(&retry, statement) == SQLITE_ROW
|
||||
) {
|
||||
/* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
|
||||
const unsigned char *id;
|
||||
|
||||
const char *id;
|
||||
/* Get values */
|
||||
if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
|
||||
id = sqlite3_column_text(statement, 0);
|
||||
id = (const char *) sqlite3_column_text(statement, 0);
|
||||
else {
|
||||
WHY("Incorrect type in id column of files table");
|
||||
break;
|
||||
@ -1167,10 +1178,16 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
|
||||
WHY("Incorrect type in length column of files table");
|
||||
break;
|
||||
}
|
||||
/* Try to drop this file from storage, discarding any references that do not trump the priority
|
||||
of this request. The query done earlier should ensure this, but it doesn't hurt to be
|
||||
paranoid, and it also protects against inconsistency in the database. */
|
||||
rhizome_drop_stored_file((char *)id, group_priority + 1);
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, id) == -1)
|
||||
WHYF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
|
||||
else {
|
||||
/* Try to drop this file from storage, discarding any references that do not trump the
|
||||
* priority of this request. The query done earlier should ensure this, but it doesn't hurt
|
||||
* to be paranoid, and it also protects against inconsistency in the database.
|
||||
*/
|
||||
rhizome_drop_stored_file(&hash, group_priority + 1);
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
|
||||
@ -1187,14 +1204,12 @@ int rhizome_make_space(int group_priority, uint64_t bytes)
|
||||
/* Drop the specified file from storage, and any manifests that reference it, provided that none of
|
||||
* those manifests are being retained at a higher priority than the maximum specified here.
|
||||
*/
|
||||
int rhizome_drop_stored_file(const char *id, int maximum_priority)
|
||||
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority)
|
||||
{
|
||||
if (!rhizome_str_is_file_hash(id))
|
||||
return WHYF("invalid file hash id=%s", alloca_str_toprint(id));
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", TEXT_TOUPPER, id, END);
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", RHIZOME_FILEHASH_T, hashp, END);
|
||||
if (!statement)
|
||||
return WHYF("Could not drop stored file id=%s", id);
|
||||
return WHYF("Could not drop stored file id=%s", alloca_tohex_rhizome_filehash_t(*hashp));
|
||||
int can_drop = 1;
|
||||
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
||||
/* Find manifests for this file */
|
||||
@ -1214,9 +1229,9 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority)
|
||||
that are lower priority, and thus free up a little space. */
|
||||
int priority = rhizome_manifest_priority(&retry, &bid);
|
||||
if (priority == -1)
|
||||
WHYF("Cannot drop fileid=%s due to error, bid=%s", id, alloca_tohex_rhizome_bid_t(bid));
|
||||
WHYF("Cannot drop fileid=%s due to error, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
|
||||
else if (priority > maximum_priority) {
|
||||
WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", id, alloca_tohex_rhizome_bid_t(bid));
|
||||
WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
|
||||
can_drop = 0;
|
||||
} else {
|
||||
if (config.debug.rhizome)
|
||||
@ -1228,7 +1243,7 @@ int rhizome_drop_stored_file(const char *id, int maximum_priority)
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
if (can_drop)
|
||||
rhizome_delete_file_retry(&retry, id);
|
||||
rhizome_delete_file_retry(&retry, hashp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1272,16 +1287,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
|
||||
rhizome_manifest_to_bar(m,bar);
|
||||
|
||||
/* Store the file (but not if it is already in the database) */
|
||||
char filehash[RHIZOME_FILEHASH_STRLEN + 1];
|
||||
if (m->fileLength > 0) {
|
||||
strncpy(filehash, m->fileHexHash, sizeof filehash);
|
||||
str_toupper_inplace(filehash);
|
||||
|
||||
if (!rhizome_exists(filehash))
|
||||
return WHY("File should already be stored by now");
|
||||
} else {
|
||||
filehash[0] = '\0';
|
||||
}
|
||||
if (m->fileLength > 0 && !rhizome_exists(&m->filehash))
|
||||
return WHY("File should already be stored by now");
|
||||
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
|
||||
@ -1331,7 +1338,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
|
||||
INT64, (int64_t) gettime_ms(),
|
||||
STATIC_BLOB, bar, RHIZOME_BAR_BYTES,
|
||||
INT64, m->fileLength,
|
||||
TEXT_TOUPPER|NUL, filehash,
|
||||
RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL,
|
||||
SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author,
|
||||
STATIC_TEXT, service,
|
||||
STATIC_TEXT|NUL, name,
|
||||
@ -1627,15 +1634,28 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
|
||||
if (service == NULL)
|
||||
return WHY("Manifest has no service");
|
||||
|
||||
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
|
||||
const char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
|
||||
|
||||
|
||||
sid_t *sender = NULL;
|
||||
const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
|
||||
if (sender_field) {
|
||||
sender = (sid_t *) alloca(sizeof *sender);
|
||||
if (str_to_sid_t(sender, sender_field) == -1)
|
||||
return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
|
||||
}
|
||||
|
||||
sid_t *recipient = NULL;
|
||||
const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
|
||||
if (recipient_field) {
|
||||
recipient = (sid_t *) alloca(sizeof *recipient);
|
||||
if (str_to_sid_t(recipient, recipient_field) == -1)
|
||||
return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
|
||||
}
|
||||
|
||||
char sqlcmd[1024];
|
||||
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
|
||||
strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
|
||||
|
||||
if (m->fileLength != 0)
|
||||
strbuf_puts(b, " AND filehash = ?");
|
||||
if (name)
|
||||
@ -1644,73 +1664,60 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
strbuf_puts(b, " AND sender = ?");
|
||||
if (recipient)
|
||||
strbuf_puts(b, " AND recipient = ?");
|
||||
|
||||
if (strbuf_overrun(b))
|
||||
return WHYF("SQL command too long: %s", strbuf_str(b));
|
||||
|
||||
|
||||
int ret = 0;
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b));
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END);
|
||||
if (!statement)
|
||||
return -1;
|
||||
|
||||
int field = 1;
|
||||
sqlite3_bind_int(statement, field++, m->fileLength);
|
||||
sqlite3_bind_text(statement, field++, service, -1, SQLITE_STATIC);
|
||||
|
||||
int field = 2;
|
||||
if (m->fileLength != 0)
|
||||
sqlite3_bind_text(statement, field++, m->fileHexHash, -1, SQLITE_STATIC);
|
||||
sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END);
|
||||
if (name)
|
||||
sqlite3_bind_text(statement, field++, name, -1, SQLITE_STATIC);
|
||||
sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END);
|
||||
if (sender)
|
||||
sqlite3_bind_text(statement, field++, sender, -1, SQLITE_STATIC);
|
||||
sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END);
|
||||
if (recipient)
|
||||
sqlite3_bind_text(statement, field++, recipient, -1, SQLITE_STATIC);
|
||||
|
||||
sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END);
|
||||
|
||||
int rows = 0;
|
||||
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
||||
++rows;
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Row %d", rows);
|
||||
|
||||
rhizome_manifest *blob_m = rhizome_new_manifest();
|
||||
if (blob_m == NULL) {
|
||||
ret = WHY("Out of manifests");
|
||||
break;
|
||||
}
|
||||
|
||||
const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
|
||||
|
||||
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
|
||||
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
||||
if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
|
||||
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
|
||||
goto next;
|
||||
}
|
||||
|
||||
if (rhizome_manifest_verify(blob_m)) {
|
||||
WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
|
||||
goto next;
|
||||
}
|
||||
|
||||
const char *q_author = (const char *) sqlite3_column_text(statement, 2);
|
||||
if (q_author) {
|
||||
if (config.debug.rhizome)
|
||||
strbuf_sprintf(b, " .author=%s", q_author);
|
||||
str_to_sid_t(&blob_m->author, q_author);
|
||||
}
|
||||
|
||||
// check that we can re-author this manifest
|
||||
if (rhizome_extract_privatekey(blob_m, NULL)){
|
||||
goto next;
|
||||
}
|
||||
|
||||
*found = blob_m;
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Found duplicate payload, %s", q_manifestid);
|
||||
ret = 1;
|
||||
break;
|
||||
|
||||
next:
|
||||
if (blob_m)
|
||||
rhizome_manifest_free(blob_m);
|
||||
@ -1807,14 +1814,14 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom
|
||||
return sqlite3_changes(rhizome_db) ? 0 : 1;
|
||||
}
|
||||
|
||||
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
|
||||
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
|
||||
{
|
||||
int ret = 0;
|
||||
rhizome_delete_external(fileid);
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", TEXT_TOUPPER, fileid, END);
|
||||
rhizome_delete_external(hashp);
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
|
||||
if (!statement || sqlite_exec_retry(retry, statement) == -1)
|
||||
ret = -1;
|
||||
statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", TEXT_TOUPPER, fileid, END);
|
||||
statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
|
||||
if (!statement || sqlite_exec_retry(retry, statement) == -1)
|
||||
ret = -1;
|
||||
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
|
||||
@ -1826,7 +1833,10 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome
|
||||
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
|
||||
if (rows == -1)
|
||||
return -1;
|
||||
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, strbuf_str(fh)) == -1)
|
||||
return WHYF("invalid field FILES.id=%s", strbuf_str(fh));
|
||||
if (rows && rhizome_delete_file_retry(retry, &hash) == -1)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
@ -1887,10 +1897,10 @@ int rhizome_delete_payload(const rhizome_bid_t *bidp)
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int rhizome_delete_file(const char *fileid)
|
||||
int rhizome_delete_file(const rhizome_filehash_t *hashp)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
return rhizome_delete_file_retry(&retry, fileid);
|
||||
return rhizome_delete_file_retry(&retry, hashp);
|
||||
}
|
||||
|
||||
static int is_interesting(const char *id_hex, int64_t version)
|
||||
@ -1901,7 +1911,7 @@ static int is_interesting(const char *id_hex, int64_t version)
|
||||
// do we have this bundle [or later]?
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
||||
"SELECT filehash FROM manifests WHERE id like ? and version >= ?",
|
||||
"SELECT filehash FROM MANIFESTS WHERE id LIKE ? AND version >= ?",
|
||||
TEXT_TOUPPER, id_hex,
|
||||
INT64, version,
|
||||
END);
|
||||
@ -1910,8 +1920,14 @@ static int is_interesting(const char *id_hex, int64_t version)
|
||||
if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){
|
||||
const char *q_filehash = (const char *) sqlite3_column_text(statement, 0);
|
||||
ret=0;
|
||||
if (q_filehash && *q_filehash && !rhizome_exists(q_filehash))
|
||||
ret=1;
|
||||
if (q_filehash && *q_filehash) {
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, q_filehash) == -1) {
|
||||
WARNF("invalid field MANIFESTS.filehash=%s -- ignored", alloca_str_toprint(q_filehash));
|
||||
ret = 1;
|
||||
} else if (!rhizome_exists(&hash))
|
||||
ret = 1;
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
RETURN(ret);
|
||||
|
||||
@ -892,10 +892,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
|
||||
|
||||
/* Get filehash and size from manifest if present */
|
||||
DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
|
||||
const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0);
|
||||
DEBUGF("bundle file hash = '%s'",hash);
|
||||
int64_t filesize = rhizome_manifest_get_ll(m, "filesize");
|
||||
DEBUGF("file size = %"PRId64,filesize);
|
||||
DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
DEBUGF("file size = %"PRId64, m->fileLength);
|
||||
int64_t version = rhizome_manifest_get_ll(m, "version");
|
||||
DEBUGF("version = %"PRId64,version);
|
||||
|
||||
@ -923,7 +921,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
|
||||
+m->manifest_all_bytes
|
||||
+strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */
|
||||
+strlen(boundary)
|
||||
+filesize
|
||||
+m->fileLength
|
||||
+strlen("\r\n--")+strlen(boundary)+strlen("--\r\n");
|
||||
|
||||
/* XXX For some reason the above is four bytes out, so fix that */
|
||||
@ -961,17 +959,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
|
||||
|
||||
/* send file contents */
|
||||
{
|
||||
char filehash[SHA512_DIGEST_STRING_LENGTH];
|
||||
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, filehash) == -1)
|
||||
rhizome_filehash_t filehash;
|
||||
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1)
|
||||
goto closeit;
|
||||
|
||||
struct rhizome_read read;
|
||||
bzero(&read, sizeof read);
|
||||
if (rhizome_open_read(&read, filehash))
|
||||
if (rhizome_open_read(&read, &filehash))
|
||||
goto closeit;
|
||||
|
||||
int read_ofs;
|
||||
for(read_ofs=0;read_ofs<filesize;){
|
||||
int64_t read_ofs;
|
||||
for(read_ofs=0;read_ofs<m->fileLength;){
|
||||
unsigned char buffer[4096];
|
||||
read.offset=read_ofs;
|
||||
int bytes_read = rhizome_read(&read, buffer, sizeof buffer);
|
||||
|
||||
@ -487,7 +487,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
|
||||
slot->manifest->dataFileUnlinkOnFree = 0;
|
||||
|
||||
strbuf r = strbuf_local(slot->request, sizeof slot->request);
|
||||
strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", slot->manifest->fileHexHash);
|
||||
strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
|
||||
|
||||
if (slot->manifest->journalTail>=0){
|
||||
// if we're fetching a journal bundle, work out how many bytes we have of a previous version
|
||||
@ -509,12 +509,12 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
|
||||
}
|
||||
|
||||
strbuf_puts(r, "\r\n");
|
||||
|
||||
|
||||
if (strbuf_overrun(r))
|
||||
RETURN(WHY("request overrun"));
|
||||
slot->request_len = strbuf_len(r);
|
||||
|
||||
if (rhizome_open_write(&slot->write_state, slot->manifest->fileHexHash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT))
|
||||
if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT))
|
||||
RETURN(-1);
|
||||
} else {
|
||||
strbuf r = strbuf_local(slot->request, sizeof slot->request);
|
||||
@ -690,9 +690,9 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
|
||||
for (i = 0; i < NQUEUES; ++i) {
|
||||
struct rhizome_fetch_slot *as = &rhizome_fetch_queues[i].active;
|
||||
const rhizome_manifest *am = as->manifest;
|
||||
if (as->state != RHIZOME_FETCH_FREE && strcasecmp(m->fileHexHash, am->fileHexHash) == 0) {
|
||||
if (as->state != RHIZOME_FETCH_FREE && cmp_rhizome_filehash_t(&m->filehash, &am->filehash) == 0) {
|
||||
if (config.debug.rhizome_rx)
|
||||
DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, m->fileHexHash);
|
||||
DEBUGF(" fetch already in progress, slot=%d filehash=%s", i, alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
RETURN(SAMEPAYLOAD);
|
||||
}
|
||||
}
|
||||
@ -707,7 +707,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
|
||||
DEBUGF(" is new");
|
||||
|
||||
// If the payload is already available, no need to fetch, so import now.
|
||||
if (rhizome_exists(m->fileHexHash)){
|
||||
if (rhizome_exists(&m->filehash)){
|
||||
if (config.debug.rhizome_rx)
|
||||
DEBUGF(" fetch not started - payload already present, so importing instead");
|
||||
if (rhizome_add_manifest(m, m->ttl-1) == -1)
|
||||
@ -1138,8 +1138,7 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){
|
||||
if (start>=0 && start < slot->previous->fileLength && length>0){
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Copying %"PRId64" bytes from previous journal", length);
|
||||
rhizome_journal_pipe(&slot->write_state, slot->previous->fileHexHash,
|
||||
start, length);
|
||||
rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length);
|
||||
}
|
||||
|
||||
// and we don't need to do this again, so drop the manifest
|
||||
@ -1274,10 +1273,11 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
|
||||
}
|
||||
INFOF("Completed http request from %s:%u for file %s",
|
||||
buf, ntohs(slot->peer_ipandport.sin_port),
|
||||
slot->manifest->fileHexHash);
|
||||
alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
|
||||
} else {
|
||||
INFOF("Completed MDP request from %s for file %s",
|
||||
alloca_tohex_sid_t(slot->peer_sid), slot->manifest->fileHexHash);
|
||||
alloca_tohex_sid_t(slot->peer_sid),
|
||||
alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
|
||||
}
|
||||
} else {
|
||||
/* This was to fetch the manifest, so now fetch the file if needed */
|
||||
@ -1409,7 +1409,8 @@ int rhizome_received_content(const unsigned char *bidprefix,
|
||||
|
||||
if (m){
|
||||
if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){
|
||||
INFOF("Completed MDP transfer in one hit for file %s", m->fileHexHash);
|
||||
INFOF("Completed MDP transfer in one hit for file %s",
|
||||
alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
if (c)
|
||||
candidate_unqueue(c);
|
||||
}
|
||||
|
||||
@ -449,14 +449,15 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder, con
|
||||
if (!is_rhizome_http_enabled())
|
||||
return 1;
|
||||
|
||||
if (!rhizome_str_is_file_hash(remainder))
|
||||
rhizome_filehash_t filehash;
|
||||
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
|
||||
return -1;
|
||||
|
||||
bzero(&r->read_state, sizeof(r->read_state));
|
||||
|
||||
/* Refuse to honour HTTP request if required (used for debugging and
|
||||
testing transition from HTTP to MDP) */
|
||||
if (rhizome_open_read(&r->read_state, remainder))
|
||||
if (rhizome_open_read(&r->read_state, &filehash))
|
||||
return 1;
|
||||
|
||||
if (r->read_state.length==-1){
|
||||
|
||||
115
rhizome_store.c
115
rhizome_store.c
@ -5,22 +5,22 @@
|
||||
|
||||
#define RHIZOME_BUFFER_MAXIMUM_SIZE (1024*1024)
|
||||
|
||||
int rhizome_exists(const char *fileHash)
|
||||
int rhizome_exists(const rhizome_filehash_t *hashp)
|
||||
{
|
||||
int64_t gotfile = 0;
|
||||
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", TEXT_TOUPPER, fileHash, END) != 1)
|
||||
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", RHIZOME_FILEHASH_T, hashp, END) != 1)
|
||||
return 0;
|
||||
return gotfile;
|
||||
}
|
||||
|
||||
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority)
|
||||
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority)
|
||||
{
|
||||
write->blob_fd=-1;
|
||||
|
||||
if (expectedFileHash){
|
||||
if (rhizome_exists(expectedFileHash))
|
||||
if (expectedHashp){
|
||||
if (rhizome_exists(expectedHashp))
|
||||
return 1;
|
||||
strlcpy(write->id, expectedFileHash, SHA512_DIGEST_STRING_LENGTH);
|
||||
write->id = *expectedHashp;
|
||||
write->id_known=1;
|
||||
}else{
|
||||
write->id_known=0;
|
||||
@ -429,7 +429,7 @@ int rhizome_fail_write(struct rhizome_write *write)
|
||||
write->buffer_list=n->_next;
|
||||
free(n);
|
||||
}
|
||||
rhizome_delete_file(write->id);
|
||||
rhizome_delete_file(&write->id);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -460,39 +460,39 @@ int rhizome_finish_write(struct rhizome_write *write)
|
||||
if (write_release_lock(write))
|
||||
goto failure;
|
||||
|
||||
char hash_out[SHA512_DIGEST_STRING_LENGTH + 1];
|
||||
SHA512_End(&write->sha512_context, hash_out);
|
||||
str_toupper_inplace(hash_out);
|
||||
rhizome_filehash_t hash_out;
|
||||
SHA512_Final(hash_out.binary, &write->sha512_context);
|
||||
SHA512_End(&write->sha512_context, NULL);
|
||||
|
||||
if (write->id_known) {
|
||||
if (strcasecmp(write->id, hash_out) != 0) {
|
||||
WHYF("expected filehash=%s, got %s", write->id, hash_out);
|
||||
if (cmp_rhizome_filehash_t(&write->id, &hash_out) != 0) {
|
||||
WHYF("expected filehash=%s, got %s", alloca_tohex_rhizome_filehash_t(write->id), alloca_tohex_rhizome_filehash_t(hash_out));
|
||||
goto failure;
|
||||
}
|
||||
} else {
|
||||
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH);
|
||||
write->id = hash_out;
|
||||
}
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
rhizome_remove_file_datainvalid(&retry, write->id);
|
||||
if (rhizome_exists(write->id)) {
|
||||
rhizome_remove_file_datainvalid(&retry, &write->id);
|
||||
if (rhizome_exists(&write->id)) {
|
||||
// we've already got that payload, delete the new copy
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("File id='%s' already present, removed id='%"PRId64"'", write->id, write->temp_id);
|
||||
DEBUGF("File id=%s already present, removed id='%"PRId64"'", alloca_tohex_rhizome_filehash_t(write->id), write->temp_id);
|
||||
} else {
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
|
||||
goto dbfailure;
|
||||
|
||||
// delete any half finished records
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", STATIC_TEXT, write->id, END);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", STATIC_TEXT, write->id, END);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", RHIZOME_FILEHASH_T, &write->id, END);
|
||||
|
||||
if (sqlite_exec_void_retry(
|
||||
&retry,
|
||||
"UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?",
|
||||
STATIC_TEXT, write->id,
|
||||
RHIZOME_FILEHASH_T, &write->id,
|
||||
INT64, gettime_ms(),
|
||||
UINT64_TOSTR, write->temp_id,
|
||||
END
|
||||
@ -507,7 +507,7 @@ int rhizome_finish_write(struct rhizome_write *write)
|
||||
WHYF("Failed to generate file path");
|
||||
goto dbfailure;
|
||||
}
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, write->id)){
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, alloca_tohex_rhizome_filehash_t(write->id))){
|
||||
WHYF("Failed to generate file path");
|
||||
goto dbfailure;
|
||||
}
|
||||
@ -521,7 +521,7 @@ int rhizome_finish_write(struct rhizome_write *write)
|
||||
if (sqlite_exec_void_retry(
|
||||
&retry,
|
||||
"UPDATE FILEBLOBS SET id = ? WHERE rowid = ?",
|
||||
STATIC_TEXT, write->id,
|
||||
RHIZOME_FILEHASH_T, &write->id,
|
||||
INT64, write->blob_rowid,
|
||||
END
|
||||
) == -1
|
||||
@ -531,7 +531,7 @@ int rhizome_finish_write(struct rhizome_write *write)
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1)
|
||||
goto dbfailure;
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Stored file %s", write->id);
|
||||
DEBUGF("Stored file %s", alloca_tohex_rhizome_filehash_t(write->id));
|
||||
}
|
||||
write->blob_rowid=-1;
|
||||
return 0;
|
||||
@ -553,7 +553,7 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
|
||||
struct rhizome_write write;
|
||||
bzero(&write, sizeof(write));
|
||||
|
||||
int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
|
||||
int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
|
||||
if (ret!=0)
|
||||
return ret;
|
||||
|
||||
@ -583,7 +583,7 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length
|
||||
struct rhizome_write write;
|
||||
bzero(&write, sizeof(write));
|
||||
|
||||
int ret=rhizome_open_write(&write, m->fileHexHash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
|
||||
int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
|
||||
if (ret!=0)
|
||||
return ret;
|
||||
|
||||
@ -624,7 +624,7 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
|
||||
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
|
||||
|
||||
if (m->fileLength == 0){
|
||||
m->fileHexHash[0] = '\0';
|
||||
m->filehash = RHIZOME_FILEHASH_NONE;
|
||||
rhizome_manifest_del(m, "filehash");
|
||||
}
|
||||
return 0;
|
||||
@ -676,8 +676,8 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
|
||||
if (rhizome_finish_write(&write))
|
||||
goto failure;
|
||||
|
||||
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
|
||||
rhizome_manifest_set(m, "filehash", m->fileHexHash);
|
||||
m->filehash = write.id;
|
||||
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
return 0;
|
||||
|
||||
failure:
|
||||
@ -687,26 +687,24 @@ failure:
|
||||
|
||||
/* Return -1 on error, 0 if file blob found, 1 if not found.
|
||||
*/
|
||||
int rhizome_open_read(struct rhizome_read *read, const char *fileid)
|
||||
int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp)
|
||||
{
|
||||
strncpy(read->id, fileid, sizeof read->id);
|
||||
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
|
||||
str_toupper_inplace(read->id);
|
||||
read->id = *hashp;
|
||||
read->blob_rowid = -1;
|
||||
read->blob_fd = -1;
|
||||
if (sqlite_exec_int64(&read->blob_rowid,
|
||||
if (sqlite_exec_int64(&read->blob_rowid,
|
||||
"SELECT FILEBLOBS.rowid "
|
||||
"FROM FILEBLOBS, FILES "
|
||||
"WHERE FILEBLOBS.id = FILES.id"
|
||||
" AND FILES.id = ?"
|
||||
" AND FILES.datavalid != 0", STATIC_TEXT, read->id, END) == -1)
|
||||
" AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1)
|
||||
return -1;
|
||||
if (read->blob_rowid != -1) {
|
||||
read->length = -1; // discover the length on opening the db BLOB
|
||||
} else {
|
||||
// No row in FILEBLOBS, look for an external blob file.
|
||||
char blob_path[1024];
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, read->id))
|
||||
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, alloca_tohex_rhizome_filehash_t(read->id)))
|
||||
return -1;
|
||||
read->blob_fd = open(blob_path, O_RDONLY);
|
||||
if (read->blob_fd == -1) {
|
||||
@ -787,13 +785,13 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
|
||||
read_state->hash_offset += bytes_read;
|
||||
// if we hash everything and the has doesn't match, we need to delete the payload
|
||||
if (read_state->hash_offset>=read_state->length){
|
||||
char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
|
||||
SHA512_End(&read_state->sha512_context, hash_out);
|
||||
|
||||
if (strcasecmp(read_state->id, hash_out)){
|
||||
rhizome_filehash_t hash_out;
|
||||
SHA512_Final(hash_out.binary, &read_state->sha512_context);
|
||||
SHA512_End(&read_state->sha512_context, NULL);
|
||||
if (cmp_rhizome_filehash_t(&read_state->id, &hash_out) != 0) {
|
||||
// hash failure, mark the payload as invalid
|
||||
read_state->invalid = 1;
|
||||
RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out));
|
||||
RETURN(WHYF("Expected hash=%s, got %s", alloca_tohex_rhizome_filehash_t(read_state->id), alloca_tohex_rhizome_filehash_t(hash_out)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -874,7 +872,7 @@ int rhizome_read_close(struct rhizome_read *read)
|
||||
read->blob_fd = -1;
|
||||
if (read->invalid){
|
||||
// delete payload!
|
||||
rhizome_delete_file(read->id);
|
||||
rhizome_delete_file(&read->id);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -980,8 +978,7 @@ int rhizome_cache_count()
|
||||
}
|
||||
|
||||
// read a block of data, caching meta data for reuse
|
||||
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout,
|
||||
uint64_t fileOffset, unsigned char *buffer, int length)
|
||||
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length)
|
||||
{
|
||||
// look for a cached entry
|
||||
struct cache_entry **ptr = find_entry_location(&root, bidp, version);
|
||||
@ -989,15 +986,13 @@ int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t t
|
||||
|
||||
// if we don't have one yet, create one and open it
|
||||
if (!entry){
|
||||
char filehash[SHA512_DIGEST_STRING_LENGTH];
|
||||
if (rhizome_database_filehash_from_id(bidp, version, filehash) == -1)
|
||||
rhizome_filehash_t filehash;
|
||||
if (rhizome_database_filehash_from_id(bidp, version, &filehash) == -1)
|
||||
return -1;
|
||||
|
||||
entry = emalloc_zero(sizeof(struct cache_entry));
|
||||
|
||||
if (rhizome_open_read(&entry->read_state, filehash)){
|
||||
if (rhizome_open_read(&entry->read_state, &filehash)){
|
||||
free(entry);
|
||||
return WHYF("Payload %s not found", filehash);
|
||||
return WHYF("Payload %s not found", alloca_tohex_rhizome_filehash_t(filehash));
|
||||
}
|
||||
entry->bundle_id = *bidp;
|
||||
entry->version = version;
|
||||
@ -1073,8 +1068,9 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){
|
||||
int ret = rhizome_open_read(read_state, m->fileHexHash);
|
||||
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state)
|
||||
{
|
||||
int ret = rhizome_open_read(read_state, &m->filehash);
|
||||
if (ret == 0)
|
||||
ret = read_derive_key(m, bsk, read_state);
|
||||
return ret;
|
||||
@ -1100,12 +1096,12 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
|
||||
*
|
||||
* Returns -1 on error, 0 if dumped successfully, 1 if not found.
|
||||
*/
|
||||
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
|
||||
int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length)
|
||||
{
|
||||
struct rhizome_read read_state;
|
||||
bzero(&read_state, sizeof read_state);
|
||||
|
||||
int ret = rhizome_open_read(&read_state, id);
|
||||
int ret = rhizome_open_read(&read_state, hashp);
|
||||
|
||||
if (ret == 0) {
|
||||
ret = write_file(&read_state, filepath);
|
||||
@ -1141,13 +1137,12 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint64_t start_offset, uint64_t length)
|
||||
int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length)
|
||||
{
|
||||
struct rhizome_read read_state;
|
||||
bzero(&read_state, sizeof read_state);
|
||||
if (rhizome_open_read(&read_state, fileHash))
|
||||
if (rhizome_open_read(&read_state, hashp))
|
||||
return -1;
|
||||
|
||||
read_state.offset = start_offset;
|
||||
int ret = rhizome_pipe(&read_state, write, length);
|
||||
rhizome_read_close(&read_state);
|
||||
@ -1182,7 +1177,7 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m,
|
||||
|
||||
if (copy_length>0){
|
||||
// note that we don't need to bother decrypting the existing journal payload
|
||||
ret = rhizome_journal_pipe(write, m->fileHexHash, advance_by, copy_length);
|
||||
ret = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);
|
||||
if (ret)
|
||||
goto failure;
|
||||
}
|
||||
@ -1218,8 +1213,8 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
|
||||
if (ret)
|
||||
goto failure;
|
||||
|
||||
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
|
||||
rhizome_manifest_set(m, "filehash", m->fileHexHash);
|
||||
m->filehash = write.id;
|
||||
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
return 0;
|
||||
|
||||
failure:
|
||||
@ -1250,8 +1245,8 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t
|
||||
if (ret)
|
||||
goto failure;
|
||||
|
||||
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
|
||||
rhizome_manifest_set(m, "filehash", m->fileHexHash);
|
||||
m->filehash = write.id;
|
||||
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
|
||||
|
||||
return 0;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user