From f9b828c3dd9e08027e26a13a119a7811431e9289 Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Wed, 21 Aug 2013 15:45:18 +0930 Subject: [PATCH] Remove payload if hash doesn't match when reading back --- meshms.c | 4 +-- rhizome.h | 8 ++--- rhizome_database.c | 23 ++++++++++---- rhizome_direct_http.c | 2 +- rhizome_http.c | 4 +-- rhizome_store.c | 71 ++++++++++++++++++------------------------- tests/rhizomeops | 16 ++++++++++ 7 files changed, 71 insertions(+), 57 deletions(-) diff --git a/meshms.c b/meshms.c index b0e295bb..8bc839f7 100644 --- a/meshms.c +++ b/meshms.c @@ -221,7 +221,7 @@ static int ply_read_open(struct ply_read *ply, const char *id, rhizome_manifest DEBUGF("Opening ply %s", id); if (rhizome_retrieve_manifest(id, m)) return -1; - if (rhizome_open_decrypt_read(m, NULL, &ply->read, 0)) + if (rhizome_open_decrypt_read(m, NULL, &ply->read)) return -1; ply->read.offset = ply->read.length = m->fileLength; return 0; @@ -468,7 +468,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct rhizome_read_buffer buff; bzero(&buff, sizeof(buff)); - int ret = rhizome_open_decrypt_read(m, NULL, &read, 0); + int ret = rhizome_open_decrypt_read(m, NULL, &read); if (ret<0) goto end; diff --git a/rhizome.h b/rhizome.h index b3767fa5..c2930403 100644 --- a/rhizome.h +++ b/rhizome.h @@ -432,8 +432,9 @@ struct rhizome_read{ unsigned char key[RHIZOME_CRYPT_KEY_BYTES]; unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES]; - int hash; + int64_t hash_offset; SHA512_CTX sha512_context; + char invalid; int64_t blob_rowid; int blob_fd; @@ -718,12 +719,11 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, const unsigned char *key, const unsigned char *nonce); -int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash); +int rhizome_open_read(struct rhizome_read *read, const char *fileid); int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length); int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len); int rhizome_read_close(struct rhizome_read *read); -int rhizome_store_delete(const char *id); -int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash); +int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state); int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk); int rhizome_dump_file(const char *id, const char *filepath, int64_t *length); int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout, diff --git a/rhizome_database.c b/rhizome_database.c index c5506c22..cd52a5c6 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. static char rhizome_thisdatastore_path[256]; +static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid); + const char *rhizome_datastore_path() { if (!rhizome_thisdatastore_path[0]) @@ -716,11 +718,20 @@ int rhizome_database_filehash_from_id(const char *id, uint64_t version, char has OUT(); } +static int rhizome_delete_external(const char *fileid) +{ + // attempt to remove any external blob + char blob_path[1024]; + if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid)) + return -1; + return unlink(blob_path); +} + static int rhizome_cleanup_external(sqlite_retry_state *retry, sqlite3_stmt *statement){ int ret=0; while (sqlite_step_retry(retry, statement) == SQLITE_ROW) { const char *id = (const char *) sqlite3_column_text(statement, 0); - if (rhizome_store_delete(id)==0) + if (rhizome_delete_external(id)==0) ret++; } return ret; @@ -851,11 +862,8 @@ int rhizome_drop_stored_file(const char *id,int maximum_priority) } } sqlite3_finalize(statement); - if (can_drop) { - sqlite_exec_void_retry(&retry, "delete from files where id='%s';",id); - rhizome_store_delete(id); - sqlite_exec_void_retry(&retry, "delete from fileblobs where id='%s';",id); - } + if (can_drop) + rhizome_delete_file_retry(&retry, id); return 0; } @@ -1394,6 +1402,9 @@ int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifes static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid) { int ret = 0; + + rhizome_delete_external(fileid); + sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?"); if (!statement) ret = -1; diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index c24b4d76..7e4c314a 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -976,7 +976,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r) struct rhizome_read read; bzero(&read, sizeof read); - if (rhizome_open_read(&read, filehash, 0)) + if (rhizome_open_read(&read, filehash)) goto closeit; int read_ofs; diff --git a/rhizome_http.c b/rhizome_http.c index 88e6646a..c38cb1c0 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -620,7 +620,7 @@ int rhizome_server_parse_http_request(rhizome_http_request *r) } else { str_toupper_inplace(id); bzero(&r->read_state, sizeof(r->read_state)); - if (rhizome_open_read(&r->read_state, id, 1)) + if (rhizome_open_read(&r->read_state, id)) rhizome_server_simple_http_response(r, 404, "

Payload not found

\r\n"); else{ if (r->read_state.length==-1){ @@ -635,8 +635,6 @@ int rhizome_server_parse_http_request(rhizome_http_request *r) if (range){ sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset); DEBUGF("Found range header %"PRId64,r->read_state.offset); - if (r->read_state.offset) - r->read_state.hash=0; } if (r->read_state.length - r->read_state.offset>0){ diff --git a/rhizome_store.c b/rhizome_store.c index 053026c7..1157f933 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -410,25 +410,12 @@ end: return ret; } -int rhizome_store_delete(const char *id){ - char blob_path[1024]; - if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id)) - return -1; - if (unlink(blob_path)){ - if (config.debug.externalblobs) - DEBUG_perror("unlink"); - return -1; - } - return 0; -} - int rhizome_fail_write(struct rhizome_write *write){ if (write->blob_fd>=0){ if (config.debug.externalblobs) DEBUGF("Closing and removing fd %d", write->blob_fd); close(write->blob_fd); write->blob_fd=-1; - rhizome_store_delete(write->id); } write_release_lock(write); while(write->buffer_list){ @@ -436,16 +423,7 @@ int rhizome_fail_write(struct rhizome_write *write){ write->buffer_list=n->_next; free(n); } - // don't worry too much about sql failures. - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; - if (write->blob_rowid>=0){ - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, - "DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid); - write->blob_rowid=-1; - } - sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, - "DELETE FROM FILES WHERE id='%s'", - write->id); + rhizome_delete_file(write->id); return 0; } @@ -689,7 +667,7 @@ failure: /* Return -1 on error, 0 if file blob found, 1 if not found. */ -int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash) +int rhizome_open_read(struct rhizome_read *read, const char *fileid) { strncpy(read->id, fileid, sizeof read->id); read->id[RHIZOME_FILEHASH_STRLEN] = '\0'; @@ -716,19 +694,22 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash) if (config.debug.externalblobs) DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length); } - read->hash = hash; read->offset = 0; - if (hash) - SHA512_Init(&read->sha512_context); + read->hash_offset = 0; + SHA512_Init(&read->sha512_context); return 0; // file opened } /* Read content from the store, hashing and decrypting as we go. - Random access is supported, but hashing requires reads to be sequential though we don't enforce this. */ + Random access is supported, but hashing requires all payload contents to be read sequentially. */ // returns the number of bytes read int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length) { IN(); + // hash check failed, just return an error + if (read_state->invalid) + RETURN(-1); + int bytes_read = 0; if (read_state->blob_fd >= 0) { if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1) @@ -774,19 +755,24 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf } while (1); } else RETURN(WHY("file not open")); - if (read_state->hash){ - if (buffer && bytes_read>0) - SHA512_Update(&read_state->sha512_context, buffer, bytes_read); - - if (read_state->offset + bytes_read>=read_state->length){ + + // hash the payload as we go, but only if we happen to read the payload data in order + if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){ + SHA512_Update(&read_state->sha512_context, buffer, bytes_read); + read_state->hash_offset += bytes_read; + // if we hash everything and the has doesn't match, we need to delete the payload + if (read_state->hash_offset>=read_state->length){ char hash_out[SHA512_DIGEST_STRING_LENGTH+1]; SHA512_End(&read_state->sha512_context, hash_out); + if (strcasecmp(read_state->id, hash_out)){ + // hash failure, mark the payload as invalid + read_state->invalid = 1; RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out)); } - read_state->hash=0; } } + if (read_state->crypt && buffer && bytes_read>0){ if(rhizome_crypt_xor_block( buffer, bytes_read, @@ -861,6 +847,10 @@ int rhizome_read_close(struct rhizome_read *read) close(read->blob_fd); } read->blob_fd = -1; + if (read->invalid){ + // delete payload! + rhizome_delete_file(read->id); + } return 0; } @@ -970,7 +960,7 @@ int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t ti entry = emalloc_zero(sizeof(struct cache_entry)); - if (rhizome_open_read(&entry->read_state, filehash, 0)){ + if (rhizome_open_read(&entry->read_state, filehash)){ free(entry); return WHYF("Payload %s not found", filehash); } @@ -1048,9 +1038,8 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom return 0; } -int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash){ - // for now, always hash the file - int ret = rhizome_open_read(read_state, m->fileHexHash, hash); +int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){ + int ret = rhizome_open_read(read_state, m->fileHexHash); if (ret == 0) ret = read_derive_key(m, bsk, read_state); return ret; @@ -1065,7 +1054,7 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t { struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - int ret = rhizome_open_decrypt_read(m, bsk, &read_state, 1); + int ret = rhizome_open_decrypt_read(m, bsk, &read_state); if (ret == 0) ret = write_file(&read_state, filepath); rhizome_read_close(&read_state); @@ -1081,7 +1070,7 @@ int rhizome_dump_file(const char *id, const char *filepath, int64_t *length) struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - int ret = rhizome_open_read(&read_state, id, 1); + int ret = rhizome_open_read(&read_state, id); if (ret == 0) { ret = write_file(&read_state, filepath); @@ -1121,7 +1110,7 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint { struct rhizome_read read_state; bzero(&read_state, sizeof read_state); - if (rhizome_open_read(&read_state, fileHash, start_offset>0?0:1)) + if (rhizome_open_read(&read_state, fileHash)) return -1; read_state.offset = start_offset; diff --git a/tests/rhizomeops b/tests/rhizomeops index 46848b19..d9b9f822 100755 --- a/tests/rhizomeops +++ b/tests/rhizomeops @@ -311,6 +311,22 @@ test_ExtractManifestFileFromExtBlob() { assert diff file2 file2x } +doc_CorruptExternalBlob="A corrupted payload should fail to export" +setup_CorruptExternalBlob() { + setup_servald + setup_rhizome + executeOk_servald config set rhizome.external_blobs 1 + echo "A test file" >file1 + executeOk_servald rhizome add file $SIDB1 file1 file1.manifest + extract_manifest_id manifestid file1.manifest + extract_manifest_filehash filehash file1.manifest + echo "Replacement" >$SERVALINSTANCE_PATH/$filehash +} +test_CorruptExternalBlob() { + execute --exit-status=255 $servald rhizome extract file $manifestid file1a + tfw_cat --stderr +} + doc_ExtractManifestToStdout="Export manifest to output field" setup_ExtractManifestToStdout() { setup_servald