ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-20 05:37:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove payload if hash doesn't match when reading back

Browse Source
This commit is contained in:
Jeremy Lakeman 2013-08-21 15:45:18 +09:30
parent 1468d2deb0
commit f9b828c3dd
7 changed files with 71 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
meshms.c
View File

@ -221,7 +221,7 @@ static int ply_read_open(struct ply_read *ply, const char *id, rhizome_manifest
DEBUGF("Opening ply %s", id);
if (rhizome_retrieve_manifest(id, m))
return -1;
if (rhizome_open_decrypt_read(m, NULL, &ply->read, 0))
if (rhizome_open_decrypt_read(m, NULL, &ply->read))
return -1;
ply->read.offset = ply->read.length = m->fileLength;
return 0;
@ -468,7 +468,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
struct rhizome_read_buffer buff;
bzero(&buff, sizeof(buff));
int ret = rhizome_open_decrypt_read(m, NULL, &read, 0);
int ret = rhizome_open_decrypt_read(m, NULL, &read);
if (ret<0)
goto end;

8
rhizome.h
View File

@ -432,8 +432,9 @@ struct rhizome_read{
unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
int hash;
int64_t hash_offset;
SHA512_CTX sha512_context;
char invalid;
int64_t blob_rowid;
int blob_fd;
@ -718,12 +719,11 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);
int rhizome_open_read(struct rhizome_read *read, const char *fileid);
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
int rhizome_read_close(struct rhizome_read *read);
int rhizome_store_delete(const char *id);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length);
int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout,

23
rhizome_database.c
View File

@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
static char rhizome_thisdatastore_path[256];
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid);
const char *rhizome_datastore_path()
{
if (!rhizome_thisdatastore_path[0])
@ -716,11 +718,20 @@ int rhizome_database_filehash_from_id(const char *id, uint64_t version, char has
OUT();
}
static int rhizome_delete_external(const char *fileid)
{
// attempt to remove any external blob
char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid))
return -1;
return unlink(blob_path);
}
static int rhizome_cleanup_external(sqlite_retry_state *retry, sqlite3_stmt *statement){
int ret=0;
while (sqlite_step_retry(retry, statement) == SQLITE_ROW) {
const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_store_delete(id)==0)
if (rhizome_delete_external(id)==0)
ret++;
}
return ret;
@ -851,11 +862,8 @@ int rhizome_drop_stored_file(const char *id,int maximum_priority)
}
}
sqlite3_finalize(statement);
if (can_drop) {
sqlite_exec_void_retry(&retry, "delete from files where id='%s';",id);
rhizome_store_delete(id);
sqlite_exec_void_retry(&retry, "delete from fileblobs where id='%s';",id);
}
if (can_drop)
rhizome_delete_file_retry(&retry, id);
return 0;
}
@ -1394,6 +1402,9 @@ int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifes
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
{
int ret = 0;
rhizome_delete_external(fileid);
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
if (!statement)
ret = -1;

2
rhizome_direct_http.c
View File

@ -976,7 +976,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
struct rhizome_read read;
bzero(&read, sizeof read);
if (rhizome_open_read(&read, filehash, 0))
if (rhizome_open_read(&read, filehash))
goto closeit;
int read_ofs;

4
rhizome_http.c
View File

@ -620,7 +620,7 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
} else {
str_toupper_inplace(id);
bzero(&r->read_state, sizeof(r->read_state));
if (rhizome_open_read(&r->read_state, id, 1))
if (rhizome_open_read(&r->read_state, id))
rhizome_server_simple_http_response(r, 404, "<html><h1>Payload not found</h1></html>\r\n");
else{
if (r->read_state.length==-1){
@ -635,8 +635,6 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
if (range){
sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset);
DEBUGF("Found range header %"PRId64,r->read_state.offset);
if (r->read_state.offset)
r->read_state.hash=0;
}
if (r->read_state.length - r->read_state.offset>0){

71
rhizome_store.c
View File

@ -410,25 +410,12 @@ end:
return ret;
}
int rhizome_store_delete(const char *id){
char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id))
return -1;
if (unlink(blob_path)){
if (config.debug.externalblobs)
DEBUG_perror("unlink");
return -1;
}
return 0;
}
int rhizome_fail_write(struct rhizome_write *write){
if (write->blob_fd>=0){
if (config.debug.externalblobs)
DEBUGF("Closing and removing fd %d", write->blob_fd);
close(write->blob_fd);
write->blob_fd=-1;
rhizome_store_delete(write->id);
}
write_release_lock(write);
while(write->buffer_list){
@ -436,16 +423,7 @@ int rhizome_fail_write(struct rhizome_write *write){
write->buffer_list=n->_next;
free(n);
}
// don't worry too much about sql failures.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (write->blob_rowid>=0){
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
write->blob_rowid=-1;
}
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILES WHERE id='%s'",
write->id);
rhizome_delete_file(write->id);
return 0;
}
@ -689,7 +667,7 @@ failure:
/* Return -1 on error, 0 if file blob found, 1 if not found.
*/
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
int rhizome_open_read(struct rhizome_read *read, const char *fileid)
{
strncpy(read->id, fileid, sizeof read->id);
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
@ -716,19 +694,22 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
if (config.debug.externalblobs)
DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length);
}
read->hash = hash;
read->offset = 0;
if (hash)
SHA512_Init(&read->sha512_context);
read->hash_offset = 0;
SHA512_Init(&read->sha512_context);
return 0; // file opened
}
/* Read content from the store, hashing and decrypting as we go.
Random access is supported, but hashing requires reads to be sequential though we don't enforce this. */
Random access is supported, but hashing requires all payload contents to be read sequentially. */
// returns the number of bytes read
int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length)
{
IN();
// hash check failed, just return an error
if (read_state->invalid)
RETURN(-1);
int bytes_read = 0;
if (read_state->blob_fd >= 0) {
if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1)
@ -774,19 +755,24 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
} while (1);
} else
RETURN(WHY("file not open"));
if (read_state->hash){
if (buffer && bytes_read>0)
SHA512_Update(&read_state->sha512_context, buffer, bytes_read);
if (read_state->offset + bytes_read>=read_state->length){
// hash the payload as we go, but only if we happen to read the payload data in order
if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){
SHA512_Update(&read_state->sha512_context, buffer, bytes_read);
read_state->hash_offset += bytes_read;
// if we hash everything and the has doesn't match, we need to delete the payload
if (read_state->hash_offset>=read_state->length){
char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
SHA512_End(&read_state->sha512_context, hash_out);
if (strcasecmp(read_state->id, hash_out)){
// hash failure, mark the payload as invalid
read_state->invalid = 1;
RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out));
}
read_state->hash=0;
}
}
if (read_state->crypt && buffer && bytes_read>0){
if(rhizome_crypt_xor_block(
buffer, bytes_read,
@ -861,6 +847,10 @@ int rhizome_read_close(struct rhizome_read *read)
close(read->blob_fd);
}
read->blob_fd = -1;
if (read->invalid){
// delete payload!
rhizome_delete_file(read->id);
}
return 0;
}
@ -970,7 +960,7 @@ int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t ti
entry = emalloc_zero(sizeof(struct cache_entry));
if (rhizome_open_read(&entry->read_state, filehash, 0)){
if (rhizome_open_read(&entry->read_state, filehash)){
free(entry);
return WHYF("Payload %s not found", filehash);
}
@ -1048,9 +1038,8 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
return 0;
}
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash){
// for now, always hash the file
int ret = rhizome_open_read(read_state, m->fileHexHash, hash);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){
int ret = rhizome_open_read(read_state, m->fileHexHash);
if (ret == 0)
ret = read_derive_key(m, bsk, read_state);
return ret;
@ -1065,7 +1054,7 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
{
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
int ret = rhizome_open_decrypt_read(m, bsk, &read_state, 1);
int ret = rhizome_open_decrypt_read(m, bsk, &read_state);
if (ret == 0)
ret = write_file(&read_state, filepath);
rhizome_read_close(&read_state);
@ -1081,7 +1070,7 @@ int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
int ret = rhizome_open_read(&read_state, id, 1);
int ret = rhizome_open_read(&read_state, id);
if (ret == 0) {
ret = write_file(&read_state, filepath);
@ -1121,7 +1110,7 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint
{
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
if (rhizome_open_read(&read_state, fileHash, start_offset>0?0:1))
if (rhizome_open_read(&read_state, fileHash))
return -1;
read_state.offset = start_offset;

16
tests/rhizomeops
View File

@ -311,6 +311,22 @@ test_ExtractManifestFileFromExtBlob() {
assert diff file2 file2x
}
doc_CorruptExternalBlob="A corrupted payload should fail to export"
setup_CorruptExternalBlob() {
setup_servald
setup_rhizome
executeOk_servald config set rhizome.external_blobs 1
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_manifest_id manifestid file1.manifest
extract_manifest_filehash filehash file1.manifest
echo "Replacement" >$SERVALINSTANCE_PATH/$filehash
}
test_CorruptExternalBlob() {
execute --exit-status=255 $servald rhizome extract file $manifestid file1a
tfw_cat --stderr
}
doc_ExtractManifestToStdout="Export manifest to output field"
setup_ExtractManifestToStdout() {
setup_servald