ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rhizome extract file will now log an error on hash mismatch

Browse Source
This commit is contained in:
Jeremy Lakeman 2012-12-28 11:34:22 +10:30
parent 3f45623d04
commit 57439b2162
3 changed files with 170 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
rhizome.h
View File

@ -607,15 +607,32 @@ struct rhizome_write{
int64_t file_offset;
int64_t file_length;
int crypt;
unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
// note the last 8 bytes will be reset with the current file_offest
unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
int crypt;
SHA512_CTX sha512_context;
int64_t blob_rowid;
};
struct rhizome_read{
char id[SHA512_DIGEST_STRING_LENGTH+1];
int crypt;
unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
// note the last 8 bytes will be reset with the current file_offest
unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
int hash;
SHA512_CTX sha512_context;
int64_t blob_rowid;
int64_t offset;
int64_t length;
};
int rhizome_exists(const char *fileHash);
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority);
int rhizome_flush(struct rhizome_write *write);
@ -627,5 +644,7 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
const unsigned char *key, unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
#endif //__SERVALDNA__RHIZOME_H

132
rhizome_database.c
View File

@ -1359,99 +1359,53 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
*/
int rhizome_retrieve_file(const char *fileid, const char *filepath, const unsigned char *key)
{
if (rhizome_update_file_priority(fileid) == -1) {
WHY("Failed to update file priority");
int ret;
if (rhizome_update_file_priority(fileid) == -1)
return WHY("Failed to update file priority");
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
// for now, always hash the file
if (rhizome_open_read(&read_state, fileid, 1))
return 0;
int fd=-1;
if (filepath&&filepath[0]) {
fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
if (fd == -1)
return WHY_perror("open");
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT rowid FROM fileblobs WHERE (SELECT 1 FROM files WHERE FILEBLOBS.id = FILES.id AND id = ? AND datavalid != 0)");
if (!statement)
return -1;
int ret = 0;
char fileIdUpper[RHIZOME_FILEHASH_STRLEN + 1];
strncpy(fileIdUpper, fileid, sizeof fileIdUpper);
fileIdUpper[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(fileIdUpper);
sqlite3_bind_text(statement, 1, fileIdUpper, -1, SQLITE_STATIC);
int stepcode = sqlite_step_retry(&retry, statement);
if (stepcode != SQLITE_ROW) {
ret = 0; // no files found
} else if (!( sqlite3_column_count(statement) == 1
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER
)) {
ret = WHY("Incorrect statement column");
} else {
long long length;
int64_t rowid = sqlite3_column_int64(statement, 0);
sqlite3_blob *blob = NULL;
int code;
do code = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(code) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(code)) {
ret = WHY("Could not open blob for reading");
} else {
length = sqlite3_blob_bytes(blob);
cli_puts("filehash"); cli_delim(":");
cli_puts(fileIdUpper); cli_delim("\n");
cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", length); cli_delim("\n");
ret = 1;
if (filepath&&filepath[0]) {
int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
if (fd == -1) {
WHY_perror("open");
ret = WHYF("Cannot open %s for write/create", filepath);
} else {
/* read from blob and write to disk, decrypting if necessary as we go. Each 4KB block of
data has a nonce which is fed with the key into crypto_stream_xsalsa20(). The nonce is
the file address divided by 4KB. This approach is used as it allows us to append to
files easily, without having to get the XOR stream for the whole file, and without the
cipher on existing bytes having to change. Both of these are important properties for
journal bundles, such as will be used by MeshMS. For non-journal bundles where it is
important that changing the payload changes the encryption key (so that the XOR between
any two versions of the payload cannot be easily obtained). We will do this by having
journal manifests identified, causing the key to be locked, rather than based on the
version number. But anyway, we are supplied with the key here, so all we need to do is
do the block counting and call crypto_stream_xsalsa20().
*/
long long offset;
unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
bzero(nonce, crypto_stream_xsalsa20_NONCEBYTES);
unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
for (offset = 0; offset < length; offset += RHIZOME_CRYPT_PAGE_SIZE) {
long long count=length-offset;
if (count>RHIZOME_CRYPT_PAGE_SIZE) count=RHIZOME_CRYPT_PAGE_SIZE;
if(sqlite3_blob_read(blob,&buffer[0],count,offset)!=SQLITE_OK) {
ret = 0;
WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
WHYF("Error reading %lld bytes of data from blob at offset 0x%llx", count, offset);
break;
}
if (key) {
if(rhizome_crypt_xor_block(buffer, count, offset, key, nonce)){
ret=0;
break;
}
}
if (write(fd,buffer,count)!=count) {
ret =0;
WHY("Failed to write data to file");
break;
}
}
sqlite3_blob_close(blob);
blob = NULL;
}
if (fd != -1 && close(fd) == -1) {
WHY_perror("close");
WHYF("Error flushing to %s ", filepath);
ret = 0;
}
if (key){
bcopy(key, read_state.key, sizeof(read_state.key));
read_state.crypt=1;
}
unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
while((ret=rhizome_read(&read_state, buffer, sizeof(buffer)))>0){
if (fd!=-1){
if (write(fd,buffer,ret)!=ret) {
ret = WHY("Failed to write data to file");
break;
}
if (blob)
sqlite3_blob_close(blob);
}
}
sqlite3_finalize(statement);
if (fd!=-1){
if (close(fd)==-1)
ret=WHY_perror("close");
}
if (ret>=0){
cli_puts("filehash"); cli_delim(":");
cli_puts(read_state.id); cli_delim("\n");
cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", read_state.length); cli_delim("\n");
return 1;
}
return ret;
}

130
rhizome_store.c
View File

@ -111,7 +111,8 @@ int rhizome_flush(struct rhizome_write *write){
return WHY("No content supplied");
if (write->crypt){
rhizome_crypt_xor_block(write->buffer, write->data_size, write->file_offset, write->key, write->nonce);
if (rhizome_crypt_xor_block(write->buffer, write->data_size, write->file_offset, write->key, write->nonce))
return -1;
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
@ -120,7 +121,7 @@ int rhizome_flush(struct rhizome_write *write){
sqlite3_blob *blob=NULL;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", write->blob_rowid, 1 /* read/write */, &blob);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
if (sqlite_code_busy(ret))
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_open() failed: %s",
@ -131,7 +132,7 @@ int rhizome_flush(struct rhizome_write *write){
ret=sqlite3_blob_write(blob, write->buffer, write->data_size,
write->file_offset);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
if (sqlite_code_busy(ret))
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_write() failed: %s",
@ -142,7 +143,7 @@ int rhizome_flush(struct rhizome_write *write){
ret = sqlite3_blob_close(blob);
blob=NULL;
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
if (sqlite_code_busy(ret))
goto again;
else if (ret==SQLITE_OK)
break;
@ -153,7 +154,7 @@ int rhizome_flush(struct rhizome_write *write){
again:
if (blob) sqlite3_blob_close(blob);
if (_sqlite_retry(__WHENCE__, &retry, "sqlite3_blob_write")==0)
if (sqlite_retry(&retry, "sqlite3_blob_write")==0)
return -1;
}while(1);
@ -343,27 +344,110 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
return 0;
}
/*
int rhizome_open_append(struct rhizome_write *write, int64_t size, const char *expectedFileHash, const char *existingFileHash){
}
struct rhizome_read{
};
int rhizome_open_read(struct rhizome_read *read, ){
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
strncpy(read->id, fileid, sizeof read->id);
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(read->id);
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = ? AND FILES.datavalid != 0");
if (!statement)
return -1;
sqlite3_bind_text(statement, 1, read->id, -1, SQLITE_STATIC);
int ret = sqlite_step_retry(&retry, statement);
if (ret != SQLITE_ROW){
WHYF("Failed to open file blob: %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return -1;
}
if (!(sqlite3_column_count(statement) == 1
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER)) {
sqlite3_finalize(statement);
return WHY("Incorrect statement column");
}
read->blob_rowid = sqlite3_column_int64(statement, 0);
read->hash=hash;
read->offset=0;
read->length=-1;
sqlite3_finalize(statement);
if (hash)
SHA512_Init(&read->sha512_context);
return 0;
}
// returns the number of bytes read
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length){
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
do{
sqlite3_blob *blob = NULL;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read->blob_rowid, 0 /* read only */, &blob);
if (sqlite_code_busy(ret))
goto again;
else if(ret!=SQLITE_OK)
return WHYF("sqlite3_blob_open failed: %s",sqlite3_errmsg(rhizome_db));
if (read->length==-1)
read->length=sqlite3_blob_bytes(blob);
if (!buffer){
sqlite3_blob_close(blob);
return 0;
}
int count = read->length - read->offset;
if (count>buffer_length)
count=buffer_length;
if (count>0){
ret = sqlite3_blob_read(blob, buffer, count, read->offset);
if (sqlite_code_busy(ret))
goto again;
else if(ret!=SQLITE_OK){
WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
return -1;
}
if (read->hash){
SHA512_Update(&read->sha512_context, buffer, count);
if (read->offset + count>=read->length){
char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
SHA512_End(&read->sha512_context, hash_out);
if (strcasecmp(read->id, hash_out)){
sqlite3_blob_close(blob);
WHYF("Expected hash=%s, got %s", read->id, hash_out);
}
}
}
if (read->crypt){
if(rhizome_crypt_xor_block(buffer, count, read->offset, read->key, read->nonce))
return -1;
}
read->offset+=count;
}
sqlite3_blob_close(blob);
return count;
again:
if (blob) sqlite3_blob_close(blob);
if (sqlite_retry(&retry, "sqlite3_blob_open")==0)
return -1;
}while (1);
}
int rhizome_seek(struct rhizome_read *read, int64_t offset){
}
*/