From 2de6bb025ad0ddbf93170c356759e6c9ead0deaf Mon Sep 17 00:00:00 2001 From: gardners Date: Sat, 26 May 2012 13:42:33 +0930 Subject: [PATCH] rhizome_extract_file can now decrypt a file, and also uses progressive blob operations so that we can extract files of unlimited size. --- commandline.c | 7 +++- rhizome.h | 7 +++- rhizome_database.c | 98 +++++++++++++++++++++++++++++++++------------- 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/commandline.c b/commandline.c index 93d2fa62..bff133fe 100644 --- a/commandline.c +++ b/commandline.c @@ -1323,8 +1323,11 @@ int app_rhizome_extract_file(int argc, const char *const *argv, struct command_l return -1; if (rhizome_opendb() == -1) return -1; - /* Extract the file from the database */ - int ret = rhizome_retrieve_file(fileid, filepath); + /* Extract the file from the database. + We don't provide a decryption key here, because we don't know it. + (We probably should allow the user to provide one). + */ + int ret = rhizome_retrieve_file(fileid, filepath,NULL); switch (ret) { case 0: ret = 1; break; case 1: ret = 0; break; diff --git a/rhizome.h b/rhizome.h index d7a49fed..516dbeca 100644 --- a/rhizome.h +++ b/rhizome.h @@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define RHIZOME_FILEHASH_BYTES SHA512_DIGEST_LENGTH #define RHIZOME_FILEHASH_STRLEN (RHIZOME_FILEHASH_BYTES * 2) +#define RHIZOME_CRYPT_PAGE_SIZE 4096 + #define RHIZOME_HTTP_PORT 4110 extern long long rhizome_voice_timeout; @@ -248,7 +250,7 @@ int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r); double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); int chartonybl(int c); int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); -int rhizome_update_file_priority(char *fileid); +int rhizome_update_file_priority(const char *fileid); int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, int checkVersionP); int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar); @@ -256,7 +258,8 @@ char nybltochar_upper(int n); int rhizome_queue_manifest_import(rhizome_manifest *m, struct sockaddr_in *peerip, int *manifest_kept); int rhizome_list_manifests(const char *service, const char *sender_sid, const char *recipient_sid, int limit, int offset); int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp); -int rhizome_retrieve_file(const char *fileid, const char *filepath); +int rhizome_retrieve_file(const char *fileid, const char *filepath, + const unsigned char *key); #define RHIZOME_DONTVERIFY 0 #define RHIZOME_VERIFY 1 diff --git a/rhizome_database.c b/rhizome_database.c index 097f2883..044a31f7 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -863,7 +863,7 @@ void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCoun out[i] = '\0'; } -int rhizome_update_file_priority(char *fileid) +int rhizome_update_file_priority(const char *fileid) { /* Drop if no references */ int referrers=sqlite_exec_int64("SELECT COUNT(*) FROM FILEMANIFESTS WHERE fileid='%s';",fileid); @@ -1141,8 +1141,10 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp) * Returns 0 if file is not found. * Returns -1 on error. */ -int rhizome_retrieve_file(const char *fileid, const char *filepath) +int rhizome_retrieve_file(const char *fileid, const char *filepath, + const unsigned char *key) { + sqlite3_blob *blob=NULL; rhizome_update_file_priority(fileid); long long count=sqlite_exec_int64("SELECT COUNT(*) FROM files WHERE id = '%s' AND datavalid != 0",fileid); if (count<1) { @@ -1152,7 +1154,7 @@ int rhizome_retrieve_file(const char *fileid, const char *filepath) WARNF("There is more than one file in the database with ID=%s",fileid); } char sqlcmd[1024]; - int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT id, data, length FROM files WHERE id = ? AND datavalid != 0"); + int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT id, rowid, length FROM files WHERE id = ? AND datavalid != 0"); if (n >= sizeof(sqlcmd)) { WHY("SQL command too long"); return 0; } sqlite3_stmt *statement; @@ -1172,41 +1174,83 @@ int rhizome_retrieve_file(const char *fileid, const char *filepath) ret = 0; /* no files returned */ } else if (!( sqlite3_column_count(statement) == 3 && sqlite3_column_type(statement, 0) == SQLITE_TEXT - && sqlite3_column_type(statement, 1) == SQLITE_BLOB + && sqlite3_column_type(statement, 1) == SQLITE_INTEGER && sqlite3_column_type(statement, 2) == SQLITE_INTEGER )) { WHY("Incorrect statement column"); ret = 0; /* no files returned */ } else { #warning This won't work for large blobs. It also won't allow for decryption - const char *fileblob = (char *) sqlite3_column_blob(statement, 1); - size_t fileblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() long long length = sqlite3_column_int64(statement, 2); - if (fileblobsize != length) { - ret = 0; WHY("File length does not match blob size"); - } else { - cli_puts("filehash"); cli_delim(":"); - cli_puts((const char *)sqlite3_column_text(statement, 0)); cli_delim("\n"); - cli_puts("filesize"); cli_delim(":"); - cli_printf("%lld", length); cli_delim("\n"); - ret = 1; - if (filepath&&filepath[0]) { - int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775); - if (fd == -1) { - WHY_perror("open"); - ret = WHYF("Cannot open %s for write/create", filepath); - } else if (write(fd, fileblob, length) != length) { - WHY_perror("write"); - ret = WHYF("Error writing %lld bytes to %s ", (long long) length, filepath); - } - if (fd != -1 && close(fd) == -1) { - WHY_perror("close"); - ret = 0; WHYF("Error flushing to %s ", filepath); - } + long long rowid = sqlite3_column_int64(statement, 1); + if (sqlite3_blob_open(rhizome_db,"main","files","data",rowid, + 0 /* read only */,&blob)!=SQLITE_OK) { + ret =0; + WHY("Could not open blob for reading"); + } + + cli_puts("filehash"); cli_delim(":"); + cli_puts((const char *)sqlite3_column_text(statement, 0)); cli_delim("\n"); + cli_puts("filesize"); cli_delim(":"); + cli_printf("%lld", length); cli_delim("\n"); + ret = 1; + if (filepath&&filepath[0]) { + int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775); + if (fd == -1) { + WHY_perror("open"); + ret = WHYF("Cannot open %s for write/create", filepath); + } else { + /* read from blob and write to disk, decrypting if necessary as + we go. + Each 4KB block of data has a nonce which is fed with the key + into crypto_stream_xsalsa20(). The nonce is the file address + divided by 4KB. This approach is used as it allows us to append + to files easily, without having to get the XOR stream for the whole + file, and without the cipher on existing bytes having to change. + Both of these are important properties for journal bundles, such as + will be used by MeshMS. For non-journal bundles where it is important + that changing the payload changes the encryption key (so that the XOR + between any two versions of the payload cannot be easily obtained). + We will do this by having journal manifests identified, causing the + key to be locked, rather than based on the version number. + But anyway, we are supplied with the key here, so all we need to do + is do the block counting and call crypto_stream_xsalsa20(). + */ + long long offset; + unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES]; + bzero(nonce,crypto_stream_xsalsa20_NONCEBYTES); + unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE]; + for(offset=0;offsetRHIZOME_CRYPT_PAGE_SIZE) count=RHIZOME_CRYPT_PAGE_SIZE; + if(sqlite3_blob_read(blob,&buffer[0],count,offset)!=SQLITE_OK) { + ret =0; + WHYF("Error reading %lld bytes of data from blob at offset 0x%llx", + count, offset); + WHYF("sqlite says: %s",sqlite3_errmsg(rhizome_db)); + } + if (key) { + /* calculate block nonce */ + int i; for(i=0;i<8;i++) nonce[i]=(offset>>(i*8))&0xff; + crypto_stream_xsalsa20_xor(&buffer[0],&buffer[0],count, + nonce,key); + } + if (write(fd,buffer,count)!=count) { + ret =0; + WHY("Failed to write data to file"); + } + } + sqlite3_blob_close(blob); blob=NULL; + } + if (fd != -1 && close(fd) == -1) { + WHY_perror("close"); + ret = 0; WHYF("Error flushing to %s ", filepath); } } } } + if (blob) sqlite3_blob_close(blob); sqlite3_finalize(statement); return ret; }