From c0ac693957384542a33a0dcfeff6c90fa604687e Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Fri, 25 May 2012 14:29:55 +0930 Subject: [PATCH] Remove old payload when updating a manifest with a new payload --- rhizome.c | 16 +++++++-- rhizome.h | 5 ++- rhizome_database.c | 87 ++++++++++++++++++++++++++++++++++++++-------- tests/dna_rhizome | 52 +++++++++++++++++++-------- 4 files changed, 127 insertions(+), 33 deletions(-) diff --git a/rhizome.c b/rhizome.c index e83b91c4..c5c41130 100644 --- a/rhizome.c +++ b/rhizome.c @@ -276,6 +276,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in, /* If the manifest already has an ID */ char id[SID_STRLEN + 1]; + char ofilehash[RHIZOME_FILEHASH_STRLEN + 1]; + ofilehash[0] = '\0'; if (rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) { str_toupper_inplace(id); /* Discard the new manifest unless it is newer than the most recent known version with the same ID */ @@ -292,6 +294,10 @@ int rhizome_add_manifest(rhizome_manifest *m_in, rhizome_hex_to_bytes(id, m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2); if (rhizome_extract_privatekey(m_in, author) == 0) m_in->haveSecret=1; + strbuf b = strbuf_local(ofilehash, sizeof ofilehash); + sqlite_exec_strbuf(b, "SELECT fileid from filemanifests where manifestid='%s';", id); + if (strbuf_overrun(b)) + return WHYF("fileid too long: '%s'", strbuf_str(b)); } else { /* The manifest had no ID (256 bit random string being a public key in the NaCl CryptoSign crypto system), so create one. */ @@ -319,7 +325,7 @@ int rhizome_add_manifest(rhizome_manifest *m_in, } else { return WHY("Failed to set BK"); } - } + } } /* Add group memberships */ @@ -332,11 +338,15 @@ int rhizome_add_manifest(rhizome_manifest *m_in, /* Finish completing the manifest */ if (m_in->finalised==0) if (rhizome_manifest_finalise(m_in, signP, author)) - return WHY("Failed to finalise manifest.\n"); + return WHY("Failed to finalise manifest"); /* Okay, it is written, and can be put directly into the rhizome database now */ if (rhizome_store_bundle(m_in, filename) == -1) - return WHY("rhizome_store_bundle() failed."); + return WHY("Failed to store manifest and payload"); + + /* If there was a prior payload, remove it from the database if it is no longer needed */ + if (ofilehash[0] && strcasecmp(ofilehash, m_in->fileHexHash) != 0 && rhizome_clean_payload(ofilehash) == -1) + return WHYF("Failed to clean old payload fileid=%s", ofilehash); monitor_announce_bundle(m_in); if (m_out) *m_out = m_in; diff --git a/rhizome.h b/rhizome.h index 36fe8b1f..dbf345d1 100644 --- a/rhizome.h +++ b/rhizome.h @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include "sha2.h" +#include "strbuf.h" #include #define RHIZOME_MANIFEST_ID_BYTES crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES @@ -213,7 +214,8 @@ rhizome_manifest *_rhizome_new_manifest(const char *file,const char *func,int li int rhizome_manifest_pack_variables(rhizome_manifest *m); int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename); int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); -int rhizome_store_file(const char *file,char *hash,int priortity); +int rhizome_store_file(const char *file,char *hash,int priority); +int rhizome_clean_payload(const char *fileidhex); int rhizome_finish_sqlstatement(sqlite3_stmt *statement); int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, const char *bundle, char *groups[], int ttl, @@ -233,6 +235,7 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r); int rhizome_server_parse_http_request(int rn,rhizome_http_request *r); int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response); long long sqlite_exec_int64(char *sqlformat,...); +int sqlite_exec_strbuf(strbuf sb, char *sqlformat,...); int rhizome_server_http_response_header(rhizome_http_request *r,int result, char *mime_type,unsigned long long bytes); int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r); diff --git a/rhizome_database.c b/rhizome_database.c index 0ca90018..28e28c58 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -174,6 +174,8 @@ int rhizome_opendb() /* Convenience wrapper for executing an SQL command that returns a single int64 value + Returns -1 if an error occurs, otherwise the value of the column in the first row. + If there are no rows, return zero. */ long long sqlite_exec_int64(char *sqlformat,...) { @@ -187,8 +189,7 @@ long long sqlite_exec_int64(char *sqlformat,...) va_end(ap); sqlite3_stmt *statement; - switch (sqlite3_prepare_v2(rhizome_db,sqlstatement,-1,&statement,NULL)) - { + switch (sqlite3_prepare_v2(rhizome_db,sqlstatement,-1,&statement,NULL)) { case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW: break; default: @@ -198,19 +199,64 @@ long long sqlite_exec_int64(char *sqlformat,...) WHY(sqlstatement); WHY(sqlite3_errmsg(rhizome_db)); return WHY("Could not prepare sql statement."); + } + if (sqlite3_step(statement) == SQLITE_ROW) { + int n = sqlite3_column_count(statement); + if (n != 1) { + sqlite3_finalize(statement); + return WHYF("Incorrect column count %d (should be 1)", n); } - if (sqlite3_step(statement) == SQLITE_ROW) - { - if (sqlite3_column_count(statement)!=1) { - sqlite3_finalize(statement); - return -1; - } - long long result= sqlite3_column_int64(statement,0); - sqlite3_finalize(statement); - return result; - } - sqlite3_finalize(statement); - return 0; + long long result= sqlite3_column_int64(statement, 0); + sqlite3_finalize(statement); + return result; + } + sqlite3_finalize(statement); + return 0; +} + +/* + Convenience wrapper for executing an SQL command that returns a single text value. + Returns -1 if an error occurs, otherwise the number of rows that were found: + 0 means no rows, nothing is appended to the strbuf + 1 means exactly one row, and the its column is appended to the strbuf + 2 more than one row, and the first row's column is appended to the strbuf + @author Andrew Bettison + */ +int sqlite_exec_strbuf(strbuf sb, char *sqlformat,...) +{ + if (!rhizome_db) rhizome_opendb(); + strbuf stmt = strbuf_alloca(8192); + va_list ap; + va_start(ap, sqlformat); + strbuf_vsprintf(stmt, sqlformat, ap); + va_end(ap); + sqlite3_stmt *statement; + switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement,NULL)) { + case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW: + break; + default: + sqlite3_finalize(statement); + sqlite3_close(rhizome_db); + rhizome_db=NULL; + WHY(strbuf_str(stmt)); + WHY(sqlite3_errmsg(rhizome_db)); + return WHY("Could not prepare sql statement."); + } + int rows = 0; + if (sqlite3_step(statement) == SQLITE_ROW) { + int n = sqlite3_column_count(statement); + if (n != 1) { + sqlite3_finalize(statement); + return WHYF("Incorrect column count %d (should be 1)", n); + } + strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0)); + sqlite3_finalize(statement); + ++rows; + } + if (sqlite3_step(statement) == SQLITE_ROW) + ++rows; + sqlite3_finalize(statement); + return rows; } long long rhizome_database_used_bytes() @@ -766,6 +812,19 @@ int rhizome_store_file(const char *file,char *hash,int priority) return 0; } +int rhizome_clean_payload(const char *fileidhex) +{ + /* See if the file has any referents, and if not, delete it */ + int count = sqlite_exec_int64("SELECT COUNT(*) FROM FILEMANIFESTS WHERE fileid='%s';", fileidhex); + if (count == -1) + return -1; + if (count == 0) { + if (sqlite_exec_int64("DELETE FROM FILES WHERE id='%s';", fileidhex) == -1) + return -1; + } + return 0; +} + void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount) { int i=0; diff --git a/tests/dna_rhizome b/tests/dna_rhizome index 80d52e4b..0c2edc38 100755 --- a/tests/dna_rhizome +++ b/tests/dna_rhizome @@ -65,10 +65,7 @@ unpack_manifest_for_grep() { local filename="$1" re_service='[A-Za-z0-9_]\+' re_size=$(( $(cat "$filename" | wc -c) + 0 )) - re_filehash=$($servald rhizome hash file "$filename") - if [ -z "$re_filehash" ]; then - error "Could not compute rhizome hash of $filename" - fi + compute_filehash re_filehash "$filename" re_manifestid='[0-9a-fA-F]\{64\}' re_name="${filename##*/}" # TODO should escape grep metacharacters # If there is a manifest file that looks like it matches this payload @@ -141,6 +138,15 @@ extract_manifest_version() { extract_manifest "$1" "$2" version '[0-9]\{1,\}' } +compute_filehash() { + local _var="$1" + local _file="$2" + local _hash=$($servald rhizome hash file "$_file") || error "$servald failed to compute file hash" + [ -z "${_hash//[0-9a-fA-F]/}" ] || error "file hash contains non-hex: $_hash" + [ "${#_hash}" -eq 128 ] || error "file hash incorrect length: $_hash" + [ -n "$_var" ] && eval $_var=$_hash +} + doc_InitialEmptyList="Initial list is empty" setup_InitialEmptyList() { setup_servald_rhizome @@ -474,24 +480,40 @@ test_AddMeshMSCreate() { assert diff file1 file1x } -doc_AddMeshMSGrow="Second add MeshMS updates manifest" +doc_AddMeshMSGrow="Subsequent add MeshMS updates manifest and removes old payload" setup_AddMeshMSGrow() { setup_servald_rhizome echo "Message1" >file1 - echo "Message2" >file2 echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest - executeOk $servald rhizome add file $sid '' file1 file1.manifest - extract_manifest_id id file1.manifest - extract_manifest_BK bk file1.manifest - echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$sid\nrecipient=$sid1" >file2.manifest } test_AddMeshMSGrow() { - executeOk $servald rhizome add file $sid '' file2 file2.manifest - extract_manifest_filehash filehash file2.manifest + executeOk $servald rhizome add file $sid '' file1 file1.manifest executeOk $servald rhizome list - assert_rhizome_list file2 - executeOk $servald rhizome extract file $filehash file2x - assert diff file2 file2x + assert_rhizome_list file1 + extract_manifest_id id file1.manifest + extract_manifest_filehash filehash file1.manifest + extract_manifest_BK bk file1.manifest + local -a ofilehashes=() + for m in 2 3 4 5; do + ofilehashes+=("$filehash") + echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest + echo "Message$m" >>file1 + executeOk $servald rhizome add file $sid '' file1 file1.manifest + executeOk $servald rhizome list + assert_rhizome_list file1 + extract_manifest_id idx file1.manifest + extract_manifest_filehash filehashx file1.manifest + extract_manifest_BK bkx file1.manifest + compute_filehash filehash file1 + assert --message="manifest ID remains the same" [ "$idx" = "$id" ] + assert --message="manifest BK remains the same" [ "$bkx" = "$bk" ] + assert --message="filehash is for new file" [ "$filehash" = "$filehashx" ] + executeOk $servald rhizome extract file "$filehash" file1x + assert --message="extracted payload is correct" diff file1 file1x + for ofilehash in "${ofilehashes[@]}"; do + execute --exit-status=1 --stderr $servald rhizome extract file "$ofilehash" + done + done } doc_AddMeshMSMissingSender="Add MeshMS without sender fails"