Remove old payload when updating a manifest with a new payload

rhizome.c

@@ -276,6 +276,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in,
 
   /* If the manifest already has an ID */
   char id[SID_STRLEN + 1];
+  char ofilehash[RHIZOME_FILEHASH_STRLEN + 1];
+  ofilehash[0] = '\0';
   if (rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) {
     str_toupper_inplace(id);
     /* Discard the new manifest unless it is newer than the most recent known version with the same ID */
@@ -292,6 +294,10 @@ int rhizome_add_manifest(rhizome_manifest *m_in,
     rhizome_hex_to_bytes(id, m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2); 
     if (rhizome_extract_privatekey(m_in, author) == 0)
       m_in->haveSecret=1;
+    strbuf b = strbuf_local(ofilehash, sizeof ofilehash);
+    sqlite_exec_strbuf(b, "SELECT fileid from filemanifests where manifestid='%s';", id);
+    if (strbuf_overrun(b))
+      return WHYF("fileid too long: '%s'", strbuf_str(b));
   } else {
     /* The manifest had no ID (256 bit random string being a public key in the NaCl CryptoSign
        crypto system), so create one. */
@@ -319,7 +325,7 @@ int rhizome_add_manifest(rhizome_manifest *m_in,
       } else {
 	return WHY("Failed to set BK");
       }
-    }   
+    }
   }
 
   /* Add group memberships */
@@ -332,11 +338,15 @@ int rhizome_add_manifest(rhizome_manifest *m_in,
   /* Finish completing the manifest */
   if (m_in->finalised==0)
     if (rhizome_manifest_finalise(m_in, signP, author))
-      return WHY("Failed to finalise manifest.\n");
+      return WHY("Failed to finalise manifest");
 
   /* Okay, it is written, and can be put directly into the rhizome database now */
   if (rhizome_store_bundle(m_in, filename) == -1)
-    return WHY("rhizome_store_bundle() failed.");
+    return WHY("Failed to store manifest and payload");
+
+  /* If there was a prior payload, remove it from the database if it is no longer needed */
+  if (ofilehash[0] && strcasecmp(ofilehash, m_in->fileHexHash) != 0 && rhizome_clean_payload(ofilehash) == -1)
+    return WHYF("Failed to clean old payload fileid=%s", ofilehash);
 
   monitor_announce_bundle(m_in);
   if (m_out) *m_out = m_in;

rhizome.h

@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include <sqlite3.h>
 #include "sha2.h"
+#include "strbuf.h"
 #include <sys/stat.h>
 
 #define RHIZOME_MANIFEST_ID_BYTES       crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES
@@ -213,7 +214,8 @@ rhizome_manifest *_rhizome_new_manifest(const char *file,const char *func,int li
 int rhizome_manifest_pack_variables(rhizome_manifest *m);
 int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename);
 int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
-int rhizome_store_file(const char *file,char *hash,int priortity);
+int rhizome_store_file(const char *file,char *hash,int priority);
+int rhizome_clean_payload(const char *fileidhex);
 int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
 int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, const char *bundle,
 			  char *groups[], int ttl,
@@ -233,6 +235,7 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r);
 int rhizome_server_parse_http_request(int rn,rhizome_http_request *r);
 int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response);
 long long sqlite_exec_int64(char *sqlformat,...);
+int sqlite_exec_strbuf(strbuf sb, char *sqlformat,...);
 int rhizome_server_http_response_header(rhizome_http_request *r,int result,
 					char *mime_type,unsigned long long bytes);
 int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r);

rhizome_database.c

@@ -174,6 +174,8 @@ int rhizome_opendb()
 
 /* 
    Convenience wrapper for executing an SQL command that returns a single int64 value 
+   Returns -1 if an error occurs, otherwise the value of the column in the first row.
+   If there are no rows, return zero.
  */
 long long sqlite_exec_int64(char *sqlformat,...)
 {
@@ -187,8 +189,7 @@ long long sqlite_exec_int64(char *sqlformat,...)
   va_end(ap);
 
   sqlite3_stmt *statement;
-  switch (sqlite3_prepare_v2(rhizome_db,sqlstatement,-1,&statement,NULL))
-    {
+  switch (sqlite3_prepare_v2(rhizome_db,sqlstatement,-1,&statement,NULL)) {
     case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW:
       break;
     default:
@@ -198,19 +199,64 @@ long long sqlite_exec_int64(char *sqlformat,...)
       WHY(sqlstatement);
       WHY(sqlite3_errmsg(rhizome_db));
       return WHY("Could not prepare sql statement.");
+  }
+  if (sqlite3_step(statement) == SQLITE_ROW) {
+    int n = sqlite3_column_count(statement);
+    if (n != 1) {
+      sqlite3_finalize(statement);
+      return WHYF("Incorrect column count %d (should be 1)", n);
     }
-   if (sqlite3_step(statement) == SQLITE_ROW)
-     {
-       if (sqlite3_column_count(statement)!=1) {
-	 sqlite3_finalize(statement);
-	 return -1;
-       }
-       long long result= sqlite3_column_int64(statement,0);
-       sqlite3_finalize(statement);
-       return result;
-     }
-   sqlite3_finalize(statement);
-   return 0;
+    long long result= sqlite3_column_int64(statement, 0);
+    sqlite3_finalize(statement);
+    return result;
+  }
+  sqlite3_finalize(statement);
+  return 0;
+}
+
+/* 
+   Convenience wrapper for executing an SQL command that returns a single text value.
+   Returns -1 if an error occurs, otherwise the number of rows that were found:
+    0 means no rows, nothing is appended to the strbuf
+    1 means exactly one row, and the its column is appended to the strbuf
+    2 more than one row, and the first row's column is appended to the strbuf
+   @author Andrew Bettison <andrew@servalproject.com>
+ */
+int sqlite_exec_strbuf(strbuf sb, char *sqlformat,...)
+{
+  if (!rhizome_db) rhizome_opendb();
+  strbuf stmt = strbuf_alloca(8192);
+  va_list ap;
+  va_start(ap, sqlformat);
+  strbuf_vsprintf(stmt, sqlformat, ap);
+  va_end(ap);
+  sqlite3_stmt *statement;
+  switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement,NULL)) {
+    case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW:
+      break;
+    default:
+      sqlite3_finalize(statement);
+      sqlite3_close(rhizome_db);
+      rhizome_db=NULL;
+      WHY(strbuf_str(stmt));
+      WHY(sqlite3_errmsg(rhizome_db));
+      return WHY("Could not prepare sql statement.");
+  }
+  int rows = 0;
+  if (sqlite3_step(statement) == SQLITE_ROW) {
+    int n = sqlite3_column_count(statement);
+    if (n != 1) {
+      sqlite3_finalize(statement);
+      return WHYF("Incorrect column count %d (should be 1)", n);
+    }
+    strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
+    sqlite3_finalize(statement);
+    ++rows;
+  }
+  if (sqlite3_step(statement) == SQLITE_ROW)
+    ++rows;
+  sqlite3_finalize(statement);
+  return rows;
 }
 
 long long rhizome_database_used_bytes()
@@ -766,6 +812,19 @@ int rhizome_store_file(const char *file,char *hash,int priority)
   return 0;
 }
 
+int rhizome_clean_payload(const char *fileidhex)
+{
+  /* See if the file has any referents, and if not, delete it */
+  int count = sqlite_exec_int64("SELECT COUNT(*) FROM FILEMANIFESTS WHERE fileid='%s';", fileidhex); 
+  if (count == -1)
+    return -1;
+  if (count == 0) {
+    if (sqlite_exec_int64("DELETE FROM FILES WHERE id='%s';", fileidhex) == -1)
+      return -1;
+  }
+  return 0;
+}
+
 void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
 {
   int i=0;

tests/dna_rhizome

@@ -65,10 +65,7 @@ unpack_manifest_for_grep() {
    local filename="$1"
    re_service='[A-Za-z0-9_]\+'
    re_size=$(( $(cat "$filename" | wc -c) + 0 ))
-   re_filehash=$($servald rhizome hash file "$filename")
-   if [ -z "$re_filehash" ]; then
-      error "Could not compute rhizome hash of $filename"
-   fi
+   compute_filehash re_filehash "$filename"
    re_manifestid='[0-9a-fA-F]\{64\}'
    re_name="${filename##*/}" # TODO should escape grep metacharacters
    # If there is a manifest file that looks like it matches this payload
@@ -141,6 +138,15 @@ extract_manifest_version() {
    extract_manifest "$1" "$2" version '[0-9]\{1,\}'
 }
 
+compute_filehash() {
+   local _var="$1"
+   local _file="$2"
+   local _hash=$($servald rhizome hash file "$_file") || error "$servald failed to compute file hash"
+   [ -z "${_hash//[0-9a-fA-F]/}" ] || error "file hash contains non-hex: $_hash"
+   [ "${#_hash}" -eq 128 ] || error "file hash incorrect length: $_hash"
+   [ -n "$_var" ] && eval $_var=$_hash
+}
+
 doc_InitialEmptyList="Initial list is empty"
 setup_InitialEmptyList() {
    setup_servald_rhizome
@@ -474,24 +480,40 @@ test_AddMeshMSCreate() {
    assert diff file1 file1x
 }
 
-doc_AddMeshMSGrow="Second add MeshMS updates manifest"
+doc_AddMeshMSGrow="Subsequent add MeshMS updates manifest and removes old payload"
 setup_AddMeshMSGrow() {
    setup_servald_rhizome
    echo "Message1" >file1
-   echo "Message2" >file2
    echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
-   executeOk $servald rhizome add file $sid '' file1 file1.manifest
-   extract_manifest_id id file1.manifest
-   extract_manifest_BK bk file1.manifest
-   echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$sid\nrecipient=$sid1" >file2.manifest
 }
 test_AddMeshMSGrow() {
-   executeOk $servald rhizome add file $sid '' file2 file2.manifest
-   extract_manifest_filehash filehash file2.manifest
+   executeOk $servald rhizome add file $sid '' file1 file1.manifest
    executeOk $servald rhizome list
-   assert_rhizome_list file2
-   executeOk $servald rhizome extract file $filehash file2x
-   assert diff file2 file2x
+   assert_rhizome_list file1
+   extract_manifest_id id file1.manifest
+   extract_manifest_filehash filehash file1.manifest
+   extract_manifest_BK bk file1.manifest
+   local -a ofilehashes=()
+   for m in 2 3 4 5; do
+      ofilehashes+=("$filehash")
+      echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
+      echo "Message$m" >>file1
+      executeOk $servald rhizome add file $sid '' file1 file1.manifest
+      executeOk $servald rhizome list
+      assert_rhizome_list file1
+      extract_manifest_id idx file1.manifest
+      extract_manifest_filehash filehashx file1.manifest
+      extract_manifest_BK bkx file1.manifest
+      compute_filehash filehash file1
+      assert --message="manifest ID remains the same" [ "$idx" = "$id" ]
+      assert --message="manifest BK remains the same" [ "$bkx" = "$bk" ]
+      assert --message="filehash is for new file" [ "$filehash" = "$filehashx" ]
+      executeOk $servald rhizome extract file "$filehash" file1x
+      assert --message="extracted payload is correct" diff file1 file1x
+      for ofilehash in "${ofilehashes[@]}"; do
+         execute --exit-status=1 --stderr $servald rhizome extract file "$ofilehash"
+      done
+   done
 }
 
 doc_AddMeshMSMissingSender="Add MeshMS without sender fails"