Fetch file payloads if they are missing, even if we already have the manifest

rhizome.h

@@ -331,6 +331,7 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
 int64_t rhizome_bar_version(const unsigned char *bar);
 uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar);
 int rhizome_is_bar_interesting(unsigned char *bar);
+int rhizome_is_manifest_interesting(rhizome_manifest *m);
 int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, 
 			   const char *sender_sid, const char *recipient_sid, 
 			   int limit, int offset, char count_rows);
@@ -345,8 +346,6 @@ int rhizome_delete_file(const char *fileid);
 #define RHIZOME_VERIFY 1
 
 int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
-int rhizome_manifest_version_cache_lookup(rhizome_manifest *m);
-int rhizome_manifest_version_cache_store(rhizome_manifest *m);
 int monitor_announce_bundle(rhizome_manifest *m);
 int rhizome_find_secret(const unsigned char *authorSid, int *rs_len, const unsigned char **rs);
 int rhizome_bk_xor_stream(

rhizome_database.c

@@ -1496,18 +1496,15 @@ int rhizome_delete_file(const char *fileid)
   return rhizome_delete_file_retry(&retry, fileid);
 }
 
-int rhizome_is_bar_interesting(unsigned char *bar){
+static int is_interesting(const char *id_hex, int64_t version)
+{
   IN();
   int ret=1;
-  int64_t version = rhizome_bar_version(bar);
-  char id_hex[RHIZOME_MANIFEST_ID_STRLEN];
-  tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES);
-  strcat(id_hex, "%");
 
   // do we have this bundle [or later]?
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   sqlite3_stmt *statement = sqlite_prepare(&retry, 
-    "SELECT id, version FROM manifests WHERE id like ? and version >= ?");
+    "SELECT filehash FROM manifests WHERE id like ? and version >= ?");
 
   if (!statement)
     RETURN(-1);
@@ -1516,15 +1513,32 @@ int rhizome_is_bar_interesting(unsigned char *bar){
   sqlite3_bind_int64(statement, 2, version);
   
   if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){
-    if (0){
-      const char *q_id = (const char *) sqlite3_column_text(statement, 0);
-      int64_t q_version = sqlite3_column_int64(statement, 1);
-      DEBUGF("Already have %s, %"PRId64" (vs %s, %"PRId64")", q_id, q_version, id_hex, version);
-    }
+    const char *q_filehash = (const char *) sqlite3_column_text(statement, 0);
     ret=0;
+    if (q_filehash && !rhizome_exists(q_filehash))
+      ret=1;
   }  
   sqlite3_finalize(statement);
 
   RETURN(ret);
   OUT();
 }
+
+int rhizome_is_bar_interesting(unsigned char *bar)
+{
+  int64_t version = rhizome_bar_version(bar);
+  char id_hex[RHIZOME_MANIFEST_ID_STRLEN];
+  tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES);
+  strcat(id_hex, "%");
+  return is_interesting(id_hex, version);
+}
+
+int rhizome_is_manifest_interesting(rhizome_manifest *m)
+{
+  char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
+  if (!rhizome_manifest_get(m, "id", id, sizeof id))
+    // dodgy manifest, we don't want to receive it
+    return WHY("Ignoring bad manifest (no ID field)");
+  str_toupper_inplace(id);
+  return is_interesting(id, m->version);
+}

rhizome_fetch.c

@@ -357,25 +357,6 @@ int rhizome_any_fetch_queued()
   return 0;
 }
 
-int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
-{
-  char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
-  if (!rhizome_manifest_get(m, "id", id, sizeof id))
-    // dodgy manifest, we don't want to receive it
-    return WHY("Ignoring bad manifest (no ID field)");
-  str_toupper_inplace(id);
-  m->version = rhizome_manifest_get_ll(m, "version");
-  
-  int64_t dbVersion = -1;
-  if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id='%s';", id) == -1)
-    return WHY("Select failure");
-  if (dbVersion >= m->version) {
-    if (0) WHYF("We already have %s (%"PRId64" vs %"PRId64")", id, dbVersion, m->version);
-    return -1;
-  }
-  return 0;
-}
-
 typedef struct ignored_manifest {
   unsigned char bid[RHIZOME_BAR_PREFIX_BYTES];
   time_ms_t timeout;
@@ -695,7 +676,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
   }
 
   // If we already have this version or newer, do not fetch.
-  if (rhizome_manifest_version_cache_lookup(m)) {
+  if (!rhizome_is_manifest_interesting(m)) {
     if (config.debug.rhizome_rx)
       DEBUG("   fetch not started -- already have that version or newer");
     RETURN(SUPERSEDED);
@@ -859,7 +840,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
   if (config.debug.rhizome_rx)
     DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", bid, m->version, m->fileLength, priority);
 
-  if (rhizome_manifest_version_cache_lookup(m)) {
+  if (!rhizome_is_manifest_interesting(m)) {
     if (config.debug.rhizome_rx)
       DEBUG("   already have that version or newer");
     rhizome_manifest_free(m);

rhizome_packetformats.c

@@ -369,7 +369,7 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
       }
 
       /* Manifest is okay, so see if it is worth storing */
-      if (rhizome_manifest_version_cache_lookup(m)) {
+      if (!rhizome_is_manifest_interesting(m)) {
 	/* We already have this version or newer */
 	if (config.debug.rhizome_ads)
 	  DEBUG("We already have that manifest or newer.");

rhizome_store.c

@@ -514,6 +514,9 @@ int rhizome_finish_write(struct rhizome_write *write){
   if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
     goto dbfailure;
   write->blob_rowid=-1;
+  
+  if (config.debug.rhizome)
+    DEBUGF("Stored file %s", hash_out);
   return 0;
   
 dbfailure:

tests/rhizomeprotocol

@@ -435,6 +435,25 @@ test_FileTransferDelete() {
    assert_rhizome_received file1_2
 }
 
+doc_CorruptPayload="A corrupted payload should be re-fetched"
+setup_CorruptPayload() {
+   setup_common
+   set_instance +A
+   executeOk_servald config set rhizome.external_blobs 1
+   rhizome_add_file file1 1024
+   create_file file2 1024
+   start_servald_instances +A +B
+   wait_until bundle_received_by $BID:$VERSION +B
+}
+test_CorruptPayload() {
+   set_instance +A
+   cp file2 $SERVALINSTANCE_PATH/$FILEHASH
+   execute --exit-status=255 $servald rhizome extract file $BID file1a
+   stop_servald_server
+   start_servald_server
+   wait_until grep -i "Stored file $FILEHASH" $LOGA
+}
+
 doc_HttpFetchRange="Fetch a file range using HTTP GET."
 setup_HttpFetchRange() {
    setup_curl_7