From 643b21d6a7f3f1c8b616c386cc3bb50b0aff0676 Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Thu, 22 Aug 2013 15:14:21 +0930 Subject: [PATCH] Fetch file payloads if they are missing, even if we already have the manifest --- rhizome.h | 3 +-- rhizome_database.c | 36 +++++++++++++++++++++++++----------- rhizome_fetch.c | 23 ++--------------------- rhizome_packetformats.c | 2 +- rhizome_store.c | 3 +++ tests/rhizomeprotocol | 19 +++++++++++++++++++ 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/rhizome.h b/rhizome.h index c2930403..ce61980e 100644 --- a/rhizome.h +++ b/rhizome.h @@ -331,6 +331,7 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar); int64_t rhizome_bar_version(const unsigned char *bar); uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar); int rhizome_is_bar_interesting(unsigned char *bar); +int rhizome_is_manifest_interesting(rhizome_manifest *m); int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, const char *sender_sid, const char *recipient_sid, int limit, int offset, char count_rows); @@ -345,8 +346,6 @@ int rhizome_delete_file(const char *fileid); #define RHIZOME_VERIFY 1 int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax); -int rhizome_manifest_version_cache_lookup(rhizome_manifest *m); -int rhizome_manifest_version_cache_store(rhizome_manifest *m); int monitor_announce_bundle(rhizome_manifest *m); int rhizome_find_secret(const unsigned char *authorSid, int *rs_len, const unsigned char **rs); int rhizome_bk_xor_stream( diff --git a/rhizome_database.c b/rhizome_database.c index cd52a5c6..fee21328 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1496,18 +1496,15 @@ int rhizome_delete_file(const char *fileid) return rhizome_delete_file_retry(&retry, fileid); } -int rhizome_is_bar_interesting(unsigned char *bar){ +static int is_interesting(const char *id_hex, int64_t version) +{ IN(); int ret=1; - int64_t version = rhizome_bar_version(bar); - char id_hex[RHIZOME_MANIFEST_ID_STRLEN]; - tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES); - strcat(id_hex, "%"); // do we have this bundle [or later]? sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite3_stmt *statement = sqlite_prepare(&retry, - "SELECT id, version FROM manifests WHERE id like ? and version >= ?"); + "SELECT filehash FROM manifests WHERE id like ? and version >= ?"); if (!statement) RETURN(-1); @@ -1516,15 +1513,32 @@ int rhizome_is_bar_interesting(unsigned char *bar){ sqlite3_bind_int64(statement, 2, version); if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){ - if (0){ - const char *q_id = (const char *) sqlite3_column_text(statement, 0); - int64_t q_version = sqlite3_column_int64(statement, 1); - DEBUGF("Already have %s, %"PRId64" (vs %s, %"PRId64")", q_id, q_version, id_hex, version); - } + const char *q_filehash = (const char *) sqlite3_column_text(statement, 0); ret=0; + if (q_filehash && !rhizome_exists(q_filehash)) + ret=1; } sqlite3_finalize(statement); RETURN(ret); OUT(); } + +int rhizome_is_bar_interesting(unsigned char *bar) +{ + int64_t version = rhizome_bar_version(bar); + char id_hex[RHIZOME_MANIFEST_ID_STRLEN]; + tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES); + strcat(id_hex, "%"); + return is_interesting(id_hex, version); +} + +int rhizome_is_manifest_interesting(rhizome_manifest *m) +{ + char id[RHIZOME_MANIFEST_ID_STRLEN + 1]; + if (!rhizome_manifest_get(m, "id", id, sizeof id)) + // dodgy manifest, we don't want to receive it + return WHY("Ignoring bad manifest (no ID field)"); + str_toupper_inplace(id); + return is_interesting(id, m->version); +} \ No newline at end of file diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 733b81a8..9db30fab 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -357,25 +357,6 @@ int rhizome_any_fetch_queued() return 0; } -int rhizome_manifest_version_cache_lookup(rhizome_manifest *m) -{ - char id[RHIZOME_MANIFEST_ID_STRLEN + 1]; - if (!rhizome_manifest_get(m, "id", id, sizeof id)) - // dodgy manifest, we don't want to receive it - return WHY("Ignoring bad manifest (no ID field)"); - str_toupper_inplace(id); - m->version = rhizome_manifest_get_ll(m, "version"); - - int64_t dbVersion = -1; - if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id='%s';", id) == -1) - return WHY("Select failure"); - if (dbVersion >= m->version) { - if (0) WHYF("We already have %s (%"PRId64" vs %"PRId64")", id, dbVersion, m->version); - return -1; - } - return 0; -} - typedef struct ignored_manifest { unsigned char bid[RHIZOME_BAR_PREFIX_BYTES]; time_ms_t timeout; @@ -695,7 +676,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct } // If we already have this version or newer, do not fetch. - if (rhizome_manifest_version_cache_lookup(m)) { + if (!rhizome_is_manifest_interesting(m)) { if (config.debug.rhizome_rx) DEBUG(" fetch not started -- already have that version or newer"); RETURN(SUPERSEDED); @@ -859,7 +840,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock if (config.debug.rhizome_rx) DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", bid, m->version, m->fileLength, priority); - if (rhizome_manifest_version_cache_lookup(m)) { + if (!rhizome_is_manifest_interesting(m)) { if (config.debug.rhizome_rx) DEBUG(" already have that version or newer"); rhizome_manifest_free(m); diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c index db71182b..da225de3 100644 --- a/rhizome_packetformats.c +++ b/rhizome_packetformats.c @@ -369,7 +369,7 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st } /* Manifest is okay, so see if it is worth storing */ - if (rhizome_manifest_version_cache_lookup(m)) { + if (!rhizome_is_manifest_interesting(m)) { /* We already have this version or newer */ if (config.debug.rhizome_ads) DEBUG("We already have that manifest or newer."); diff --git a/rhizome_store.c b/rhizome_store.c index 1157f933..e6c503b6 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -514,6 +514,9 @@ int rhizome_finish_write(struct rhizome_write *write){ if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1) goto dbfailure; write->blob_rowid=-1; + + if (config.debug.rhizome) + DEBUGF("Stored file %s", hash_out); return 0; dbfailure: diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol index 1741281c..8fbec7b9 100755 --- a/tests/rhizomeprotocol +++ b/tests/rhizomeprotocol @@ -435,6 +435,25 @@ test_FileTransferDelete() { assert_rhizome_received file1_2 } +doc_CorruptPayload="A corrupted payload should be re-fetched" +setup_CorruptPayload() { + setup_common + set_instance +A + executeOk_servald config set rhizome.external_blobs 1 + rhizome_add_file file1 1024 + create_file file2 1024 + start_servald_instances +A +B + wait_until bundle_received_by $BID:$VERSION +B +} +test_CorruptPayload() { + set_instance +A + cp file2 $SERVALINSTANCE_PATH/$FILEHASH + execute --exit-status=255 $servald rhizome extract file $BID file1a + stop_servald_server + start_servald_server + wait_until grep -i "Stored file $FILEHASH" $LOGA +} + doc_HttpFetchRange="Fetch a file range using HTTP GET." setup_HttpFetchRange() { setup_curl_7