From 2967d1e00f9474a153b0578539c5e3dd92fc0c0d Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Thu, 3 Jan 2013 11:18:30 +1030 Subject: [PATCH] Don't check for duplicates if an existing id has been supplied --- rhizome.c | 9 +++++++-- rhizome.h | 6 ++++-- rhizome_bundle.c | 6 +----- rhizome_crypto.c | 10 +++++----- rhizome_database.c | 14 ++++---------- rhizome_fetch.c | 2 ++ 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/rhizome.c b/rhizome.c index 27fa26f4..bb7dc8ba 100644 --- a/rhizome.c +++ b/rhizome.c @@ -202,14 +202,19 @@ int rhizome_manifest_bind_id(rhizome_manifest *m_in) } /* Check if a manifest is already stored for the same payload with the same details. - This catches the case of "dna rhizome add file " on the same file more than once. + This catches the case of "rhizome add file " on the same file more than once. (Debounce!) */ int rhizome_manifest_check_duplicate(rhizome_manifest *m_in, rhizome_manifest **m_out) { + // if a manifest was supplied with an ID, don't bother to check for a duplicate. + // we only want to filter out added files with no existing manifest. + if (m_in->haveSecret!=NEW_BUNDLE_ID) + return 0; + if (config.debug.rhizome) DEBUG("Checking for duplicate"); if (m_out) *m_out = NULL; rhizome_manifest *dupm = NULL; - if (rhizome_find_duplicate(m_in, &dupm,0 /* version doesn't matter */) == -1) + if (rhizome_find_duplicate(m_in, &dupm) == -1) return WHY("Errors encountered searching for duplicate manifest"); if (dupm) { /* If the caller wants the duplicate manifest, it must be finalised, otherwise discarded. */ diff --git a/rhizome.h b/rhizome.h index 6c679769..d54afd42 100644 --- a/rhizome.h +++ b/rhizome.h @@ -73,6 +73,9 @@ extern time_ms_t rhizome_voice_timeout; #define RHIZOME_IDLE_TIMEOUT 10000 +#define EXISTING_BUNDLE_ID 1 +#define NEW_BUNDLE_ID 2 + typedef struct rhizome_signature { unsigned char signature[crypto_sign_edwards25519sha512batch_BYTES +crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES+1]; @@ -298,8 +301,7 @@ int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,... double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); int rhizome_update_file_priority(const char *fileid); -int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, - int checkVersionP); +int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found); int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar); long long rhizome_bar_version(unsigned char *bar); unsigned long long rhizome_bar_bidprefix_ll(unsigned char *bar); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index bcdb19be..21092f15 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -644,12 +644,8 @@ int rhizome_manifest_dump(rhizome_manifest *m, const char *msg) int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout) { - /* Add the manifest and its associated file to the Rhizome database, - generating an "id" in the process. - PGS @20121003 - Hang on, didn't we create the ID above? Presumably the - following does NOT in fact generate a bundle ID. - */ int ret=0; + if (rhizome_manifest_check_duplicate(m, mout) == 2) { /* duplicate found -- verify it so that we can write it out later */ rhizome_manifest_verify(*mout); diff --git a/rhizome_crypto.c b/rhizome_crypto.c index 42d61321..97fbef3d 100644 --- a/rhizome_crypto.c +++ b/rhizome_crypto.c @@ -34,7 +34,7 @@ unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m) int rhizome_manifest_createid(rhizome_manifest *m) { - m->haveSecret=1; + m->haveSecret=NEW_BUNDLE_ID; int r=crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic,m->cryptoSignSecret); if (!r) return 0; return WHY("Failed to create keypair for manifest ID."); @@ -225,7 +225,7 @@ int rhizome_extract_privatekey(rhizome_manifest *m) bkBytes,m->cryptoSignSecret); if (result == 0) { - m->haveSecret=1; + m->haveSecret=EXISTING_BUNDLE_ID; RETURN(0); // bingo } memset(m->cryptoSignSecret, 0, sizeof m->cryptoSignSecret); @@ -275,7 +275,7 @@ int rhizome_find_bundle_author(rhizome_manifest *m) if (!rhizome_bk2secret(m,m->cryptoSignPublic,rs,rs_len, bkBytes,m->cryptoSignSecret)) { memcpy(m->author, authorSid, sizeof m->author); - m->haveSecret=1; + m->haveSecret=EXISTING_BUNDLE_ID; if (config.debug.rhizome) DEBUGF("found bundle author sid=%s", alloca_tohex_sid(m->author)); @@ -333,8 +333,8 @@ int rhizome_verify_bundle_privatekey(rhizome_manifest *m, if (config.debug.rhizome) DEBUGF("We have the private key for this bundle."); if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic) { - DEBUGF("Set haveSecret=1 in manifest"); - m->haveSecret=1; + DEBUGF("Set haveSecret=%d in manifest",EXISTING_BUNDLE_ID); + m->haveSecret=EXISTING_BUNDLE_ID; } RETURN(0); } diff --git a/rhizome_database.c b/rhizome_database.c index 8b4e473b..c5069ffa 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1032,8 +1032,10 @@ int rhizome_update_file_priority(const char *fileid) @author Andrew Bettison */ -int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, int checkVersionP) +int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) { + // TODO, add service, name, sender & recipient to manifests table so we can simply query them. + const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *name = NULL; const char *sender = NULL; @@ -1058,8 +1060,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, strbuf_puts(b, "filehash = ?"); } else strbuf_puts(b, "filesize = 0"); - if (checkVersionP) - strbuf_puts(b, " AND version = ?"); if (strbuf_overrun(b)) return WHYF("SQL command too long: %s", strbuf_str(b)); int ret = 0; @@ -1076,8 +1076,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, DEBUGF("filehash=\"%s\"", filehash); sqlite3_bind_text(statement, field++, filehash, -1, SQLITE_STATIC); } - if (checkVersionP) - sqlite3_bind_int64(statement, field++, m->version); size_t rows = 0; while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { ++rows; @@ -1128,7 +1126,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, WARNF("MANIFESTS row id=%s has inconsistent blob with id=%s -- skipped", q_manifestid, blob_id); ++inconsistent; } - if (checkVersionP && blob_version != q_version) { + if (blob_version != q_version) { WARNF("MANIFESTS row id=%s has inconsistent blob: manifests.version=%lld, blob.version=%lld -- skipped", q_manifestid, q_version, blob_version); ++inconsistent; @@ -1151,10 +1149,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, ++inconsistent; } } - if (checkVersionP && q_version != m->version) { - WARNF("SELECT query with version=%lld returned incorrect row: manifests.version=%lld -- skipped", m->version, q_version); - ++inconsistent; - } if (blob_service == NULL) { WARNF("MANIFESTS row id=%s has blob with no 'service' -- skipped", q_manifestid, blob_id); ++inconsistent; diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 84bec7fb..2deb0b68 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -305,6 +305,8 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m) str_toupper_inplace(id); m->version = rhizome_manifest_get_ll(m, "version"); + // TODO, work out why the cache was failing and fix it, then prove that it is faster than accessing the database. + // skip the cache for now long long dbVersion = -1; if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id='%s';", id) == -1)