ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-20 17:33:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed various manifest synchronisation bugs, including probably

Browse Source 
the main one that was stopping meshms (large manifests would
practically never get advertised under certain (common) conditions).
This commit is contained in:
gardners 2012-05-27 08:30:51 +02:00
parent b3819835ca
commit 30e53f963c
6 changed files with 71 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
overlay_interface.c
View File

@ -826,7 +826,11 @@ int overlay_tick_interface(int i, long long now)
*/
overlay_stuff_packet_from_queue(i,e,OQ_MESH_MANAGEMENT,now,pax,&frame_pax,MAX_FRAME_PAX);
ob_limitsize(e,overlay_interfaces[i].mtu*3/4);
/* We previously limited manifest space to 3/4 of MTU, but that causes problems for
MeshMS journal manifests, at least until we move to a compact binary format.
So for now, allow allow rest of packet to get used */
#warning reduce to <= mtu*3/4 once we have compacty binary canonical manifest format
ob_limitsize(e,overlay_interfaces[i].mtu*4/4);
/* Add advertisements for ROUTES not Rhizome bundles.
Rhizome bundle advertisements are lower priority */

2
rhizome.c
View File

@ -318,6 +318,8 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
if (rhizome_store_bundle(m_in) == -1)
return WHY("rhizome_store_bundle() failed.");
WHYF("Announcing arrival of manifest %s* version %lld",
overlay_render_sid_prefix(m_in->cryptoSignPublic,8),m_in->version);
monitor_announce_bundle(m_in);
return 0;
}

3
rhizome.h
View File

@ -201,7 +201,7 @@ int rhizome_strn_is_file_hash(const char *text);
int rhizome_str_is_file_hash(const char *text);
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(char *id,int maximum_priority);
int rhizome_drop_stored_file(const char *id,int maximum_priority);
int rhizome_manifest_priority(char *id);
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize);
int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out);
@ -224,6 +224,7 @@ int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out,
const char *bundle, int ttl);
int rhizome_manifest_verify(rhizome_manifest *m);
int rhizome_manifest_check_sanity(rhizome_manifest *m_in);
int rhizome_manifest_check_file(rhizome_manifest *m_in);
int rhizome_manifest_check_duplicate(rhizome_manifest *m_in,rhizome_manifest **m_out);

8
rhizome_database.c
View File

@ -371,7 +371,7 @@ int rhizome_make_space(int group_priority, long long bytes)
/* Drop the specified file from storage, and any manifests that reference it,
provided that none of those manifests are being retained at a higher priority
than the maximum specified here. */
int rhizome_drop_stored_file(char *id,int maximum_priority)
int rhizome_drop_stored_file(const char *id,int maximum_priority)
{
char sql[1024];
sqlite3_stmt *statement;
@ -1160,7 +1160,11 @@ int rhizome_retrieve_file(const char *fileid, const char *filepath,
rhizome_update_file_priority(fileid);
long long count=sqlite_exec_int64("SELECT COUNT(*) FROM files WHERE id = '%s' AND datavalid != 0",fileid);
if (count<1) {
WHY("No such file ID in the database");
char id[9];
int i;
for(i=0;i<8;i++) id[i]=fileid[i];
id[8]=0;
WHYF("No such file ID %s* in the database",id);
return 0; /* 0 files returned */
} else if (count>1) {
WARNF("There is more than one file in the database with ID=%s",fileid);

19
rhizome_fetch.c
View File

@ -141,7 +141,8 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
if (!rhizome_manifest_get(m, "id", id, sizeof id))
return -3; // dodgy manifest, we don't want to receive it
// dodgy manifest, we don't want to receive it
return WHY("Ignoring bad manifest (no ID field)");
str_toupper_inplace(id);
/* Work out bin number in cache */
@ -159,13 +160,14 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
for(i=0;i<24;i++)
{
int byte=
(chartonybl(id[(i*2)+RHIZOME_VERSION_CACHE_NYBLS])<<4)
|chartonybl(id[(i*2)+RHIZOME_VERSION_CACHE_NYBLS+1]);
(chartonybl(id[(i*2)])<<4)
|chartonybl(id[(i*2)+1]);
if (byte!=entry->idprefix[i]) break;
}
if (i==24) {
/* Entries match -- so check version */
unsigned long long rev = rhizome_manifest_get_ll(m,"version");
WHYF("cached version same or newer (%lld)",entry->version);
if (rev<entry->version) {
/* the presented manifest is older than we have.
This allows the caller to know that they can tell whoever gave them the
@ -179,9 +181,12 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
/* the presented manifest is newer than we have */
return 0;
}
} else {
}
}
WHY("Not in manifest cache");
/* Not in cache, so all is well, well, maybe.
What we do know is that it is unlikely to be in the database, so it probably
doesn't hurt to try to receive it.
@ -208,9 +213,12 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
id,manifest_version)>0) {
/* Okay, so we have a stored version which is newer, so update the cache
using a random replacement strategy. */
long long stored_version = sqlite_exec_int64("select version from manifests where id='%s'", id);
if (stored_version == -1)
return -4; // database is broken, we can't receive it
return WHY("database error reading stored manifest version"); // database is broken, we can't confirm that it is here
WHYF("stored version=%lld, manifest_version=%lld (not fetching; remembering in cache)",
stored_version,manifest_version);
slot=random()%RHIZOME_VERSION_CACHE_ASSOCIATIVITY;
rhizome_manifest_version_cache_slot *entry
=&rhizome_manifest_version_cache[bin][slot];
@ -223,7 +231,8 @@ int rhizome_manifest_version_cache_lookup(rhizome_manifest *m)
/* Finally, say that it isn't worth RXing this manifest */
return stored_version > manifest_version ? -2 : -1;
}
/* At best we hold an older version of this manifest */
/* At best we hold an older version of this manifest, and at worst we
don't hold any copy. */
return 0;
}

64
rhizome_packetformats.c
View File

@ -146,7 +146,7 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
bundle_offset[0]=0;
if (bundles_available==-1||(bundle_offset[1]>=bundles_available))
bundle_offset[1]=0;
if(0)
if(1)
DEBUGF("%d bundles in database (%d %d), slots=%d.",bundles_available,
bundle_offset[0],bundle_offset[1],slots);
@ -166,6 +166,7 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
bundle_offset[pass],slots);
break;
}
WHYF("sql query: %s",query);
switch (sqlite3_prepare_v2(rhizome_db,query,-1,&statement,NULL))
{
@ -204,8 +205,9 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
/* Only include manifests that are <=1KB inline.
Longer ones are only advertised by BAR */
if (blob_bytes>1024) {
WARN("blob>1k - ignoring");
if (0) WARN("blob>1k - ignoring");
sqlite3_blob_close(blob); blob=NULL;
bundle_offset[pass]++;
continue;
}
@ -219,12 +221,21 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
e,e->bytes,e->length,e->allocSize);
if (ob_makespace(e,overhead+2+blob_bytes)) {
if (0&&debug&DEBUG_RHIZOME)
DEBUGF("Stopped cramming %s into Rhizome advertisement frame.",
pass?"BARs":"manifests");
if (0||debug&DEBUG_RHIZOME) {
rhizome_manifest *m=rhizome_new_manifest();
char mdata[blob_bytes]; mdata[0]=0; mdata[1]=0;
sqlite3_blob_read(blob,&mdata[0],blob_bytes,0);
rhizome_read_manifest_file(m,mdata, blob_bytes);
long long version = rhizome_manifest_get_ll(m, "version");
DEBUGF("Stop cramming %s advertisements: not enough space for %s*:v%lld (%d bytes, size limit=%d, used=%d)",
pass?"BARs":"manifests",
overlay_render_sid_prefix(m->cryptoSignPublic,8),
version,
blob_bytes,e->sizeLimit,e->length);
rhizome_manifest_free(m);
}
frameFull=1;
}
if (!pass) {
} else if (!pass) {
/* put manifest length field and manifest ID */
/* XXX why on earth is this being done this way, instead of
with ob_append_byte() ??? */
@ -244,19 +255,24 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
}
if (sqlite3_blob_read(blob,&e->bytes[e->length+overhead],blob_bytes,0)
!=SQLITE_OK) {
if (!pass) {
if (0) {
DEBUG(" Manifest:");
int i;
for(i=0;i<blob_bytes;i++) DEBUGF(" %c",e->bytes[e->length+overhead+i]);
}
}
if (debug&DEBUG_RHIZOME) DEBUG("Couldn't read from blob");
sqlite3_blob_close(blob); blob=NULL;
dump("buffer (225)",(unsigned char *)e,sizeof(*e));
continue;
}
/* debug: show which BID/version combos we are advertising */
if (0&&(!pass)) {
rhizome_manifest *m=rhizome_new_manifest();
rhizome_read_manifest_file
(m, (char *)&e->bytes[e->length+overhead], blob_bytes);
long long version = rhizome_manifest_get_ll(m, "version");
WHYF("Advertising manifest %s* version %lld",
overlay_render_sid_prefix(m->cryptoSignPublic,8),
version);
rhizome_manifest_free(m);
}
e->length+=overhead+blob_bytes;
if (e->length>e->allocSize) {
WHY("e->length > e->size");
@ -265,7 +281,8 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
}
bytes_used+=overhead+blob_bytes;
bundles_advertised++;
bundle_offset[pass]=sqlite3_column_int64(statement,1);
bundle_offset[pass]++;
// bundle_offset[pass]=sqlite3_column_int64(statement,1);
sqlite3_blob_close(blob); blob=NULL;
}
@ -345,14 +362,17 @@ int overlay_rhizome_saw_advertisements(int i,overlay_frame *f, long long now)
rhizome_manifest_free(m);
return 0;
}
char manifest_id[RHIZOME_MANIFEST_ID_STRLEN + 1];
if (rhizome_manifest_get(m, "id", manifest_id, sizeof manifest_id) == NULL) {
char manifest_id_prefix[RHIZOME_MANIFEST_ID_STRLEN + 1];
if (rhizome_manifest_get(m, "id", manifest_id_prefix, sizeof manifest_id_prefix) == NULL) {
WHY("Manifest does not contain 'id' field");
rhizome_manifest_free(m);
return 0;
}
/* trim manifest ID to a prefix for ease of debugging
(that is the only use of this */
manifest_id_prefix[8]=0;
long long version = rhizome_manifest_get_ll(m, "version");
if (debug & DEBUG_RHIZOMESYNC) DEBUGF("manifest id=%s version=%lld", manifest_id, version);
if (0||debug & DEBUG_RHIZOMESYNC) DEBUGF("manifest id=%s* version=%lld", manifest_id_prefix, version);
/* Crude signature presence test */
for(i=m->manifest_all_bytes-1;i>0;i--)
@ -371,7 +391,7 @@ int overlay_rhizome_saw_advertisements(int i,overlay_frame *f, long long now)
if (rhizome_ignore_manifest_check(m,(struct sockaddr_in *)f->recvaddr))
{
/* Ignoring manifest that has caused us problems recently */
if (0) WARNF("Ignoring manifest with errors: %s", manifest_id);
if (1) WARNF("Ignoring manifest with errors: %s*", manifest_id_prefix);
}
else if (m&&(!m->errors))
{
@ -409,12 +429,12 @@ int overlay_rhizome_saw_advertisements(int i,overlay_frame *f, long long now)
rhizome_manifest_free(m);
m = NULL;
} else if (m->errors) {
if (debug&DEBUG_RHIZOME) DEBUGF("Verifying manifest %s revealed errors -- not storing.", manifest_id);
if (debug&DEBUG_RHIZOME) DEBUGF("Verifying manifest %s* revealed errors -- not storing.", manifest_id_prefix);
rhizome_queue_ignore_manifest(m,(struct sockaddr_in*)f->recvaddr,60000);
rhizome_manifest_free(m);
m = NULL;
} else {
if (debug&DEBUG_RHIZOME) DEBUGF("Verifying manifest %s revealed no errors -- will try to store.", manifest_id);
if (debug&DEBUG_RHIZOME) DEBUGF("Verifying manifest %s* revealed no errors -- will try to store.", manifest_id_prefix);
/* Add manifest to import queue. We need to know originating IPv4 address
so that we can transfer by HTTP. */
if (0) DEBUG("Suggesting fetching of a bundle");