ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-01-18 10:46:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Switched to new BAR format with 15 bytes of BID prefix, and TTL

Browse Source 
at the end, and log2(filesize) instead of filesize.  Equally
importantly BAR construction and parsing now uses #defines for
field sizes and offsets instead of it being hardwired without
meaningful documentation.
WILL BREAK BACKWARD COMPATIBILITY WITH PREVIOUS BUILDS.
YOU MUST DELETE AND REBUILD YOUR RHIZOME DATABASE AS OLD-FORMAT
BIDs WILL BE IN THERE AND GET SENT, AND STRANGE THINGS WILL HAPPEN.
This break with backwards compatibility is only reasonable to
consider because we have not yet had an official build using the
new Rhizome with old BAR format.  0.08 uses old Rhizome.  #9
This commit is contained in:
gardners 2012-10-02 19:24:35 +09:30
parent d4fe995206
commit e015f0670b
4 changed files with 90 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
rhizome.h
View File

@ -63,6 +63,13 @@ typedef struct rhizome_signature {
} rhizome_signature;
#define RHIZOME_BAR_BYTES 32
#define RHIZOME_BAR_COMPARE_BYTES 31
#define RHIZOME_BAR_PREFIX_BYTES 15
#define RHIZOME_BAR_PREFIX_OFFSET 0
#define RHIZOME_BAR_FILESIZE_OFFSET 15
#define RHIZOME_BAR_VERSION_OFFSET 16
#define RHIZOME_BAR_GEOBOX_OFFSET 23
#define RHIZOME_BAR_TTL_OFFSET 31
#define MAX_MANIFEST_VARS 256
#define MAX_MANIFEST_BYTES 8192
@ -268,7 +275,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
int checkVersionP);
int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
long long rhizome_bar_version(unsigned char *bar);
unsigned long long rhizome_bar_bidprefix(unsigned char *bar);
unsigned long long rhizome_bar_bidprefix_ll(unsigned char *bar);
int rhizome_queue_manifest_import(rhizome_manifest *m, struct sockaddr_in *peerip, int *manifest_kept);
int rhizome_list_manifests(const char *service, const char *sender_sid, const char *recipient_sid, int limit, int offset);
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp);

68
rhizome_direct.c
View File

@ -275,10 +275,11 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
int them_count=(size-10)/RHIZOME_BAR_BYTES;
/* We need to get a list of BARs that will fit into max_response_bytes when we
have summarised them into 9-byte PUSH/PULL hints.
have summarised them into (1+RHIZOME_BAR_PREFIX_BYTES)-byte PUSH/PULL hints.
So we need an intermediate buffer that is somewhat larger to allow the actual
maximum response buffer to be completely filled. */
int max_intermediate_bytes=10+((max_response_bytes-10)/9)*RHIZOME_BAR_BYTES;
int max_intermediate_bytes
=10+((max_response_bytes-10)/(1+RHIZOME_BAR_PREFIX_BYTES))*RHIZOME_BAR_BYTES;
unsigned char usbuffer[max_intermediate_bytes];
rhizome_direct_bundle_cursor
*c=rhizome_direct_bundle_iterator(max_intermediate_bytes);
@ -304,18 +305,23 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
c->buffer_used=0;
/* Iterate until we are through both lists.
Note that the responses are 9-bytes each, much smaller than the 32 bytes
used by BARs, therefore the response will never be bigger than the request,
and so we don't need to worry about overflows. */
Note that the responses are (1+RHIZOME_BAR_PREFIX_BYTES)-bytes each, much
smaller than the 32 bytes used by BARs, therefore the response will never be
bigger than the request, and so we don't need to worry about overflows. */
int them=0,us=0;
DEBUGF("themcount=%d, uscount=%d",them_count,us_count);
while(them<them_count||us<us_count)
{
unsigned char *them_id=&buffer[10+them*RHIZOME_BAR_BYTES];
unsigned char *us_id=&usbuffer[10+us*RHIZOME_BAR_BYTES];
DEBUGF("them=%d, us=%d",them,us);
unsigned char *them_bar=&buffer[10+them*RHIZOME_BAR_BYTES];
unsigned char *us_bar=&usbuffer[10+us*RHIZOME_BAR_BYTES];
int relation=0;
if (them<them_count&&us<us_count)
relation=memcmp(them_id,us_id,RHIZOME_BAR_BYTES);
if (them<them_count&&us<us_count) {
relation=memcmp(them_bar,us_bar,RHIZOME_BAR_COMPARE_BYTES);
DEBUGF("relation = %d",relation);
dump("them BAR",them_bar,RHIZOME_BAR_BYTES);
dump("us BAR",us_bar,RHIZOME_BAR_BYTES);
}
else if (us==us_count) relation=-1; /* they have a bundle we don't have */
else if (them==them_count) relation=+1; /* we have a bundle they don't have */
else {
@ -325,28 +331,28 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
int who=0;
if (relation<0) {
/* They have a bundle that we don't have any version of.
Append 9-byte "please send" record consisting of 0x01 followed
Append 16-byte "please send" record consisting of 0x01 followed
by the eight-byte BID prefix from the BAR. */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x01; /* Please send */
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES],
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
8);
c->buffer_used+=1+8;
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
who=-1;
DEBUGF("They have previously unseen bundle %016llx*",
rhizome_bar_bidprefix(&buffer[10+them*RHIZOME_BAR_BYTES]));
rhizome_bar_bidprefix_ll(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else if (relation>0) {
/* We have a bundle that they don't have any version of
Append 9-byte "I have [newer]" record consisting of 0x02 followed
Append 16-byte "I have [newer]" record consisting of 0x02 followed
by the eight-byte BID prefix from the BAR. */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x02; /* I have [newer] */
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES],
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
8);
c->buffer_used+=1+8;
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
who=+1;
DEBUGF("We have previously unseen bundle %016llx*",
rhizome_bar_bidprefix(&usbuffer[10+us*RHIZOME_BAR_BYTES]));
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]));
} else {
/* We each have a version of this bundle, so see whose is newer */
long long them_version
@ -356,28 +362,28 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
if (them_version>us_version) {
/* They have the newer version of the bundle */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x01; /* Please send */
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES],
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
8);
c->buffer_used+=1+8;
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
DEBUGF("They have newer version of bundle %016llx* (%lld versus %lld)",
rhizome_bar_bidprefix(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else if (them_version<us_version) {
/* We have the newer version of the bundle */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x02; /* I have [newer] */
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES],
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
8);
c->buffer_used+=1+8;
DEBUGF("We have newer version of bundle %016llx* (%lld versus %lld)",
rhizome_bar_bidprefix(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]));
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
DEBUGF("We have newer version of bundle %016llx* (%lld versus %lld)",
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else {
DEBUGF("We both have the same version of %016llx*",
rhizome_bar_bidprefix(&buffer[10+them*RHIZOME_BAR_BYTES]));
rhizome_bar_bidprefix_ll(&buffer[10+them*RHIZOME_BAR_BYTES]));
}
}

15
rhizome_direct_http.c
View File

@ -712,9 +712,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
DEBUGF("content_length=%d",content_length);
dump("response",(unsigned char *)p,content_length);
/* We now have the list of 9-byte records that indicate the list of BAR prefixes
that differ between the two nodes. We can now action those which are relevant,
i.e., based on whether we are pushing, pulling or synchronising (both).
/* We now have the list of (1+RHIZOME_BAR_PREFIX_BYTES)-byte records that indicate
the list of BAR prefixes that differ between the two nodes. We can now action
those which are relevant, i.e., based on whether we are pushing, pulling or
synchronising (both).
I am currently undecided as to whether it is cleaner to have some general
rhizome direct function for doing that, or whether it just adds unnecessary
@ -724,12 +725,12 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
*/
DEBUGF("XXX Need to parse responses into actions");
int i;
for(i=10;i<content_length;i+=9)
for(i=10;i<content_length;i+=(1+RHIZOME_BAR_PREFIX_BYTES))
{
int type=p[i];
// unsigned char *bid_prefix=(unsigned char *)&p[i+1];
unsigned long long
bid_prefix_ll=rhizome_bar_bidprefix((unsigned char *)&p[i+1]);
bid_prefix_ll=rhizome_bar_bidprefix_ll((unsigned char *)&p[i+1]);
DEBUGF("%s %016llx*",type==1?"push":"pull",bid_prefix_ll);
if (type==2&&r->pullP) {
WARN("XXX Rhizome direct http pull yet implemented");
@ -742,7 +743,9 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* Start by getting the manifest, which is the main thing we need, and also
gives us the information we need for sending any associated file. */
rhizome_manifest *m=rhizome_direct_get_manifest((unsigned char *)&p[i+1],8);
rhizome_manifest
*m=rhizome_direct_get_manifest((unsigned char *)&p[i+1],
RHIZOME_BAR_PREFIX_BYTES);
if (!m) {
WHY("This should never happen. The manifest exists, but when I went looking for it, it doesn't appear to be there.");
goto next_item;

52
rhizome_packetformats.c
View File

@ -35,6 +35,8 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
and geographic bounding box information that is used to help manage flooding of
bundles.
Old BAR format (no longer used):
64 bits - manifest ID prefix.
56 bits - low 56 bits of version number.
8 bits - TTL of bundle in hops.
@ -43,6 +45,18 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
16 bits - min longitude (-180 - +180).
16 bits - max latitude (-90 - +90).
16 bits - max longitude (-180 - +180).
New BAR format with longer manifest ID prefix:
120 bits - manifest ID prefix.
8 bits - log2(length) of associated file.
56 bits - low 56 bits of version number.
16 bits - min latitude (-90 - +90).
16 bits - min longitude (-180 - +180).
16 bits - max latitude (-90 - +90).
16 bits - max longitude (-180 - +180).
8 bits - TTL of bundle in hops (0xff = unlimited distribution)
*/
if (!m) { RETURN(WHY("null manifest passed in")); }
@ -50,13 +64,13 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
int i;
/* Manifest prefix */
for(i=0;i<8;i++) bar[i]=m->cryptoSignPublic[i];
/* Version */
for(i=0;i<7;i++) bar[8+6-i]=(m->version>>(8*i))&0xff;
/* TTL */
if (m->ttl>0) bar[15]=m->ttl-1; else bar[15]=0;
for(i=0;i<RHIZOME_BAR_PREFIX_BYTES;i++)
bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic[i];
/* file length */
for(i=0;i<8;i++) bar[16+7-i]=(m->fileLength>>(8*i))&0xff;
bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2(m->fileLength);
/* Version */
for(i=0;i<7;i++) bar[RHIZOME_BAR_VERSION_OFFSET+6-i]=(m->version>>(8*i))&0xff;
/* geo bounding box */
double minLat=rhizome_manifest_get_double(m,"min_lat",-90);
if (minLat<-90) minLat=-90; if (minLat>90) minLat=90;
@ -65,13 +79,17 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
double maxLat=rhizome_manifest_get_double(m,"max_lat",+90);
if (maxLat<-90) maxLat=-90; if (maxLat>90) maxLat=90;
double maxLong=rhizome_manifest_get_double(m,"max_long",+180);
if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180;
if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180;
unsigned short v;
v=(minLat+90)*(65535/180); bar[24]=(v>>8)&0xff; bar[25]=(v>>0)&0xff;
v=(minLong+180)*(65535/360); bar[26]=(v>>8)&0xff; bar[27]=(v>>0)&0xff;
v=(maxLat+90)*(65535/180); bar[28]=(v>>8)&0xff; bar[29]=(v>>0)&0xff;
v=(maxLong+180)*(65535/360); bar[30]=(v>>8)&0xff; bar[31]=(v>>0)&0xff;
int o=RHIZOME_BAR_GEOBOX_OFFSET;
v=(minLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
v=(minLong+180)*(65535/360); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
v=(maxLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
v=(maxLong+180)*(65535/360); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
/* TTL */
if (m->ttl>0) bar[RHIZOME_BAR_TTL_OFFSET]=m->ttl-1;
else bar[RHIZOME_BAR_TTL_OFFSET]=0;
RETURN(0);
}
@ -81,19 +99,21 @@ long long rhizome_bar_version(unsigned char *bar)
long long version=0;
int i;
// for(i=0;i<7;i++) bar[8+6-i]=(m->version>>(8*i))&0xff;
for(i=0;i<7;i++) version|=bar[8+6-i]<<(8LL*i);
for(i=0;i<7;i++) version|=bar[RHIZOME_BAR_VERSION_OFFSET+6-i]<<(8LL*i);
return version;
}
unsigned long long rhizome_bar_bidprefix(unsigned char *bar)
/* This function only displays the first 8 bytes, and should not be used
for comparison. */
unsigned long long rhizome_bar_bidprefix_ll(unsigned char *bar)
{
long long bidprefix=0;
int i;
for(i=0;i<8;i++) bidprefix|=((unsigned long long)bar[7-i])<<(8*i);
for(i=0;i<8;i++)
bidprefix|=((unsigned long long)bar[RHIZOME_BAR_PREFIX_OFFSET+7-i])<<(8*i);
return bidprefix;
}
int bundles_available=-1;
int bundle_offset[2]={0,0};
int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)