ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-19 21:27:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed various bugs and memory leaks in rhizome file fetching.

Browse Source 
Now successfully fetches manifests for files we already have in
the database, using the existing database copy of the file.
Also added filter to stop same file being fetched more than once at
same time..
This commit is contained in:
gardners 2012-01-27 16:21:48 +10:30
parent 957262ed47
commit c0676fd530
4 changed files with 66 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
rhizome.c
View File

@ -36,19 +36,24 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
The file should be included in the specified rhizome groups, if possible.
(some groups may be closed groups that we do not have the private key for.)
*/
int rhizome_bundle_import(char *bundle,char *groups[], int ttl,
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP)
{
char filename[1024];
char manifestname[1024];
char *buffer;
snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
/* Open files */
rhizome_manifest *m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,
RHIZOME_VERIFY);
rhizome_manifest *m=m_in;
if (!m_in)
m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,RHIZOME_VERIFY);
else
if (debug&DEBUG_RHIZOMESYNC) fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP);
if (!m) return WHY("Could not read manifest file.");
char hexhash[SHA512_DIGEST_STRING_LENGTH];
@ -57,10 +62,12 @@ int rhizome_bundle_import(char *bundle,char *groups[], int ttl,
/* Keep associated file name handy for later */
m->dataFileName=strdup(filename);
struct stat stat;
if (lstat(filename,&stat)) {
return WHY("Could not stat() associated file");
m->fileLength=stat.st_size;
if (checkFileP) {
struct stat stat;
if (lstat(filename,&stat)) {
return WHY("Could not stat() associated file");
m->fileLength=stat.st_size;
}
}
if (checkFileP||signP) {

2
rhizome.h
View File

@ -177,7 +177,7 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
int rhizome_store_file(char *file,char *hash,int priortity);
char *rhizome_safe_encode(unsigned char *in,int len);
int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
int rhizome_bundle_import(char *bundle,char *groups[],int ttl,
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP);
int rhizome_manifest_finalise(rhizome_manifest *m,int signP);
char *rhizome_bytes_to_hex(unsigned char *in,int byteCount);

12
rhizome_bundle.c
View File

@ -360,14 +360,14 @@ int rhizome_manifest_finalise(rhizome_manifest *m,int signP)
if (rhizome_hash_file(m->dataFileName,m->fileHexHash))
return WHY("rhizome_hash_file() failed during finalisation of manifest.");
m->fileHashedP=1;
}
/* set fileLength */
struct stat stat;
if (lstat(m->dataFileName,&stat)) {
return WHY("Could not stat() associated file");
/* set fileLength */
struct stat stat;
if (lstat(m->dataFileName,&stat)) {
return WHY("Could not stat() associated file");
}
m->fileLength=stat.st_size;
}
m->fileLength=stat.st_size;
/* Set file hash and size information */
rhizome_manifest_set(m,"filehash",m->fileHexHash);

49
rhizome_fetch.c
View File

@ -90,15 +90,30 @@ rhizome_file_fetch_record file_fetch_queue[MAX_QUEUED_FILES];
int rhizome_queue_manifest_import(rhizome_manifest *m,
struct sockaddr_in *peerip)
{
int i;
/* Don't queue if queue slots already full */
if (rhizome_file_fetch_queue_count>=MAX_QUEUED_FILES) {
if (debug&DEBUG_RHIZOME) fprintf(stderr,"Already busy fetching files");
return -1;
}
/* Don't queue if already queued */
char *id=rhizome_manifest_get(m,"id",NULL,0);
for(i=0;i<rhizome_file_fetch_queue_count;i++) {
rhizome_file_fetch_record
*q=&file_fetch_queue[i];
if (!strcasecmp(id,rhizome_manifest_get(q->manifest,"id",NULL,0))) {
if (debug&DEBUG_RHIZOMESYNC)
fprintf(stderr,"Already have %s in the queue.\n",id);
return -1;
}
}
char *filehash=rhizome_manifest_get(m,"filehash",NULL,0);
long long filesize=rhizome_manifest_get_ll(m,"filesize");
if (debug&DEBUG_RHIZOME) fprintf(stderr,"Getting ready to fetch file %s\n",filehash);
if (debug&DEBUG_RHIZOMESYNC)
fprintf(stderr,"Getting ready to fetch file %s for manifest %s\n",filehash,rhizome_manifest_get(m,"id",NULL,0));
if (filesize>0&&(filehash!=NULL))
{
@ -196,11 +211,33 @@ int rhizome_queue_manifest_import(rhizome_manifest *m,
return WHY("Rhizome fetching via overlay not implemented");
}
}
else
{
if (debug&DEBUG_RHIZOMESYNC)
fprintf(stderr,"We already have the file for this manifest; importing from manifest alone.\n");
m->finalised=1;
m->fileHashedP=1;
m->manifest_bytes=m->manifest_all_bytes;
char filename[1024];
snprintf(filename,1024,"%s/import/manifest.%s",
rhizome_datastore_path,
rhizome_manifest_get(m,"id",NULL,0));
if (!rhizome_write_manifest_file(m,filename)) {
rhizome_bundle_import(m,rhizome_manifest_get(m,"id",NULL,0),
NULL /* no additional groups */,
m->ttl-1 /* TTL */,
1 /* do verify */,
0 /* don't check hash of file (since we are using the databse stored copy) */,
0 /* do not sign it, just keep existing
signatures */);
}
}
}
return 0;
}
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax)
{
int i;
@ -315,14 +352,16 @@ int rhizome_fetch_poll()
q->manifest->finalised=1;
q->manifest->manifest_bytes=q->manifest->manifest_all_bytes;
if (!rhizome_write_manifest_file(q->manifest,filename)) {
rhizome_bundle_import(rhizome_manifest_get(q->manifest,
"id",NULL,0),
rhizome_bundle_import(q->manifest,
rhizome_manifest_get(q->manifest,
"id",NULL,0),
NULL /* no additional groups */,
q->manifest->ttl-1 /* TTL */,
1 /* do verify */,
1 /* do check hash of file */,
0 /* do not sign it, just keep existing
signatures */);
signatures */);
q->manifest=NULL;
}
}
}