From c0676fd530308e97fbb5f9c7a10633b84cfc27dd Mon Sep 17 00:00:00 2001 From: gardners Date: Fri, 27 Jan 2012 16:21:48 +1030 Subject: [PATCH] Fixed various bugs and memory leaks in rhizome file fetching. Now successfully fetches manifests for files we already have in the database, using the existing database copy of the file. Also added filter to stop same file being fetched more than once at same time.. --- rhizome.c | 23 +++++++++++++++-------- rhizome.h | 2 +- rhizome_bundle.c | 12 ++++++------ rhizome_fetch.c | 49 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/rhizome.c b/rhizome.c index 164beeab..e5e8e3b1 100644 --- a/rhizome.c +++ b/rhizome.c @@ -36,19 +36,24 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. The file should be included in the specified rhizome groups, if possible. (some groups may be closed groups that we do not have the private key for.) */ -int rhizome_bundle_import(char *bundle,char *groups[], int ttl, +int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, int verifyP, int checkFileP, int signP) { char filename[1024]; char manifestname[1024]; char *buffer; - + + snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0; snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0; /* Open files */ - rhizome_manifest *m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */, - RHIZOME_VERIFY); + rhizome_manifest *m=m_in; + if (!m_in) + m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,RHIZOME_VERIFY); + else + if (debug&DEBUG_RHIZOMESYNC) fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP); + if (!m) return WHY("Could not read manifest file."); char hexhash[SHA512_DIGEST_STRING_LENGTH]; @@ -57,10 +62,12 @@ int rhizome_bundle_import(char *bundle,char *groups[], int ttl, /* Keep associated file name handy for later */ m->dataFileName=strdup(filename); - struct stat stat; - if (lstat(filename,&stat)) { - return WHY("Could not stat() associated file"); - m->fileLength=stat.st_size; + if (checkFileP) { + struct stat stat; + if (lstat(filename,&stat)) { + return WHY("Could not stat() associated file"); + m->fileLength=stat.st_size; + } } if (checkFileP||signP) { diff --git a/rhizome.h b/rhizome.h index 56e0297f..b63ded6f 100644 --- a/rhizome.h +++ b/rhizome.h @@ -177,7 +177,7 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); int rhizome_store_file(char *file,char *hash,int priortity); char *rhizome_safe_encode(unsigned char *in,int len); int rhizome_finish_sqlstatement(sqlite3_stmt *statement); -int rhizome_bundle_import(char *bundle,char *groups[],int ttl, +int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, int verifyP, int checkFileP, int signP); int rhizome_manifest_finalise(rhizome_manifest *m,int signP); char *rhizome_bytes_to_hex(unsigned char *in,int byteCount); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index e9826463..5b03c6b6 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -360,14 +360,14 @@ int rhizome_manifest_finalise(rhizome_manifest *m,int signP) if (rhizome_hash_file(m->dataFileName,m->fileHexHash)) return WHY("rhizome_hash_file() failed during finalisation of manifest."); m->fileHashedP=1; - } - /* set fileLength */ - struct stat stat; - if (lstat(m->dataFileName,&stat)) { - return WHY("Could not stat() associated file"); + /* set fileLength */ + struct stat stat; + if (lstat(m->dataFileName,&stat)) { + return WHY("Could not stat() associated file"); + } + m->fileLength=stat.st_size; } - m->fileLength=stat.st_size; /* Set file hash and size information */ rhizome_manifest_set(m,"filehash",m->fileHexHash); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index f6bfb750..11adf1ca 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -90,15 +90,30 @@ rhizome_file_fetch_record file_fetch_queue[MAX_QUEUED_FILES]; int rhizome_queue_manifest_import(rhizome_manifest *m, struct sockaddr_in *peerip) { + int i; + + /* Don't queue if queue slots already full */ if (rhizome_file_fetch_queue_count>=MAX_QUEUED_FILES) { if (debug&DEBUG_RHIZOME) fprintf(stderr,"Already busy fetching files"); return -1; } + /* Don't queue if already queued */ + char *id=rhizome_manifest_get(m,"id",NULL,0); + for(i=0;imanifest,"id",NULL,0))) { + if (debug&DEBUG_RHIZOMESYNC) + fprintf(stderr,"Already have %s in the queue.\n",id); + return -1; + } + } char *filehash=rhizome_manifest_get(m,"filehash",NULL,0); long long filesize=rhizome_manifest_get_ll(m,"filesize"); - if (debug&DEBUG_RHIZOME) fprintf(stderr,"Getting ready to fetch file %s\n",filehash); + if (debug&DEBUG_RHIZOMESYNC) + fprintf(stderr,"Getting ready to fetch file %s for manifest %s\n",filehash,rhizome_manifest_get(m,"id",NULL,0)); if (filesize>0&&(filehash!=NULL)) { @@ -196,11 +211,33 @@ int rhizome_queue_manifest_import(rhizome_manifest *m, return WHY("Rhizome fetching via overlay not implemented"); } } + else + { + if (debug&DEBUG_RHIZOMESYNC) + fprintf(stderr,"We already have the file for this manifest; importing from manifest alone.\n"); + m->finalised=1; + m->fileHashedP=1; + m->manifest_bytes=m->manifest_all_bytes; + char filename[1024]; + snprintf(filename,1024,"%s/import/manifest.%s", + rhizome_datastore_path, + rhizome_manifest_get(m,"id",NULL,0)); + if (!rhizome_write_manifest_file(m,filename)) { + rhizome_bundle_import(m,rhizome_manifest_get(m,"id",NULL,0), + NULL /* no additional groups */, + m->ttl-1 /* TTL */, + 1 /* do verify */, + 0 /* don't check hash of file (since we are using the databse stored copy) */, + 0 /* do not sign it, just keep existing + signatures */); + + } + } } return 0; } - + int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax) { int i; @@ -315,14 +352,16 @@ int rhizome_fetch_poll() q->manifest->finalised=1; q->manifest->manifest_bytes=q->manifest->manifest_all_bytes; if (!rhizome_write_manifest_file(q->manifest,filename)) { - rhizome_bundle_import(rhizome_manifest_get(q->manifest, - "id",NULL,0), + rhizome_bundle_import(q->manifest, + rhizome_manifest_get(q->manifest, + "id",NULL,0), NULL /* no additional groups */, q->manifest->ttl-1 /* TTL */, 1 /* do verify */, 1 /* do check hash of file */, 0 /* do not sign it, just keep existing - signatures */); + signatures */); + q->manifest=NULL; } } }