From cdf1c45da4ba0e7165c8b1442ffa028d1c04cbf4 Mon Sep 17 00:00:00 2001 From: gardners Date: Fri, 13 Jan 2012 21:13:17 +1030 Subject: [PATCH] Rhizome fetching over http now receives files and imports them. Still some wrinkles to work out. --- rhizome.h | 1 + rhizome_bundle.c | 10 ++++++++-- rhizome_database.c | 2 +- rhizome_fetch.c | 42 +++++++++++++++++++++++++++++++++++------- rhizome_http.c | 45 ++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/rhizome.h b/rhizome.h index 27b0800e..56e0297f 100644 --- a/rhizome.h +++ b/rhizome.h @@ -95,6 +95,7 @@ typedef struct rhizome_signature { #define MAX_MANIFEST_BYTES 8192 typedef struct rhizome_manifest { int manifest_bytes; + int manifest_all_bytes; unsigned char manifestdata[MAX_MANIFEST_BYTES]; unsigned char manifesthash[crypto_hash_sha512_BYTES]; diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 053ded3c..17494058 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -40,6 +40,8 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag fclose(f); } + m->manifest_all_bytes=m->manifest_bytes; + /* Parse out variables, signature etc */ int ofs=0; while((ofsmanifest_bytes)&&(m->manifestdata[ofs])) @@ -122,7 +124,8 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag fprintf(stderr, "Manifest id variable does not match first signature block.\n"); m->errors++; - } + m->selfSigned=0; + } else m->selfSigned=1; } } @@ -286,6 +289,8 @@ int rhizome_manifest_pack_variables(rhizome_manifest *m) } m->manifestdata[ofs++]=0x00; m->manifest_bytes=ofs; + if (debug&DEBUG_RHIZOME) WHY("Repacked variables in manifest."); + m->manifest_all_bytes=ofs; /* Recalculate hash */ crypto_hash_sha512(m->manifesthash,m->manifestdata,m->manifest_bytes); @@ -309,6 +314,7 @@ int rhizome_manifest_sign(rhizome_manifest *m) bcopy(&sig->signature[0],&m->manifestdata[m->manifest_bytes],sig->signatureLength); m->manifest_bytes+=sig->signatureLength; + m->manifest_all_bytes=m->manifest_bytes; free(sig); return 0; @@ -319,7 +325,7 @@ int rhizome_write_manifest_file(rhizome_manifest *m,char *filename) if (!m) return WHY("Manifest is null."); if (!m->finalised) return WHY("Manifest must be finalised before it can be written."); FILE *f=fopen(filename,"w"); - int r=fwrite(m->manifestdata,m->manifest_bytes,1,f); + int r=fwrite(m->manifestdata,m->manifest_all_bytes,1,f); fclose(f); if (r!=1) return WHY("Failed to fwrite() manifest file."); return 0; diff --git a/rhizome_database.c b/rhizome_database.c index 397293e4..7a6cf098 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -360,7 +360,7 @@ int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename) /* We don't have the secret for this manifest, so only allow updates if the self-signature is valid */ if (!m->selfSigned) { - WHY("*** Insert into manifests failed (-2)."); + WHY("*** Insert into manifests failed (-2)."); return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt."); } } diff --git a/rhizome_fetch.c b/rhizome_fetch.c index ccc9a564..4ce8c680 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -179,6 +179,7 @@ int rhizome_queue_manifest_import(rhizome_manifest *m, char filename[1024]; snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path, rhizome_manifest_get(q->manifest,"id",NULL,0)); + q->manifest->dataFileName=strdup(filename); q->file=fopen(filename,"w"); if (!q->file) { if (debug&DEBUG_RHIZOME) @@ -262,7 +263,7 @@ int rhizome_fetch_poll() } break; case RHIZOME_FETCH_RXFILE: - /* Keep reading until we have two CR/LFs in a row */ + /* Keep reading until we have the promised amount of data */ if (debug&DEBUG_RHIZOME) fprintf(stderr,"receiving rhizome fetch file body (current offset=%d)\n", q->file_ofs); @@ -291,14 +292,39 @@ int rhizome_fetch_poll() continue; } q->file_ofs+=bytes; - if (q->file_ofs>=q->file_len) + } + if (q->file_ofs>=q->file_len) + { + /* got all of file */ + q->close=1; + if (debug&DEBUG_RHIZOME) fprintf(stderr,"Received all of file via rhizome -- now to import it\n"); { - /* got all of file */ - q->close=1; - if (debug&DEBUG_RHIZOME) fprintf(stderr,"Received all of file via rhizome -- now to import it\n"); + fclose(q->file); + char filename[1024]; + snprintf(filename,1024,"%s/import/manifest.%s", + rhizome_datastore_path, + rhizome_manifest_get(q->manifest,"id",NULL,0)); + /* Do really write the manifest unchanged */ + fprintf(stderr,"manifest has %d signatories\n",q->manifest->sig_count); + fprintf(stderr,"manifest id = %s, len=%d\n", + rhizome_manifest_get(q->manifest,"id",NULL,0), + q->manifest->manifest_bytes); + dump("manifest",&q->manifest->manifestdata[0], + q->manifest->manifest_all_bytes); + q->manifest->finalised=1; + q->manifest->manifest_bytes=q->manifest->manifest_all_bytes; + if (!rhizome_write_manifest_file(q->manifest,filename)) { + rhizome_bundle_import(rhizome_manifest_get(q->manifest, + "id",NULL,0), + NULL /* no additional groups */, + q->manifest->ttl-1 /* TTL */, + 1 /* do verify */, + 1 /* do check hash of file */, + 0 /* do not sign it, just keep existing + signatures */); + } } - - } + } break; case RHIZOME_FETCH_RXHTTPHEADERS: /* Keep reading until we have two CR/LFs in a row */ @@ -388,6 +414,8 @@ int rhizome_fetch_poll() have moved out of range. */ if (!action) { if (time(0)-q->last_action>RHIZOME_IDLE_TIMEOUT) { + if (debug&DEBUG_RHIZOME) + WHY("Closing connection due to inactivity timeout."); q->close=1; continue; } diff --git a/rhizome_http.c b/rhizome_http.c index 797ec435..17df0820 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -528,7 +528,7 @@ int rhizome_server_parse_http_request(int rn,rhizome_http_request *r) long long rowid = sqlite_exec_int64("select rowid from files where id='%s';",id); sqlite3_blob *blob; if (rowid>=0) - if (sqlite3_blob_open(rhizome_db,"main","files","id",rowid,0,&blob) + if (sqlite3_blob_open(rhizome_db,"main","files","data",rowid,0,&blob) !=SQLITE_OK) rowid=-1; @@ -538,12 +538,13 @@ int rhizome_server_parse_http_request(int rn,rhizome_http_request *r) } else { r->blob_table=strdup("files"); - r->blob_column=strdup("id"); + r->blob_column=strdup("data"); r->blob_rowid=rowid; r->source_index=0; r->blob_end=sqlite3_blob_bytes(blob); rhizome_server_http_response_header(r,200,"application/binary", r->blob_end-r->source_index); + r->request_type|=RHIZOME_HTTP_REQUEST_BLOB; sqlite3_blob_close(blob); WHY("opened blob and file -- but still need to send file body."); } @@ -608,6 +609,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char */ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r) { + sqlite3_blob *blob; int bytes; fcntl(r->socket,F_SETFL,fcntl(r->socket, F_GETFL, NULL)|O_NONBLOCK); @@ -642,7 +644,7 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r) } } - switch(r->request_type) + switch(r->request_type&(~RHIZOME_HTTP_REQUEST_FROMBUFFER)) { case RHIZOME_HTTP_REQUEST_FAVICON: if (r->buffer_sizebuffer_length=r->blob_end-r->source_index; + if (r->buffer_length<=0) { + /* end of blob reached */ + r->request_type=0; break; + } + if (r->buffer_size<65536) { + free(r->buffer); r->buffer=malloc(65536); + if (!r->buffer) { + if (debug&DEBUG_RHIZOME) WHY("malloc() failed"); + r->request_type=0; break; + } + r->buffer_size=65536; + } + if (r->buffer_length>r->buffer_size) r->buffer_length=r->buffer_size; + if (sqlite3_blob_open(rhizome_db,"main",r->blob_table,r->blob_column, + r->blob_rowid,0,&blob)==SQLITE_OK) + { + if(sqlite3_blob_read(blob,&r->buffer[0],r->buffer_length,r->source_index) + ==SQLITE_OK) + { + r->request_type|=RHIZOME_HTTP_REQUEST_FROMBUFFER; + r->source_index+=r->buffer_length; + } + else + r->request_type=0; + sqlite3_blob_close(blob); + } + else + { + if (debug&DEBUG_RHIZOME) WHY("could not open blob to send more data"); + r->request_type=0; + } + break; case RHIZOME_HTTP_REQUEST_FROMBUFFER: /* This really shouldn't happen! */ @@ -672,6 +709,8 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r) break; } + if (!r->request_type) return rhizome_server_close_http_request(rn); + fcntl(r->socket,F_SETFL,fcntl(r->socket, F_GETFL, NULL)&(~O_NONBLOCK)); return 1; }