Rhizome fetching over http now receives files and imports them.

Still some wrinkles to work out.

rhizome.h

@@ -95,6 +95,7 @@ typedef struct rhizome_signature {
 #define MAX_MANIFEST_BYTES 8192
 typedef struct rhizome_manifest {
   int manifest_bytes;
+  int manifest_all_bytes;
   unsigned char manifestdata[MAX_MANIFEST_BYTES];
   unsigned char manifesthash[crypto_hash_sha512_BYTES];
 

rhizome_bundle.c

@@ -40,6 +40,8 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
     fclose(f);
   }
 
+  m->manifest_all_bytes=m->manifest_bytes;
+
   /* Parse out variables, signature etc */
   int ofs=0;
   while((ofs<m->manifest_bytes)&&(m->manifestdata[ofs]))
@@ -122,7 +124,8 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
 	      fprintf(stderr,
 		      "Manifest id variable does not match first signature block.\n");
 	    m->errors++;
-	  }
+	    m->selfSigned=0;
+	  } else m->selfSigned=1;
       }
     }
 
@@ -286,6 +289,8 @@ int rhizome_manifest_pack_variables(rhizome_manifest *m)
     }
   m->manifestdata[ofs++]=0x00;
   m->manifest_bytes=ofs;
+  if (debug&DEBUG_RHIZOME) WHY("Repacked variables in manifest.");
+  m->manifest_all_bytes=ofs;
 
   /* Recalculate hash */
   crypto_hash_sha512(m->manifesthash,m->manifestdata,m->manifest_bytes);
@@ -309,6 +314,7 @@ int rhizome_manifest_sign(rhizome_manifest *m)
   bcopy(&sig->signature[0],&m->manifestdata[m->manifest_bytes],sig->signatureLength);
 
   m->manifest_bytes+=sig->signatureLength;
+  m->manifest_all_bytes=m->manifest_bytes;
 
   free(sig);
   return 0;
@@ -319,7 +325,7 @@ int rhizome_write_manifest_file(rhizome_manifest *m,char *filename)
   if (!m) return WHY("Manifest is null.");
   if (!m->finalised) return WHY("Manifest must be finalised before it can be written.");
   FILE *f=fopen(filename,"w");
-  int r=fwrite(m->manifestdata,m->manifest_bytes,1,f);
+  int r=fwrite(m->manifestdata,m->manifest_all_bytes,1,f);
   fclose(f);
   if (r!=1) return WHY("Failed to fwrite() manifest file.");
   return 0;

rhizome_database.c

@@ -360,7 +360,7 @@ int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
     /* We don't have the secret for this manifest, so only allow updates if 
        the self-signature is valid */
     if (!m->selfSigned) {
-    WHY("*** Insert into manifests failed (-2).");
+      WHY("*** Insert into manifests failed (-2).");
       return WHY("Manifest is not signed, and I don't have the key.  Manifest might be forged or corrupt.");
     }
   }

rhizome_fetch.c

@@ -179,6 +179,7 @@ int rhizome_queue_manifest_import(rhizome_manifest *m,
 	    char filename[1024];
 	    snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,
 		     rhizome_manifest_get(q->manifest,"id",NULL,0));
+	    q->manifest->dataFileName=strdup(filename);
 	    q->file=fopen(filename,"w");
 	    if (!q->file) {
 	      if (debug&DEBUG_RHIZOME)
@@ -262,7 +263,7 @@ int rhizome_fetch_poll()
 	  }
 	  break;
 	case RHIZOME_FETCH_RXFILE:
-	  /* Keep reading until we have two CR/LFs in a row */
+	  /* Keep reading until we have the promised amount of data */
 	  if (debug&DEBUG_RHIZOME) 
 	    fprintf(stderr,"receiving rhizome fetch file body (current offset=%d)\n",
 		    q->file_ofs);
@@ -291,14 +292,39 @@ int rhizome_fetch_poll()
 		continue;
 	      }
 	    q->file_ofs+=bytes;
-	    if (q->file_ofs>=q->file_len)
+	  }	  
+	  if (q->file_ofs>=q->file_len)
+	    {
+	      /* got all of file */
+	      q->close=1;
+	      if (debug&DEBUG_RHIZOME) fprintf(stderr,"Received all of file via rhizome -- now to import it\n");
 	      {
-		/* got all of file */
-		q->close=1;
-		if (debug&DEBUG_RHIZOME) fprintf(stderr,"Received all of file via rhizome -- now to import it\n");
+		fclose(q->file);
+		char filename[1024];
+		snprintf(filename,1024,"%s/import/manifest.%s",
+			 rhizome_datastore_path,
+			 rhizome_manifest_get(q->manifest,"id",NULL,0));
+		/* Do really write the manifest unchanged */
+		fprintf(stderr,"manifest has %d signatories\n",q->manifest->sig_count);
+		fprintf(stderr,"manifest id = %s, len=%d\n",
+			rhizome_manifest_get(q->manifest,"id",NULL,0),
+			q->manifest->manifest_bytes);
+		dump("manifest",&q->manifest->manifestdata[0],
+		     q->manifest->manifest_all_bytes);
+		q->manifest->finalised=1;
+		q->manifest->manifest_bytes=q->manifest->manifest_all_bytes;
+		if (!rhizome_write_manifest_file(q->manifest,filename)) {
+		    rhizome_bundle_import(rhizome_manifest_get(q->manifest,
+							       "id",NULL,0),
+					  NULL /* no additional groups */,
+					  q->manifest->ttl-1 /* TTL */,
+					  1 /* do verify */,
+					  1 /* do check hash of file */,
+					  0 /* do not sign it, just keep existing
+					       signatures */);		  
+		  }
 	      }
-	    
-	  }
+	    }
 	  break;
 	case RHIZOME_FETCH_RXHTTPHEADERS:
 	  /* Keep reading until we have two CR/LFs in a row */
@@ -388,6 +414,8 @@ int rhizome_fetch_poll()
 	     have moved out of range. */
 	  if (!action) {
 	    if (time(0)-q->last_action>RHIZOME_IDLE_TIMEOUT) {
+	      if (debug&DEBUG_RHIZOME) 
+		WHY("Closing connection due to inactivity timeout.");
 	      q->close=1;
 	      continue;
 	    }

rhizome_http.c

@@ -528,7 +528,7 @@ int rhizome_server_parse_http_request(int rn,rhizome_http_request *r)
 	  long long rowid = sqlite_exec_int64("select rowid from files where id='%s';",id);
 	  sqlite3_blob *blob;
 	  if (rowid>=0) 
-	    if (sqlite3_blob_open(rhizome_db,"main","files","id",rowid,0,&blob)
+	    if (sqlite3_blob_open(rhizome_db,"main","files","data",rowid,0,&blob)
 		!=SQLITE_OK)
 	      rowid=-1;
 
@@ -538,12 +538,13 @@ int rhizome_server_parse_http_request(int rn,rhizome_http_request *r)
 	  }
 	  else {
 	    r->blob_table=strdup("files");
-	    r->blob_column=strdup("id");
+	    r->blob_column=strdup("data");
 	    r->blob_rowid=rowid;
 	    r->source_index=0;	    
 	    r->blob_end=sqlite3_blob_bytes(blob);
 	    rhizome_server_http_response_header(r,200,"application/binary",
 						r->blob_end-r->source_index);
+	    r->request_type|=RHIZOME_HTTP_REQUEST_BLOB;
 	    sqlite3_blob_close(blob);
 	    WHY("opened blob and file -- but still need to send file body.");
 	  }
@@ -608,6 +609,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char
 */
 int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r)
 {
+  sqlite3_blob *blob;
   int bytes;
   fcntl(r->socket,F_SETFL,fcntl(r->socket, F_GETFL, NULL)|O_NONBLOCK);
 
@@ -642,7 +644,7 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r)
       }
     }
 
-  switch(r->request_type)
+  switch(r->request_type&(~RHIZOME_HTTP_REQUEST_FROMBUFFER))
     {
     case RHIZOME_HTTP_REQUEST_FAVICON:
       if (r->buffer_size<favicon_len) {
@@ -662,6 +664,41 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r)
       }
       
       break;
+    case RHIZOME_HTTP_REQUEST_BLOB:
+      /* Get more data from the file and put it in the buffer */
+      r->buffer_length=r->blob_end-r->source_index;
+      if (r->buffer_length<=0) {
+	/* end of blob reached */
+	r->request_type=0; break;
+      }
+      if (r->buffer_size<65536) {
+	free(r->buffer); r->buffer=malloc(65536);
+	if (!r->buffer) {
+	  if (debug&DEBUG_RHIZOME) WHY("malloc() failed");
+	  r->request_type=0; break;
+	}
+	r->buffer_size=65536;
+      }
+      if (r->buffer_length>r->buffer_size) r->buffer_length=r->buffer_size;
+      if (sqlite3_blob_open(rhizome_db,"main",r->blob_table,r->blob_column,
+			    r->blob_rowid,0,&blob)==SQLITE_OK)
+	{
+	  if(sqlite3_blob_read(blob,&r->buffer[0],r->buffer_length,r->source_index)
+	     ==SQLITE_OK)
+	    {
+	      r->request_type|=RHIZOME_HTTP_REQUEST_FROMBUFFER;
+	      r->source_index+=r->buffer_length;
+	    }
+	  else
+	    r->request_type=0;
+	  sqlite3_blob_close(blob);
+	}
+      else
+	{
+	  if (debug&DEBUG_RHIZOME) WHY("could not open blob to send more data");
+	  r->request_type=0;
+	}
+      break;
     case RHIZOME_HTTP_REQUEST_FROMBUFFER:
       /* This really shouldn't happen! */
       
@@ -672,6 +709,8 @@ int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r)
       break;
     }
 
+  if (!r->request_type) return rhizome_server_close_http_request(rn);	  
+
   fcntl(r->socket,F_SETFL,fcntl(r->socket, F_GETFL, NULL)&(~O_NONBLOCK));
   return 1;
 }