Remove payload if hash doesn't match when reading back

meshms.c

@@ -221,7 +221,7 @@ static int ply_read_open(struct ply_read *ply, const char *id, rhizome_manifest
     DEBUGF("Opening ply %s", id);
   if (rhizome_retrieve_manifest(id, m))
     return -1;
-  if (rhizome_open_decrypt_read(m, NULL, &ply->read, 0))
+  if (rhizome_open_decrypt_read(m, NULL, &ply->read))
     return -1;
   ply->read.offset = ply->read.length = m->fileLength;
   return 0;
@@ -468,7 +468,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
   struct rhizome_read_buffer buff;
   bzero(&buff, sizeof(buff));
   
-  int ret = rhizome_open_decrypt_read(m, NULL, &read, 0);
+  int ret = rhizome_open_decrypt_read(m, NULL, &read);
   if (ret<0)
     goto end;
   

rhizome.h

@@ -432,8 +432,9 @@ struct rhizome_read{
   unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
   unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
   
-  int hash;
+  int64_t hash_offset;
   SHA512_CTX sha512_context;
+  char invalid;
   
   int64_t blob_rowid;
   int blob_fd;
@@ -718,12 +719,11 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint
 
 int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, 
 			    const unsigned char *key, const unsigned char *nonce);
-int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);
+int rhizome_open_read(struct rhizome_read *read, const char *fileid);
 int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
 int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
 int rhizome_read_close(struct rhizome_read *read);
-int rhizome_store_delete(const char *id);
+int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
-int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash);
 int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
 int rhizome_dump_file(const char *id, const char *filepath, int64_t *length);
 int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout, 

rhizome_database.c

@@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 static char rhizome_thisdatastore_path[256];
 
+static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid);
+
 const char *rhizome_datastore_path()
 {
   if (!rhizome_thisdatastore_path[0])
@@ -716,11 +718,20 @@ int rhizome_database_filehash_from_id(const char *id, uint64_t version, char has
   OUT();
 }
 
+static int rhizome_delete_external(const char *fileid)
+{
+  // attempt to remove any external blob
+  char blob_path[1024];
+  if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid))
+    return -1;
+  return unlink(blob_path);
+}
+
 static int rhizome_cleanup_external(sqlite_retry_state *retry, sqlite3_stmt *statement){
   int ret=0;
   while (sqlite_step_retry(retry, statement) == SQLITE_ROW) {
     const char *id = (const char *) sqlite3_column_text(statement, 0);
-    if (rhizome_store_delete(id)==0)
+    if (rhizome_delete_external(id)==0)
       ret++;
   }
   return ret;
@@ -851,11 +862,8 @@ int rhizome_drop_stored_file(const char *id,int maximum_priority)
     }
   }
   sqlite3_finalize(statement);
-  if (can_drop) {
+  if (can_drop)
-    sqlite_exec_void_retry(&retry, "delete from files where id='%s';",id);
+    rhizome_delete_file_retry(&retry, id);
-    rhizome_store_delete(id);
-    sqlite_exec_void_retry(&retry, "delete from fileblobs where id='%s';",id);
-  }
   return 0;
 }
 
@@ -1394,6 +1402,9 @@ int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifes
 static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
 {
   int ret = 0;
+  
+  rhizome_delete_external(fileid);
+  
   sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
   if (!statement)
     ret = -1;

rhizome_direct_http.c

@@ -976,7 +976,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  
 	  struct rhizome_read read;
 	  bzero(&read, sizeof read);
-	  if (rhizome_open_read(&read, filehash, 0))
+	  if (rhizome_open_read(&read, filehash))
 	    goto closeit;
 	
 	  int read_ofs;

rhizome_http.c

@@ -620,7 +620,7 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
 	} else {
 	  str_toupper_inplace(id);
 	  bzero(&r->read_state, sizeof(r->read_state));
-	  if (rhizome_open_read(&r->read_state, id, 1))
+	  if (rhizome_open_read(&r->read_state, id))
 	    rhizome_server_simple_http_response(r, 404, "<html><h1>Payload not found</h1></html>\r\n");
 	  else{
 	    if (r->read_state.length==-1){
@@ -635,8 +635,6 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
 	    if (range){
 	      sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset);
 	      DEBUGF("Found range header %"PRId64,r->read_state.offset);
-	      if (r->read_state.offset)
-		r->read_state.hash=0;
 	    }
 	    
 	    if (r->read_state.length - r->read_state.offset>0){

rhizome_store.c

@@ -410,25 +410,12 @@ end:
   return ret;
 }
 
-int rhizome_store_delete(const char *id){
-  char blob_path[1024];
-  if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id))
-    return -1;
-  if (unlink(blob_path)){
-    if (config.debug.externalblobs)
-      DEBUG_perror("unlink");
-    return -1;
-  }
-  return 0;
-}
-
 int rhizome_fail_write(struct rhizome_write *write){
   if (write->blob_fd>=0){
     if (config.debug.externalblobs)
       DEBUGF("Closing and removing fd %d", write->blob_fd);
     close(write->blob_fd);
     write->blob_fd=-1;
-    rhizome_store_delete(write->id);
   }
   write_release_lock(write);
   while(write->buffer_list){
@@ -436,16 +423,7 @@ int rhizome_fail_write(struct rhizome_write *write){
     write->buffer_list=n->_next;
     free(n);
   }
-  // don't worry too much about sql failures.
+  rhizome_delete_file(write->id);
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  if (write->blob_rowid>=0){
-    sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
-			   "DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
-    write->blob_rowid=-1;
-  }
-  sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
-			 "DELETE FROM FILES WHERE id='%s'",
-			 write->id);
   return 0;
 }
 
@@ -689,7 +667,7 @@ failure:
 
 /* Return -1 on error, 0 if file blob found, 1 if not found.
  */
-int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
+int rhizome_open_read(struct rhizome_read *read, const char *fileid)
 {
   strncpy(read->id, fileid, sizeof read->id);
   read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
@@ -716,19 +694,22 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
     if (config.debug.externalblobs)
       DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length);
   }
-  read->hash = hash;
   read->offset = 0;
-  if (hash)
+  read->hash_offset = 0;
-    SHA512_Init(&read->sha512_context);
+  SHA512_Init(&read->sha512_context);
   return 0; // file opened
 }
 
 /* Read content from the store, hashing and decrypting as we go. 
- Random access is supported, but hashing requires reads to be sequential though we don't enforce this. */
+ Random access is supported, but hashing requires all payload contents to be read sequentially. */
 // returns the number of bytes read
 int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length)
 {
   IN();
+  // hash check failed, just return an error
+  if (read_state->invalid)
+    RETURN(-1);
+    
   int bytes_read = 0;
   if (read_state->blob_fd >= 0) {
     if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1)
@@ -774,19 +755,24 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
     } while (1);
   } else
     RETURN(WHY("file not open"));
-  if (read_state->hash){
-    if (buffer && bytes_read>0)
-      SHA512_Update(&read_state->sha512_context, buffer, bytes_read);
   
-    if (read_state->offset + bytes_read>=read_state->length){
+  // hash the payload as we go, but only if we happen to read the payload data in order
+  if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){
+    SHA512_Update(&read_state->sha512_context, buffer, bytes_read);
+    read_state->hash_offset += bytes_read;
+    // if we hash everything and the has doesn't match, we need to delete the payload
+    if (read_state->hash_offset>=read_state->length){
       char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
       SHA512_End(&read_state->sha512_context, hash_out);
+      
       if (strcasecmp(read_state->id, hash_out)){
+	// hash failure, mark the payload as invalid
+	read_state->invalid = 1;
 	RETURN(WHYF("Expected hash=%s, got %s", read_state->id, hash_out));
       }
-      read_state->hash=0;
     }
   }
+  
   if (read_state->crypt && buffer && bytes_read>0){
     if(rhizome_crypt_xor_block(
 	buffer, bytes_read, 
@@ -861,6 +847,10 @@ int rhizome_read_close(struct rhizome_read *read)
     close(read->blob_fd);
   }
   read->blob_fd = -1;
+  if (read->invalid){
+    // delete payload!
+    rhizome_delete_file(read->id);
+  }
   return 0;
 }
 
@@ -970,7 +960,7 @@ int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t ti
     
     entry = emalloc_zero(sizeof(struct cache_entry));
     
-    if (rhizome_open_read(&entry->read_state, filehash, 0)){
+    if (rhizome_open_read(&entry->read_state, filehash)){
       free(entry);
       return WHYF("Payload %s not found", filehash);
     }
@@ -1048,9 +1038,8 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
   return 0;
 }
 
-int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state, int hash){
+int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){
-  // for now, always hash the file
+  int ret = rhizome_open_read(read_state, m->fileHexHash);
-  int ret = rhizome_open_read(read_state, m->fileHexHash, hash);
   if (ret == 0)
     ret = read_derive_key(m, bsk, read_state);
   return ret;
@@ -1065,7 +1054,7 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
 {
   struct rhizome_read read_state;
   bzero(&read_state, sizeof read_state);
-  int ret = rhizome_open_decrypt_read(m, bsk, &read_state, 1);
+  int ret = rhizome_open_decrypt_read(m, bsk, &read_state);
   if (ret == 0)
     ret = write_file(&read_state, filepath);
   rhizome_read_close(&read_state);
@@ -1081,7 +1070,7 @@ int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
   struct rhizome_read read_state;
   bzero(&read_state, sizeof read_state);
 
-  int ret = rhizome_open_read(&read_state, id, 1);
+  int ret = rhizome_open_read(&read_state, id);
   
   if (ret == 0) {
     ret = write_file(&read_state, filepath);
@@ -1121,7 +1110,7 @@ int rhizome_journal_pipe(struct rhizome_write *write, const char *fileHash, uint
 {
   struct rhizome_read read_state;
   bzero(&read_state, sizeof read_state);
-  if (rhizome_open_read(&read_state, fileHash, start_offset>0?0:1))
+  if (rhizome_open_read(&read_state, fileHash))
     return -1;
 
   read_state.offset = start_offset;

tests/rhizomeops

@@ -311,6 +311,22 @@ test_ExtractManifestFileFromExtBlob() {
    assert diff file2 file2x
 }
 
+doc_CorruptExternalBlob="A corrupted payload should fail to export"
+setup_CorruptExternalBlob() {
+   setup_servald
+   setup_rhizome
+   executeOk_servald config set rhizome.external_blobs 1
+   echo "A test file" >file1
+   executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
+   extract_manifest_id manifestid file1.manifest
+   extract_manifest_filehash filehash file1.manifest
+   echo "Replacement" >$SERVALINSTANCE_PATH/$filehash
+}
+test_CorruptExternalBlob() {
+   execute --exit-status=255 $servald rhizome extract file $manifestid file1a
+   tfw_cat --stderr
+}
+
 doc_ExtractManifestToStdout="Export manifest to output field"
 setup_ExtractManifestToStdout() {
    setup_servald