New stress test, rhizome add while transferring

rhizome_database.c

@@ -303,15 +303,16 @@ int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const
   ++retry->busytries;
   if (retry->start == -1)
     retry->start = now;
-  else
-    retry->elapsed += now - retry->start;
-  INFOF("%s on try %u after %.3f seconds (%.3f elapsed): %s",
+  retry->elapsed = now - retry->start;
+  
+  INFOF("%s on try %u after %.3f seconds (limit %.3f): %s",
       sqlite3_errmsg(rhizome_db),
       retry->busytries,
-      (now - retry->start) / 1e3,
-      retry->elapsed / 1e3,
+      (retry->elapsed) / 1e3,
+      (retry->limit) / 1e3,
       action
     );
+  
   if (retry->elapsed >= retry->limit) {
     // reset ready for next query
     retry->busytries = 0;
@@ -319,6 +320,7 @@ int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const
       retry->start = -1;
     return 0; // tell caller to stop trying
   }
+  
   if (retry->sleep)
     sleep_ms(retry->sleep);
   return 1; // tell caller to try again
@@ -328,10 +330,10 @@ void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry,
 {
   if (retry->busytries) {
     time_ms_t now = gettime_ms();
-    INFOF("succeeded on try %u after %.3f seconds (%.3f elapsed): %s",
+    INFOF("succeeded on try %u after %.3f seconds (limit %.3f): %s",
 	retry->busytries + 1,
 	(now - retry->start) / 1e3,
-	retry->elapsed / 1e3,
+	(retry->limit) / 1e3,
 	action
       );
   }
@@ -773,6 +775,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
 
   // we might need to leave the old file around for a bit
   // clean out unreferenced files first
+  
+  // FIXME where is the ? parameter bound????
   if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT  1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);")) {
     WHYF("delete failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
     goto rollback;

rhizome_fetch.c

@@ -1245,25 +1245,58 @@ void rhizome_fetch_write(struct rhizome_fetch_slot *slot)
 
 int rhizome_fetch_flush_blob_buffer(struct rhizome_fetch_slot *slot)
 {
-  sqlite3_blob *blob;
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  do{
+    sqlite3_blob *blob=NULL;
+    
     int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", slot->rowid, 1 /* read/write */, &blob);
-  if (ret!=SQLITE_OK) {
-    if (blob) sqlite3_blob_close(blob);
-    return -1;
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
+      goto again;
+    else if (ret!=SQLITE_OK){
+      WHYF("sqlite3_blob_open() failed, %s", 
+	   sqlite3_errmsg(rhizome_db));
+      goto failed;
     }
+    
     ret=sqlite3_blob_write(blob, slot->blob_buffer, slot->blob_buffer_bytes, 
 			   slot->file_ofs-slot->blob_buffer_bytes);
-  sqlite3_blob_close(blob); blob=NULL;
     
-  if (ret!=SQLITE_OK) {
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
+      goto again;
+    else if (ret!=SQLITE_OK) {
       WHYF("sqlite3_blob_write(,,%d,%lld) failed, %s", 
 	   slot->blob_buffer_bytes,slot->file_ofs-slot->blob_buffer_bytes,
 	   sqlite3_errmsg(rhizome_db));
-    rhizome_fetch_close(slot);
-    return -1;
+      goto failed;
     }
+    
+    ret=sqlite3_blob_close(blob); 
+    blob=NULL;
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
+      goto again;
+    else if (ret!=SQLITE_OK) {
+      WHYF("sqlite3_blob_close() failed, %s", 
+	   sqlite3_errmsg(rhizome_db));
+      goto failed;
+    }
+    
     slot->blob_buffer_bytes=0;
     return 0;
+    
+  failed:
+    if (blob) sqlite3_blob_close(blob); 
+    rhizome_fetch_close(slot);
+    return -1;
+    
+  again:
+    if (blob)
+      sqlite3_blob_close(blob); 
+    blob=NULL;
+    
+    if (_sqlite_retry(__WHENCE__, &retry, "sqlite3_blob_write")==0)
+      return -1;
+    
+  }while(1);
 }
 
 int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int bytes)
@@ -1315,8 +1348,10 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
       slot->blob_buffer_bytes+=count;
       slot->file_ofs+=count;
       buffer+=count; bytesRemaining-=count;
-      if (slot->blob_buffer_bytes==slot->blob_buffer_size)
-	rhizome_fetch_flush_blob_buffer(slot);
+      if (slot->blob_buffer_bytes==slot->blob_buffer_size){
+	if (rhizome_fetch_flush_blob_buffer(slot))
+	  RETURN (-1);
+      }
     }
   }
 
@@ -1330,8 +1365,10 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
 
       char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
       SHA512_End(&slot->sha512_context, (char *)hash_out);      
-      if (slot->blob_buffer_bytes) rhizome_fetch_flush_blob_buffer(slot);
-      
+      if (slot->blob_buffer_bytes){
+	if (rhizome_fetch_flush_blob_buffer(slot))
+	  RETURN (-1);
+      }
       sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
       if (strcasecmp(hash_out,slot->manifest->fileHexHash)) {
 	if (config.debug.rhizome_rx)

rhizome_store.c

@@ -114,18 +114,26 @@ int rhizome_flush(struct rhizome_write *write){
     rhizome_crypt_xor_block(write->buffer, write->data_size, write->file_offset, write->key, write->nonce);
   }
   
-  sqlite3_blob *blob;
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  
+  do{
+    sqlite3_blob *blob=NULL;
+    
     int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", write->blob_rowid, 1 /* read/write */, &blob);
-  if (ret!=SQLITE_OK) {
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
+      goto again;
+    else if (ret!=SQLITE_OK) {
       WHYF("sqlite3_blob_open() failed: %s", 
 	sqlite3_errmsg(rhizome_db));
       if (blob) sqlite3_blob_close(blob);
       return -1;
     }
+    
     ret=sqlite3_blob_write(blob, write->buffer, write->data_size, 
 			   write->file_offset);
-  
-  if (ret!=SQLITE_OK) {
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED) 
+      goto again;
+    else if (ret!=SQLITE_OK) {
       WHYF("sqlite3_blob_write() failed: %s", 
 	   sqlite3_errmsg(rhizome_db));
       if (blob) sqlite3_blob_close(blob);
@@ -133,15 +141,30 @@ int rhizome_flush(struct rhizome_write *write){
     }
     
     ret = sqlite3_blob_close(blob);
-  if (ret!=SQLITE_OK)
-    return WHYF("sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db));
     blob=NULL;
+    if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
+      goto again;
+    else if (ret==SQLITE_OK)
+      break;
+    
+    WHYF("sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db));
+    if (blob) sqlite3_blob_close(blob);
+    return -1;
+    
+  again:
+    if (blob) sqlite3_blob_close(blob);
+    if (_sqlite_retry(__WHENCE__, &retry, "sqlite3_blob_write")==0)
+      return -1;
+    
+  }while(1);
+  
   SHA512_Update(&write->sha512_context, write->buffer, write->data_size);
   write->file_offset+=write->data_size;
   if (config.debug.rhizome)
     DEBUGF("Written %lld of %lld", write->file_offset, write->file_length);
   write->data_size=0;
   return 0;
+  
 }
 
 /* Expects file to be at least file_length in size */

tests/rhizomestress

@@ -82,4 +82,35 @@ test_FileTransferStress() {
    done
 }
 
+setup_AddStressTest() {
+   setup_servald
+   assert_no_servald_processes
+   foreach_instance +A +B +C +D +E +F +G +H create_single_identity
+   start_servald_instances +A +B +C +D +E +F +G +H
+}
+
+ammendfile() {
+   echo "XXX  ${instance_name}  XXX" >> $instance_dir/file
+   executeOk_servald rhizome add file "" "" $instance_dir/file 
+}
+
+alldone() {
+   executeOk_servald rhizome list ''
+   local lineCount=$(( $(cat $_tfw_tmp/stdout | wc -l) + 0 ))
+   tfw_log "lines $lineCount"
+# total files = 26 * 8 + 2
+   if [ "$lineCount" -eq "210" ]; then return 0; fi
+   return 1
+}
+
+doc_AddStressTest="Add files while transferring"
+test_AddStressTest() {
+   for i in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+   do
+      foreach_instance +A +B +C +D +E +F +G +H ammendfile
+   done
+   foreach_instance +A +B +C +D +E +F +G +H wait_until --timeout=120 alldone 
+   sleep 10
+}
+
 runTests "$@"