ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-20 21:53:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

New stress test, rhizome add while transferring

Browse Source
This commit is contained in:
Jeremy Lakeman 2012-12-27 15:15:23 +10:30
parent 9e3d4d10ee
commit 3f45623d04
4 changed files with 141 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
rhizome_database.c
View File

@ -303,15 +303,16 @@ int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const
++retry->busytries;
if (retry->start == -1)
retry->start = now;
else
retry->elapsed += now - retry->start;
INFOF("%s on try %u after %.3f seconds (%.3f elapsed): %s",
retry->elapsed = now - retry->start;
INFOF("%s on try %u after %.3f seconds (limit %.3f): %s",
sqlite3_errmsg(rhizome_db),
retry->busytries,
(now - retry->start) / 1e3,
retry->elapsed / 1e3,
(retry->elapsed) / 1e3,
(retry->limit) / 1e3,
action
);
if (retry->elapsed >= retry->limit) {
// reset ready for next query
retry->busytries = 0;
@ -319,6 +320,7 @@ int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const
retry->start = -1;
return 0; // tell caller to stop trying
}
if (retry->sleep)
sleep_ms(retry->sleep);
return 1; // tell caller to try again
@ -328,10 +330,10 @@ void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry,
{
if (retry->busytries) {
time_ms_t now = gettime_ms();
INFOF("succeeded on try %u after %.3f seconds (%.3f elapsed): %s",
INFOF("succeeded on try %u after %.3f seconds (limit %.3f): %s",
retry->busytries + 1,
(now - retry->start) / 1e3,
retry->elapsed / 1e3,
(retry->limit) / 1e3,
action
);
}
@ -773,6 +775,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
// we might need to leave the old file around for a bit
// clean out unreferenced files first
// FIXME where is the ? parameter bound????
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);")) {
WHYF("delete failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
goto rollback;

77
rhizome_fetch.c
View File

@ -1245,25 +1245,58 @@ void rhizome_fetch_write(struct rhizome_fetch_slot *slot)
int rhizome_fetch_flush_blob_buffer(struct rhizome_fetch_slot *slot)
{
sqlite3_blob *blob;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", slot->rowid, 1 /* read/write */, &blob);
if (ret!=SQLITE_OK) {
if (blob) sqlite3_blob_close(blob);
return -1;
}
ret=sqlite3_blob_write(blob, slot->blob_buffer, slot->blob_buffer_bytes,
slot->file_ofs-slot->blob_buffer_bytes);
sqlite3_blob_close(blob); blob=NULL;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
do{
sqlite3_blob *blob=NULL;
if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_write(,,%d,%lld) failed, %s",
slot->blob_buffer_bytes,slot->file_ofs-slot->blob_buffer_bytes,
sqlite3_errmsg(rhizome_db));
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", slot->rowid, 1 /* read/write */, &blob);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret!=SQLITE_OK){
WHYF("sqlite3_blob_open() failed, %s",
sqlite3_errmsg(rhizome_db));
goto failed;
}
ret=sqlite3_blob_write(blob, slot->blob_buffer, slot->blob_buffer_bytes,
slot->file_ofs-slot->blob_buffer_bytes);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_write(,,%d,%lld) failed, %s",
slot->blob_buffer_bytes,slot->file_ofs-slot->blob_buffer_bytes,
sqlite3_errmsg(rhizome_db));
goto failed;
}
ret=sqlite3_blob_close(blob);
blob=NULL;
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_close() failed, %s",
sqlite3_errmsg(rhizome_db));
goto failed;
}
slot->blob_buffer_bytes=0;
return 0;
failed:
if (blob) sqlite3_blob_close(blob);
rhizome_fetch_close(slot);
return -1;
}
slot->blob_buffer_bytes=0;
return 0;
again:
if (blob)
sqlite3_blob_close(blob);
blob=NULL;
if (_sqlite_retry(__WHENCE__, &retry, "sqlite3_blob_write")==0)
return -1;
}while(1);
}
int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int bytes)
@ -1315,8 +1348,10 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
slot->blob_buffer_bytes+=count;
slot->file_ofs+=count;
buffer+=count; bytesRemaining-=count;
if (slot->blob_buffer_bytes==slot->blob_buffer_size)
rhizome_fetch_flush_blob_buffer(slot);
if (slot->blob_buffer_bytes==slot->blob_buffer_size){
if (rhizome_fetch_flush_blob_buffer(slot))
RETURN (-1);
}
}
}
@ -1330,8 +1365,10 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
char hash_out[SHA512_DIGEST_STRING_LENGTH+1];
SHA512_End(&slot->sha512_context, (char *)hash_out);
if (slot->blob_buffer_bytes) rhizome_fetch_flush_blob_buffer(slot);
if (slot->blob_buffer_bytes){
if (rhizome_fetch_flush_blob_buffer(slot))
RETURN (-1);
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (strcasecmp(hash_out,slot->manifest->fileHexHash)) {
if (config.debug.rhizome_rx)

57
rhizome_store.c
View File

@ -114,34 +114,57 @@ int rhizome_flush(struct rhizome_write *write){
rhizome_crypt_xor_block(write->buffer, write->data_size, write->file_offset, write->key, write->nonce);
}
sqlite3_blob *blob;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", write->blob_rowid, 1 /* read/write */, &blob);
if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_open() failed: %s",
sqlite3_errmsg(rhizome_db));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
do{
sqlite3_blob *blob=NULL;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", write->blob_rowid, 1 /* read/write */, &blob);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_open() failed: %s",
sqlite3_errmsg(rhizome_db));
if (blob) sqlite3_blob_close(blob);
return -1;
}
ret=sqlite3_blob_write(blob, write->buffer, write->data_size,
write->file_offset);
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_write() failed: %s",
sqlite3_errmsg(rhizome_db));
if (blob) sqlite3_blob_close(blob);
return -1;
}
ret = sqlite3_blob_close(blob);
blob=NULL;
if (ret==SQLITE_BUSY || ret==SQLITE_LOCKED)
goto again;
else if (ret==SQLITE_OK)
break;
WHYF("sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db));
if (blob) sqlite3_blob_close(blob);
return -1;
}
ret=sqlite3_blob_write(blob, write->buffer, write->data_size,
write->file_offset);
if (ret!=SQLITE_OK) {
WHYF("sqlite3_blob_write() failed: %s",
sqlite3_errmsg(rhizome_db));
again:
if (blob) sqlite3_blob_close(blob);
return -1;
}
if (_sqlite_retry(__WHENCE__, &retry, "sqlite3_blob_write")==0)
return -1;
}while(1);
ret = sqlite3_blob_close(blob);
if (ret!=SQLITE_OK)
return WHYF("sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db));
blob=NULL;
SHA512_Update(&write->sha512_context, write->buffer, write->data_size);
write->file_offset+=write->data_size;
if (config.debug.rhizome)
DEBUGF("Written %lld of %lld", write->file_offset, write->file_length);
write->data_size=0;
return 0;
}
/* Expects file to be at least file_length in size */

31
tests/rhizomestress
View File

@ -82,4 +82,35 @@ test_FileTransferStress() {
done
}
setup_AddStressTest() {
setup_servald
assert_no_servald_processes
foreach_instance +A +B +C +D +E +F +G +H create_single_identity
start_servald_instances +A +B +C +D +E +F +G +H
}
ammendfile() {
echo "XXX ${instance_name} XXX" >> $instance_dir/file
executeOk_servald rhizome add file "" "" $instance_dir/file
}
alldone() {
executeOk_servald rhizome list ''
local lineCount=$(( $(cat $_tfw_tmp/stdout | wc -l) + 0 ))
tfw_log "lines $lineCount"
# total files = 26 * 8 + 2
if [ "$lineCount" -eq "210" ]; then return 0; fi
return 1
}
doc_AddStressTest="Add files while transferring"
test_AddStressTest() {
for i in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
do
foreach_instance +A +B +C +D +E +F +G +H ammendfile
done
foreach_instance +A +B +C +D +E +F +G +H wait_until --timeout=120 alldone
sleep 10
}
runTests "$@"