mirror of
https://github.com/servalproject/serval-dna.git
synced 2025-02-06 11:09:13 +00:00
simplify external/internal blob storage to a boolean flag instead
of a threshold.
This commit is contained in:
parent
bb8b3d6376
commit
944486c048
@ -312,7 +312,8 @@ STRUCT(rhizome)
|
|||||||
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
|
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
|
||||||
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
|
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
|
||||||
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
|
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
|
||||||
ATOM(uint64_t, max_internal_blob_size, 10000000000, cf_opt_uint64_scaled,, "Size of largest bundle to store internally in database.")
|
ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.")
|
||||||
|
|
||||||
ATOM(uint64_t, rhizome_mdp_block_size, 512, cf_opt_uint64_scaled,, "Rhizome MDP block size.")
|
ATOM(uint64_t, rhizome_mdp_block_size, 512, cf_opt_uint64_scaled,, "Rhizome MDP block size.")
|
||||||
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, cf_opt_uint64_scaled,, "Rhizome transfer timeout if no data received.")
|
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, cf_opt_uint64_scaled,, "Rhizome transfer timeout if no data received.")
|
||||||
ATOM(uint32_t, fetch_delay_ms, 50, cf_opt_uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
|
ATOM(uint32_t, fetch_delay_ms, 50, cf_opt_uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
|
||||||
|
@ -724,6 +724,8 @@ rhizome_blob_handle *rhizome_database_open_blob_byrowid(int row_id,int writeP)
|
|||||||
if (config.debug.externalblobs)
|
if (config.debug.externalblobs)
|
||||||
DEBUGF("Opening blob for rowid #%d",row_id);
|
DEBUGF("Opening blob for rowid #%d",row_id);
|
||||||
|
|
||||||
|
if (!config.rhizome.external_blobs)
|
||||||
|
{
|
||||||
// Try opening as internal blob.
|
// Try opening as internal blob.
|
||||||
// If column is not a blob, then this will fail.
|
// If column is not a blob, then this will fail.
|
||||||
int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data",
|
int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data",
|
||||||
@ -732,9 +734,12 @@ rhizome_blob_handle *rhizome_database_open_blob_byrowid(int row_id,int writeP)
|
|||||||
blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob);
|
blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob);
|
||||||
RETURN(blob);
|
RETURN(blob);
|
||||||
}
|
}
|
||||||
|
free(blob);
|
||||||
// Try opening as an external file
|
RETURN(NULL);
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
|
// Try opening as an external file
|
||||||
char *blobfile=rhizome_database_get_blob_filename(row_id);
|
char *blobfile=rhizome_database_get_blob_filename(row_id);
|
||||||
errno=0;
|
errno=0;
|
||||||
blob->fd_blob=open(blobfile,O_RDWR);
|
blob->fd_blob=open(blobfile,O_RDWR);
|
||||||
@ -1315,7 +1320,7 @@ int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
|
|||||||
sqlite3_int64 fileblob_rowid=sqlite3_last_insert_rowid(rhizome_db);
|
sqlite3_int64 fileblob_rowid=sqlite3_last_insert_rowid(rhizome_db);
|
||||||
|
|
||||||
sqlite3_stmt *statement=NULL;
|
sqlite3_stmt *statement=NULL;
|
||||||
if (fileLength<config.rhizome.max_internal_blob_size) {
|
if (!config.rhizome.external_blobs) {
|
||||||
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",filehashhex_or_tempid);
|
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",filehashhex_or_tempid);
|
||||||
if (!statement)
|
if (!statement)
|
||||||
goto insert_row_fail;
|
goto insert_row_fail;
|
||||||
|
@ -243,7 +243,7 @@ doc_ExtractManifestFileFromExtBlob="Extract manifest and file from external blob
|
|||||||
setup_ExtractManifestFileFromExtBlob() {
|
setup_ExtractManifestFileFromExtBlob() {
|
||||||
setup_servald
|
setup_servald
|
||||||
setup_rhizome
|
setup_rhizome
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 1
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
echo "A test file" >file1
|
echo "A test file" >file1
|
||||||
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
|
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
|
||||||
executeOk_servald rhizome list
|
executeOk_servald rhizome list
|
||||||
|
@ -288,9 +288,11 @@ setup_FileTransferBigMDPExtBlob() {
|
|||||||
setup_common
|
setup_common
|
||||||
set_instance +B
|
set_instance +B
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +A
|
set_instance +A
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
dd if=/dev/urandom of=file1 bs=1k count=16k 2>&1
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
|
dd if=/dev/urandom of=file1 bs=1k count=4k 2>&1
|
||||||
echo x >>file1
|
echo x >>file1
|
||||||
ls -l file1
|
ls -l file1
|
||||||
rhizome_add_file file1
|
rhizome_add_file file1
|
||||||
@ -311,19 +313,19 @@ setup_FileTransferMultiMDPExtBlob() {
|
|||||||
setup_common
|
setup_common
|
||||||
set_instance +A
|
set_instance +A
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 0
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +B
|
set_instance +B
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 0
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +C
|
set_instance +C
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 0
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +D
|
set_instance +D
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 0
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +E
|
set_instance +E
|
||||||
executeOk_servald config set rhizome.http.enable 0
|
executeOk_servald config set rhizome.http.enable 0
|
||||||
executeOk_servald config set rhizome.max_internal_blob_size 0
|
executeOk_servald config set rhizome.external_blobs 1
|
||||||
set_instance +A
|
set_instance +A
|
||||||
rhizome_add_file file1
|
rhizome_add_file file1
|
||||||
start_servald_instances +A +B +C +D +E
|
start_servald_instances +A +B +C +D +E
|
||||||
|
Loading…
x
Reference in New Issue
Block a user