simplify external/internal blob storage to a boolean flag instead

of a threshold.
This commit is contained in:
gardners 2013-02-18 15:09:03 +10:30 committed by Jeremy Lakeman
parent bb8b3d6376
commit 944486c048
4 changed files with 42 additions and 34 deletions

View File

@ -312,7 +312,8 @@ STRUCT(rhizome)
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting") ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory") STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes") ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
ATOM(uint64_t, max_internal_blob_size, 10000000000, cf_opt_uint64_scaled,, "Size of largest bundle to store internally in database.") ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.")
ATOM(uint64_t, rhizome_mdp_block_size, 512, cf_opt_uint64_scaled,, "Rhizome MDP block size.") ATOM(uint64_t, rhizome_mdp_block_size, 512, cf_opt_uint64_scaled,, "Rhizome MDP block size.")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, cf_opt_uint64_scaled,, "Rhizome transfer timeout if no data received.") ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, cf_opt_uint64_scaled,, "Rhizome transfer timeout if no data received.")
ATOM(uint32_t, fetch_delay_ms, 50, cf_opt_uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch") ATOM(uint32_t, fetch_delay_ms, 50, cf_opt_uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")

View File

@ -724,6 +724,8 @@ rhizome_blob_handle *rhizome_database_open_blob_byrowid(int row_id,int writeP)
if (config.debug.externalblobs) if (config.debug.externalblobs)
DEBUGF("Opening blob for rowid #%d",row_id); DEBUGF("Opening blob for rowid #%d",row_id);
if (!config.rhizome.external_blobs)
{
// Try opening as internal blob. // Try opening as internal blob.
// If column is not a blob, then this will fail. // If column is not a blob, then this will fail.
int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data", int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data",
@ -732,9 +734,12 @@ rhizome_blob_handle *rhizome_database_open_blob_byrowid(int row_id,int writeP)
blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob); blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob);
RETURN(blob); RETURN(blob);
} }
free(blob);
// Try opening as an external file RETURN(NULL);
}
else
{ {
// Try opening as an external file
char *blobfile=rhizome_database_get_blob_filename(row_id); char *blobfile=rhizome_database_get_blob_filename(row_id);
errno=0; errno=0;
blob->fd_blob=open(blobfile,O_RDWR); blob->fd_blob=open(blobfile,O_RDWR);
@ -1315,7 +1320,7 @@ int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
sqlite3_int64 fileblob_rowid=sqlite3_last_insert_rowid(rhizome_db); sqlite3_int64 fileblob_rowid=sqlite3_last_insert_rowid(rhizome_db);
sqlite3_stmt *statement=NULL; sqlite3_stmt *statement=NULL;
if (fileLength<config.rhizome.max_internal_blob_size) { if (!config.rhizome.external_blobs) {
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",filehashhex_or_tempid); statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",filehashhex_or_tempid);
if (!statement) if (!statement)
goto insert_row_fail; goto insert_row_fail;

View File

@ -243,7 +243,7 @@ doc_ExtractManifestFileFromExtBlob="Extract manifest and file from external blob
setup_ExtractManifestFileFromExtBlob() { setup_ExtractManifestFileFromExtBlob() {
setup_servald setup_servald
setup_rhizome setup_rhizome
executeOk_servald config set rhizome.max_internal_blob_size 1 executeOk_servald config set rhizome.external_blobs 1
echo "A test file" >file1 echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome list executeOk_servald rhizome list

View File

@ -288,9 +288,11 @@ setup_FileTransferBigMDPExtBlob() {
setup_common setup_common
set_instance +B set_instance +B
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +A set_instance +A
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
dd if=/dev/urandom of=file1 bs=1k count=16k 2>&1 executeOk_servald config set rhizome.external_blobs 1
dd if=/dev/urandom of=file1 bs=1k count=4k 2>&1
echo x >>file1 echo x >>file1
ls -l file1 ls -l file1
rhizome_add_file file1 rhizome_add_file file1
@ -311,19 +313,19 @@ setup_FileTransferMultiMDPExtBlob() {
setup_common setup_common
set_instance +A set_instance +A
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0 executeOk_servald config set rhizome.external_blobs 1
set_instance +B set_instance +B
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0 executeOk_servald config set rhizome.external_blobs 1
set_instance +C set_instance +C
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0 executeOk_servald config set rhizome.external_blobs 1
set_instance +D set_instance +D
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0 executeOk_servald config set rhizome.external_blobs 1
set_instance +E set_instance +E
executeOk_servald config set rhizome.http.enable 0 executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0 executeOk_servald config set rhizome.external_blobs 1
set_instance +A set_instance +A
rhizome_add_file file1 rhizome_add_file file1
start_servald_instances +A +B +C +D +E start_servald_instances +A +B +C +D +E