ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-01-29 15:43:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

simplify external/internal blob storage to a boolean flag instead

Browse Source 
of a threshold.
This commit is contained in:
gardners 2013-02-18 15:09:03 +10:30 committed by Jeremy Lakeman
parent bb8b3d6376
commit 944486c048
4 changed files with 42 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
conf_schema.h
View File

@ -312,7 +312,8 @@ STRUCT(rhizome)
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
ATOM(uint64_t, max_internal_blob_size, 10000000000, cf_opt_uint64_scaled,, "Size of largest bundle to store internally in database.")
ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.")
ATOM(uint64_t, rhizome_mdp_block_size, 512, cf_opt_uint64_scaled,, "Rhizome MDP block size.")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, cf_opt_uint64_scaled,, "Rhizome transfer timeout if no data received.")
ATOM(uint32_t, fetch_delay_ms, 50, cf_opt_uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")

57
rhizome_database.c
View File

@ -724,32 +724,37 @@ rhizome_blob_handle *rhizome_database_open_blob_byrowid(int row_id,int writeP)
if (config.debug.externalblobs)
DEBUGF("Opening blob for rowid #%d",row_id);
// Try opening as internal blob.
// If column is not a blob, then this will fail.
int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data",
row_id, writeP, &blob->sqlite_blob);
if (ret==SQLITE_OK) {
blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob);
RETURN(blob);
if (!config.rhizome.external_blobs)
{
// Try opening as internal blob.
// If column is not a blob, then this will fail.
int ret=sqlite3_blob_open(rhizome_db, "main", "fileblobs", "data",
row_id, writeP, &blob->sqlite_blob);
if (ret==SQLITE_OK) {
blob->blob_bytes=sqlite3_blob_bytes(blob->sqlite_blob);
RETURN(blob);
}
free(blob);
RETURN(NULL);
}
else
{
// Try opening as an external file
char *blobfile=rhizome_database_get_blob_filename(row_id);
errno=0;
blob->fd_blob=open(blobfile,O_RDWR);
if (blob->fd_blob==-1&&writeP) blob->fd_blob=open(blobfile,O_CREAT|O_RDWR,0664);
if (blob->fd_blob>-1) {
// File is stored externally
blob->blob_bytes=lseek(blob->fd_blob,0,SEEK_END);
DEBUGF("Opened fileblobs blob file '%s' (%lld bytes)",
blobfile,blob->blob_bytes);
RETURN(blob);
}
DEBUGF("Could not open fileblobs blob file '%s', will try sqlite blob",
blobfile);
// WHY_perror("open");
}
// Try opening as an external file
{
char *blobfile=rhizome_database_get_blob_filename(row_id);
errno=0;
blob->fd_blob=open(blobfile,O_RDWR);
if (blob->fd_blob==-1&&writeP) blob->fd_blob=open(blobfile,O_CREAT|O_RDWR,0664);
if (blob->fd_blob>-1) {
// File is stored externally
blob->blob_bytes=lseek(blob->fd_blob,0,SEEK_END);
DEBUGF("Opened fileblobs blob file '%s' (%lld bytes)",
blobfile,blob->blob_bytes);
RETURN(blob);
}
DEBUGF("Could not open fileblobs blob file '%s', will try sqlite blob",
blobfile);
// WHY_perror("open");
}
// Couldn't open, so fail
free(blob);
@ -1315,7 +1320,7 @@ int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
sqlite3_int64 fileblob_rowid=sqlite3_last_insert_rowid(rhizome_db);
sqlite3_stmt *statement=NULL;
if (fileLength<config.rhizome.max_internal_blob_size) {
if (!config.rhizome.external_blobs) {
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",filehashhex_or_tempid);
if (!statement)
goto insert_row_fail;

2
tests/rhizomeops
View File

@ -243,7 +243,7 @@ doc_ExtractManifestFileFromExtBlob="Extract manifest and file from external blob
setup_ExtractManifestFileFromExtBlob() {
setup_servald
setup_rhizome
executeOk_servald config set rhizome.max_internal_blob_size 1
executeOk_servald config set rhizome.external_blobs 1
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome list

14
tests/rhizomeprotocol
View File

@ -288,9 +288,11 @@ setup_FileTransferBigMDPExtBlob() {
setup_common
set_instance +B
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +A
executeOk_servald config set rhizome.http.enable 0
dd if=/dev/urandom of=file1 bs=1k count=16k 2>&1
executeOk_servald config set rhizome.external_blobs 1
dd if=/dev/urandom of=file1 bs=1k count=4k 2>&1
echo x >>file1
ls -l file1
rhizome_add_file file1
@ -311,19 +313,19 @@ setup_FileTransferMultiMDPExtBlob() {
setup_common
set_instance +A
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +B
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +C
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +D
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +E
executeOk_servald config set rhizome.http.enable 0
executeOk_servald config set rhizome.max_internal_blob_size 0
executeOk_servald config set rhizome.external_blobs 1
set_instance +A
rhizome_add_file file1
start_servald_instances +A +B +C +D +E