ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-04-17 07:28:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rhizome cleanup will now expire payloads and vacuum the db file.

Browse Source
This commit is contained in:
Jeremy Lakeman 2014-06-19 13:58:01 +09:30
parent f948f8e12e
commit 47f051917d
3 changed files with 51 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
rhizome.h
View File

@ -318,18 +318,6 @@ int rhizome_opendb();
int rhizome_close_db();
void verify_bundles();
struct rhizome_cleanup_report {
unsigned deleted_stale_incoming_files;
unsigned deleted_orphan_files;
unsigned deleted_orphan_fileblobs;
unsigned deleted_orphan_manifests;
};
int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
typedef struct sqlite_retry_state {
unsigned int limit; // do not retry once elapsed >= limit
unsigned int sleep; // number of milliseconds to sleep between retries
@ -343,6 +331,20 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
#define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)
struct rhizome_cleanup_report {
unsigned deleted_stale_incoming_files;
unsigned deleted_expired_files;
unsigned deleted_orphan_files;
unsigned deleted_orphan_fileblobs;
unsigned deleted_orphan_manifests;
};
int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_store_cleanup(struct rhizome_cleanup_report *report);
void rhizome_vacuum_db(sqlite_retry_state *retry);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
struct rhizome_manifest_summary {
rhizome_bid_t bid;
uint64_t version;

16
rhizome_database.c
View File

@ -1174,6 +1174,13 @@ int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t versio
OUT();
}
void rhizome_vacuum_db(sqlite_retry_state *retry){
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "PRAGMA incremental_vacuum;", END);
if (!statement)
return;
sqlite_exec_retry(retry, statement);
}
int rhizome_cleanup(struct rhizome_cleanup_report *report)
{
IN();
@ -1183,6 +1190,9 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
bzero(report, sizeof *report);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// make sure we are under our database size limit
rhizome_store_cleanup(report);
/* For testing, it helps to speed up the cleanup process. */
const char *orphan_payload_persist_ms = getenv("SERVALD_ORPHAN_PAYLOAD_PERSIST_MS");
time_ms_t now = gettime_ms();
@ -1214,18 +1224,20 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
// in an incremental background task. See GitHub issue #50.
// Remove payload blobs that are no longer referenced.
int ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
int ret = sqlite_exec_void_retry(&retry,
"DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
END);
if (ret > 0 && report)
report->deleted_orphan_fileblobs += ret;
// delete manifests that no longer have payload files
ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
ret = sqlite_exec_void_retry(&retry,
"DELETE FROM MANIFESTS WHERE filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);", END);
if (report && ret > 0)
report->deleted_orphan_manifests += ret;
rhizome_vacuum_db(&retry);
if (config.debug.rhizome && report)
DEBUGF("report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u deleted_orphan_manifests=%u",
report->deleted_stale_incoming_files,

34
rhizome_store.c
View File

@ -151,12 +151,13 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp)
}
// TODO readonly version?
static enum rhizome_payload_status store_make_space(uint64_t bytes)
static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizome_cleanup_report *report)
{
uint64_t external_bytes;
uint64_t db_page_size;
uint64_t db_page_count;
uint64_t db_free_page_count;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// TODO limit based on free space?
@ -166,10 +167,10 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes)
// TODO index external_bytes calculation and/or cache result
if ( sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL
|| sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL
|| sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END) == -1LL
|| sqlite_exec_uint64(&external_bytes,
if ( sqlite_exec_uint64_retry(&retry, &db_page_size, "PRAGMA page_size;", END) == -1LL
|| sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END) == -1LL
|| sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END) == -1LL
|| sqlite_exec_uint64_retry(&retry, &external_bytes,
"SELECT SUM(length) "
"FROM FILES "
"WHERE NOT EXISTS( "
@ -193,6 +194,10 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes)
return RHIZOME_PAYLOAD_STATUS_TOO_BIG;
}
// vacuum database pages if we're already using too much free space
if (external_bytes + db_page_size * db_page_count > limit)
rhizome_vacuum_db(&retry);
// If there is enough space, do nothing
if (db_used + bytes <= limit)
return RHIZOME_PAYLOAD_STATUS_NEW;
@ -200,7 +205,6 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes)
// penalise new things by 10 minutes to reduce churn
time_ms_t cost = gettime_ms() - 60000 - bytes;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// query files by age, penalise larger files so they are removed earlier
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, length, inserttime FROM FILES ORDER BY (inserttime - length)",
@ -233,12 +237,15 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes)
if (s)
sqlite_exec_retry(&retry, s);
sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END);
sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END);
sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END);
sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END);
if (report)
report->deleted_expired_files++;
db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
}
sqlite3_finalize(statement);
rhizome_vacuum_db(&retry);
if (db_used + bytes <= limit)
return RHIZOME_PAYLOAD_STATUS_NEW;
@ -250,6 +257,11 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes)
return RHIZOME_PAYLOAD_STATUS_UNINITERESTING;
}
int rhizome_store_cleanup(struct rhizome_cleanup_report *report)
{
return store_make_space(0, report);
}
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length)
{
if (file_length == 0)
@ -268,7 +280,7 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons
}
if (file_length!=RHIZOME_SIZE_UNSET){
enum rhizome_payload_status status = store_make_space(file_length);
enum rhizome_payload_status status = store_make_space(file_length, NULL);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
}
@ -632,7 +644,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
if (config.debug.rhizome_store)
DEBUGF("Wrote %"PRIu64" bytes, set file_length", write->file_offset);
write->file_length = write->file_offset;
status = store_make_space(write->file_length);
status = store_make_space(write->file_length, NULL);
if (status!=RHIZOME_PAYLOAD_STATUS_NEW)
goto failure;
}