From 47f051917d9f2d21738932679a270589720d7e57 Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Thu, 19 Jun 2014 13:58:01 +0930 Subject: [PATCH] Rhizome cleanup will now expire payloads and vacuum the db file. --- rhizome.h | 26 ++++++++++++++------------ rhizome_database.c | 16 ++++++++++++++-- rhizome_store.c | 34 +++++++++++++++++++++++----------- 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/rhizome.h b/rhizome.h index 3911f2d5..ed937701 100644 --- a/rhizome.h +++ b/rhizome.h @@ -318,18 +318,6 @@ int rhizome_opendb(); int rhizome_close_db(); void verify_bundles(); -struct rhizome_cleanup_report { - unsigned deleted_stale_incoming_files; - unsigned deleted_orphan_files; - unsigned deleted_orphan_fileblobs; - unsigned deleted_orphan_manifests; -}; - -int rhizome_cleanup(struct rhizome_cleanup_report *report); - -int rhizome_manifest_createid(rhizome_manifest *m); -int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed); - typedef struct sqlite_retry_state { unsigned int limit; // do not retry once elapsed >= limit unsigned int sleep; // number of milliseconds to sleep between retries @@ -343,6 +331,20 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int #define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1) +struct rhizome_cleanup_report { + unsigned deleted_stale_incoming_files; + unsigned deleted_expired_files; + unsigned deleted_orphan_files; + unsigned deleted_orphan_fileblobs; + unsigned deleted_orphan_manifests; +}; + +int rhizome_cleanup(struct rhizome_cleanup_report *report); +int rhizome_store_cleanup(struct rhizome_cleanup_report *report); +void rhizome_vacuum_db(sqlite_retry_state *retry); +int rhizome_manifest_createid(rhizome_manifest *m); +int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed); + struct rhizome_manifest_summary { rhizome_bid_t bid; uint64_t version; diff --git a/rhizome_database.c b/rhizome_database.c index 9792a159..ce1e9c1e 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1174,6 +1174,13 @@ int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t versio OUT(); } +void rhizome_vacuum_db(sqlite_retry_state *retry){ + sqlite3_stmt *statement = sqlite_prepare_bind(retry, "PRAGMA incremental_vacuum;", END); + if (!statement) + return; + sqlite_exec_retry(retry, statement); +} + int rhizome_cleanup(struct rhizome_cleanup_report *report) { IN(); @@ -1183,6 +1190,9 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) bzero(report, sizeof *report); sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; + // make sure we are under our database size limit + rhizome_store_cleanup(report); + /* For testing, it helps to speed up the cleanup process. */ const char *orphan_payload_persist_ms = getenv("SERVALD_ORPHAN_PAYLOAD_PERSIST_MS"); time_ms_t now = gettime_ms(); @@ -1214,18 +1224,20 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report) // in an incremental background task. See GitHub issue #50. // Remove payload blobs that are no longer referenced. - int ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, + int ret = sqlite_exec_void_retry(&retry, "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );", END); if (ret > 0 && report) report->deleted_orphan_fileblobs += ret; // delete manifests that no longer have payload files - ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, + ret = sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);", END); if (report && ret > 0) report->deleted_orphan_manifests += ret; + rhizome_vacuum_db(&retry); + if (config.debug.rhizome && report) DEBUGF("report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u deleted_orphan_manifests=%u", report->deleted_stale_incoming_files, diff --git a/rhizome_store.c b/rhizome_store.c index 0637beae..23cd57f0 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -151,12 +151,13 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp) } // TODO readonly version? -static enum rhizome_payload_status store_make_space(uint64_t bytes) +static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizome_cleanup_report *report) { uint64_t external_bytes; uint64_t db_page_size; uint64_t db_page_count; uint64_t db_free_page_count; + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; // TODO limit based on free space? @@ -166,10 +167,10 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes) // TODO index external_bytes calculation and/or cache result - if ( sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL - || sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL - || sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END) == -1LL - || sqlite_exec_uint64(&external_bytes, + if ( sqlite_exec_uint64_retry(&retry, &db_page_size, "PRAGMA page_size;", END) == -1LL + || sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END) == -1LL + || sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END) == -1LL + || sqlite_exec_uint64_retry(&retry, &external_bytes, "SELECT SUM(length) " "FROM FILES " "WHERE NOT EXISTS( " @@ -193,6 +194,10 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes) return RHIZOME_PAYLOAD_STATUS_TOO_BIG; } + // vacuum database pages if we're already using too much free space + if (external_bytes + db_page_size * db_page_count > limit) + rhizome_vacuum_db(&retry); + // If there is enough space, do nothing if (db_used + bytes <= limit) return RHIZOME_PAYLOAD_STATUS_NEW; @@ -200,7 +205,6 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes) // penalise new things by 10 minutes to reduce churn time_ms_t cost = gettime_ms() - 60000 - bytes; - sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; // query files by age, penalise larger files so they are removed earlier sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id, length, inserttime FROM FILES ORDER BY (inserttime - length)", @@ -233,12 +237,15 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes) if (s) sqlite_exec_retry(&retry, s); - sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END); - sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END); - + sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END); + sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END); + if (report) + report->deleted_expired_files++; db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count); } sqlite3_finalize(statement); + + rhizome_vacuum_db(&retry); if (db_used + bytes <= limit) return RHIZOME_PAYLOAD_STATUS_NEW; @@ -250,6 +257,11 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes) return RHIZOME_PAYLOAD_STATUS_UNINITERESTING; } +int rhizome_store_cleanup(struct rhizome_cleanup_report *report) +{ + return store_make_space(0, report); +} + enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length) { if (file_length == 0) @@ -268,7 +280,7 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons } if (file_length!=RHIZOME_SIZE_UNSET){ - enum rhizome_payload_status status = store_make_space(file_length); + enum rhizome_payload_status status = store_make_space(file_length, NULL); if (status != RHIZOME_PAYLOAD_STATUS_NEW) return status; } @@ -632,7 +644,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write) if (config.debug.rhizome_store) DEBUGF("Wrote %"PRIu64" bytes, set file_length", write->file_offset); write->file_length = write->file_offset; - status = store_make_space(write->file_length); + status = store_make_space(write->file_length, NULL); if (status!=RHIZOME_PAYLOAD_STATUS_NEW) goto failure; }