mirror of
https://github.com/servalproject/serval-dna.git
synced 2025-02-05 02:29:57 +00:00
Refactor storage space measurement to expose new status API
Change semantics of fake space limit to exclude used space, otherwise the result will keep changing on each call
This commit is contained in:
Jeremy Lakeman
parent
c2526446ed
commit
4aeb6545ad
@ -1139,6 +1139,37 @@ The import logic proceeds in the following steps:
|
||||
returns status [201 Created][201] and the [bundle status
|
||||
code](#bundle-status-code) for “new”.
|
||||
|
||||
### GET /restful/rhizome/storestatus.json
|
||||
|
||||
Fetch on the current disk usage of the rhizome store.
|
||||
|
||||
The results will be a single json object with the following fields;
|
||||
|
||||
* `external_bytes` - the total size of all payloads larger than
|
||||
rhizome.max_blob_size, that have been stored outside of sqlite, in the
|
||||
rhizome blob folder.
|
||||
|
||||
* `db_page_size` - the size of disk pages returned by sqlite.
|
||||
|
||||
* `db_total_pages` - the number of disk pages in the sqlite database file.
|
||||
|
||||
* `db_available_pages` - the number of disk pages in the sqlite database file
|
||||
that have been allocated but are not currently in use.
|
||||
|
||||
* `content_bytes` - the total bytes of space used in the sqlite database, and
|
||||
in payloads stored outside of sqlite. This should be equal to;
|
||||
db_page_size * (db_total_pages - db_available_pages) + external_bytes
|
||||
|
||||
* `content_limit_bytes` - the calculated storage limit that is being applied.
|
||||
This will be the smallest of the configured rhizome.database_size or the
|
||||
maximum we can store while keeping rhizome.min_free_space available for
|
||||
other uses.
|
||||
|
||||
* `filesystem_bytes` - the measured total size of the filesystem where the
|
||||
rhizome store is located.
|
||||
|
||||
* `filesystem_free_bytes` - the measured free space of the filesystem.
|
||||
|
||||
-----
|
||||
**Copyright 2015-2017 Serval Project Inc.**
|
||||
|
||||
|
||||
4
httpd.h
4
httpd.h
@ -263,6 +263,10 @@ typedef struct httpd_request
|
||||
size_t offset;
|
||||
}
|
||||
file;
|
||||
|
||||
struct {
|
||||
struct rhizome_space_report rhizome_space;
|
||||
} status;
|
||||
} u;
|
||||
|
||||
} httpd_request;
|
||||
|
||||
30
rhizome.h
30
rhizome.h
@ -367,16 +367,34 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
|
||||
|
||||
#define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)
|
||||
|
||||
struct rhizome_space_report {
|
||||
uint64_t file_count;
|
||||
uint64_t internal_bytes;
|
||||
uint64_t external_bytes;
|
||||
|
||||
uint64_t db_page_size;
|
||||
uint64_t db_total_pages;
|
||||
uint64_t db_available_pages;
|
||||
|
||||
uint64_t content_bytes;
|
||||
uint64_t content_limit_bytes;
|
||||
|
||||
uint64_t filesystem_bytes;
|
||||
uint64_t filesystem_free_bytes;
|
||||
};
|
||||
|
||||
struct rhizome_cleanup_report {
|
||||
unsigned deleted_stale_incoming_files;
|
||||
unsigned deleted_expired_files;
|
||||
unsigned deleted_orphan_files;
|
||||
unsigned deleted_orphan_fileblobs;
|
||||
unsigned deleted_orphan_manifests;
|
||||
struct rhizome_space_report space_used;
|
||||
unsigned deleted_stale_incoming_files;
|
||||
unsigned deleted_expired_files;
|
||||
unsigned deleted_orphan_files;
|
||||
unsigned deleted_orphan_fileblobs;
|
||||
unsigned deleted_orphan_manifests;
|
||||
};
|
||||
|
||||
int rhizome_cleanup(struct rhizome_cleanup_report *report);
|
||||
int rhizome_store_cleanup(struct rhizome_cleanup_report *report);
|
||||
enum rhizome_payload_status rhizome_store_space_usage(struct rhizome_space_report *space);
|
||||
void rhizome_vacuum_db(sqlite_retry_state *retry);
|
||||
int rhizome_manifest_createid(rhizome_manifest *m);
|
||||
struct rhizome_bundle_result rhizome_private_bundle(rhizome_manifest *m, const sign_keypair_t *keypair);
|
||||
@ -911,8 +929,6 @@ int rhizome_cache_close();
|
||||
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);
|
||||
|
||||
void rhizome_process_added_bundles(uint64_t up_to_rowid);
|
||||
void rhizome_sync_status();
|
||||
|
||||
DECLARE_ALARM(rhizome_fetch_status);
|
||||
|
||||
/* Rhizome triggers */
|
||||
|
||||
@ -472,11 +472,15 @@ static int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_contex
|
||||
DEFINE_CMD(app_rhizome_clean, 0,
|
||||
"Remove stale and orphaned content from the Rhizome store",
|
||||
"rhizome","clean","[verify]" KEYRING_PIN_OPTIONS);
|
||||
DEFINE_CMD(app_rhizome_clean, 0,
|
||||
"Report on the space usage of the Rhizome store",
|
||||
"rhizome","status" KEYRING_PIN_OPTIONS);
|
||||
static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context *context)
|
||||
{
|
||||
DEBUG_cli_parsed(verbose, parsed);
|
||||
int verify = cli_arg(parsed, "verify", NULL, NULL, NULL) == 0;
|
||||
|
||||
int clean = strcasecmp(parsed->args[1], "clean")==0;
|
||||
|
||||
/* Ensure the Rhizome database exists and is open */
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
@ -488,16 +492,36 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context
|
||||
verify_bundles();
|
||||
}
|
||||
struct rhizome_cleanup_report report;
|
||||
if (rhizome_cleanup(&report) == -1)
|
||||
if (clean){
|
||||
if (rhizome_cleanup(&report) == -1)
|
||||
return -1;
|
||||
cli_field_name(context, "deleted_stale_incoming_files", ":");
|
||||
cli_put_long(context, report.deleted_stale_incoming_files, "\n");
|
||||
cli_field_name(context, "deleted_orphan_files", ":");
|
||||
cli_put_long(context, report.deleted_orphan_files, "\n");
|
||||
cli_field_name(context, "deleted_orphan_fileblobs", ":");
|
||||
cli_put_long(context, report.deleted_orphan_fileblobs, "\n");
|
||||
cli_field_name(context, "deleted_orphan_manifests", ":");
|
||||
cli_put_long(context, report.deleted_orphan_manifests, "\n");
|
||||
}
|
||||
if (rhizome_store_space_usage(&report.space_used)!=RHIZOME_PAYLOAD_STATUS_EMPTY)
|
||||
return -1;
|
||||
cli_field_name(context, "deleted_stale_incoming_files", ":");
|
||||
cli_put_long(context, report.deleted_stale_incoming_files, "\n");
|
||||
cli_field_name(context, "deleted_orphan_files", ":");
|
||||
cli_put_long(context, report.deleted_orphan_files, "\n");
|
||||
cli_field_name(context, "deleted_orphan_fileblobs", ":");
|
||||
cli_put_long(context, report.deleted_orphan_fileblobs, "\n");
|
||||
cli_field_name(context, "deleted_orphan_manifests", ":");
|
||||
cli_put_long(context, report.deleted_orphan_manifests, "\n");
|
||||
cli_field_name(context, "file_count", ":");
|
||||
cli_put_long(context, report.space_used.file_count, "\n");
|
||||
cli_field_name(context, "file_size_bytes", ":");
|
||||
int64_t used = report.space_used.internal_bytes + report.space_used.external_bytes;
|
||||
cli_put_long(context, used, "\n");
|
||||
cli_field_name(context, "overhead_bytes", ":");
|
||||
cli_put_long(context, report.space_used.content_bytes - used, "\n");
|
||||
cli_field_name(context, "used_bytes", ":");
|
||||
cli_put_long(context, report.space_used.content_bytes, "\n");
|
||||
cli_field_name(context, "available_space_bytes", ":");
|
||||
cli_put_long(context, report.space_used.content_limit_bytes -
|
||||
((report.space_used.content_limit_bytes == UINT64_MAX) ? 0 : report.space_used.content_bytes), "\n");
|
||||
cli_field_name(context, "reclaimable_bytes", ":");
|
||||
cli_put_long(context, report.space_used.db_available_pages * report.space_used.db_page_size, "\n");
|
||||
cli_field_name(context, "free_space_bytes", ":");
|
||||
cli_put_long(context, report.space_used.filesystem_free_bytes, "\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -1388,8 +1388,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
|
||||
if (report && ret > 0)
|
||||
report->deleted_orphan_manifests += ret;
|
||||
|
||||
rhizome_vacuum_db(&retry);
|
||||
|
||||
if (report)
|
||||
DEBUGF(rhizome, "report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u deleted_orphan_manifests=%u",
|
||||
report->deleted_stale_incoming_files,
|
||||
|
||||
@ -34,6 +34,7 @@ DECLARE_HANDLER("/restful/rhizome/newsince/", restful_rhizome_newsince);
|
||||
DECLARE_HANDLER("/restful/rhizome/insert", restful_rhizome_insert);
|
||||
DECLARE_HANDLER("/restful/rhizome/import", restful_rhizome_import);
|
||||
DECLARE_HANDLER("/restful/rhizome/append", restful_rhizome_append);
|
||||
DECLARE_HANDLER("/restful/rhizome/storestatus.json", restful_rhizome_disk_status);
|
||||
DECLARE_HANDLER("/restful/rhizome/", restful_rhizome_);
|
||||
|
||||
static HTTP_RENDERER render_status_and_manifest_headers;
|
||||
@ -1243,3 +1244,60 @@ static void render_status_and_import_headers(const struct http_request *hr, strb
|
||||
render_import_headers(r, sb);
|
||||
}
|
||||
}
|
||||
|
||||
static int rhizome_disk_status_content_chunk(struct http_request *hr, strbuf b)
|
||||
{
|
||||
httpd_request *r = (httpd_request *) hr;
|
||||
strbuf_puts(b, "{\n");
|
||||
|
||||
strbuf_puts(b, "\"rhizome_dir\":");
|
||||
strbuf_json_string(b, rhizome_database.dir_path);
|
||||
strbuf_puts(b, ",\n");
|
||||
|
||||
strbuf_puts(b, "\"rhizome_uuid\":");
|
||||
strbuf_json_string(b, alloca_uuid_str(rhizome_database.uuid));
|
||||
strbuf_puts(b, ",\n");
|
||||
|
||||
strbuf_sprintf(b, "\"external_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.external_bytes);
|
||||
strbuf_sprintf(b, "\"db_page_size\":%"PRIu64",\n", r->u.status.rhizome_space.db_page_size);
|
||||
strbuf_sprintf(b, "\"db_total_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_total_pages);
|
||||
strbuf_sprintf(b, "\"db_available_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_available_pages);
|
||||
strbuf_sprintf(b, "\"content_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_bytes);
|
||||
strbuf_sprintf(b, "\"content_limit_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_limit_bytes);
|
||||
strbuf_sprintf(b, "\"filesystem_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.filesystem_bytes);
|
||||
strbuf_sprintf(b, "\"filesystem_free_bytes\":%"PRIu64"\n}", r->u.status.rhizome_space.filesystem_free_bytes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int rhizome_disk_status_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
|
||||
{
|
||||
return generate_http_content_from_strbuf_chunks(hr, (char *)buf, bufsz, result, rhizome_disk_status_content_chunk);
|
||||
}
|
||||
|
||||
static int restful_rhizome_disk_status(httpd_request *r, const char *remainder)
|
||||
{
|
||||
if (*remainder || !is_rhizome_http_enabled())
|
||||
return 404;
|
||||
int ret = authorize_restful(&r->http);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
enum rhizome_payload_status p = rhizome_store_space_usage(&r->u.status.rhizome_space);
|
||||
switch (p) {
|
||||
case RHIZOME_PAYLOAD_STATUS_EMPTY:
|
||||
break;
|
||||
case RHIZOME_PAYLOAD_STATUS_STORED:
|
||||
case RHIZOME_PAYLOAD_STATUS_NEW:
|
||||
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
|
||||
case RHIZOME_PAYLOAD_STATUS_BUSY:
|
||||
case RHIZOME_PAYLOAD_STATUS_ERROR:
|
||||
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
|
||||
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
|
||||
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
|
||||
case RHIZOME_PAYLOAD_STATUS_EVICTED:
|
||||
return http_request_rhizome_response(r, 500, "Failed to measure storage space");
|
||||
}
|
||||
http_request_response_generated(&r->http, 200, &CONTENT_TYPE_JSON, rhizome_disk_status_content);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
223
rhizome_store.c
223
rhizome_store.c
@ -186,98 +186,142 @@ int rhizome_delete_file(const rhizome_filehash_t *filehash)
|
||||
return rhizome_delete_file_retry(&retry, filehash);
|
||||
}
|
||||
|
||||
static uint64_t store_get_free_space()
|
||||
{
|
||||
const char *fake_space = getenv("SERVALD_FREE_SPACE");
|
||||
uint64_t space = UINT64_MAX;
|
||||
if (fake_space)
|
||||
space = atol(fake_space);
|
||||
#if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H))
|
||||
else {
|
||||
struct statvfs stats;
|
||||
if (statvfs(rhizome_database.dir_path, &stats)==-1)
|
||||
WARNF_perror("statvfs(%s)", alloca_str_toprint(rhizome_database.dir_path));
|
||||
else
|
||||
space = stats.f_frsize * (uint64_t)stats.f_bavail;
|
||||
}
|
||||
#endif
|
||||
if (IF_DEBUG(rhizome)) {
|
||||
// Automated tests depend on this message; do not alter.
|
||||
DEBUGF(rhizome, "RHIZOME SPACE FREE bytes=%"PRIu64" (%sB)", space, alloca_double_scaled_binary(space));
|
||||
}
|
||||
return space;
|
||||
}
|
||||
static enum rhizome_payload_status store_space_report(sqlite_retry_state *retry, struct rhizome_space_report *space){
|
||||
int stepcode = sqlite_exec_uint64_retry(retry, &space->db_page_size, "PRAGMA page_size;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
stepcode = sqlite_exec_uint64_retry(retry, &space->db_total_pages, "PRAGMA page_count;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
stepcode = sqlite_exec_uint64_retry(retry, &space->db_available_pages, "PRAGMA freelist_count;", END);
|
||||
if (sqlite_code_ok(stepcode)){
|
||||
sqlite3_stmt *statement = sqlite_prepare_bind(retry,
|
||||
"SELECT CASE WHEN B.ID IS NULL THEN 0 ELSE 1 END, SUM(length), count(*) "
|
||||
"FROM FILES F "
|
||||
"LEFT JOIN FILEBLOBS B "
|
||||
"ON F.ID = B.ID "
|
||||
"GROUP BY CASE WHEN B.ID IS NULL THEN 0 ELSE 1 END;",
|
||||
END);
|
||||
if (statement == NULL)
|
||||
return RHIZOME_PAYLOAD_STATUS_ERROR;
|
||||
|
||||
static uint64_t store_space_limit(uint64_t current_size)
|
||||
{
|
||||
uint64_t limit = config.rhizome.database_size;
|
||||
|
||||
if (config.rhizome.min_free_space!=0){
|
||||
uint64_t free_space = store_get_free_space();
|
||||
if (free_space < config.rhizome.min_free_space){
|
||||
if (current_size + free_space < config.rhizome.min_free_space)
|
||||
limit = 0;
|
||||
else
|
||||
limit = current_size + free_space - config.rhizome.min_free_space;
|
||||
space->file_count=0;
|
||||
space->internal_bytes=0;
|
||||
space->external_bytes=0;
|
||||
while((stepcode = sqlite_step_retry(retry, statement)) == SQLITE_ROW) {
|
||||
int64_t type = sqlite3_column_int64(statement, 0);
|
||||
int64_t len = sqlite3_column_int64(statement, 1);
|
||||
int64_t count = sqlite3_column_int64(statement, 2);
|
||||
|
||||
space->file_count += count;
|
||||
if (type==1){
|
||||
space->internal_bytes = len;
|
||||
}else{
|
||||
space->external_bytes = len;
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
}
|
||||
return limit;
|
||||
}
|
||||
|
||||
// TODO readonly version?
|
||||
static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizome_cleanup_report *report)
|
||||
{
|
||||
uint64_t external_bytes=0;
|
||||
uint64_t db_page_size=0;
|
||||
uint64_t db_page_count=0;
|
||||
uint64_t db_free_page_count=0;
|
||||
|
||||
// No limit?
|
||||
if (config.rhizome.database_size==UINT64_MAX && config.rhizome.min_free_space==0)
|
||||
return RHIZOME_PAYLOAD_STATUS_NEW;
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
int stepcode = sqlite_exec_uint64_retry(&retry, &db_page_size, "PRAGMA page_size;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
stepcode = sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
stepcode = sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
// TODO index and/or cache result?
|
||||
stepcode = sqlite_exec_uint64_retry(&retry, &external_bytes,
|
||||
"SELECT SUM(length) "
|
||||
"FROM FILES "
|
||||
"WHERE NOT EXISTS( "
|
||||
"SELECT 1 "
|
||||
"FROM FILEBLOBS "
|
||||
"WHERE FILES.ID = FILEBLOBS.ID "
|
||||
");", END);
|
||||
|
||||
if (sqlite_code_busy(stepcode))
|
||||
return RHIZOME_PAYLOAD_STATUS_BUSY;
|
||||
if (!sqlite_code_ok(stepcode))
|
||||
return RHIZOME_PAYLOAD_STATUS_ERROR;
|
||||
|
||||
uint64_t db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
|
||||
const uint64_t limit = store_space_limit(db_used);
|
||||
space->content_bytes = space->external_bytes + space->db_page_size * (space->db_total_pages - space->db_available_pages);
|
||||
|
||||
// Automated tests depend on this message; do not alter.
|
||||
DEBUGF(rhizome, "RHIZOME SPACE USED bytes=%"PRIu64" (%sB), LIMIT bytes=%"PRIu64" (%sB)",
|
||||
db_used, alloca_double_scaled_binary(db_used),
|
||||
limit, alloca_double_scaled_binary(limit));
|
||||
|
||||
if (bytes && bytes >= limit){
|
||||
// Measure filesystem free space
|
||||
space->filesystem_bytes = UINT64_MAX;
|
||||
space->filesystem_free_bytes = UINT64_MAX;
|
||||
|
||||
#if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H))
|
||||
{
|
||||
struct statvfs stats;
|
||||
if (statvfs(rhizome_database.dir_path, &stats)==-1)
|
||||
WARNF_perror("statvfs(%s)", rhizome_database.dir_path);
|
||||
else{
|
||||
space->filesystem_bytes = stats.f_frsize * (uint64_t)stats.f_blocks;
|
||||
space->filesystem_free_bytes = stats.f_frsize * (uint64_t)stats.f_bavail;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// Fake limit for reproducible testing
|
||||
const char *fake_space = getenv("SERVALD_FAKE_SPACE_LIMIT");
|
||||
if (fake_space){
|
||||
uint64_t space_limit;
|
||||
// subtrace measured space used to give the same result as we add and remove content
|
||||
if (str_to_uint64_scaled(fake_space, 10, &space_limit, NULL)==1
|
||||
&& space_limit < space->filesystem_free_bytes + space->content_bytes)
|
||||
space->filesystem_free_bytes = space_limit - space->content_bytes;
|
||||
}
|
||||
|
||||
// Calculate storage limit
|
||||
space->content_limit_bytes = config.rhizome.database_size;
|
||||
|
||||
if (config.rhizome.min_free_space !=0){
|
||||
uint64_t space_limit;
|
||||
if (space->content_bytes + space->filesystem_free_bytes < config.rhizome.min_free_space)
|
||||
space_limit = 0;
|
||||
else
|
||||
space_limit = space->content_bytes + space->filesystem_free_bytes - config.rhizome.min_free_space;
|
||||
if (space_limit < space->content_limit_bytes)
|
||||
space->content_limit_bytes = space_limit;
|
||||
}
|
||||
|
||||
DEBUGF(rhizome, "RHIZOME SPACE USED bytes=%"PRIu64" (%sB), FREE bytes=%"PRIu64" (%sB), LIMIT bytes=%"PRIu64" (%sB)",
|
||||
space->content_bytes, alloca_double_scaled_binary(space->content_bytes),
|
||||
space->filesystem_free_bytes, alloca_double_scaled_binary(space->filesystem_free_bytes),
|
||||
space->content_limit_bytes, alloca_double_scaled_binary(space->content_limit_bytes));
|
||||
|
||||
return RHIZOME_PAYLOAD_STATUS_EMPTY;
|
||||
}
|
||||
|
||||
enum rhizome_payload_status rhizome_store_space_usage(struct rhizome_space_report *space)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
return store_space_report(&retry, space);
|
||||
}
|
||||
|
||||
|
||||
static enum rhizome_payload_status sqlite_vacuum(sqlite_retry_state *retry, struct rhizome_space_report *space){
|
||||
if (space->db_available_pages == 0)
|
||||
return RHIZOME_PAYLOAD_STATUS_EMPTY;
|
||||
|
||||
// vacuum database pages if more than 1/4 of the db is free or we're already over the limit
|
||||
if (space->db_available_pages > (space->db_total_pages>>2)+1 || space->external_bytes + space->db_page_size * space->db_total_pages > space->content_limit_bytes){
|
||||
rhizome_vacuum_db(retry);
|
||||
|
||||
int stepcode = sqlite_exec_uint64_retry(retry, &space->db_total_pages, "PRAGMA page_count;", END);
|
||||
if (sqlite_code_ok(stepcode))
|
||||
stepcode = sqlite_exec_uint64_retry(retry, &space->db_available_pages, "PRAGMA freelist_count;", END);
|
||||
|
||||
if (sqlite_code_busy(stepcode))
|
||||
return RHIZOME_PAYLOAD_STATUS_BUSY;
|
||||
if (!sqlite_code_ok(stepcode))
|
||||
return RHIZOME_PAYLOAD_STATUS_ERROR;
|
||||
}
|
||||
return RHIZOME_PAYLOAD_STATUS_EMPTY;
|
||||
}
|
||||
|
||||
// TODO readonly version?
|
||||
static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizome_cleanup_report *report)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
struct rhizome_space_report *space = (report ? &report->space_used : alloca(sizeof *space));
|
||||
int stepcode;
|
||||
|
||||
enum rhizome_payload_status r;
|
||||
if ((r = store_space_report(&retry, space)) != RHIZOME_PAYLOAD_STATUS_EMPTY)
|
||||
return r;
|
||||
|
||||
if (bytes && bytes >= space->content_limit_bytes){
|
||||
DEBUGF(rhizome, "Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64,
|
||||
bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
|
||||
bytes, space->content_bytes, space->external_bytes, space->db_page_size, space->db_total_pages, space->db_available_pages, space->content_limit_bytes);
|
||||
return RHIZOME_PAYLOAD_STATUS_TOO_BIG;
|
||||
}
|
||||
|
||||
// vacuum database pages if more than 1/4 of the db is free or we're already over the limit
|
||||
if (db_free_page_count > (db_page_count>>2)+1 || external_bytes + db_page_size * db_page_count > limit)
|
||||
rhizome_vacuum_db(&retry);
|
||||
|
||||
|
||||
if ((r = sqlite_vacuum(&retry, space)) != RHIZOME_PAYLOAD_STATUS_EMPTY)
|
||||
return r;
|
||||
|
||||
// If there is enough space, do nothing
|
||||
if (db_used + bytes <= limit)
|
||||
if (space->content_bytes + bytes <= space->content_limit_bytes)
|
||||
return RHIZOME_PAYLOAD_STATUS_NEW;
|
||||
|
||||
// penalise new things by 10 minutes to reduce churn
|
||||
@ -290,7 +334,7 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
|
||||
if (!statement)
|
||||
return RHIZOME_PAYLOAD_STATUS_ERROR;
|
||||
|
||||
while (db_used + bytes > limit && (stepcode=sqlite_step_retry(&retry, statement)) == SQLITE_ROW) {
|
||||
while (space->content_bytes + bytes > space->content_limit_bytes && (stepcode=sqlite_step_retry(&retry, statement)) == SQLITE_ROW) {
|
||||
const char *id=(const char *) sqlite3_column_text(statement, 0);
|
||||
uint64_t length = sqlite3_column_int(statement, 1);
|
||||
time_ms_t inserttime = sqlite3_column_int64(statement, 2);
|
||||
@ -307,25 +351,29 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
|
||||
rhizome_filehash_t hash;
|
||||
if (str_to_rhizome_filehash_t(&hash, id)!=-1
|
||||
&& rhizome_delete_external(&hash)==0)
|
||||
external_bytes -= length;
|
||||
space->external_bytes -= length;
|
||||
|
||||
int rowcount=0;
|
||||
sqlite3_stmt *s = sqlite_prepare_bind(&retry, "DELETE FROM fileblobs WHERE id = ?", STATIC_TEXT, id, END);
|
||||
if (s && !sqlite_code_ok(stepcode = sqlite_exec_code_retry(&retry, s, &rowcount)))
|
||||
break;
|
||||
if (rowcount>0)
|
||||
space->internal_bytes -= length;
|
||||
|
||||
s = sqlite_prepare_bind(&retry, "DELETE FROM files WHERE id = ?", STATIC_TEXT, id, END);
|
||||
if (s && !sqlite_code_ok(stepcode = sqlite_exec_code_retry(&retry, s, &rowcount)))
|
||||
break;
|
||||
if (rowcount>0)
|
||||
space->file_count --;
|
||||
|
||||
if (!sqlite_code_ok(stepcode = sqlite_exec_uint64_retry(&retry, &db_page_count, "PRAGMA page_count;", END)))
|
||||
if (!sqlite_code_ok(stepcode = sqlite_exec_uint64_retry(&retry, &space->db_total_pages, "PRAGMA page_count;", END)))
|
||||
break;
|
||||
if (!sqlite_code_ok(stepcode = sqlite_exec_uint64_retry(&retry, &db_free_page_count, "PRAGMA freelist_count;", END)))
|
||||
if (!sqlite_code_ok(stepcode = sqlite_exec_uint64_retry(&retry, &space->db_available_pages, "PRAGMA freelist_count;", END)))
|
||||
break;
|
||||
|
||||
if (report)
|
||||
report->deleted_expired_files++;
|
||||
db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
|
||||
space->content_bytes = space->external_bytes + space->db_page_size * (space->db_total_pages - space->db_available_pages);
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
|
||||
@ -334,13 +382,14 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
|
||||
if (!sqlite_code_ok(stepcode))
|
||||
return RHIZOME_PAYLOAD_STATUS_ERROR;
|
||||
|
||||
rhizome_vacuum_db(&retry);
|
||||
if ((r = sqlite_vacuum(&retry, space)) != RHIZOME_PAYLOAD_STATUS_EMPTY)
|
||||
return r;
|
||||
|
||||
if (db_used + bytes <= limit)
|
||||
if (space->content_bytes + bytes <= space->content_limit_bytes)
|
||||
return RHIZOME_PAYLOAD_STATUS_NEW;
|
||||
|
||||
DEBUGF(rhizome, "Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64,
|
||||
bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
|
||||
bytes, space->content_bytes, space->external_bytes, space->db_page_size, space->db_total_pages, space->db_available_pages, space->content_limit_bytes);
|
||||
|
||||
return RHIZOME_PAYLOAD_STATUS_EVICTED;
|
||||
}
|
||||
|
||||
@ -1747,12 +1747,11 @@ doc_payloadUninteresting="Fail to insert a payload that is uninteresting"
|
||||
setup_payloadUninteresting() {
|
||||
setup_servald
|
||||
setup_rhizome
|
||||
executeOk_servald config set rhizome.clean_on_open yes
|
||||
# Create Rhizome database, discover fixed space overhead.
|
||||
executeOk_servald rhizome list
|
||||
bytes_used=$($SED -n -e '/.*RHIZOME SPACE.*USED bytes=\([0-9]\+\).*/{s//\1/p;q}' "$TFWSTDERR")
|
||||
assert [ -n "$bytes_used" ]
|
||||
executeOk_servald config set rhizome.database_size $((bytes_used + 65535))
|
||||
# get size of empty rhizome db
|
||||
executeOk_servald rhizome status
|
||||
extract_stdout_keyvalue used_bytes 'used_bytes' '[0-9]\+'
|
||||
assert [ -n "$used_bytes" ]
|
||||
executeOk_servald config set rhizome.database_size $((used_bytes + 65535))
|
||||
}
|
||||
test_payloadUninteresting(){
|
||||
create_file file1 32K
|
||||
@ -1802,9 +1801,12 @@ setup_evictFreeSpace() {
|
||||
test_evictFreeSpace() {
|
||||
executeOk_servald config \
|
||||
set rhizome.min_free_space 1M
|
||||
executeOk_servald rhizome status
|
||||
tfw_cat --stdout
|
||||
# only 640K free...
|
||||
export SERVALD_FREE_SPACE=655360
|
||||
export SERVALD_FAKE_SPACE_LIMIT=2M
|
||||
rhizome_clean
|
||||
tfw_cat --stdout
|
||||
assert [ $deleted_files = 0 ]
|
||||
assert [ $deleted_fileblobs = 0 ]
|
||||
assert [ $deleted_manifests = 1 ]
|
||||
|
||||
@ -1155,4 +1155,16 @@ test_RhizomeAppendNonJournalForbidden() {
|
||||
assert_rhizome_list file1
|
||||
}
|
||||
|
||||
doc_RhizomeStatus="REST API Rhizome storage status"
|
||||
setup_RhizomeStatus() {
|
||||
export SERVALD_FAKE_SPACE_LIMIT=2M
|
||||
setup
|
||||
}
|
||||
test_RhizomeStatus() {
|
||||
rest_request GET "/restful/rhizome/storestatus.json"
|
||||
assertJq response.json 'keys==(["rhizome_dir","rhizome_uuid","external_bytes","db_page_size","db_total_pages","db_available_pages","content_bytes","content_limit_bytes","filesystem_bytes","filesystem_free_bytes"]|sort)'
|
||||
assertJq response.json 'contains({"external_bytes":2048})'
|
||||
assertJq response.json 'contains({"filesystem_free_bytes":2041856})'
|
||||
}
|
||||
|
||||
runTests "$@"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user