ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-20 17:33:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Reinstate rhizome database storage limit

Browse Source 
- old / large payloads should be evicted to fit more payloads
- if there isn't enough space, new payloads will not be added
This commit is contained in:
Jeremy Lakeman 2014-06-18 17:25:15 +09:30
parent f4e6841ed7
commit 7ff89afcf4
11 changed files with 362 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
commandline.c
View File

@ -1760,6 +1760,11 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
status = RHIZOME_BUNDLE_STATUS_DONOTWANT;
WHY("Insufficient space to store payload");
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
status = RHIZOME_BUNDLE_STATUS_ERROR;
break;
@ -1804,6 +1809,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
case RHIZOME_BUNDLE_STATUS_ERROR:
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_FAKE:
case RHIZOME_BUNDLE_STATUS_DONOTWANT:
break;
default:
FATALF("status=%d", status);

2
conf_schema.h
View File

@ -429,7 +429,7 @@ ATOM(bool_t, fetch, 1, boolean,, "If false, no new bundl
ATOM(bool_t, clean_on_open, 0, boolean,, "If true, Rhizome database is cleaned at start of every command")
ATOM(bool_t, clean_on_start, 1, boolean,, "If true, Rhizome database is cleaned at start of daemon")
STRING(256, datastore_path, "", str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, uint64_scaled,, "Size of database in bytes")
ATOM(uint64_t, database_size, 0, uint64_scaled,, "Maximum size of database in bytes")
ATOM(uint32_t, max_blob_size, 128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
ATOM(uint64_t, rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")

32
meshms.c
View File

@ -334,7 +334,6 @@ static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshm
case 0:
break;
case -1:
status = MESHMS_STATUS_ERROR;
goto end;
default:
status = MESHMS_STATUS_PROTOCOL_FAULT;
@ -346,16 +345,14 @@ static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshm
goto end;
}
} else if (create_ply(my_sid, conv, m) == -1) {
status = MESHMS_STATUS_ERROR;
goto end;
}
assert(m->haveSecret);
assert(m->authorship == AUTHOR_AUTHENTIC);
enum rhizome_payload_status pstatus = rhizome_append_journal_buffer(m, 0, buffer, len);
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW) {
status = MESHMS_STATUS_ERROR;
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
goto end;
}
enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1);
if (config.debug.meshms)
DEBUGF("bstatus=%d", bstatus);
@ -664,17 +661,20 @@ static enum meshms_status write_known_conversations(rhizome_manifest *m, struct
rhizome_manifest_set_filehash(m, NULL);
enum rhizome_payload_status pstatus = rhizome_write_open_manifest(&write, m);
if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) {
unsigned char version=1;
if (rhizome_write_buffer(&write, &version, 1) == -1)
goto end;
if (write_conversation(&write, conv) == -1)
goto end;
pstatus = rhizome_finish_write(&write);
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
goto end;
rhizome_manifest_set_filehash(m, &write.id);
}
if (pstatus!=RHIZOME_PAYLOAD_STATUS_NEW)
// TODO log something?
goto end;
unsigned char version=1;
if (rhizome_write_buffer(&write, &version, 1) == -1)
goto end;
if (write_conversation(&write, conv) == -1)
goto end;
pstatus = rhizome_finish_write(&write);
if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
goto end;
rhizome_manifest_set_filehash(m, &write.id);
enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1);
switch (bstatus) {
case RHIZOME_BUNDLE_STATUS_ERROR:

4
rhizome.c
View File

@ -167,6 +167,10 @@ enum rhizome_bundle_status rhizome_bundle_import_files(rhizome_manifest *m, rhiz
if (rhizome_store_manifest(m) == -1)
return -1;
break;
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
status = RHIZOME_BUNDLE_STATUS_DONOTWANT;
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
return -1;

23
rhizome.h
View File

@ -45,14 +45,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
extern time_ms_t rhizome_voice_timeout;
#define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE
#define RHIZOME_PRIORITY_SERVAL_CORE 5
#define RHIZOME_PRIORITY_SUBSCRIBED 4
#define RHIZOME_PRIORITY_SERVAL_OPTIONAL 3
#define RHIZOME_PRIORITY_DEFAULT 2
#define RHIZOME_PRIORITY_SERVAL_BULK 1
#define RHIZOME_PRIORITY_NOTINTERESTED 0
#define RHIZOME_IDLE_TIMEOUT 20000
typedef struct rhizome_signature {
@ -182,8 +174,6 @@ typedef struct rhizome_manifest
AUTHOR_AUTHENTIC // a local identity is the verified author
} authorship;
int fileHighestPriority;
/* Absolute path of the file associated with the manifest */
const char *dataFileName;
@ -370,6 +360,7 @@ enum rhizome_bundle_status {
RHIZOME_BUNDLE_STATUS_INVALID = 4, // manifest is invalid
RHIZOME_BUNDLE_STATUS_FAKE = 5, // manifest signature not valid
RHIZOME_BUNDLE_STATUS_INCONSISTENT = 6, // manifest filesize/filehash does not match supplied payload
RHIZOME_BUNDLE_STATUS_DONOTWANT = 7, // Wont fit or we already have more important bundles
};
enum rhizome_payload_status {
@ -380,12 +371,12 @@ enum rhizome_payload_status {
RHIZOME_PAYLOAD_STATUS_WRONG_SIZE = 3, // payload's size does not match manifest
RHIZOME_PAYLOAD_STATUS_WRONG_HASH = 4, // payload's hash does not match manifest
RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL = 5, // cannot encrypt/decrypt (payload key unknown)
RHIZOME_PAYLOAD_STATUS_TOO_BIG = 6, // payload will never fit in our store
RHIZOME_PAYLOAD_STATUS_UNINITERESTING = 7, // other payloads in our store are more interesting
};
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_validate(rhizome_manifest *m);
int rhizome_manifest_parse(rhizome_manifest *m);
@ -399,7 +390,6 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
int rhizome_store_manifest(rhizome_manifest *m);
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath);
@ -469,6 +459,7 @@ sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc, int log_level, sqlite_ret
int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, int log_level, const char *sqltext, ...);
int _sqlite_exec_void_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
int _sqlite_exec_uint64(struct __sourceloc, uint64_t *result, const char *sqltext, ...);
@ -530,7 +521,6 @@ int _sqlite_blob_close(struct __sourceloc, int log_level, sqlite3_blob *blob);
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
int rhizome_update_file_priority(const char *fileid);
enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
int rhizome_manifest_to_bar(rhizome_manifest *m, rhizome_bar_t *bar);
int rhizome_is_bar_interesting(const rhizome_bar_t *bar);
@ -541,6 +531,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
int rhizome_delete_bundle(const rhizome_bid_t *bidp);
int rhizome_delete_manifest(const rhizome_bid_t *bidp);
int rhizome_delete_payload(const rhizome_bid_t *bidp);
int rhizome_delete_file_id(const char *id);
int rhizome_delete_file(const rhizome_filehash_t *hashp);
#define RHIZOME_DONTVERIFY 0
@ -633,7 +624,6 @@ struct rhizome_write
rhizome_filehash_t id;
uint64_t temp_id;
char id_known;
int priority;
uint64_t tail;
uint64_t file_offset;
uint64_t written_offset;
@ -793,6 +783,7 @@ enum rhizome_start_fetch_result {
SUPERSEDED,
OLDERBUNDLE,
NEWERBUNDLE,
DONOTWANT,
IMPORTED,
SLOTBUSY
};
@ -809,7 +800,7 @@ int rhizome_fetch_has_queue_space(unsigned char log2_size);
/* rhizome storage methods */
int rhizome_exists(const rhizome_filehash_t *hashp);
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size);
enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);

433
rhizome_database.c
View File

@ -32,8 +32,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "server.h"
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
static int create_rhizome_store_dir()
{
@ -49,23 +47,6 @@ static int create_rhizome_store_dir()
sqlite3 *rhizome_db = NULL;
serval_uuid_t rhizome_db_uuid;
/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
{
uint64_t result = 0;
if (sqlite_exec_uint64_retry(retry, &result,
"SELECT max(grouplist.priorty) FROM GROUPLIST,MANIFESTS,GROUPMEMBERSHIPS"
" WHERE MANIFESTS.id = ?"
" AND GROUPLIST.id = GROUPMEMBERSHIPS.groupid"
" AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id;",
RHIZOME_BID_T, bidp,
END
) == -1
)
return -1;
return (int) result;
}
int is_debug_rhizome()
{
return config.debug.rhizome;
@ -226,6 +207,11 @@ int rhizome_opendb()
if (!FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db"))
RETURN(-1);
struct file_meta meta;
if (get_file_meta(dbpath, &meta) == -1)
RETURN(-1);
if (sqlite3_open(dbpath,&rhizome_db)){
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
}
@ -246,13 +232,38 @@ int rhizome_opendb()
if (version<1){
/* Create tables as required */
sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;", END);
if ( sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);", END) == -1
|| sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, version integer,inserttime integer, filesize integer, filehash text, author text, bar blob, manifest blob);", END) == -1
|| sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, length integer, highestpriority integer, datavalid integer, inserttime integer);", END) == -1
|| sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILEBLOBS(id text not null primary key, data blob);", END) == -1
|| sqlite_exec_void_retry(&retry, "DROP TABLE IF EXISTS FILEMANIFESTS;", END) == -1
|| sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);", END) == -1
|| sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);", END) == -1
if ( sqlite_exec_void_retry(&retry,
"CREATE TABLE IF NOT EXISTS MANIFESTS("
"id text not null primary key, "
"version integer, "
"inserttime integer, "
"filesize integer, "
"filehash text, "
"author text, "
"bar blob, "
"manifest blob, "
"service text, "
"name text, "
"sender text collate nocase, "
"recipient text collate nocase, "
"tail integer"
");", END) == -1
|| sqlite_exec_void_retry(&retry,
"CREATE TABLE IF NOT EXISTS FILES("
"id text not null primary key, "
"length integer, "
"datavalid integer, "
"inserttime integer"
");", END) == -1
|| sqlite_exec_void_retry(&retry,
"CREATE TABLE IF NOT EXISTS FILEBLOBS("
"id text not null primary key, "
"data blob"
");", END) == -1
|| sqlite_exec_void_retry(&retry,
"CREATE TABLE IF NOT EXISTS IDENTITY("
"uuid text not null"
"); ", END) == -1
) {
RETURN(WHY("Failed to create schema"));
}
@ -261,12 +272,14 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
}
if (version<2){
if (version<2 && meta.mtime.tv_sec != -1){
// we need to populate these fields on upgrade from very old versions, we can simply re-insert all old manifests
// at some point we may deprecate upgrading the database and simply drop it and create a new one
// if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;", END);
// if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
verify_bundles();
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
}
@ -274,14 +287,30 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
}
if (version<4){
if (version<4 && meta.mtime.tv_sec != -1){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
}
if (version<5){
if (version<5 && meta.mtime.tv_sec != -1){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
}
if (version<6){
if (meta.mtime.tv_sec != -1){
// we've always been at war with eurasia
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPLIST; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPMEMBERSHIPS; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS VERIFICATIONS; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS FILEMANIFESTS;", END);
}
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=6;", END);
}
// TODO recreate tables with collate nocase on all hex columns
/* Future schema updates should be performed here.
The above schema can be assumed to exist, no matter which version we upgraded from.
All changes should attempt to preserve all existing interesting data */
char buf[UUID_STRLEN + 1];
int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
@ -306,12 +335,6 @@ int rhizome_opendb()
DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
}
// TODO recreate tables with collate nocase on hex columns
/* Future schema updates should be performed here.
The above schema can be assumed to exist.
All changes should attempt to preserve any existing data */
// We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open)
rhizome_cleanup(NULL);
@ -877,7 +900,7 @@ int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
if (!statement)
return -1;
@ -1138,21 +1161,6 @@ int _sqlite_blob_close(struct __sourceloc __whence, int log_level, sqlite3_blob
return 0;
}
static uint64_t rhizome_database_used_bytes()
{
uint64_t db_page_size;
uint64_t db_page_count;
uint64_t db_free_page_count;
if ( sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL
|| sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL
|| sqlite_exec_uint64(&db_free_page_count, "PRAGMA free_count;", END) == -1LL
) {
WHY("Cannot measure database used bytes");
return UINT64_MAX;
}
return db_page_size * (db_page_count - db_free_page_count);
}
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp)
{
IN();
@ -1166,47 +1174,6 @@ int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t versio
OUT();
}
static int rhizome_delete_external(const char *id)
{
// attempt to remove any external blob
char blob_path[1024];
if (!FORMF_RHIZOME_STORE_PATH(blob_path, "%s/%s", RHIZOME_BLOB_SUBDIR, id))
return -1;
if (unlink(blob_path) == -1) {
if (errno != ENOENT)
return WHYF_perror("unlink(%s)", alloca_str_toprint(blob_path));
return 1;
}
if (config.debug.rhizome_store)
DEBUGF("Deleted blob file %s", blob_path);
return 0;
}
static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
{
return sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
"DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
END);
}
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
{
int ret = 0;
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
"DELETE FROM FILES WHERE id = ? and datavalid = 0;",
RHIZOME_FILEHASH_T, hashp, END
) == -1
)
ret = -1;
if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
"DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
RHIZOME_FILEHASH_T, hashp, END
) == -1
)
ret = -1;
return ret;
}
int rhizome_cleanup(struct rhizome_cleanup_report *report)
{
IN();
@ -1218,20 +1185,15 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
/* For testing, it helps to speed up the cleanup process. */
const char *orphan_payload_persist_ms = getenv("SERVALD_ORPHAN_PAYLOAD_PERSIST_MS");
const char *invalid_payload_persist_ms = getenv("SERVALD_INVALID_PAYLOAD_PERSIST_MS");
time_ms_t now = gettime_ms();
time_ms_t insert_horizon_no_manifest = now - (orphan_payload_persist_ms ? atoi(orphan_payload_persist_ms) : 1000); // 1 second ago
time_ms_t insert_horizon_not_valid = now - (invalid_payload_persist_ms ? atoi(invalid_payload_persist_ms) : 300000); // 5 minutes ago
// Remove external payload files for stale, incomplete payloads.
unsigned candidates = 0;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 0;",
INT64, insert_horizon_not_valid, END);
"SELECT id FROM FILES WHERE datavalid = 0;", END);
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
candidates++;
const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_delete_external(id) == 0 && report)
if (rhizome_delete_file_id(id)==0 && report)
++report->deleted_stale_incoming_files;
}
sqlite3_finalize(statement);
@ -1241,9 +1203,8 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
"SELECT id FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
INT64, insert_horizon_no_manifest, END);
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
candidates++;
const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_delete_external(id) == 0 && report)
if (rhizome_delete_file_id(id)==0 && report)
++report->deleted_orphan_files;
}
sqlite3_finalize(statement);
@ -1252,29 +1213,16 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
// referenced or are stale. This could take a long time, so for scalability should be done
// in an incremental background task. See GitHub issue #50.
// Remove payload records that are stale and incomplete or old and unreferenced.
int ret;
if (candidates) {
ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILES WHERE inserttime < ? AND datavalid = 0;",
INT64, insert_horizon_not_valid, END);
if (report && ret > 0)
report->deleted_stale_incoming_files += ret;
ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILES WHERE inserttime < ? AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
INT64, insert_horizon_no_manifest, END);
if (report && ret > 0)
report->deleted_orphan_files += ret;
}
// Remove payload blobs that are no longer referenced.
if ((ret = rhizome_delete_orphan_fileblobs_retry(&retry)) > 0 && report)
int ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
END);
if (ret > 0 && report)
report->deleted_orphan_fileblobs += ret;
// delete manifests that no longer have payload files
ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM MANIFESTS WHERE inserttime < ? AND filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);",
INT64, insert_horizon_no_manifest, END);
"DELETE FROM MANIFESTS WHERE filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);", END);
if (report && ret > 0)
report->deleted_orphan_manifests += ret;
@ -1289,118 +1237,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
OUT();
}
int rhizome_make_space(int group_priority, uint64_t bytes)
{
/* Asked for impossibly large amount */
const size_t margin = 65536;
const uint64_t limit = config.rhizome.database_size > margin ? config.rhizome.database_size - margin : 1;
if (bytes >= limit)
return WHYF("bytes=%"PRIu64" is too large", bytes);
uint64_t db_used = rhizome_database_used_bytes();
if (db_used == UINT64_MAX)
return -1;
rhizome_cleanup(NULL);
/* If there is already enough space now, then do nothing more */
if (db_used + bytes <= limit)
return 0;
/* Okay, not enough space, so free up some. */
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id,length FROM FILES WHERE highestpriority < ? ORDER BY DESCENDING LENGTH",
INT, group_priority, END);
if (!statement)
return -1;
while (db_used + bytes > limit && sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
/* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
const char *id;
/* Get values */
if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
id = (const char *) sqlite3_column_text(statement, 0);
else {
WHY("Incorrect type in id column of files table");
break;
}
if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER)
; //length=sqlite3_column_int(statement, 1);
else {
WHY("Incorrect type in length column of files table");
break;
}
rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, id) == -1)
WHYF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
else {
/* Try to drop this file from storage, discarding any references that do not trump the
* priority of this request. The query done earlier should ensure this, but it doesn't hurt
* to be paranoid, and it also protects against inconsistency in the database.
*/
rhizome_drop_stored_file(&hash, group_priority + 1);
if ((db_used = rhizome_database_used_bytes()) == UINT64_MAX)
break;
}
}
sqlite3_finalize(statement);
//int64_t equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM
//FILES WHERE highestpriority = ? and length > ?", INT, group_priority, INT64, bytes, END);
/* XXX Get rid of any equal priority files that are larger than this one */
/* XXX Get rid of any higher priority files that are not relevant in this time or location */
/* Couldn't make space */
return WHY("Incomplete");
}
/* Drop the specified file from storage, and any manifests that reference it, provided that none of
* those manifests are being retained at a higher priority than the maximum specified here.
*/
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement)
return WHYF("Could not drop stored file id=%s", alloca_tohex_rhizome_filehash_t(*hashp));
int can_drop = 1;
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
/* Find manifests for this file */
if (sqlite3_column_type(statement, 0) != SQLITE_TEXT) {
WHYF("Incorrect type in id column of manifests table");
break;
}
const char *q_id = (char *) sqlite3_column_text(statement, 0);
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, q_id) == -1) {
WARNF("malformed column value MANIFESTS.id = %s -- skipped", alloca_str_toprint(q_id));
continue;
}
/* Check that manifest is not part of a higher priority group.
If so, we cannot drop the manifest or the file.
However, we will keep iterating, as we can still drop any other manifests pointing to this file
that are lower priority, and thus free up a little space. */
int priority = rhizome_manifest_priority(&retry, &bid);
if (priority == -1)
WHYF("Cannot drop fileid=%s due to error, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
else if (priority > maximum_priority) {
WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
can_drop = 0;
} else {
if (config.debug.rhizome)
DEBUGF("removing stale manifests, groupmemberships");
sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &bid, END);
sqlite_exec_void_retry(&retry, "DELETE FROM KEYPAIRS WHERE public = ?;", RHIZOME_BID_T, &bid, END);
sqlite_exec_void_retry(&retry, "DELETE FROM GROUPMEMBERSHIPS WHERE manifestid = ?;", RHIZOME_BID_T, &bid, END);
}
}
sqlite3_finalize(statement);
if (can_drop)
rhizome_delete_file_retry(&retry, hashp);
return 0;
}
/*
Store the specified manifest into the sqlite database.
We assume that sufficient space has been made for us.
@ -1419,8 +1255,7 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
The trick is to insert the blob as all zeroes using a special function, and then
substitute bytes in the blog progressively.
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES,
and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES tables.
*/
int rhizome_store_manifest(rhizome_manifest *m)
{
@ -1493,52 +1328,6 @@ int rhizome_store_manifest(rhizome_manifest *m)
rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db));
rhizome_manifest_set_inserttime(m, now);
// if (serverMode)
// rhizome_sync_bundle_inserted(bar);
// TODO remove old payload?
#if 0
if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
int closed=rhizome_manifest_get_ll(m,"closedgroup");
if (closed<1) closed=0;
int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
if (ciphered<1) ciphered=0;
if ((stmt = sqlite_prepare_bind(&retry,
"INSERT OR REPLACE INTO GROUPLIST(id,closed,ciphered,priority) VALUES (?,?,?,?);",
RHIZOME_BID_T, &m->cryptoSignPublic,
INT, closed,
INT, ciphered,
INT, RHIZOME_PRIORITY_DEFAULT,
END
)
) == NULL
)
goto rollback;
if (sqlite_step_retry(&retry, stmt) == -1)
goto rollback;
sqlite3_finalize(stmt);
stmt = NULL;
}
#endif
#if 0
if (m->group_count > 0) {
if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS (manifestid, groupid) VALUES (?, ?);")) == NULL)
goto rollback;
unsigned i;
for (i=0;i<m->group_count;i++){
if (sqlite_bind(&retry, stmt, RHIZOME_BID_T, &m->cryptoSignPublic, TEXT, m->groups[i]) == -1)
goto rollback;
if (sqlite_step_retry(&retry, stmt) == -1)
goto rollback;
sqlite3_reset(stmt);
}
sqlite3_finalize(stmt);
stmt = NULL;
}
#endif
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
// This message used in tests; do not modify or remove.
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRIu64,
@ -1753,29 +1542,6 @@ void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCoun
(void) tohex(out, byteCount * 2, in);
}
int rhizome_update_file_priority(const char *fileid)
{
/* work out the highest priority of any referrer */
uint64_t highestPriority = 0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_uint64_retry(&retry, &highestPriority,
"SELECT max(grouplist.priority) FROM MANIFESTS, GROUPMEMBERSHIPS, GROUPLIST"
" WHERE MANIFESTS.filehash = ?"
" AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id"
" AND GROUPMEMBERSHIPS.groupid = GROUPLIST.id;",
TEXT_TOUPPER, fileid, END
) == -1
)
return -1;
if (sqlite_exec_void_retry(&retry,
"UPDATE files SET highestPriority = ? WHERE id = ?;",
INT64, highestPriority, TEXT_TOUPPER, fileid, END
) == -1
)
return WHYF("cannot update priority for fileid=%s", fileid);
return 0;
}
/* Search the database for a manifest having the same name and payload content, and if the version
* is known, having the same version. Returns RHIZOME_BUNDLE_STATUS_DUPLICATE if a duplicate is found
* (setting *found to point to the duplicate's manifest), returns RHIZOME_BUNDLE_STATUS_NEW if no
@ -1961,33 +1727,6 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom
return sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
{
int ret = 0;
rhizome_delete_external(alloca_tohex_rhizome_filehash_t(*hashp));
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1;
statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1;
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
{
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
if (rows == -1)
return -1;
rhizome_filehash_t hash;
if (str_to_rhizome_filehash_t(&hash, strbuf_str(fh)) == -1)
return WHYF("invalid field FILES.id=%s", strbuf_str(fh));
if (rows && rhizome_delete_file_retry(retry, &hash) == -1)
return -1;
return 0;
}
/* Remove a manifest and its bundle from the database, given its manifest ID.
*
* Returns 0 if manifest is found and removed and bundle was either absent or removed
@ -1998,10 +1737,9 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome
*/
int rhizome_delete_bundle(const rhizome_bid_t *bidp)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_delete_payload_retry(&retry, bidp) == -1)
if (rhizome_delete_payload(bidp) == -1)
return -1;
if (rhizome_delete_manifest_retry(&retry, bidp) == -1)
if (rhizome_delete_manifest(bidp) == -1)
return -1;
return 0;
}
@ -2021,35 +1759,6 @@ int rhizome_delete_manifest(const rhizome_bid_t *bidp)
return rhizome_delete_manifest_retry(&retry, bidp);
}
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
* untouched if present.
*
* Returns 0 if manifest is found, its payload is found and removed
* Returns 1 if manifest or payload is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_payload(const rhizome_bid_t *bidp)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_payload_retry(&retry, bidp);
}
/* Remove a file from the database, given its file hash.
*
* Returns 0 if file is found and removed
* Returns 1 if file is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_file(const rhizome_filehash_t *hashp)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_retry(&retry, hashp);
}
static int is_interesting(const char *id_hex, uint64_t version)
{
IN();

3
rhizome_direct_http.c
View File

@ -119,6 +119,9 @@ static int rhizome_direct_import_end(struct http_request *hr)
case RHIZOME_BUNDLE_STATUS_FAKE:
http_request_simple_response(&r->http, 403, "Manifest not signed");
return 0;
case RHIZOME_BUNDLE_STATUS_DONOTWANT:
http_request_simple_response(&r->http, 403, "Not enough space");
return 0;
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_ERROR:
break;

7
rhizome_fetch.c
View File

@ -552,12 +552,14 @@ schedule_fetch(struct rhizome_fetch_slot *slot)
slot->request_len = strbuf_len(r);
enum rhizome_payload_status status = rhizome_open_write(&slot->write_state,
&slot->manifest->filehash,
slot->manifest->filesize,
RHIZOME_PRIORITY_DEFAULT);
slot->manifest->filesize);
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_STORED:
RETURN(IMPORTED);
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
RETURN(DONOTWANT);
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
@ -824,6 +826,7 @@ static void rhizome_start_next_queued_fetch(struct rhizome_fetch_slot *slot)
case SAMEBUNDLE:
case SAMEPAYLOAD:
case SUPERSEDED:
case DONOTWANT:
case NEWERBUNDLE:
default:
// Discard the candidate fetch and loop to try the next in queue.

4
rhizome_restful.c
View File

@ -495,6 +495,10 @@ static int restful_rhizome_insert_end(struct http_request *hr)
http_request_simple_response(&r->http, 403, strbuf_str(msg));
return 403;
}
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
http_request_simple_response(&r->http, 403, "Not enough space");
return 403;
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest");
return 403;

205
rhizome_store.c
View File

@ -76,14 +76,187 @@ static uint64_t rhizome_create_fileblob(sqlite_retry_state *retry, uint64_t id,
return rowid;
}
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority)
static int rhizome_delete_external(const char *id)
{
// attempt to remove any external blob
char blob_path[1024];
if (!FORMF_RHIZOME_STORE_PATH(blob_path, "%s/%s", RHIZOME_BLOB_SUBDIR, id))
return -1;
if (unlink(blob_path) == -1) {
if (errno != ENOENT)
return WHYF_perror("unlink(%s)", alloca_str_toprint(blob_path));
return 1;
}
if (config.debug.rhizome_store)
DEBUGF("Deleted blob file %s", blob_path);
return 0;
}
static int rhizome_delete_file_id_retry(sqlite_retry_state *retry, const char *id)
{
int ret = 0;
rhizome_delete_external(id);
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", STATIC_TEXT, id, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1;
statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", STATIC_TEXT, id, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1;
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
{
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
if (rows == -1)
return -1;
if (rows && rhizome_delete_file_id_retry(retry, strbuf_str(fh)) == -1)
return -1;
return 0;
}
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
* untouched if present.
*
* Returns 0 if manifest is found, its payload is found and removed
* Returns 1 if manifest or payload is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_payload(const rhizome_bid_t *bidp)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_payload_retry(&retry, bidp);
}
int rhizome_delete_file_id(const char *id)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_id_retry(&retry, id);
}
/* Remove a file from the database, given its file hash.
*
* Returns 0 if file is found and removed
* Returns 1 if file is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_file(const rhizome_filehash_t *hashp)
{
return rhizome_delete_file_id(alloca_tohex_rhizome_filehash_t(*hashp));
}
// TODO readonly version?
static enum rhizome_payload_status store_make_space(uint64_t bytes)
{
uint64_t external_bytes;
uint64_t db_page_size;
uint64_t db_page_count;
uint64_t db_free_page_count;
// TODO limit based on free space?
// No limit?
if (config.rhizome.database_size==0)
return RHIZOME_PAYLOAD_STATUS_NEW;
// TODO index external_bytes calculation and/or cache result
if ( sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL
|| sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL
|| sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END) == -1LL
|| sqlite_exec_uint64(&external_bytes,
"SELECT SUM(length) "
"FROM FILES "
"WHERE NOT EXISTS( "
"SELECT 1 "
"FROM FILEBLOBS "
"WHERE FILES.ID = FILEBLOBS.ID "
");", END) == -1LL
)
return WHY("Cannot measure database used bytes");
if (config.rhizome.database_size < db_page_size*10)
return WHYF("rhizome.database_size is too small to store anything");
const uint64_t limit = config.rhizome.database_size - db_page_size*4;
uint64_t db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
if (bytes >= limit){
if (config.debug.rhizome)
DEBUGF("Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64,
bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
return RHIZOME_PAYLOAD_STATUS_TOO_BIG;
}
// If there is enough space, do nothing
if (db_used + bytes <= limit)
return RHIZOME_PAYLOAD_STATUS_NEW;
// penalise new things by 10 minutes to reduce churn
time_ms_t cost = gettime_ms() - 60000 - bytes;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// query files by age, penalise larger files so they are removed earlier
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, length, inserttime FROM FILES ORDER BY (inserttime - length)",
END);
if (!statement)
return RHIZOME_PAYLOAD_STATUS_ERROR;
while (db_used + bytes > limit && sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
const char *id=(const char *) sqlite3_column_text(statement, 0);
uint64_t length = sqlite3_column_int(statement, 1);
time_ms_t inserttime = sqlite3_column_int64(statement, 2);
time_ms_t cost_existing = inserttime - length;
if (config.debug.rhizome)
DEBUGF("Considering dropping file %s, size %"PRId64" cost %"PRId64" vs %"PRId64" to add %"PRId64" new bytes",
id, length, cost, cost_existing, bytes);
// don't allow the new file, we've got more important things to store
if (cost < cost_existing)
break;
// drop the existing content and recalculate used space
if (rhizome_delete_external(id)==0)
external_bytes -= length;
sqlite3_stmt *s = sqlite_prepare_bind(&retry, "DELETE FROM fileblobs WHERE id = ?", STATIC_TEXT, id, END);
if (s)
sqlite_exec_retry(&retry, s);
s = sqlite_prepare_bind(&retry, "DELETE FROM files WHERE id = ?", STATIC_TEXT, id, END);
if (s)
sqlite_exec_retry(&retry, s);
sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END);
sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END);
db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
}
sqlite3_finalize(statement);
if (db_used + bytes <= limit)
return RHIZOME_PAYLOAD_STATUS_NEW;
if (config.debug.rhizome)
DEBUGF("Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64,
bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
return RHIZOME_PAYLOAD_STATUS_UNINITERESTING;
}
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length)
{
if (file_length == 0)
return RHIZOME_PAYLOAD_STATUS_EMPTY;
write->blob_fd=-1;
write->sql_blob=NULL;
write->priority = priority;
if (expectedHashp){
if (rhizome_exists(expectedHashp))
@ -93,6 +266,13 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons
}else{
write->id_known=0;
}
if (file_length!=RHIZOME_SIZE_UNSET){
enum rhizome_payload_status status = store_make_space(file_length);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
}
time_ms_t now = gettime_ms();
static uint64_t last_id=0;
write->temp_id = now;
@ -452,6 +632,9 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
if (config.debug.rhizome_store)
DEBUGF("Wrote %"PRIu64" bytes, set file_length", write->file_offset);
write->file_length = write->file_offset;
status = store_make_space(write->file_length);
if (status!=RHIZOME_PAYLOAD_STATUS_NEW)
goto failure;
}
// flush out any remaining buffered pieces to disk
@ -533,7 +716,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
rhizome_remove_file_datainvalid(&retry, &write->id);
if (rhizome_exists(&write->id)) {
// we've already got that payload, delete the new copy
if (write->blob_rowid){
@ -552,10 +735,9 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
if (sqlite_exec_void_retry(
&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES(?,?,?,1,?);",
"INSERT OR REPLACE INTO FILES(id,length,datavalid,inserttime) VALUES(?,?,1,?);",
RHIZOME_FILEHASH_T, &write->id,
INT64, write->file_length,
INT, write->priority,
INT64, gettime_ms(),
END
) == -1
@ -615,7 +797,7 @@ enum rhizome_payload_status rhizome_import_payload_from_file(rhizome_manifest *m
struct rhizome_write write;
bzero(&write, sizeof(write));
enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
@ -644,7 +826,7 @@ enum rhizome_payload_status rhizome_import_buffer(rhizome_manifest *m, unsigned
struct rhizome_write write;
bzero(&write, sizeof(write));
enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
@ -714,8 +896,7 @@ enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *wr
enum rhizome_payload_status status = rhizome_open_write(
write,
m->has_filehash ? &m->filehash : NULL,
m->filesize,
RHIZOME_PRIORITY_DEFAULT
m->filesize
);
if (status == RHIZOME_PAYLOAD_STATUS_NEW)
status = rhizome_write_derive_key(m, write);
@ -735,6 +916,8 @@ enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, cons
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
@ -765,6 +948,8 @@ enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, cons
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
break;
default:
FATALF("status = %d", status);
@ -1329,7 +1514,7 @@ enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *wri
if (advance_by > 0)
rhizome_manifest_set_tail(m, m->tail + advance_by);
rhizome_manifest_set_version(m, m->filesize);
enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT);
enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize);
if (status == RHIZOME_PAYLOAD_STATUS_NEW && copy_length > 0) {
// note that we don't need to bother decrypting the existing journal payload
enum rhizome_payload_status rstatus = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);

51
tests/rhizomeops
View File

@ -1198,7 +1198,7 @@ test_DeleteManifest() {
assert diff file2 file2x
rhizome_clean
assert [ $deleted_files = 1 ]
assert [ $deleted_fileblobs = 1 ]
assert [ $deleted_fileblobs = 0 ]
assert [ $deleted_manifests = 0 ]
}
@ -1257,4 +1257,53 @@ test_DeleteFile() {
assert_rhizome_list file{2..4}
}
doc_payloadTooBig="Fail to insert a payload that is larger than the database"
setup_payloadTooBig() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 32K
}
test_payloadTooBig(){
create_file file1 32K
execute $servald rhizome add file $SIDA file1 file1.manifest
assertExitStatus '==' 7
}
doc_payloadUninteresting="Fail to insert a payload that is uninteresting"
setup_payloadUninteresting() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 64K
}
test_payloadUninteresting(){
create_file file1 32K
create_file file2 32K
executeOk_servald rhizome add file $SIDA file1 file1.manifest
execute $servald rhizome add file $SIDA file2 file2.manifest
assertExitStatus '==' 7
}
doc_evictUninteresting="Evict a large payload to make room for smaller payloads"
setup_evictUninteresting() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 1M
}
test_evictUninteresting(){
create_file file1 512K
create_file file2 256K
create_file file3 128K
create_file file4 128K
executeOk_servald rhizome add file $SIDA file1 file1.manifest
executeOk_servald rhizome add file $SIDA file2 file2.manifest
executeOk_servald rhizome add file $SIDA file3 file3.manifest
executeOk_servald rhizome add file $SIDA file4 file4.manifest
rhizome_clean
executeOk_servald rhizome list
assert_rhizome_list file{2,3,4}
}
runTests "$@"