ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-20 05:37:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use config to set and remember rhizome database location

Browse Source
This commit is contained in:
Jeremy Lakeman 2018-04-10 10:50:06 +09:30
parent 57767e444d
commit 59b6a70374
14 changed files with 129 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
instance.c
View File

@ -105,17 +105,13 @@ void set_instance_path(const char *path)
know_instancepath = 1; know_instancepath = 1;
} }
static int vformf_path(struct __sourceloc __whence, strbuf b, const char *syspath, const char *fmt, va_list ap) static int vformf_basepath(struct __sourceloc __whence, strbuf b, const char *basepath, const char *fmt, va_list ap)
{ {
if (fmt) if (fmt)
strbuf_va_vprintf(b, fmt, ap); strbuf_va_vprintf(b, fmt, ap);
if (!strbuf_overrun(b) && (strbuf_len(b) == 0 || strbuf_str(b)[0] != '/')) { if (!strbuf_overrun(b) && (strbuf_len(b) == 0 || strbuf_str(b)[0] != '/')) {
strbuf_reset(b); strbuf_reset(b);
const char *ipath = instance_path(); strbuf_puts(b, basepath);
#ifdef ANDROID
assert(ipath != NULL);
#endif
strbuf_puts(b, ipath ? ipath : syspath);
if (fmt) { if (fmt) {
if (strbuf_substr(b, -1)[0] != '/') if (strbuf_substr(b, -1)[0] != '/')
strbuf_putc(b, '/'); strbuf_putc(b, '/');
@ -131,6 +127,23 @@ static int vformf_path(struct __sourceloc __whence, strbuf b, const char *syspat
return 0; return 0;
} }
int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_basepath(__whence, strbuf_local(buf, bufsiz), basepath, fmt, ap);
va_end(ap);
return ret;
}
static int vformf_path(struct __sourceloc __whence, strbuf b, const char *syspath, const char *fmt, va_list ap)
{
const char *ipath = instance_path();
if (!ipath)
ipath=syspath;
return vformf_basepath(__whence, b, ipath, fmt, ap);
}
int _formf_serval_etc_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...) int _formf_serval_etc_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...)
{ {
va_list ap; va_list ap;
@ -231,7 +244,5 @@ int create_serval_instance_dir()
ret = -1; ret = -1;
if (FORMF_SERVALD_PROC_PATH(path, NULL) && emkdirs_info(path, 0755) == -1) if (FORMF_SERVALD_PROC_PATH(path, NULL) && emkdirs_info(path, 0755) == -1)
ret = -1; ret = -1;
if (FORMF_RHIZOME_STORE_PATH(path, NULL) && emkdirs_info(path, 0700) == -1)
ret = -1;
return ret; return ret;
} }

5
instance.h
View File

@ -31,6 +31,8 @@ void set_instance_path(const char *path);
int create_serval_instance_dir(); int create_serval_instance_dir();
int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,5,6)));
int _formf_serval_etc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_serval_etc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _vformf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list); int _vformf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list);
@ -39,6 +41,8 @@ int _formf_serval_tmp_path(struct __sourceloc, char *buf, size_t bufsiz, const c
int _formf_servald_proc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_servald_proc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_rhizome_store_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_rhizome_store_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
#define formf_path(buf,bufsz,basepath,fmt,...) _formf_path(__WHENCE__, buf, bufsz, basepath, fmt, ##__VA_ARGS__)
#define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) #define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) #define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap) #define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap)
@ -57,7 +61,6 @@ int _formf_rhizome_store_path(struct __sourceloc, char *buf, size_t bufsiz, cons
#define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) #define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) #define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) #define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_rhizome_store_path((buf), sizeof(buf), (fmt), ##__VA_ARGS__)
strbuf strbuf_system_log_path(strbuf sb); strbuf strbuf_system_log_path(strbuf sb);
strbuf strbuf_serval_log_path(strbuf sb); strbuf strbuf_serval_log_path(strbuf sb);

10
meshmb_restful.c
View File

@ -453,8 +453,8 @@ static int restful_meshmb_newsince_list(httpd_request *r, const char *remainder)
static char *find_token_to_str(char *buf, uint64_t rowid) static char *find_token_to_str(char *buf, uint64_t rowid)
{ {
struct iovec iov[2]; struct iovec iov[2];
iov[0].iov_base = rhizome_db_uuid.u.binary; iov[0].iov_base = rhizome_database.uuid.u.binary;
iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; iov[0].iov_len = sizeof rhizome_database.uuid.u.binary;
iov[1].iov_base = &rowid; iov[1].iov_base = &rowid;
iov[1].iov_len = sizeof rowid; iov[1].iov_len = sizeof rowid;
size_t n = base64url_encodev(buf, iov, 2); size_t n = base64url_encodev(buf, iov, 2);
@ -465,11 +465,11 @@ static char *find_token_to_str(char *buf, uint64_t rowid)
static int strn_to_find_token(const char *str, uint64_t *rowidp, const char **afterp) static int strn_to_find_token(const char *str, uint64_t *rowidp, const char **afterp)
{ {
unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; unsigned char token[sizeof rhizome_database.uuid.u.binary + sizeof *rowidp];
if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token
&& cmp_uuid_t(&rhizome_db_uuid, (serval_uuid_t *) &token) == 0 && cmp_uuid_t(&rhizome_database.uuid, (serval_uuid_t *) &token) == 0
&& **afterp=='/'){ && **afterp=='/'){
memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); memcpy(rowidp, token + sizeof rhizome_database.uuid.u.binary, sizeof *rowidp);
(*afterp)++; (*afterp)++;
}else{ }else{
// don't skip the token // don't skip the token

6
rhizome.c
View File

@ -69,14 +69,14 @@ int is_rhizome_http_enabled()
{ {
return config.rhizome.enable return config.rhizome.enable
&& config.rhizome.http.enable && config.rhizome.http.enable
&& rhizome_db; && rhizome_database.db;
} }
int is_rhizome_mdp_enabled() int is_rhizome_mdp_enabled()
{ {
return config.rhizome.enable return config.rhizome.enable
&& config.rhizome.mdp.enable && config.rhizome.mdp.enable
&& rhizome_db; && rhizome_database.db;
} }
int is_rhizome_mdp_server_running() int is_rhizome_mdp_server_running()
@ -88,7 +88,7 @@ int is_rhizome_advertise_enabled()
{ {
return config.rhizome.enable return config.rhizome.enable
&& config.rhizome.advertise.enable && config.rhizome.advertise.enable
&& rhizome_db && rhizome_database.db
&& (is_httpd_server_running() || is_rhizome_mdp_server_running()); && (is_httpd_server_running() || is_rhizome_mdp_server_running());
} }

11
rhizome.h
View File

@ -340,8 +340,15 @@ int rhizome_fetch_delay_ms();
#define RHIZOME_BLOB_SUBDIR "blob" #define RHIZOME_BLOB_SUBDIR "blob"
#define RHIZOME_HASH_SUBDIR "hash" #define RHIZOME_HASH_SUBDIR "hash"
extern __thread sqlite3 *rhizome_db; struct rhizome_database{
extern serval_uuid_t rhizome_db_uuid; sqlite3 *db;
serval_uuid_t uuid;
char folder[1024];
};
extern __thread struct rhizome_database rhizome_database;
#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_path((buf), sizeof(buf), rhizome_database.folder, (fmt), ##__VA_ARGS__)
int rhizome_opendb(); int rhizome_opendb();
int rhizome_close_db(); int rhizome_close_db();

91
rhizome_database.c
View File

@ -37,17 +37,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
static int create_rhizome_store_dir() __thread struct rhizome_database rhizome_database = {NULL,{{{0}}},{0}};
{
char rdpath[1024];
if (!formf_rhizome_store_path(rdpath, sizeof rdpath, "%s", config.rhizome.datastore_path))
return -1;
DEBUGF(rhizome, "Rhizome datastore path = %s", alloca_str_toprint(rdpath));
return emkdirs_info(rdpath, 0700);
}
__thread sqlite3 *rhizome_db = NULL;
serval_uuid_t rhizome_db_uuid;
static time_ms_t rhizomeRetryLimit = -1; static time_ms_t rhizomeRetryLimit = -1;
int is_debug_rhizome() int is_debug_rhizome()
@ -238,8 +228,8 @@ void verify_bundles()
int rhizome_opendb() int rhizome_opendb()
{ {
if (rhizome_db) { if (rhizome_database.db) {
assert(serval_uuid_is_valid(&rhizome_db_uuid)); assert(serval_uuid_is_valid(&rhizome_database.uuid));
return 0; return 0;
} }
@ -248,8 +238,11 @@ int rhizome_opendb()
if (sodium_init()==-1) if (sodium_init()==-1)
RETURN(WHY("Failed to initialise libsodium")); RETURN(WHY("Failed to initialise libsodium"));
if (create_rhizome_store_dir() == -1) if (!formf_rhizome_store_path(rhizome_database.folder, sizeof rhizome_database.folder, "%s", config.rhizome.datastore_path))
RETURN(-1); RETURN (-1);
DEBUGF(rhizome, "Rhizome datastore path = %s", alloca_str_toprint(rhizome_database.folder));
if (emkdirs_info(rhizome_database.folder, 0700)==-1)
RETURN (-1);
char dbpath[1024]; char dbpath[1024];
if (!FORMF_RHIZOME_STORE_PATH(dbpath, RHIZOME_BLOB_SUBDIR)) if (!FORMF_RHIZOME_STORE_PATH(dbpath, RHIZOME_BLOB_SUBDIR))
RETURN(-1); RETURN(-1);
@ -275,13 +268,13 @@ int rhizome_opendb()
if (get_file_meta(dbpath, &meta) == -1) if (get_file_meta(dbpath, &meta) == -1)
RETURN(-1); RETURN(-1);
if (sqlite3_open(dbpath,&rhizome_db)){ if (sqlite3_open(dbpath,&rhizome_database.db)){
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db))); RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_database.db)));
} }
sqlite3_trace_v2(rhizome_db, SQLITE_TRACE_STMT, sqlite_trace_callback, NULL); sqlite3_trace_v2(rhizome_database.db, SQLITE_TRACE_STMT, sqlite_trace_callback, NULL);
int loglevel = IF_DEBUG(rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT; int loglevel = IF_DEBUG(rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;
const char *env = getenv("SERVALD_RHIZOME_DB_RETRY_LIMIT_MS"); const char *env = getenv("SERVALD_rhizome_database.db_RETRY_LIMIT_MS");
rhizomeRetryLimit = env ? atoi(env) : -1; rhizomeRetryLimit = env ? atoi(env) : -1;
/* Read Rhizome configuration */ /* Read Rhizome configuration */
@ -394,20 +387,20 @@ int rhizome_opendb()
if (r == -1) if (r == -1)
RETURN(-1); RETURN(-1);
if (r) { if (r) {
if (!str_to_serval_uuid(buf, &rhizome_db_uuid, NULL)) { if (!str_to_serval_uuid(buf, &rhizome_database.uuid, NULL)) {
WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf)); WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf));
if (serval_uuid_generate_random(&rhizome_db_uuid) == -1) if (serval_uuid_generate_random(&rhizome_database.uuid) == -1)
RETURN(WHY("Cannot generate new UUID for Rhizome database")); RETURN(WHY("Cannot generate new UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", SERVAL_UUID_T, &rhizome_db_uuid, END) == -1) if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", SERVAL_UUID_T, &rhizome_database.uuid, END) == -1)
RETURN(WHY("Failed to update new UUID in Rhizome database")); RETURN(WHY("Failed to update new UUID in Rhizome database"));
DEBUGF(rhizome, "Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid)); DEBUGF(rhizome, "Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_database.uuid));
} }
} else if (r == 0) { } else if (r == 0) {
if (serval_uuid_generate_random(&rhizome_db_uuid) == -1) if (serval_uuid_generate_random(&rhizome_database.uuid) == -1)
RETURN(WHY("Cannot generate UUID for Rhizome database")); RETURN(WHY("Cannot generate UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", SERVAL_UUID_T, &rhizome_db_uuid, END) == -1) if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", SERVAL_UUID_T, &rhizome_database.uuid, END) == -1)
RETURN(WHY("Failed to insert UUID into Rhizome database")); RETURN(WHY("Failed to insert UUID into Rhizome database"));
DEBUGF(rhizome, "Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid)); DEBUGF(rhizome, "Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_database.uuid));
} }
// We can't delete a file that is being transferred in another process at this very moment... // We can't delete a file that is being transferred in another process at this very moment...
@ -420,7 +413,7 @@ int rhizome_opendb()
"FROM manifests", END); "FROM manifests", END);
} }
INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_db_uuid)); INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_database.uuid));
RETURN(0); RETURN(0);
OUT(); OUT();
} }
@ -428,23 +421,23 @@ int rhizome_opendb()
int rhizome_close_db() int rhizome_close_db()
{ {
IN(); IN();
if (rhizome_db) { if (rhizome_database.db) {
rhizome_cache_close(); rhizome_cache_close();
if (!sqlite3_get_autocommit(rhizome_db)){ if (!sqlite3_get_autocommit(rhizome_database.db)){
WHY("Uncommitted transaction!"); WHY("Uncommitted transaction!");
sqlite_exec_void("ROLLBACK;", END); sqlite_exec_void("ROLLBACK;", END);
} }
sqlite3_stmt *stmt = NULL; sqlite3_stmt *stmt = NULL;
while ((stmt = sqlite3_next_stmt(rhizome_db, stmt))) { while ((stmt = sqlite3_next_stmt(rhizome_database.db, stmt))) {
const char *sql = sqlite3_sql(stmt); const char *sql = sqlite3_sql(stmt);
WARNF("closing Rhizome db with unfinalised statement: %s", sql ? sql : "BLOB"); WARNF("closing Rhizome db with unfinalised statement: %s", sql ? sql : "BLOB");
} }
int r = sqlite3_close(rhizome_db); int r = sqlite3_close(rhizome_database.db);
if (r != SQLITE_OK) if (r != SQLITE_OK)
RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_db))); RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_database.db)));
} }
rhizome_db=NULL; rhizome_database.db=NULL;
RETURN(0); RETURN(0);
OUT(); OUT();
} }
@ -519,7 +512,7 @@ int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const
retry->elapsed = now - retry->start; retry->elapsed = now - retry->start;
INFOF("%s on try %u after %.3f seconds (limit %.3f): %s", INFOF("%s on try %u after %.3f seconds (limit %.3f): %s",
sqlite3_errmsg(rhizome_db), sqlite3_errmsg(rhizome_database.db),
retry->busytries, retry->busytries,
(retry->elapsed) / 1e3, (retry->elapsed) / 1e3,
(retry->limit) / 1e3, (retry->limit) / 1e3,
@ -576,9 +569,9 @@ sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite
{ {
IN(); IN();
sqlite3_stmt *statement = NULL; sqlite3_stmt *statement = NULL;
assert(rhizome_db); assert(rhizome_database.db);
while (1) { while (1) {
switch (sqlite3_prepare_v2(rhizome_db, sqltext, -1, &statement, NULL)) { switch (sqlite3_prepare_v2(rhizome_database.db, sqltext, -1, &statement, NULL)) {
case SQLITE_OK: case SQLITE_OK:
sqlite_trace_done = 0; sqlite_trace_done = 0;
RETURN(statement); RETURN(statement);
@ -589,7 +582,7 @@ sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite
} }
// fall through... // fall through...
default: default:
LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_db), sqltext); LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_database.db), sqltext);
sqlite3_finalize(statement); sqlite3_finalize(statement);
RETURN(NULL); RETURN(NULL);
} }
@ -605,7 +598,7 @@ sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite
*/ */
int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap) int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap)
{ {
const int index_limit = sqlite3_limit(rhizome_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1); const int index_limit = sqlite3_limit(rhizome_database.db, SQLITE_LIMIT_VARIABLE_NUMBER, -1);
unsigned argnum = 0; unsigned argnum = 0;
int index_counter = 0; int index_counter = 0;
enum sqlbind_type typ; enum sqlbind_type typ;
@ -657,7 +650,7 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
continue; \ continue; \
FALLTHROUGH; \ FALLTHROUGH; \
default: \ default: \
LOGF(log_level, #FUNC "(%d) failed, %s: %s", index, sqlite3_errmsg(rhizome_db), sqlite3_sql(statement)); \ LOGF(log_level, #FUNC "(%d) failed, %s: %s", index, sqlite3_errmsg(rhizome_database.db), sqlite3_sql(statement)); \
sqlite3_finalize(statement); \ sqlite3_finalize(statement); \
return -1; \ return -1; \
} \ } \
@ -962,7 +955,7 @@ int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state
} }
// fall through... // fall through...
default: default:
LOGF(log_level, "query failed (%d), %s: %s", ret, sqlite3_errmsg(rhizome_db), sqlite3_sql(statement)); LOGF(log_level, "query failed (%d), %s: %s", ret, sqlite3_errmsg(rhizome_database.db), sqlite3_sql(statement));
statement = NULL; statement = NULL;
break; break;
} }
@ -982,7 +975,7 @@ int _sqlite_exec_code(struct __sourceloc __whence, int log_level, sqlite_retry_s
++(*rowcount); ++(*rowcount);
sqlite3_finalize(statement); sqlite3_finalize(statement);
if (sqlite_trace_func()) if (sqlite_trace_func())
_DEBUGF("rowcount=%d changes=%d", *rowcount, sqlite3_changes(rhizome_db)); _DEBUGF("rowcount=%d changes=%d", *rowcount, sqlite3_changes(rhizome_database.db));
return stepcode; return stepcode;
} }
@ -1019,7 +1012,7 @@ static int _sqlite_vexec_void_code(struct __sourceloc __whence, int log_level, s
return SQLITE_ERROR; return SQLITE_ERROR;
int stepcode = _sqlite_exec_code(__whence, log_level, retry, statement, rowcount); int stepcode = _sqlite_exec_code(__whence, log_level, retry, statement, rowcount);
if (sqlite_code_ok(stepcode)){ if (sqlite_code_ok(stepcode)){
*changes = sqlite3_changes(rhizome_db); *changes = sqlite3_changes(rhizome_database.db);
} }
return stepcode; return stepcode;
} }
@ -1095,7 +1088,7 @@ static int _sqlite_vexec_uint64(struct __sourceloc __whence, sqlite_retry_state
FATALF("query unexpectedly returned %d rows", rows); FATALF("query unexpectedly returned %d rows", rows);
sqlite3_finalize(statement); sqlite3_finalize(statement);
if (sqlite_trace_func()) if (sqlite_trace_func())
_DEBUGF("rowcount=%d changes=%d result=%"PRIu64, rows, sqlite3_changes(rhizome_db), *result); _DEBUGF("rowcount=%d changes=%d result=%"PRIu64, rows, sqlite3_changes(rhizome_database.db), *result);
if (sqlite_code_ok(stepcode) && rows>0) if (sqlite_code_ok(stepcode) && rows>0)
return SQLITE_ROW; return SQLITE_ROW;
return stepcode; return stepcode;
@ -1189,7 +1182,7 @@ int _sqlite_blob_open_retry(
{ {
IN(); IN();
while (1) { while (1) {
int code = sqlite3_blob_open(rhizome_db, dbname, tablename, colname, rowid, flags, blobp); int code = sqlite3_blob_open(rhizome_database.db, dbname, tablename, colname, rowid, flags, blobp);
switch (code) { switch (code) {
case SQLITE_OK: case SQLITE_OK:
if (retry) if (retry)
@ -1205,7 +1198,7 @@ int _sqlite_blob_open_retry(
break; // back to sqlite3_blob_open() break; // back to sqlite3_blob_open()
// fall through... // fall through...
default: default:
LOGF(log_level, "sqlite3_blob_open() failed (%d), %s", code, sqlite3_errmsg(rhizome_db)); LOGF(log_level, "sqlite3_blob_open() failed (%d), %s", code, sqlite3_errmsg(rhizome_database.db));
RETURN(-1); RETURN(-1);
} }
} }
@ -1241,7 +1234,7 @@ int _sqlite_blob_write_retry(
break; // back to sqlite3_blob_open() break; // back to sqlite3_blob_open()
// fall through... // fall through...
default: default:
LOGF(log_level, "sqlite3_blob_write() failed (%d), %s", code, sqlite3_errmsg(rhizome_db)); LOGF(log_level, "sqlite3_blob_write() failed (%d), %s", code, sqlite3_errmsg(rhizome_database.db));
RETURN(-1); RETURN(-1);
} }
} }
@ -1253,7 +1246,7 @@ int _sqlite_blob_close(struct __sourceloc __whence, int log_level, sqlite3_blob
{ {
int code = sqlite3_blob_close(blob); int code = sqlite3_blob_close(blob);
if (code != SQLITE_OK) if (code != SQLITE_OK)
LOGF(log_level, "sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_db)); LOGF(log_level, "sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_database.db));
return 0; return 0;
} }
@ -1504,7 +1497,7 @@ enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rh
goto rollback; goto rollback;
sqlite3_finalize(stmt); sqlite3_finalize(stmt);
stmt = NULL; stmt = NULL;
rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db)); rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_database.db));
rhizome_manifest_set_inserttime(m, now); rhizome_manifest_set_inserttime(m, now);
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){ if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
@ -2019,7 +2012,7 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom
return -1; return -1;
if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1) if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
return -1; return -1;
return sqlite3_changes(rhizome_db) ? 0 : 1; return sqlite3_changes(rhizome_database.db) ? 0 : 1;
} }
/* Remove a manifest and its bundle from the database, given its manifest ID. /* Remove a manifest and its bundle from the database, given its manifest ID.

4
rhizome_direct.c
View File

@ -616,11 +616,11 @@ int rhizome_direct_get_bars(const rhizome_bid_t *bidp_low,
break; break;
} }
int64_t rowid = sqlite3_column_int64(statement, 1); int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar", do ret = sqlite3_blob_open(rhizome_database.db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob); rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open")); while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) { if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db)); WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_database.db));
continue; continue;
} }
sqlite_retry_done(&retry, "sqlite3_blob_open"); sqlite_retry_done(&retry, "sqlite3_blob_open");

4
rhizome_direct_http.c
View File

@ -839,11 +839,11 @@ static rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,
{ {
int ret; int ret;
int64_t rowid = sqlite3_column_int64(statement, 1); int64_t rowid = sqlite3_column_int64(statement, 1);
do ret = sqlite3_blob_open(rhizome_db, "main", "manifests", "bar", do ret = sqlite3_blob_open(rhizome_database.db, "main", "manifests", "bar",
rowid, 0 /* read only */, &blob); rowid, 0 /* read only */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open")); while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (!sqlite_code_ok(ret)) { if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db)); WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_database.db));
sqlite3_finalize(statement); sqlite3_finalize(statement);
return NULL; return NULL;

2
rhizome_packetformats.c
View File

@ -182,7 +182,7 @@ int overlay_rhizome_saw_advertisements(struct decode_context *context, struct ov
if (!f) if (!f)
RETURN(-1); RETURN(-1);
if (!(rhizome_db && config.rhizome.fetch)) if (!(rhizome_database.db && config.rhizome.fetch))
RETURN(0); RETURN(0);
int ad_frame_type=ob_get(f->payload); int ad_frame_type=ob_get(f->payload);

10
rhizome_restful.c
View File

@ -69,8 +69,8 @@ static void finalise_union_rhizome_list(httpd_request *r)
static char *list_token_to_str(char *buf, uint64_t rowid) static char *list_token_to_str(char *buf, uint64_t rowid)
{ {
struct iovec iov[2]; struct iovec iov[2];
iov[0].iov_base = rhizome_db_uuid.u.binary; iov[0].iov_base = rhizome_database.uuid.u.binary;
iov[0].iov_len = sizeof rhizome_db_uuid.u.binary; iov[0].iov_len = sizeof rhizome_database.uuid.u.binary;
iov[1].iov_base = &rowid; iov[1].iov_base = &rowid;
iov[1].iov_len = sizeof rowid; iov[1].iov_len = sizeof rowid;
size_t n = base64url_encodev(buf, iov, 2); size_t n = base64url_encodev(buf, iov, 2);
@ -81,11 +81,11 @@ static char *list_token_to_str(char *buf, uint64_t rowid)
static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp) static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp)
{ {
unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp]; unsigned char token[sizeof rhizome_database.uuid.u.binary + sizeof *rowidp];
if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) == sizeof token
&& cmp_serval_uuid_t(&rhizome_db_uuid, (serval_uuid_t *) &token) == 0 && cmp_serval_uuid_t(&rhizome_database.uuid, (serval_uuid_t *) &token) == 0
&& **afterp=='/'){ && **afterp=='/'){
memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp); memcpy(rowidp, token + sizeof rhizome_database.uuid.u.binary, sizeof *rowidp);
(*afterp)++; (*afterp)++;
}else{ }else{
// don't skip the token // don't skip the token

32
rhizome_store.c
View File

@ -42,10 +42,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
uint64_t rhizome_copy_file_to_blob(int fd, uint64_t id, size_t size); uint64_t rhizome_copy_file_to_blob(int fd, uint64_t id, size_t size);
static int form_store_blob_path(char *buff, size_t buff_size, const char *subdir, const rhizome_filehash_t *hash){ #define FORM_BLOB_PATH(BUFF,SUBDIR,HASH) FORMF_RHIZOME_STORE_PATH((BUFF),"%s/%02X/%02X/%s", (SUBDIR), (HASH)->binary[0], (HASH)->binary[1], alloca_tohex(&(HASH)->binary[2], sizeof((HASH)->binary)-2))
return formf_rhizome_store_path(buff, buff_size, "%s/%02X/%02X/%s", subdir, hash->binary[0], hash->binary[1], alloca_tohex(&hash->binary[2], sizeof(hash->binary)-2));
}
#define FORM_BLOB_PATH(BUFF,SUBDIR,HASH) form_store_blob_path((BUFF),sizeof(BUFF),(SUBDIR),(HASH))
enum rhizome_payload_status rhizome_exists(const rhizome_filehash_t *hashp) enum rhizome_payload_status rhizome_exists(const rhizome_filehash_t *hashp)
{ {
@ -73,8 +70,10 @@ enum rhizome_payload_status rhizome_exists(const rhizome_filehash_t *hashp)
struct stat st; struct stat st;
if (stat(legacy_path, &st) == 0 if (stat(legacy_path, &st) == 0
&& emkdirsn(legacy_path, strrchr(legacy_path,'/') - legacy_path, 0700)!=-1 && emkdirsn(legacy_path, strrchr(legacy_path,'/') - legacy_path, 0700)!=-1
&& rename(legacy_path, blob_path) != -1) && rename(legacy_path, blob_path) != -1){
INFOF("Moved %s to %s", legacy_path, blob_path);
return RHIZOME_PAYLOAD_STATUS_STORED; return RHIZOME_PAYLOAD_STATUS_STORED;
}
} }
uint64_t blob_rowid = 0; uint64_t blob_rowid = 0;
@ -107,7 +106,7 @@ static uint64_t rhizome_create_fileblob(sqlite_retry_state *retry, uint64_t id,
WHYF("Failed to create blob, size=%zu, id=%"PRIu64, size, id); WHYF("Failed to create blob, size=%zu, id=%"PRIu64, size, id);
return 0; return 0;
} }
uint64_t rowid = sqlite3_last_insert_rowid(rhizome_db); uint64_t rowid = sqlite3_last_insert_rowid(rhizome_database.db);
DEBUGF(rhizome_store, "Inserted fileblob rowid=%"PRId64" for id='%"PRIu64"'", rowid, id); DEBUGF(rhizome_store, "Inserted fileblob rowid=%"PRId64" for id='%"PRIu64"'", rowid, id);
return rowid; return rowid;
} }
@ -139,7 +138,7 @@ static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_fi
statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, filehash, END); statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, filehash, END);
if (!statement || sqlite_exec_retry(retry, statement) == -1) if (!statement || sqlite_exec_retry(retry, statement) == -1)
ret = -1; ret = -1;
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1; return ret == -1 ? -1 : sqlite3_changes(rhizome_database.db) ? 0 : 1;
} }
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp) static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
@ -195,14 +194,11 @@ static uint64_t store_get_free_space()
space = atol(fake_space); space = atol(fake_space);
#if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H)) #if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H))
else { else {
char store_path[1024]; struct statvfs stats;
if (FORMF_RHIZOME_STORE_PATH(store_path, "rhizome.db")) { if (statvfs(rhizome_database.folder, &stats)==-1)
struct statvfs stats; WARNF_perror("statvfs(%s)", rhizome_database.folder);
if (statvfs(store_path, &stats)==-1) else
WARNF_perror("statvfs(%s)", store_path); space = stats.f_frsize * (uint64_t)stats.f_bavail;
else
space = stats.f_frsize * (uint64_t)stats.f_bavail;
}
} }
#endif #endif
if (IF_DEBUG(rhizome)) { if (IF_DEBUG(rhizome)) {
@ -1125,8 +1121,10 @@ enum rhizome_payload_status rhizome_open_read(struct rhizome_read *read, const r
struct stat st; struct stat st;
if (stat(legacy_path, &st) == 0 if (stat(legacy_path, &st) == 0
&& emkdirsn(legacy_path, strrchr(legacy_path,'/') - legacy_path, 0700)!=-1 && emkdirsn(legacy_path, strrchr(legacy_path,'/') - legacy_path, 0700)!=-1
&& rename(legacy_path, blob_path) != -1) && rename(legacy_path, blob_path) != -1){
INFOF("Moved %s to %s", legacy_path, blob_path);
fd = open(blob_path, O_RDONLY); fd = open(blob_path, O_RDONLY);
}
} }
} }
@ -1204,7 +1202,7 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset); ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset);
} while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read")); } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read"));
if (ret != SQLITE_OK) { if (ret != SQLITE_OK) {
WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_db)); WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_database.db));
sqlite_blob_close(blob); sqlite_blob_close(blob);
RETURN(-1); RETURN(-1);
} }

2
rhizome_sync.c
View File

@ -546,7 +546,7 @@ DEFINE_TRIGGER(nbr_change, neighbour_changed);
DEFINE_BINDING(MDP_PORT_RHIZOME_SYNC, overlay_mdp_service_rhizome_sync); DEFINE_BINDING(MDP_PORT_RHIZOME_SYNC, overlay_mdp_service_rhizome_sync);
static int overlay_mdp_service_rhizome_sync(struct internal_mdp_header *header, struct overlay_buffer *payload) static int overlay_mdp_service_rhizome_sync(struct internal_mdp_header *header, struct overlay_buffer *payload)
{ {
if (!config.rhizome.enable || !rhizome_db) if (!config.rhizome.enable || !rhizome_database.db)
return 0; return 0;
struct rhizome_sync *state = header->source->sync_state; struct rhizome_sync *state = header->source->sync_state;

4
server.c
View File

@ -614,7 +614,7 @@ void server_watchdog(struct sched_ent *alarm)
DEFINE_ALARM(rhizome_clean_db); DEFINE_ALARM(rhizome_clean_db);
void rhizome_clean_db(struct sched_ent *alarm) void rhizome_clean_db(struct sched_ent *alarm)
{ {
if (!config.rhizome.enable || !rhizome_db) if (!config.rhizome.enable || !rhizome_database.db)
return; return;
time_ms_t now = gettime_ms(); time_ms_t now = gettime_ms();
@ -647,7 +647,7 @@ static void server_on_config_change()
if (config.rhizome.enable){ if (config.rhizome.enable){
rhizome_opendb(); rhizome_opendb();
RESCHEDULE(&ALARM_STRUCT(rhizome_clean_db), now + 30*60*1000, TIME_MS_NEVER_WILL, TIME_MS_NEVER_WILL); RESCHEDULE(&ALARM_STRUCT(rhizome_clean_db), now + 30*60*1000, TIME_MS_NEVER_WILL, TIME_MS_NEVER_WILL);
}else if(rhizome_db){ }else if(rhizome_database.db){
rhizome_close_db(); rhizome_close_db();
} }
} }

19
tests/rhizomeops
View File

@ -61,6 +61,25 @@ set_rhizome_config() {
set log.console.level debug set log.console.level debug
} }
doc_DatabaseFolder="Database is created based on config"
setup_DatabaseFolder() {
set_rhizome_config() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_manifest on \
set debug.verbose on \
set log.console.level debug \
set rhizome.datastore_path "$instance_dir/moved_rhizome"
}
setup_servald
setup_rhizome
executeOk_servald rhizome list
}
test_DatabaseFolder() {
executeOk find "$instance_dir"
assert [ -e "$instance_dir/moved_rhizome/rhizome.db" ]
}
doc_InitialEmptyList="Initial list is empty" doc_InitialEmptyList="Initial list is empty"
setup_InitialEmptyList() { setup_InitialEmptyList() {
setup_servald setup_servald