ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-19 05:07:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor rhizome db creation and execution functions

Browse Source
This commit is contained in:
Andrew Bettison 2012-06-12 18:12:36 +09:30
parent b2ffc6e72c
commit 56cb12f507
4 changed files with 186 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
rhizome.h
View File

@ -248,7 +248,11 @@ int rhizome_server_close_http_request(int i);
int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r); int rhizome_server_http_send_bytes(int rn,rhizome_http_request *r);
int rhizome_server_parse_http_request(int rn,rhizome_http_request *r); int rhizome_server_parse_http_request(int rn,rhizome_http_request *r);
int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response); int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response);
int sqlite_prepare(sqlite3_stmt **statement, const strbuf stmt);
int sqlite_prepare_loglevel(int log_level, sqlite3_stmt **statement, const strbuf stmt);
int sqlite_exec_void(const char *sqlformat,...); int sqlite_exec_void(const char *sqlformat,...);
int sqlite_exec_void_loglevel(int log_level, const char *sqlformat, ...);
int sqlite_exec_void_strbuf_loglevel(int log_level, const strbuf stmt);
int sqlite_exec_int64(long long *result, const char *sqlformat,...); int sqlite_exec_int64(long long *result, const char *sqlformat,...);
int sqlite_exec_strbuf(strbuf sb, const char *sqlformat,...); int sqlite_exec_strbuf(strbuf sb, const char *sqlformat,...);
int rhizome_server_http_response_header(rhizome_http_request *r,int result, int rhizome_server_http_response_header(rhizome_http_request *r,int result,

290
rhizome_database.c
View File

@ -146,89 +146,108 @@ int rhizome_opendb()
DEBUGF("serval.conf:rhizome_kb=%.f", rhizome_kb); DEBUGF("serval.conf:rhizome_kb=%.f", rhizome_kb);
DEBUGF("Rhizome will use %lldB of storage for its database.", rhizome_space); DEBUGF("Rhizome will use %lldB of storage for its database.", rhizome_space);
} }
/* Create tables as required */
/* Create tables if required */ if ( sqlite_exec_void("PRAGMA auto_vacuum=2;") == -1
if (sqlite3_exec(rhizome_db, || sqlite_exec_void("CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);") == -1
"PRAGMA auto_vacuum=2;" || sqlite_exec_void("CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer,inserttime integer, bar blob);") == -1
"CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);" || sqlite_exec_void("CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, data blob, length integer, highestpriority integer, datavalid integer);") == -1
|| sqlite_exec_void("DROP TABLE IF EXISTS FILEMANIFESTS;") == -1
"CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer,inserttime integer, bar blob);" || sqlite_exec_void("CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);") == -1
|| sqlite_exec_void("CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);") == -1
"CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, data blob, length integer, highestpriority integer, datavalid integer);" ) {
return WHY("Failed to create schema");
"DROP TABLE IF EXISTS FILEMANIFESTS;" }
"CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);" // No easy way to tell if these columns already exist, should probably create some kind of schema
"CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);" // version table. Running these a second time will fail.
sqlite_exec_void_loglevel(LOG_LEVEL_INFO, "ALTER TABLE MANIFESTS ADD COLUMN filesize text;");
,NULL,NULL,NULL)) sqlite_exec_void_loglevel(LOG_LEVEL_INFO, "ALTER TABLE MANIFESTS ADD COLUMN filehash text;");
{ sqlite_exec_void_loglevel(LOG_LEVEL_INFO, "ALTER TABLE FILES ADD inserttime integer;");
return WHYF("Failed to create required schema: %s", sqlite3_errmsg(rhizome_db)); /* Upgrade schema */
} if ( sqlite_exec_void("CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);") == -1
// no easy way to tell if these columns already exist, should probably create some kind of schema version table || sqlite_exec_void("DELETE FROM MANIFESTS WHERE filehash IS NULL;") == -1
// running this a second time will fail. || sqlite_exec_void("DELETE FROM FILES WHERE NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);") == -1
sqlite3_exec(rhizome_db, || sqlite_exec_void("DELETE FROM MANIFESTS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);") == -1
"ALTER TABLE MANIFESTS ADD COLUMN filesize text;" ) {
"ALTER TABLE MANIFESTS ADD COLUMN filehash text;" return WHY("Failed to create schema");
"ALTER TABLE FILES ADD inserttime integer;"
,NULL,NULL,NULL);
if (sqlite3_exec(rhizome_db,
"CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);"
"DELETE FROM MANIFESTS WHERE filehash IS NULL;"
"DELETE FROM FILES WHERE NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);"
"DELETE FROM MANIFESTS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);"
,NULL,NULL,NULL)){
return WHYF("Failed to create required schema: %s", sqlite3_errmsg(rhizome_db));
} }
return 0; return 0;
} }
/* /*
Convenience wrapper for executing an SQL command that returns a no value. Convenience wrapper for preparing an SQL command.
Returns -1 if an error occurs, otherwise zero. Returns -1 if an error occurs (logged as an error), otherwise zero with the prepared
statement in *statement.
*/ */
int sqlite_exec_void(const char *sqlformat,...) int sqlite_prepare(sqlite3_stmt **statement, const strbuf stmt)
{
return sqlite_prepare_loglevel(LOG_LEVEL_ERROR, statement, stmt);
}
int sqlite_prepare_loglevel(int log_level, sqlite3_stmt **statement, const strbuf stmt)
{ {
if (!rhizome_db) rhizome_opendb();
strbuf stmt = strbuf_alloca(8192);
va_list ap;
va_start(ap, sqlformat);
strbuf_vsprintf(stmt, sqlformat, ap);
va_end(ap);
if (strbuf_overrun(stmt)) if (strbuf_overrun(stmt))
return WHYF("Sql statement overrun: %s", strbuf_str(stmt)); return WHYF("Sql statement overrun: %s", strbuf_str(stmt));
sqlite3_stmt *statement; if (!rhizome_db && rhizome_opendb() == -1)
return -1;
switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement, NULL)) { switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, statement, NULL)) {
case SQLITE_OK: case SQLITE_DONE: case SQLITE_OK: case SQLITE_DONE:
break; break;
default: default:
WHY(strbuf_str(stmt)); LOGF(log_level, "%s in %s", sqlite3_errmsg(rhizome_db), strbuf_str(stmt));
WHY(sqlite3_errmsg(rhizome_db)); sqlite3_finalize(*statement);
sqlite3_finalize(statement);
return -1; return -1;
} }
int stepcode;
int rows = 0;
while ((stepcode = sqlite3_step(statement)) == SQLITE_ROW)
++rows;
if (rows) WARNF("query unexpectedly returned %d row%s", rows, rows == 1 ? "" : "s");
switch (stepcode) {
case SQLITE_OK:
case SQLITE_DONE:
case SQLITE_ROW:
break;
default:
WHY(strbuf_str(stmt));
WHY(sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return -1;
}
sqlite3_finalize(statement);
return 0; return 0;
} }
/*
Convenience wrapper for executing an SQL command that returns a no value.
If an error occurs then logs it at error level and returns -1. Otherwise returns zero.
*/
int sqlite_exec_void(const char *sqlformat, ...)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_va_printf(stmt, sqlformat);
return sqlite_exec_void_strbuf_loglevel(LOG_LEVEL_ERROR, stmt);
}
int sqlite_exec_void_loglevel(int log_level, const char *sqlformat, ...)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_va_printf(stmt, sqlformat);
return sqlite_exec_void_strbuf_loglevel(log_level, stmt);
}
/*
Convenience wrapper for executing an SQL command that returns a no value.
If an error occurs then logs it at the given level and returns -1. Otherwise returns zero.
*/
int sqlite_exec_void_strbuf_loglevel(int log_level, const strbuf stmt)
{
sqlite3_stmt *statement;
int ret = sqlite_prepare_loglevel(log_level, &statement, stmt);
if (ret != -1) {
int stepcode;
int rows = 0;
while ((stepcode = sqlite3_step(statement)) == SQLITE_ROW)
++rows;
if (rows) WARNF("void query unexpectedly returned %d row%s", rows, rows == 1 ? "" : "s");
switch (stepcode) {
case SQLITE_OK:
case SQLITE_DONE:
case SQLITE_ROW:
ret = 0;
break;
default:
ret = -1;
LOGF(log_level, "%s in %s", sqlite3_errmsg(rhizome_db), strbuf_str(stmt));
break;
}
sqlite3_finalize(statement);
}
return ret;
}
/* /*
Convenience wrapper for executing an SQL command that returns a single int64 value. Convenience wrapper for executing an SQL command that returns a single int64 value.
Returns -1 if an error occurs. Returns -1 if an error occurs.
@ -239,48 +258,37 @@ int sqlite_exec_void(const char *sqlformat,...)
*/ */
int sqlite_exec_int64(long long *result, const char *sqlformat,...) int sqlite_exec_int64(long long *result, const char *sqlformat,...)
{ {
if (!rhizome_db) rhizome_opendb();
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
va_list ap; strbuf_va_printf(stmt, sqlformat);
va_start(ap, sqlformat);
strbuf_vsprintf(stmt, sqlformat, ap);
va_end(ap);
if (strbuf_overrun(stmt))
return WHYF("sql statement too long: %s", strbuf_str(stmt));
sqlite3_stmt *statement; sqlite3_stmt *statement;
switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement, NULL)) { int ret = sqlite_prepare(&statement, stmt);
case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW: if (ret != -1) {
break; int rowcount = 0;
default: int stepcode = sqlite3_step(statement);
WHY(strbuf_str(stmt)); if (stepcode == SQLITE_ROW) {
WHY(sqlite3_errmsg(rhizome_db)); int n = sqlite3_column_count(statement);
sqlite3_finalize(statement); if (n != 1)
return -1; ret = WHYF("Incorrect column count %d (should be 1): %s", n, strbuf_str(stmt));
} else {
int stepcode; rowcount = 1;
int rowcount = 0; *result = sqlite3_column_int64(statement, 0);
if ((stepcode = sqlite3_step(statement)) == SQLITE_ROW) { while ((stepcode = sqlite3_step(statement)) == SQLITE_ROW)
int n = sqlite3_column_count(statement); ++rowcount;
if (n != 1) { }
sqlite3_finalize(statement);
return WHYF("Incorrect column count %d (should be 1): %s", n, strbuf_str(stmt));
} }
*result = sqlite3_column_int64(statement, 0); if (ret != -1) {
rowcount = 1; switch (stepcode) {
while ((stepcode = sqlite3_step(statement)) == SQLITE_ROW) case SQLITE_OK: case SQLITE_DONE:
++rowcount; ret = rowcount;
break;
default:
ret = WHYF("%s in %s", sqlite3_errmsg(rhizome_db), strbuf_str(stmt));
break;
}
}
sqlite3_finalize(statement);
} }
switch (stepcode) { return ret;
case SQLITE_OK: case SQLITE_DONE:
break;
default:
WHY(strbuf_str(stmt));
WHY(sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return -1;
}
sqlite3_finalize(statement);
return rowcount;
} }
/* /*
@ -293,47 +301,52 @@ int sqlite_exec_int64(long long *result, const char *sqlformat,...)
*/ */
int sqlite_exec_strbuf(strbuf sb, const char *sqlformat,...) int sqlite_exec_strbuf(strbuf sb, const char *sqlformat,...)
{ {
if (!rhizome_db) rhizome_opendb();
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
va_list ap; strbuf_va_printf(stmt, sqlformat);
va_start(ap, sqlformat);
strbuf_vsprintf(stmt, sqlformat, ap);
va_end(ap);
if (strbuf_overrun(stmt))
return WHYF("sql statement too long: %s", strbuf_str(stmt));
sqlite3_stmt *statement; sqlite3_stmt *statement;
switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement,NULL)) { int ret = sqlite_prepare(&statement, stmt);
case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW: if (ret != -1) {
break; int rowcount = 0;
default: int stepcode = sqlite3_step(statement);
sqlite3_finalize(statement); if (stepcode == SQLITE_ROW) {
WHY(strbuf_str(stmt)); int n = sqlite3_column_count(statement);
WHY(sqlite3_errmsg(rhizome_db)); if (n != 1)
return WHY("Could not prepare sql statement."); ret = WHYF("Incorrect column count %d (should be 1): %s", n, strbuf_str(stmt));
} else {
int rows = 0; rowcount = 1;
if (sqlite3_step(statement) == SQLITE_ROW) { strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
int n = sqlite3_column_count(statement); while ((stepcode = sqlite3_step(statement)) == SQLITE_ROW)
if (n != 1) { ++rowcount;
sqlite3_finalize(statement); }
return WHYF("Incorrect column count %d (should be 1)", n); }
if (ret != -1) {
switch (stepcode) {
case SQLITE_OK: case SQLITE_DONE:
ret = rowcount;
break;
default:
ret = -1;
WHY(strbuf_str(stmt));
WHY(sqlite3_errmsg(rhizome_db));
break;
}
} }
strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
sqlite3_finalize(statement); sqlite3_finalize(statement);
++rows;
} }
if (sqlite3_step(statement) == SQLITE_ROW) return ret;
++rows;
sqlite3_finalize(statement);
return rows;
} }
long long rhizome_database_used_bytes() long long rhizome_database_used_bytes()
{ {
long long db_page_size=sqlite_exec_void("PRAGMA page_size;"); long long db_page_size;
long long db_page_count=sqlite_exec_void("PRAGMA page_count;"); long long db_page_count;
long long db_free_page_count=sqlite_exec_void("PRAGMA free_count;"); long long db_free_page_count;
return db_page_size*(db_page_count-db_free_page_count); if ( sqlite_exec_int64(&db_page_size, "PRAGMA page_size;") == -1LL
|| sqlite_exec_int64(&db_page_count, "PRAGMA page_count;") == -1LL
|| sqlite_exec_int64(&db_free_page_count, "PRAGMA free_count;") == -1LL
)
return WHY("Cannot measure database used bytes");
return db_page_size * (db_page_count - db_free_page_count);
} }
int rhizome_make_space(int group_priority, long long bytes) int rhizome_make_space(int group_priority, long long bytes)
@ -343,7 +356,8 @@ int rhizome_make_space(int group_priority, long long bytes)
/* Asked for impossibly large amount */ /* Asked for impossibly large amount */
if (bytes>=(rhizome_space-65536)) return -1; if (bytes>=(rhizome_space-65536)) return -1;
long long db_used=rhizome_database_used_bytes(); long long db_used = rhizome_database_used_bytes();
if (db_used == -1)
/* If there is already enough space now, then do nothing more */ /* If there is already enough space now, then do nothing more */
if (db_used<=(rhizome_space-bytes-65536)) return 0; if (db_used<=(rhizome_space-bytes-65536)) return 0;

1
serval.h
View File

@ -754,6 +754,7 @@ typedef struct overlay_txqueue {
#define OQ_MAX 5 #define OQ_MAX 5
extern overlay_txqueue overlay_tx[OQ_MAX]; extern overlay_txqueue overlay_tx[OQ_MAX];
#define LOG_LEVEL_SILENT (-1)
#define LOG_LEVEL_DEBUG (0) #define LOG_LEVEL_DEBUG (0)
#define LOG_LEVEL_INFO (1) #define LOG_LEVEL_INFO (1)
#define LOG_LEVEL_WARN (2) #define LOG_LEVEL_WARN (2)

33
strbuf.h
View File

@ -85,7 +85,7 @@ typedef const struct strbuf *const_strbuf;
*/ */
#define SIZEOF_STRBUF (sizeof(struct strbuf)) #define SIZEOF_STRBUF (sizeof(struct strbuf))
/** Convenience function for allocating a strbuf and its backing buffer on the /** Convenience macro for allocating a strbuf and its backing buffer on the
* stack within the calling function. The returned strbuf is only valid for * stack within the calling function. The returned strbuf is only valid for
* the duration of the function, so it must not be returned. See alloca(3) for * the duration of the function, so it must not be returned. See alloca(3) for
* more information. * more information.
@ -96,21 +96,46 @@ typedef const struct strbuf *const_strbuf;
* strbuf_puts(b, " some more text"); * strbuf_puts(b, " some more text");
* printf("%s\n", strbuf_str(b)); * printf("%s\n", strbuf_str(b));
* } * }
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
#define strbuf_alloca(size) strbuf_make(alloca(SIZEOF_STRBUF + size), SIZEOF_STRBUF + size) #define strbuf_alloca(size) strbuf_make(alloca(SIZEOF_STRBUF + size), SIZEOF_STRBUF + size)
/** Allocate a strbuf for use within the calling function, using a /** Convenience macro for filling a strbuf from the calling function's
* caller-supplied backing buffer. The returned strbuf is only valid for the * printf(3)-like variadic arguments. The returned strbuf is only valid for
* duration of the function, so it must not be returned. See alloca(3) for * the duration of the function, so it must not be returned. See alloca(3) for
* more information. * more information.
* *
* #include <stdarg.h>
*
* void funcf(const char *format, ...) {
* strbuf b = strbuf_alloca_fmtargs(1024, format);
* ...
* }
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
#define strbuf_va_printf(sb,fmt) do { \
va_list __strbuf_ap; \
va_start(__strbuf_ap, fmt); \
strbuf_vsprintf(sb, fmt, __strbuf_ap); \
va_end(__strbuf_ap); \
} while (0)
/** Convenience macro to allocate a strbuf for use within the calling function,
* based on a caller-supplied backing buffer. The returned strbuf is only valid
* for the duration of the function, so it must not be returned. See alloca(3)
* for more information. However, the backing buffer may have any scope.
*
* void func(char *buf, size_t len) { * void func(char *buf, size_t len) {
* strbuf b = strbuf_local(buf, len); * strbuf b = strbuf_local(buf, len);
* strbuf_puts(b, "some text"); * strbuf_puts(b, "some text");
* strbuf_puts(b, " some more text"); * strbuf_puts(b, " some more text");
* printf("%s\n", strbuf_str(b)); * printf("%s\n", strbuf_str(b));
* } * }
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
#define strbuf_local(buf,len) strbuf_init(alloca(SIZEOF_STRBUF), (buf), (len)) #define strbuf_local(buf,len) strbuf_init(alloca(SIZEOF_STRBUF), (buf), (len))