ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-01-29 15:43:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add 'rhizome delete' and 'rhizome clean' commands

Browse Source 
sqlite_void_exec() and its ilk now return the count of changed rows, not
just zero, on success

sqlite_exec_prepared() and its ilk now return the count of rows (number
of step results SQLITE_ROW), instead of just zero, on success

rhizome_clean() function now produces an optional report of its changes

rhizome_fail_write() and rhizome_finish_write() now log WARNings not
ERRORs if the SQL DELETE FROM FILES or DELETE FROM FILEBLOBS statements
fail

Refactor rhizome_open_read() to use sqlite_exec_int64()

Ensure that 'rhizome extract' and 'rhizome dump' commands return exit
status of 1 in "not found" conditions, not 255, which is reserved for
errors

Test cases for four new commands: 'rhizome delete bundle', 'rhizome
delete manifest', 'rhizome delete payload' and 'rhizome delete file'
(no test case for 'rhizome clean' yet)
This commit is contained in:
Andrew Bettison 2013-02-20 14:44:29 +10:30
parent cf13a4b51e
commit 390655580d
8 changed files with 487 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
commandline.c
View File

@ -1395,7 +1395,69 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, void *context)
return ret; return ret;
} }
int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context) int app_rhizome_delete(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *manifestid, *fileid;
if (cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, NULL) == -1)
return -1;
if (cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1;
/* Ensure the Rhizome database exists and is open */
if (create_serval_instance_dir() == -1)
return -1;
if (rhizome_opendb() == -1)
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
int ret=0;
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
if (!fileid)
return WHY("missing <fileid> argument");
unsigned char filehash[RHIZOME_FILEHASH_BYTES];
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1)
return WHY("Invalid file ID");
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1];
tohex(fileIDUpper, filehash, RHIZOME_FILEHASH_BYTES);
ret = rhizome_delete_file(fileIDUpper);
} else {
if (!manifestid)
return WHY("missing <manifestid> argument");
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
return WHY("Invalid manifest ID");
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0)
ret = rhizome_delete_bundle(manifestIdUpper);
else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0)
ret = rhizome_delete_manifest(manifestIdUpper);
else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0)
ret = rhizome_delete_payload(manifestIdUpper);
else
return WHY("unrecognised command");
}
return ret;
}
int app_rhizome_clean(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
struct rhizome_cleanup_report report;
if (rhizome_cleanup(&report) == -1)
return -1;
cli_field_name("deleted_stale_incoming_files", ":");
cli_put_long(report.deleted_stale_incoming_files, "\n");
cli_field_name("deleted_orphan_files", ":");
cli_put_long(report.deleted_orphan_files, "\n");
cli_field_name("deleted_orphan_fileblobs", ":");
cli_put_long(report.deleted_orphan_fileblobs, "\n");
return 0;
}
int app_rhizome_extract(const struct cli_parsed *parsed, void *context)
{ {
if (config.debug.verbose) if (config.debug.verbose)
DEBUG_cli_parsed(parsed); DEBUG_cli_parsed(parsed);
@ -1484,13 +1546,10 @@ int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
} }
} }
} }
if (retfile) if (retfile)
ret=retfile; ret = retfile == -1 ? -1 : 1;
if (m) if (m)
rhizome_manifest_free(m); rhizome_manifest_free(m);
return ret; return ret;
} }
@ -1502,24 +1561,20 @@ int app_rhizome_dump_file(const struct cli_parsed *parsed, void *context)
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1 if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1) || cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1; return -1;
if (create_serval_instance_dir() == -1) if (create_serval_instance_dir() == -1)
return -1; return -1;
if (rhizome_opendb() == -1) if (rhizome_opendb() == -1)
return -1; return -1;
if (!rhizome_exists(fileid)) if (!rhizome_exists(fileid))
return 1; return 1;
int64_t length; int64_t length;
if (rhizome_dump_file(fileid, filepath, &length)) int ret = rhizome_dump_file(fileid, filepath, &length);
return -1; if (ret)
return ret == -1 ? -1 : 1;
cli_puts("filehash"); cli_delim(":"); cli_puts("filehash"); cli_delim(":");
cli_puts(fileid); cli_delim("\n"); cli_puts(fileid); cli_delim("\n");
cli_puts("filesize"); cli_delim(":"); cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", length); cli_delim("\n"); cli_printf("%lld", length); cli_delim("\n");
return 0; return 0;
} }
@ -2188,28 +2243,40 @@ struct cli_schema command_line_options[]={
"Add a file to Rhizome and optionally write its manifest to the given path"}, "Add a file to Rhizome and optionally write its manifest to the given path"},
{app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE, {app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE,
"Import a payload/manifest pair into Rhizome"}, "Import a payload/manifest pair into Rhizome"},
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE, {app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,
"List all manifests and files in Rhizome"}, "[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
{app_rhizome_extract_bundle,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS, "List all manifests and files in Rhizome"},
{app_rhizome_extract,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest and decrypted file to the given paths."}, "Extract a manifest and decrypted file to the given paths."},
{app_rhizome_extract_bundle,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS, {app_rhizome_extract,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest from Rhizome and write it to the given path"}, "Extract a manifest from Rhizome and write it to the given path"},
{app_rhizome_extract_bundle,{"rhizome","extract","file" KEYRING_PIN_OPTIONS, {app_rhizome_extract,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
"<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path"}, "Extract a file from Rhizome and write it to the given path"},
{app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE, {app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path without attempting decryption"}, "Extract a file from Rhizome and write it to the given path without attempting decryption"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\manifest",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."}, "Remove the manifest for the given bundle from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\payload",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."}, "Remove the payload for the given bundle from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\bundle",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."}, "Remove the manifest and payload for the given bundle from the Rhizome store"},
{app_rhizome_delete,{"rhizome","delete","\\file",
"<fileid>",NULL},CLIFLAG_STANDALONE,
"Remove the file with the given hash from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."},
{app_rhizome_clean,{"rhizome","clean",NULL},CLIFLAG_STANDALONE,
"Remove stale and orphaned content from the Rhizome store"},
{app_keyring_create,{"keyring","create",NULL},0, {app_keyring_create,{"keyring","create",NULL},0,
"Create a new keyring file."}, "Create a new keyring file."},
{app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE, {app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE,

35
rhizome.h
View File

@ -197,8 +197,16 @@ extern sqlite3 *rhizome_db;
int rhizome_opendb(); int rhizome_opendb();
int rhizome_close_db(); int rhizome_close_db();
int rhizome_manifest_createid(rhizome_manifest *m);
struct rhizome_cleanup_report {
int deleted_stale_incoming_files;
int deleted_orphan_files;
int deleted_orphan_fileblobs;
};
int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_strn_is_manifest_id(const char *text); int rhizome_strn_is_manifest_id(const char *text);
int rhizome_str_is_manifest_id(const char *text); int rhizome_str_is_manifest_id(const char *text);
int rhizome_strn_is_bundle_key(const char *text); int rhizome_strn_is_bundle_key(const char *text);
@ -276,17 +284,20 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))();
int is_debug_rhizome(); int is_debug_rhizome();
int is_debug_rhizome_ads(); int is_debug_rhizome_ads();
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...); sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, strbuf stmt); sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt);
int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action); int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action); void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement); int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...); int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...);
int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...); int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...);
int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...); int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat,...); int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat,...); int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...); int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap);
#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) #define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb)) #define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb))
@ -297,9 +308,11 @@ int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...
#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__) #define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__) #define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) #define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__) #define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__) #define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__) #define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__)
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
@ -312,6 +325,10 @@ int rhizome_list_manifests(const char *service, const char *name,
const char *sender_sid, const char *recipient_sid, const char *sender_sid, const char *recipient_sid,
int limit, int offset, char count_rows); int limit, int offset, char count_rows);
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m); int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
int rhizome_delete_bundle(const char *manifestid);
int rhizome_delete_manifest(const char *manifestid);
int rhizome_delete_payload(const char *manifestid);
int rhizome_delete_file(const char *fileid);
#define RHIZOME_DONTVERIFY 0 #define RHIZOME_DONTVERIFY 0
#define RHIZOME_VERIFY 1 #define RHIZOME_VERIFY 1

303
rhizome_database.c
View File

@ -255,8 +255,8 @@ int rhizome_opendb()
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
long long version; long long version;
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;")<0) if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;") == -1)
RETURN(WHY("Failed to check schema version")); RETURN(-1);
if (version<1){ if (version<1){
/* Create tables as required */ /* Create tables as required */
@ -296,7 +296,7 @@ int rhizome_opendb()
// We can't delete a file that is being transferred in another process at this very moment... // We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open) if (config.rhizome.clean_on_open)
rhizome_cleanup(); rhizome_cleanup(NULL);
RETURN(0); RETURN(0);
} }
@ -491,13 +491,21 @@ int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_
} }
/* /*
Convenience wrapper for executing a prepared SQL statement that returns no value. If an error * Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
occurs then logs it at the given level and returns -1. If 'retry' is non-NULL and the BUSY error * Always finalises the statement before returning.
occurs (indicating the database is locked, ie, currently in use by another process), then resets *
the statement and retries while sqlite_retry() returns true. If sqlite_retry() returns false * If an error occurs then logs it at the given level and returns -1.
then returns -1. Otherwise returns zero. Always finalises the statement before returning. *
* If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
* currently in use by another process), then resets the statement and retries while sqlite_retry()
* returns true. If sqlite_retry() returns false then returns -1.
*
* Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
* was SQLITE_OK or SQLITE_DONE.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{ {
if (!statement) if (!statement)
return -1; return -1;
@ -505,22 +513,29 @@ static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level
int stepcode; int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW) while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW)
++rowcount; ++rowcount;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
sqlite3_finalize(statement); sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) ? 0 : -1; if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return sqlite_code_ok(stepcode) ? rowcount : -1;
} }
static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap) static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap)
{ {
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap); strbuf_vsprintf(stmt, sqlformat, ap);
return _sqlite_exec_void_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt)); int rowcount = _sqlite_exec_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
if (rowcount == -1)
return -1;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
return sqlite3_changes(rhizome_db);
} }
/* Convenience wrapper for executing an SQL command that returns no value. /* Convenience wrapper for executing an SQL command that returns no value.
If an error occurs then logs it at ERROR level and returns -1. Otherwise returns zero. * If an error occurs then logs it at ERROR level and returns -1. Otherwise returns the number of
@author Andrew Bettison <andrew@servalproject.com> * rows changed by the command.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...) int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
{ {
@ -533,7 +548,8 @@ int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
} }
/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR. /* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...) int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...)
{ {
@ -546,10 +562,11 @@ int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const
} }
/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is /* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at * instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately. * all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...) int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...)
{ {
@ -560,6 +577,19 @@ int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *ret
return ret; return ret;
} }
/* Same as sqlite_exec_void_retry(), but logs any error at the given level instead of ERROR.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void_retry_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_void(__whence, log_level, retry, sqlformat, ap);
va_end(ap);
return ret;
}
static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap) static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap)
{ {
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
@ -580,16 +610,22 @@ static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *
if (rowcount > 1) if (rowcount > 1)
WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount); WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
sqlite3_finalize(statement); sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1; if (!sqlite_code_ok(stepcode) || ret == -1)
return -1;
if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return rowcount;
} }
/* /*
Convenience wrapper for executing an SQL command that returns a single int64 value. * Convenience wrapper for executing an SQL command that returns a single int64 value.
Logs an error and returns -1 if an error occurs. * Logs an error and returns -1 if an error occurs.
If no row is found, then returns 0 and does not alter *result. * If no row is found, then returns 0 and does not alter *result.
If exactly one row is found, the assigns its value to *result and returns 1. * If exactly one row is found, the assigns its value to *result and returns 1.
If more than one row is found, then logs a warning, assigns the value of the first row to *result * If more than one row is found, then logs a warning, assigns the value of the first row to *result
and returns the number of rows. * and returns the number of rows.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...) int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...)
{ {
@ -602,10 +638,11 @@ int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const cha
} }
/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is /* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at * instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately. * all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...) int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...)
{ {
@ -616,26 +653,44 @@ int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *re
return ret; return ret;
} }
/* /* Convenience wrapper for executing an SQL command that returns a single text value.
Convenience wrapper for executing an SQL command that returns a single text value. * Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found: * 0 means no rows, nothing is appended to the strbuf
0 means no rows, nothing is appended to the strbuf * 1 means exactly one row, appends its column to the strbuf
1 means exactly one row, appends its column to the strbuf * 2 more than one row, logs a warning and appends the first row's column to the strbuf
2 more than one row, logs a warning and appends the first row's column to the strbuf *
@author Andrew Bettison <andrew@servalproject.com> * @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...) int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
{ {
strbuf stmt = strbuf_alloca(8192);
strbuf_va_printf(stmt, sqlformat);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, &retry, stmt); va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap);
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt);
if (!statement) if (!statement)
return -1; return -1;
int ret = 0; int ret = 0;
int rowcount = 0; int rowcount = 0;
int stepcode; int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) { while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
int columncount = sqlite3_column_count(statement); int columncount = sqlite3_column_count(statement);
if (columncount != 1) if (columncount != 1)
ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement)); ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
@ -661,21 +716,22 @@ long long rhizome_database_used_bytes()
return db_page_size * (db_page_count - db_free_page_count); return db_page_size * (db_page_count - db_free_page_count);
} }
void rhizome_cleanup() int rhizome_cleanup(struct rhizome_cleanup_report *report)
{ {
IN(); IN();
// clean out unreferenced files // clean out unreferenced files
// TODO keep updating inserttime for *very* long transfers? // TODO keep updating inserttime for *very* long transfers?
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000)) { int ret;
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db)); ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
} if (report)
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000)) { report->deleted_stale_incoming_files = ret;
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db)); ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
} if (report)
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) { report->deleted_orphan_files = ret;
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db)); ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
} if (report)
OUT(); report->deleted_orphan_fileblobs = ret;
RETURN(0);
} }
int rhizome_make_space(int group_priority, long long bytes) int rhizome_make_space(int group_priority, long long bytes)
@ -688,7 +744,7 @@ int rhizome_make_space(int group_priority, long long bytes)
if (db_used == -1) if (db_used == -1)
return -1; return -1;
rhizome_cleanup(); rhizome_cleanup(NULL);
/* If there is already enough space now, then do nothing more */ /* If there is already enough space now, then do nothing more */
if (db_used<=(config.rhizome.database_size-bytes-65536)) if (db_used<=(config.rhizome.database_size-bytes-65536))
@ -841,7 +897,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction"); return WHY("Failed to begin transaction");
sqlite3_stmt *stmt; sqlite3_stmt *stmt;
@ -909,7 +965,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt); sqlite3_finalize(stmt);
stmt = NULL; stmt = NULL;
} }
if (sqlite_exec_void_retry(&retry, "COMMIT;") == SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1){
// This message used in tests; do not modify or remove. // This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld", INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
@ -1049,7 +1105,6 @@ int rhizome_list_manifests(const char *service, const char *name,
const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0); const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
long long blob_date = rhizome_manifest_get_ll(m, "date"); long long blob_date = rhizome_manifest_get_ll(m, "date");
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0); const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
int from_here = 0; int from_here = 0;
unsigned char senderSid[SID_SIZE]; unsigned char senderSid[SID_SIZE];
unsigned char recipientSid[SID_SIZE]; unsigned char recipientSid[SID_SIZE];
@ -1070,7 +1125,6 @@ int rhizome_list_manifests(const char *service, const char *name,
int cn = 0, in = 0, kp = 0; int cn = 0, in = 0, kp = 0;
from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid); from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid);
} }
if (config.debug.rhizome) DEBUGF("manifest payload size = %lld", blob_filesize);
cli_put_long(rowid, ":"); cli_put_long(rowid, ":");
cli_put_string(blob_service, ":"); cli_put_string(blob_service, ":");
@ -1107,13 +1161,12 @@ cleanup:
RETURN(ret); RETURN(ret);
} }
int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength, int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength, int priority)
int priority)
{ {
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction"); return WHY("Failed to begin transaction");
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer). /* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
@ -1125,15 +1178,12 @@ int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength,
int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
*/ */
int ret=sqlite_exec_void_retry(&retry, int ret = sqlite_exec_void_retry(&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);", "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
hashhex, (long long)fileLength, priority, (long long)gettime_ms() hashhex, (long long)fileLength, priority, (long long)gettime_ms()
); );
if (ret!=SQLITE_OK) { if (ret == -1)
DEBUGF("insert or replace into files ... failed: %s",
sqlite3_errmsg(rhizome_db));
goto insert_row_fail; goto insert_row_fail;
}
sqlite3_stmt *statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",hashhex); sqlite3_stmt *statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",hashhex);
if (!statement) if (!statement)
@ -1146,7 +1196,7 @@ int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength,
goto insert_row_fail; goto insert_row_fail;
} }
/* Do actual insert, and abort if it fails */ /* Do actual insert, and abort if it fails */
if (_sqlite_exec_void_prepared(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement) == -1) { if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement) == -1) {
insert_row_fail: insert_row_fail:
WHYF("Failed to insert row for fileid=%s", hashhex); WHYF("Failed to insert row for fileid=%s", hashhex);
sqlite_exec_void_retry(&retry, "ROLLBACK;"); sqlite_exec_void_retry(&retry, "ROLLBACK;");
@ -1157,7 +1207,7 @@ insert_row_fail:
int64_t rowid = sqlite3_last_insert_rowid(rhizome_db); int64_t rowid = sqlite3_last_insert_rowid(rhizome_db);
ret = sqlite_exec_void_retry(&retry, "COMMIT;"); ret = sqlite_exec_void_retry(&retry, "COMMIT;");
if (ret!=SQLITE_OK){ if (ret == -1) {
sqlite_exec_void_retry(&retry, "ROLLBACK;"); sqlite_exec_void_retry(&retry, "ROLLBACK;");
return WHYF("Failed to commit transaction"); return WHYF("Failed to commit transaction");
} }
@ -1182,8 +1232,8 @@ int rhizome_update_file_priority(const char *fileid)
" AND groupmemberships.groupid=grouplist.id;", " AND groupmemberships.groupid=grouplist.id;",
fileid) == -1) fileid) == -1)
return -1; return -1;
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) != 0) if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) == -1)
WHYF("cannot update priority for fileid=%s", fileid); return WHYF("cannot update priority for fileid=%s", fileid);
return 0; return 0;
} }
@ -1349,7 +1399,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
if (!inconsistent) { if (!inconsistent) {
*found = blob_m; *found = blob_m;
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s", if (config.debug.rhizome)
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : "" blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : ""
); );
ret = 1; ret = 1;
@ -1370,7 +1421,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
* Returns -1 on error * Returns -1 on error
* Caller is responsible for allocating and freeing rhizome_manifest * Caller is responsible for allocating and freeing rhizome_manifest
*/ */
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m){ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m)
{
int ret=0; int ret=0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
@ -1410,3 +1462,108 @@ done:
sqlite3_finalize(statement); sqlite3_finalize(statement);
return ret; return ret;
} }
int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid)
{
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?");
if (!statement)
return -1;
sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
return -1;
return sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
{
int ret = 0;
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}
int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid)
{
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = '%s'", manifestid);
if (rows == -1)
return -1;
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
return -1;
return 0;
}
/* Remove a manifest and its bundle from the database, given its manifest ID.
*
* Returns 0 if manifest is found and removed and bundle was either absent or removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_bundle(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_delete_payload_retry(&retry, manifestid) == -1)
return -1;
if (rhizome_delete_manifest_retry(&retry, manifestid) == -1)
return -1;
return 0;
}
/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
* untouched if present.
*
* Returns 0 if manifest is found and removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_manifest(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_manifest_retry(&retry, manifestid);
}
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
* untouched if present.
*
* Returns 0 if manifest is found, its payload is found and removed
* Returns 1 if manifest or payload is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_payload(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_payload_retry(&retry, manifestid);
}
/* Remove a file from the database, given its file hash.
*
* Returns 0 if file is found and removed
* Returns 1 if file is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_file(const char *fileid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_retry(&retry, fileid);
}

14
rhizome_fetch.c
View File

@ -1371,20 +1371,16 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
DEBUGF("Hash mismatch -- dropping row from table."); DEBUGF("Hash mismatch -- dropping row from table.");
WARNF("Expected hash=%s, got %s", WARNF("Expected hash=%s, got %s",
slot->manifest->fileHexHash,hash_out); slot->manifest->fileHexHash,hash_out);
sqlite_exec_void_retry(&retry, sqlite_exec_void_retry(&retry, "DELETE FROM FILEBLOBS WHERE rowid=%lld",slot->rowid);
"DELETE FROM FILEBLOBS WHERE rowid=%lld",slot->rowid); sqlite_exec_void_retry(&retry, "DELETE FROM FILES WHERE id='%s'", slot->manifest->fileHexHash);
sqlite_exec_void_retry(&retry,
"DELETE FROM FILES WHERE id='%s'",
slot->manifest->fileHexHash);
rhizome_fetch_close(slot); rhizome_fetch_close(slot);
RETURN(-1); RETURN(-1);
} else { } else {
int ret=sqlite_exec_void_retry(&retry, int ret = sqlite_exec_void_retry(&retry,
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'", "UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
gettime_ms(), slot->manifest->fileHexHash); gettime_ms(), slot->manifest->fileHexHash);
if (ret!=SQLITE_OK) if (ret == -1 && config.debug.rhizome_rx)
if (config.debug.rhizome_rx) DEBUGF("error marking row valid: %s",sqlite3_errmsg(rhizome_db));
DEBUGF("error marking row valid: %s",sqlite3_errmsg(rhizome_db));
} }
if (!rhizome_import_received_bundle(slot->manifest)){ if (!rhizome_import_received_bundle(slot->manifest)){

124
rhizome_store.c
View File

@ -29,7 +29,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction"); return WHY("Failed to begin transaction");
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer). /* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
@ -42,13 +42,11 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
*/ */
sqlite3_stmt *statement = NULL; sqlite3_stmt *statement = NULL;
int ret=sqlite_exec_void_retry(&retry, int ret = sqlite_exec_void_retry(&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);", "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
write->id, (long long)file_length, priority, (long long)gettime_ms()); write->id, (long long)file_length, priority, (long long)gettime_ms());
if (ret!=SQLITE_OK) { if (ret == -1)
WHYF("Failed to insert into files: %s", sqlite3_errmsg(rhizome_db));
goto insert_row_fail; goto insert_row_fail;
}
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",write->id); statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",write->id);
if (!statement) { if (!statement) {
@ -85,9 +83,8 @@ insert_row_fail:
write->blob_rowid = sqlite3_last_insert_rowid(rhizome_db); write->blob_rowid = sqlite3_last_insert_rowid(rhizome_db);
DEBUGF("Got rowid %lld for %s", write->blob_rowid, write->id); DEBUGF("Got rowid %lld for %s", write->blob_rowid, write->id);
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
return WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db)); return -1;
}
write->file_length = file_length; write->file_length = file_length;
write->file_offset = 0; write->file_offset = 0;
@ -201,14 +198,9 @@ int rhizome_fail_write(struct rhizome_write *write){
if (write->buffer) if (write->buffer)
free(write->buffer); free(write->buffer);
write->buffer=NULL; write->buffer=NULL;
// don't worry too much about sql failures.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite_exec_void_retry(&retry, sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
"DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id='%s'", write->id);
sqlite_exec_void_retry(&retry,
"DELETE FROM FILES WHERE id='%s'",
write->id);
return 0; return 0;
} }
@ -225,22 +217,17 @@ int rhizome_finish_write(struct rhizome_write *write){
SHA512_End(&write->sha512_context, hash_out); SHA512_End(&write->sha512_context, hash_out);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
WHY("Failed to begin transaction");
goto failure; goto failure;
}
if (write->id_known){ if (write->id_known){
if (strcasecmp(write->id, hash_out)){ if (strcasecmp(write->id, hash_out)){
WHYF("Expected hash=%s, got %s", write->id, hash_out); WHYF("Expected hash=%s, got %s", write->id, hash_out);
goto failure; goto failure;
} }
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'", gettime_ms(), write->id) == -1)
gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
}else{ }else{
str_toupper_inplace(hash_out); str_toupper_inplace(hash_out);
@ -249,28 +236,22 @@ int rhizome_finish_write(struct rhizome_write *write){
rhizome_fail_write(write); rhizome_fail_write(write);
}else{ }else{
// delete any half finished records // delete any half finished records
sqlite_exec_void_retry(&retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
sqlite_exec_void_retry(&retry,"DELETE FROM FILES WHERE id='%s';",hash_out); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'", "UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'",
hash_out, gettime_ms(), write->id)!=SQLITE_OK){ hash_out, gettime_ms(), write->id) == -1)
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry,
"UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld", "UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld",
hash_out, write->blob_rowid)!=SQLITE_OK){ hash_out, write->blob_rowid) == -1)
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
} }
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH); strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH);
} }
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
return 0; return 0;
failure: failure:
@ -375,43 +356,24 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
return 0; return 0;
} }
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){ /* Return -1 on error, 0 if file blob found, 1 if not found.
*/
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
{
strncpy(read->id, fileid, sizeof read->id); strncpy(read->id, fileid, sizeof read->id);
read->id[RHIZOME_FILEHASH_STRLEN] = '\0'; read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(read->id); str_toupper_inplace(read->id);
int64_t rowid = -1;
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = ? AND FILES.datavalid != 0"); if (sqlite_exec_int64(&rowid, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = '%s' AND FILES.datavalid != 0", read->id) == -1)
if (!statement)
return WHYF("Failed to prepare statement: %s", sqlite3_errmsg(rhizome_db));
sqlite3_bind_text(statement, 1, read->id, -1, SQLITE_STATIC);
int ret = sqlite_step_retry(&retry, statement);
if (ret != SQLITE_ROW){
WHYF("Failed to open file blob: %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return -1; return -1;
} if (rowid == -1)
return 1;
if (!(sqlite3_column_count(statement) == 1 read->blob_rowid = rowid;
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER)) {
sqlite3_finalize(statement);
return WHY("Incorrect statement column");
}
read->blob_rowid = sqlite3_column_int64(statement, 0);
read->hash=hash; read->hash=hash;
read->offset=0; read->offset=0;
read->length=-1; read->length=-1;
sqlite3_finalize(statement);
if (hash) if (hash)
SHA512_Init(&read->sha512_context); SHA512_Init(&read->sha512_context);
return 0; return 0;
} }
@ -485,6 +447,8 @@ int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_le
}while (1); }while (1);
} }
/* Returns -1 on error, 0 on success.
*/
static int write_file(struct rhizome_read *read, const char *filepath){ static int write_file(struct rhizome_read *read, const char *filepath){
int fd=-1, ret=0; int fd=-1, ret=0;
@ -515,44 +479,44 @@ static int write_file(struct rhizome_read *read, const char *filepath){
return ret; return ret;
} }
/* Extract the file related to a manifest to the file system. /* Extract the file related to a manifest to the file system. The file will be de-crypted and
* The file will be de-crypted and verified while reading. * verified while reading. If filepath is not supplied, the file will still be checked.
* If filepath is not supplied, the file will still be checked. *
* Returns -1 on error, 0 if extracted successfully, 1 if not found.
*/ */
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk)
{
struct rhizome_read read_state; struct rhizome_read read_state;
bzero(&read_state, sizeof read_state); bzero(&read_state, sizeof read_state);
// for now, always hash the file // for now, always hash the file
if (rhizome_open_read(&read_state, m->fileHexHash, 1)) int ret = rhizome_open_read(&read_state, m->fileHexHash, 1);
return -1; if (ret != 0)
return ret;
read_state.crypt=m->payloadEncryption; read_state.crypt=m->payloadEncryption;
if (read_state.crypt){ if (read_state.crypt){
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go // if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
if (rhizome_derive_key(m, bsk)) if (rhizome_derive_key(m, bsk))
return -1; return -1;
if (config.debug.rhizome) if (config.debug.rhizome)
DEBUGF("Decrypting file contents"); DEBUGF("Decrypting file contents");
bcopy(m->payloadKey, read_state.key, sizeof(read_state.key)); bcopy(m->payloadKey, read_state.key, sizeof(read_state.key));
bcopy(m->payloadNonce, read_state.nonce, sizeof(read_state.nonce)); bcopy(m->payloadNonce, read_state.nonce, sizeof(read_state.nonce));
} }
return write_file(&read_state, filepath); return write_file(&read_state, filepath);
} }
/* dump the raw contents of a file */ /* dump the raw contents of a file
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length){ *
* Returns -1 on error, 0 if dumped successfully, 1 if not found.
*/
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
{
struct rhizome_read read_state; struct rhizome_read read_state;
bzero(&read_state, sizeof read_state); bzero(&read_state, sizeof read_state);
int ret = rhizome_open_read(&read_state, id, 1);
if (rhizome_open_read(&read_state, id, 1)) if (ret != 0)
return -1; return ret;
if (length) if (length)
*length = read_state.length; *length = read_state.length;
return write_file(&read_state, filepath); return write_file(&read_state, filepath);
} }

1
serval.h
View File

@ -538,7 +538,6 @@ int serval_packetvisualise(XPRINTF xpf, const char *message, const unsigned char
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax); int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int rhizome_opendb(); int rhizome_opendb();
void rhizome_cleanup();
int parseCommandLine(const char *argv0, int argc, const char *const *argv); int parseCommandLine(const char *argv0, int argc, const char *const *argv);

26
testdefs_rhizome.sh
View File

@ -386,14 +386,30 @@ extract_manifest_vars() {
rhizome_add_file() { rhizome_add_file() {
local name="$1" local name="$1"
local size="${2:-64}" local size="${2:-64}"
[ -e "$name" ] || create_file "$name" $size rhizome_add_files --size="$size" "$name"
local sidvar="SID$instance_name"
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "$name" --and-others
extract_manifest_vars "$name.manifest" extract_manifest_vars "$name.manifest"
} }
rhizome_add_files() {
local size=64
local sidvar="SID$instance_name"
local -a names=()
for arg; do
case "$arg" in
--size=*)
size="${arg##*=}"
;;
*)
local name="$arg"
[ -e "$name" ] || create_file "$name" $size
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
names+=("$name")
esac
done
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "${names[@]}" --and-others
}
rhizome_update_file() { rhizome_update_file() {
local orig_name="$1" local orig_name="$1"
local new_name="$2" local new_name="$2"

69
tests/rhizomeops
View File

@ -27,7 +27,7 @@ shopt -s extglob
setup_rhizome() { setup_rhizome() {
set_instance +A set_instance +A
executeOk_servald config set debug.rhizome on executeOk_servald config set debug.rhizome on
create_identities 1 create_single_identity
set_instance +B set_instance +B
executeOk_servald config set debug.rhizome on executeOk_servald config set debug.rhizome on
create_identities 4 create_identities 4
@ -833,7 +833,7 @@ test_ImportOwnBundle() {
assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
} }
doc_ImportCombinedBundle="Can generate a combined bundle, import into another instance and export again" doc_ImportCombinedBundle="Create and import combined bundle"
setup_ImportCombinedBundle() { setup_ImportCombinedBundle() {
setup_servald setup_servald
setup_rhizome setup_rhizome
@ -859,4 +859,69 @@ test_ImportCombinedBundle() {
assert diff fileA fileAx assert diff fileA fileAx
} }
setup_delete() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.clean_on_open off
rhizome_add_files file{1..4}
for i in {1..4}; do
extract_manifest_id BID$i file$i.manifest
extract_manifest_filehash HASH$i file$i.manifest
done
}
doc_DeleteManifest="Delete a manifest from store"
setup_DeleteManifest() {
setup_delete
}
test_DeleteManifest() {
executeOk_servald rhizome delete manifest "$BID2"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1,3,4}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID2"
executeOk_servald rhizome dump file "$HASH2" file2x
assert diff file2 file2x
}
doc_DeletePayload="Delete a payload from store"
setup_DeletePayload() {
setup_delete
}
test_DeletePayload() {
executeOk_servald rhizome delete payload "$BID3"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID3"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH3" file3x
}
doc_DeleteBundle="Delete a bundle from store"
setup_DeleteBundle() {
setup_delete
}
test_DeleteBundle() {
executeOk_servald rhizome delete bundle "$BID4"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..3}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID4"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH4" file4x
}
doc_DeleteFile="Delete a file from store"
setup_DeleteFile() {
setup_delete
}
test_DeleteFile() {
executeOk_servald rhizome delete file "$HASH1"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID1"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH1" file1x
}
runTests "$@" runTests "$@"