mirror of
https://github.com/servalproject/serval-dna.git
synced 2025-01-18 02:39:44 +00:00
Add 'rhizome delete' and 'rhizome clean' commands
sqlite_void_exec() and its ilk now return the count of changed rows, not just zero, on success sqlite_exec_prepared() and its ilk now return the count of rows (number of step results SQLITE_ROW), instead of just zero, on success rhizome_clean() function now produces an optional report of its changes rhizome_fail_write() and rhizome_finish_write() now log WARNings not ERRORs if the SQL DELETE FROM FILES or DELETE FROM FILEBLOBS statements fail Refactor rhizome_open_read() to use sqlite_exec_int64() Ensure that 'rhizome extract' and 'rhizome dump' commands return exit status of 1 in "not found" conditions, not 255, which is reserved for errors Test cases for four new commands: 'rhizome delete bundle', 'rhizome delete manifest', 'rhizome delete payload' and 'rhizome delete file' (no test case for 'rhizome clean' yet)
This commit is contained in:
Andrew Bettison
parent
cf13a4b51e
commit
390655580d
111
commandline.c
111
commandline.c
@ -1395,7 +1395,69 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, void *context)
|
||||
return ret;
|
||||
}
|
||||
|
||||
int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
|
||||
int app_rhizome_delete(const struct cli_parsed *parsed, void *context)
|
||||
{
|
||||
if (config.debug.verbose)
|
||||
DEBUG_cli_parsed(parsed);
|
||||
const char *manifestid, *fileid;
|
||||
if (cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, NULL) == -1)
|
||||
return -1;
|
||||
if (cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
|
||||
return -1;
|
||||
/* Ensure the Rhizome database exists and is open */
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
if (rhizome_opendb() == -1)
|
||||
return -1;
|
||||
if (!(keyring = keyring_open_instance_cli(parsed)))
|
||||
return -1;
|
||||
int ret=0;
|
||||
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
|
||||
if (!fileid)
|
||||
return WHY("missing <fileid> argument");
|
||||
unsigned char filehash[RHIZOME_FILEHASH_BYTES];
|
||||
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1)
|
||||
return WHY("Invalid file ID");
|
||||
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1];
|
||||
tohex(fileIDUpper, filehash, RHIZOME_FILEHASH_BYTES);
|
||||
ret = rhizome_delete_file(fileIDUpper);
|
||||
} else {
|
||||
if (!manifestid)
|
||||
return WHY("missing <manifestid> argument");
|
||||
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
|
||||
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
|
||||
return WHY("Invalid manifest ID");
|
||||
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
|
||||
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
|
||||
if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0)
|
||||
ret = rhizome_delete_bundle(manifestIdUpper);
|
||||
else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0)
|
||||
ret = rhizome_delete_manifest(manifestIdUpper);
|
||||
else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0)
|
||||
ret = rhizome_delete_payload(manifestIdUpper);
|
||||
else
|
||||
return WHY("unrecognised command");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int app_rhizome_clean(const struct cli_parsed *parsed, void *context)
|
||||
{
|
||||
if (config.debug.verbose)
|
||||
DEBUG_cli_parsed(parsed);
|
||||
struct rhizome_cleanup_report report;
|
||||
if (rhizome_cleanup(&report) == -1)
|
||||
return -1;
|
||||
cli_field_name("deleted_stale_incoming_files", ":");
|
||||
cli_put_long(report.deleted_stale_incoming_files, "\n");
|
||||
cli_field_name("deleted_orphan_files", ":");
|
||||
cli_put_long(report.deleted_orphan_files, "\n");
|
||||
cli_field_name("deleted_orphan_fileblobs", ":");
|
||||
cli_put_long(report.deleted_orphan_fileblobs, "\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int app_rhizome_extract(const struct cli_parsed *parsed, void *context)
|
||||
{
|
||||
if (config.debug.verbose)
|
||||
DEBUG_cli_parsed(parsed);
|
||||
@ -1484,13 +1546,10 @@ int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (retfile)
|
||||
ret=retfile;
|
||||
|
||||
ret = retfile == -1 ? -1 : 1;
|
||||
if (m)
|
||||
rhizome_manifest_free(m);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -1502,24 +1561,20 @@ int app_rhizome_dump_file(const struct cli_parsed *parsed, void *context)
|
||||
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|
||||
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
|
||||
return -1;
|
||||
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
if (rhizome_opendb() == -1)
|
||||
return -1;
|
||||
|
||||
if (!rhizome_exists(fileid))
|
||||
return 1;
|
||||
|
||||
int64_t length;
|
||||
if (rhizome_dump_file(fileid, filepath, &length))
|
||||
return -1;
|
||||
|
||||
int ret = rhizome_dump_file(fileid, filepath, &length);
|
||||
if (ret)
|
||||
return ret == -1 ? -1 : 1;
|
||||
cli_puts("filehash"); cli_delim(":");
|
||||
cli_puts(fileid); cli_delim("\n");
|
||||
cli_puts("filesize"); cli_delim(":");
|
||||
cli_printf("%lld", length); cli_delim("\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2188,28 +2243,40 @@ struct cli_schema command_line_options[]={
|
||||
"Add a file to Rhizome and optionally write its manifest to the given path"},
|
||||
{app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE,
|
||||
"Import a payload/manifest pair into Rhizome"},
|
||||
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
|
||||
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,
|
||||
"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
|
||||
"List all manifests and files in Rhizome"},
|
||||
{app_rhizome_extract_bundle,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
|
||||
{app_rhizome_extract,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
|
||||
"<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
|
||||
"Extract a manifest and decrypted file to the given paths."},
|
||||
{app_rhizome_extract_bundle,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
|
||||
{app_rhizome_extract,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
|
||||
"<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE,
|
||||
"Extract a manifest from Rhizome and write it to the given path"},
|
||||
{app_rhizome_extract_bundle,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
|
||||
{app_rhizome_extract,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
|
||||
"<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
|
||||
"Extract a file from Rhizome and write it to the given path"},
|
||||
{app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE,
|
||||
"Extract a file from Rhizome and write it to the given path without attempting decryption"},
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL},
|
||||
CLIFLAG_STANDALONE,
|
||||
{app_rhizome_delete,{"rhizome","delete","\\manifest",
|
||||
"<manifestid>",NULL},CLIFLAG_STANDALONE,
|
||||
"Remove the manifest for the given bundle from the Rhizome store"},
|
||||
{app_rhizome_delete,{"rhizome","delete","\\payload",
|
||||
"<manifestid>",NULL},CLIFLAG_STANDALONE,
|
||||
"Remove the payload for the given bundle from the Rhizome store"},
|
||||
{app_rhizome_delete,{"rhizome","delete","\\bundle",
|
||||
"<manifestid>",NULL},CLIFLAG_STANDALONE,
|
||||
"Remove the manifest and payload for the given bundle from the Rhizome store"},
|
||||
{app_rhizome_delete,{"rhizome","delete","\\file",
|
||||
"<fileid>",NULL},CLIFLAG_STANDALONE,
|
||||
"Remove the file with the given hash from the Rhizome store"},
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, CLIFLAG_STANDALONE,
|
||||
"Synchronise with the specified Rhizome Direct server. Return when done."},
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL},
|
||||
CLIFLAG_STANDALONE,
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, CLIFLAG_STANDALONE,
|
||||
"Deliver all new content to the specified Rhizome Direct server. Return when done."},
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL},
|
||||
CLIFLAG_STANDALONE,
|
||||
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, CLIFLAG_STANDALONE,
|
||||
"Fetch all new content from the specified Rhizome Direct server. Return when done."},
|
||||
{app_rhizome_clean,{"rhizome","clean",NULL},CLIFLAG_STANDALONE,
|
||||
"Remove stale and orphaned content from the Rhizome store"},
|
||||
{app_keyring_create,{"keyring","create",NULL},0,
|
||||
"Create a new keyring file."},
|
||||
{app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE,
|
||||
|
||||
29
rhizome.h
29
rhizome.h
@ -197,8 +197,16 @@ extern sqlite3 *rhizome_db;
|
||||
|
||||
int rhizome_opendb();
|
||||
int rhizome_close_db();
|
||||
int rhizome_manifest_createid(rhizome_manifest *m);
|
||||
|
||||
struct rhizome_cleanup_report {
|
||||
int deleted_stale_incoming_files;
|
||||
int deleted_orphan_files;
|
||||
int deleted_orphan_fileblobs;
|
||||
};
|
||||
|
||||
int rhizome_cleanup(struct rhizome_cleanup_report *report);
|
||||
|
||||
int rhizome_manifest_createid(rhizome_manifest *m);
|
||||
int rhizome_strn_is_manifest_id(const char *text);
|
||||
int rhizome_str_is_manifest_id(const char *text);
|
||||
int rhizome_strn_is_bundle_key(const char *text);
|
||||
@ -276,17 +284,20 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))();
|
||||
int is_debug_rhizome();
|
||||
int is_debug_rhizome_ads();
|
||||
|
||||
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...);
|
||||
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, strbuf stmt);
|
||||
int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action);
|
||||
void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action);
|
||||
int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
|
||||
sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
|
||||
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt);
|
||||
int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
|
||||
void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
|
||||
int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
|
||||
int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...);
|
||||
int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...);
|
||||
int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
|
||||
int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...);
|
||||
int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat, ...);
|
||||
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat, ...);
|
||||
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...);
|
||||
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...);
|
||||
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap);
|
||||
|
||||
#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb))
|
||||
@ -297,9 +308,11 @@ int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...
|
||||
#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__)
|
||||
#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__)
|
||||
|
||||
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
|
||||
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
|
||||
@ -312,6 +325,10 @@ int rhizome_list_manifests(const char *service, const char *name,
|
||||
const char *sender_sid, const char *recipient_sid,
|
||||
int limit, int offset, char count_rows);
|
||||
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
|
||||
int rhizome_delete_bundle(const char *manifestid);
|
||||
int rhizome_delete_manifest(const char *manifestid);
|
||||
int rhizome_delete_payload(const char *manifestid);
|
||||
int rhizome_delete_file(const char *fileid);
|
||||
|
||||
#define RHIZOME_DONTVERIFY 0
|
||||
#define RHIZOME_VERIFY 1
|
||||
|
||||
@ -255,8 +255,8 @@ int rhizome_opendb()
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
|
||||
long long version;
|
||||
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;")<0)
|
||||
RETURN(WHY("Failed to check schema version"));
|
||||
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;") == -1)
|
||||
RETURN(-1);
|
||||
|
||||
if (version<1){
|
||||
/* Create tables as required */
|
||||
@ -296,7 +296,7 @@ int rhizome_opendb()
|
||||
|
||||
// We can't delete a file that is being transferred in another process at this very moment...
|
||||
if (config.rhizome.clean_on_open)
|
||||
rhizome_cleanup();
|
||||
rhizome_cleanup(NULL);
|
||||
RETURN(0);
|
||||
}
|
||||
|
||||
@ -491,13 +491,21 @@ int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_
|
||||
}
|
||||
|
||||
/*
|
||||
Convenience wrapper for executing a prepared SQL statement that returns no value. If an error
|
||||
occurs then logs it at the given level and returns -1. If 'retry' is non-NULL and the BUSY error
|
||||
occurs (indicating the database is locked, ie, currently in use by another process), then resets
|
||||
the statement and retries while sqlite_retry() returns true. If sqlite_retry() returns false
|
||||
then returns -1. Otherwise returns zero. Always finalises the statement before returning.
|
||||
* Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
|
||||
* Always finalises the statement before returning.
|
||||
*
|
||||
* If an error occurs then logs it at the given level and returns -1.
|
||||
*
|
||||
* If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
|
||||
* currently in use by another process), then resets the statement and retries while sqlite_retry()
|
||||
* returns true. If sqlite_retry() returns false then returns -1.
|
||||
*
|
||||
* Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
|
||||
* was SQLITE_OK or SQLITE_DONE.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
|
||||
static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
|
||||
{
|
||||
if (!statement)
|
||||
return -1;
|
||||
@ -505,22 +513,29 @@ static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level
|
||||
int stepcode;
|
||||
while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW)
|
||||
++rowcount;
|
||||
if (rowcount)
|
||||
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
|
||||
sqlite3_finalize(statement);
|
||||
return sqlite_code_ok(stepcode) ? 0 : -1;
|
||||
if (sqlite_trace_func())
|
||||
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
|
||||
return sqlite_code_ok(stepcode) ? rowcount : -1;
|
||||
}
|
||||
|
||||
static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap)
|
||||
{
|
||||
strbuf stmt = strbuf_alloca(8192);
|
||||
strbuf_vsprintf(stmt, sqlformat, ap);
|
||||
return _sqlite_exec_void_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
|
||||
int rowcount = _sqlite_exec_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
|
||||
if (rowcount == -1)
|
||||
return -1;
|
||||
if (rowcount)
|
||||
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
|
||||
return sqlite3_changes(rhizome_db);
|
||||
}
|
||||
|
||||
/* Convenience wrapper for executing an SQL command that returns no value.
|
||||
If an error occurs then logs it at ERROR level and returns -1. Otherwise returns zero.
|
||||
@author Andrew Bettison <andrew@servalproject.com>
|
||||
* If an error occurs then logs it at ERROR level and returns -1. Otherwise returns the number of
|
||||
* rows changed by the command.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
|
||||
{
|
||||
@ -533,7 +548,8 @@ int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
|
||||
}
|
||||
|
||||
/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR.
|
||||
@author Andrew Bettison <andrew@servalproject.com>
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...)
|
||||
{
|
||||
@ -546,10 +562,11 @@ int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const
|
||||
}
|
||||
|
||||
/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
|
||||
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
||||
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
||||
all in the event of a busy condition, but will log it as an error and return immediately.
|
||||
@author Andrew Bettison <andrew@servalproject.com>
|
||||
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
||||
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
||||
* all in the event of a busy condition, but will log it as an error and return immediately.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...)
|
||||
{
|
||||
@ -560,6 +577,19 @@ int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Same as sqlite_exec_void_retry(), but logs any error at the given level instead of ERROR.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_void_retry_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...)
|
||||
{
|
||||
va_list ap;
|
||||
va_start(ap, sqlformat);
|
||||
int ret = _sqlite_vexec_void(__whence, log_level, retry, sqlformat, ap);
|
||||
va_end(ap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap)
|
||||
{
|
||||
strbuf stmt = strbuf_alloca(8192);
|
||||
@ -580,16 +610,22 @@ static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *
|
||||
if (rowcount > 1)
|
||||
WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
|
||||
sqlite3_finalize(statement);
|
||||
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
|
||||
if (!sqlite_code_ok(stepcode) || ret == -1)
|
||||
return -1;
|
||||
if (sqlite_trace_func())
|
||||
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
|
||||
return rowcount;
|
||||
}
|
||||
|
||||
/*
|
||||
Convenience wrapper for executing an SQL command that returns a single int64 value.
|
||||
Logs an error and returns -1 if an error occurs.
|
||||
If no row is found, then returns 0 and does not alter *result.
|
||||
If exactly one row is found, the assigns its value to *result and returns 1.
|
||||
If more than one row is found, then logs a warning, assigns the value of the first row to *result
|
||||
and returns the number of rows.
|
||||
* Convenience wrapper for executing an SQL command that returns a single int64 value.
|
||||
* Logs an error and returns -1 if an error occurs.
|
||||
* If no row is found, then returns 0 and does not alter *result.
|
||||
* If exactly one row is found, the assigns its value to *result and returns 1.
|
||||
* If more than one row is found, then logs a warning, assigns the value of the first row to *result
|
||||
* and returns the number of rows.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...)
|
||||
{
|
||||
@ -602,10 +638,11 @@ int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const cha
|
||||
}
|
||||
|
||||
/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
|
||||
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
||||
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
||||
all in the event of a busy condition, but will log it as an error and return immediately.
|
||||
@author Andrew Bettison <andrew@servalproject.com>
|
||||
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
||||
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
||||
* all in the event of a busy condition, but will log it as an error and return immediately.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...)
|
||||
{
|
||||
@ -616,26 +653,44 @@ int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *re
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
Convenience wrapper for executing an SQL command that returns a single text value.
|
||||
Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
|
||||
0 means no rows, nothing is appended to the strbuf
|
||||
1 means exactly one row, appends its column to the strbuf
|
||||
2 more than one row, logs a warning and appends the first row's column to the strbuf
|
||||
@author Andrew Bettison <andrew@servalproject.com>
|
||||
/* Convenience wrapper for executing an SQL command that returns a single text value.
|
||||
* Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
|
||||
* 0 means no rows, nothing is appended to the strbuf
|
||||
* 1 means exactly one row, appends its column to the strbuf
|
||||
* 2 more than one row, logs a warning and appends the first row's column to the strbuf
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
|
||||
{
|
||||
strbuf stmt = strbuf_alloca(8192);
|
||||
strbuf_va_printf(stmt, sqlformat);
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, &retry, stmt);
|
||||
va_list ap;
|
||||
va_start(ap, sqlformat);
|
||||
int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
|
||||
va_end(ap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
|
||||
{
|
||||
va_list ap;
|
||||
va_start(ap, sqlformat);
|
||||
int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
|
||||
va_end(ap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap)
|
||||
{
|
||||
strbuf stmt = strbuf_alloca(8192);
|
||||
strbuf_vsprintf(stmt, sqlformat, ap);
|
||||
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt);
|
||||
if (!statement)
|
||||
return -1;
|
||||
int ret = 0;
|
||||
int rowcount = 0;
|
||||
int stepcode;
|
||||
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) {
|
||||
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
|
||||
int columncount = sqlite3_column_count(statement);
|
||||
if (columncount != 1)
|
||||
ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
|
||||
@ -661,21 +716,22 @@ long long rhizome_database_used_bytes()
|
||||
return db_page_size * (db_page_count - db_free_page_count);
|
||||
}
|
||||
|
||||
void rhizome_cleanup()
|
||||
int rhizome_cleanup(struct rhizome_cleanup_report *report)
|
||||
{
|
||||
IN();
|
||||
// clean out unreferenced files
|
||||
// TODO keep updating inserttime for *very* long transfers?
|
||||
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000)) {
|
||||
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
|
||||
}
|
||||
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000)) {
|
||||
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
|
||||
}
|
||||
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) {
|
||||
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
|
||||
}
|
||||
OUT();
|
||||
int ret;
|
||||
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
|
||||
if (report)
|
||||
report->deleted_stale_incoming_files = ret;
|
||||
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
|
||||
if (report)
|
||||
report->deleted_orphan_files = ret;
|
||||
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
|
||||
if (report)
|
||||
report->deleted_orphan_fileblobs = ret;
|
||||
RETURN(0);
|
||||
}
|
||||
|
||||
int rhizome_make_space(int group_priority, long long bytes)
|
||||
@ -688,7 +744,7 @@ int rhizome_make_space(int group_priority, long long bytes)
|
||||
if (db_used == -1)
|
||||
return -1;
|
||||
|
||||
rhizome_cleanup();
|
||||
rhizome_cleanup(NULL);
|
||||
|
||||
/* If there is already enough space now, then do nothing more */
|
||||
if (db_used<=(config.rhizome.database_size-bytes-65536))
|
||||
@ -841,7 +897,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
|
||||
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK)
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
|
||||
return WHY("Failed to begin transaction");
|
||||
|
||||
sqlite3_stmt *stmt;
|
||||
@ -909,7 +965,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
|
||||
sqlite3_finalize(stmt);
|
||||
stmt = NULL;
|
||||
}
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;") == SQLITE_OK){
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1){
|
||||
// This message used in tests; do not modify or remove.
|
||||
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
|
||||
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
|
||||
@ -1049,7 +1105,6 @@ int rhizome_list_manifests(const char *service, const char *name,
|
||||
const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
long long blob_date = rhizome_manifest_get_ll(m, "date");
|
||||
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
|
||||
long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
|
||||
int from_here = 0;
|
||||
unsigned char senderSid[SID_SIZE];
|
||||
unsigned char recipientSid[SID_SIZE];
|
||||
@ -1070,7 +1125,6 @@ int rhizome_list_manifests(const char *service, const char *name,
|
||||
int cn = 0, in = 0, kp = 0;
|
||||
from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid);
|
||||
}
|
||||
if (config.debug.rhizome) DEBUGF("manifest payload size = %lld", blob_filesize);
|
||||
|
||||
cli_put_long(rowid, ":");
|
||||
cli_put_string(blob_service, ":");
|
||||
@ -1107,13 +1161,12 @@ cleanup:
|
||||
RETURN(ret);
|
||||
}
|
||||
|
||||
int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength,
|
||||
int priority)
|
||||
int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength, int priority)
|
||||
{
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK)
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
|
||||
return WHY("Failed to begin transaction");
|
||||
|
||||
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
|
||||
@ -1129,11 +1182,8 @@ int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength,
|
||||
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
|
||||
hashhex, (long long)fileLength, priority, (long long)gettime_ms()
|
||||
);
|
||||
if (ret!=SQLITE_OK) {
|
||||
DEBUGF("insert or replace into files ... failed: %s",
|
||||
sqlite3_errmsg(rhizome_db));
|
||||
if (ret == -1)
|
||||
goto insert_row_fail;
|
||||
}
|
||||
|
||||
sqlite3_stmt *statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",hashhex);
|
||||
if (!statement)
|
||||
@ -1146,7 +1196,7 @@ int64_t rhizome_database_create_blob_for(const char *hashhex,int64_t fileLength,
|
||||
goto insert_row_fail;
|
||||
}
|
||||
/* Do actual insert, and abort if it fails */
|
||||
if (_sqlite_exec_void_prepared(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement) == -1) {
|
||||
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement) == -1) {
|
||||
insert_row_fail:
|
||||
WHYF("Failed to insert row for fileid=%s", hashhex);
|
||||
sqlite_exec_void_retry(&retry, "ROLLBACK;");
|
||||
@ -1157,7 +1207,7 @@ insert_row_fail:
|
||||
int64_t rowid = sqlite3_last_insert_rowid(rhizome_db);
|
||||
|
||||
ret = sqlite_exec_void_retry(&retry, "COMMIT;");
|
||||
if (ret!=SQLITE_OK){
|
||||
if (ret == -1) {
|
||||
sqlite_exec_void_retry(&retry, "ROLLBACK;");
|
||||
return WHYF("Failed to commit transaction");
|
||||
}
|
||||
@ -1182,8 +1232,8 @@ int rhizome_update_file_priority(const char *fileid)
|
||||
" AND groupmemberships.groupid=grouplist.id;",
|
||||
fileid) == -1)
|
||||
return -1;
|
||||
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) != 0)
|
||||
WHYF("cannot update priority for fileid=%s", fileid);
|
||||
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) == -1)
|
||||
return WHYF("cannot update priority for fileid=%s", fileid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1349,6 +1399,7 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
|
||||
|
||||
if (!inconsistent) {
|
||||
*found = blob_m;
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
|
||||
blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : ""
|
||||
);
|
||||
@ -1370,7 +1421,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
|
||||
* Returns -1 on error
|
||||
* Caller is responsible for allocating and freeing rhizome_manifest
|
||||
*/
|
||||
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m){
|
||||
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m)
|
||||
{
|
||||
int ret=0;
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
@ -1410,3 +1462,108 @@ done:
|
||||
sqlite3_finalize(statement);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid)
|
||||
{
|
||||
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?");
|
||||
if (!statement)
|
||||
return -1;
|
||||
sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
|
||||
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
|
||||
return -1;
|
||||
return sqlite3_changes(rhizome_db) ? 0 : 1;
|
||||
}
|
||||
|
||||
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
|
||||
{
|
||||
int ret = 0;
|
||||
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
|
||||
if (!statement)
|
||||
ret = -1;
|
||||
else {
|
||||
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
|
||||
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
|
||||
ret = -1;
|
||||
}
|
||||
statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?");
|
||||
if (!statement)
|
||||
ret = -1;
|
||||
else {
|
||||
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
|
||||
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
|
||||
ret = -1;
|
||||
}
|
||||
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
|
||||
}
|
||||
|
||||
int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid)
|
||||
{
|
||||
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
|
||||
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = '%s'", manifestid);
|
||||
if (rows == -1)
|
||||
return -1;
|
||||
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
/* Remove a manifest and its bundle from the database, given its manifest ID.
|
||||
*
|
||||
* Returns 0 if manifest is found and removed and bundle was either absent or removed
|
||||
* Returns 1 if manifest is not found
|
||||
* Returns -1 on error
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int rhizome_delete_bundle(const char *manifestid)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
if (rhizome_delete_payload_retry(&retry, manifestid) == -1)
|
||||
return -1;
|
||||
if (rhizome_delete_manifest_retry(&retry, manifestid) == -1)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
|
||||
* untouched if present.
|
||||
*
|
||||
* Returns 0 if manifest is found and removed
|
||||
* Returns 1 if manifest is not found
|
||||
* Returns -1 on error
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int rhizome_delete_manifest(const char *manifestid)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
return rhizome_delete_manifest_retry(&retry, manifestid);
|
||||
}
|
||||
|
||||
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
|
||||
* untouched if present.
|
||||
*
|
||||
* Returns 0 if manifest is found, its payload is found and removed
|
||||
* Returns 1 if manifest or payload is not found
|
||||
* Returns -1 on error
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int rhizome_delete_payload(const char *manifestid)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
return rhizome_delete_payload_retry(&retry, manifestid);
|
||||
}
|
||||
|
||||
/* Remove a file from the database, given its file hash.
|
||||
*
|
||||
* Returns 0 if file is found and removed
|
||||
* Returns 1 if file is not found
|
||||
* Returns -1 on error
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
int rhizome_delete_file(const char *fileid)
|
||||
{
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
return rhizome_delete_file_retry(&retry, fileid);
|
||||
}
|
||||
|
||||
@ -1371,19 +1371,15 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
|
||||
DEBUGF("Hash mismatch -- dropping row from table.");
|
||||
WARNF("Expected hash=%s, got %s",
|
||||
slot->manifest->fileHexHash,hash_out);
|
||||
sqlite_exec_void_retry(&retry,
|
||||
"DELETE FROM FILEBLOBS WHERE rowid=%lld",slot->rowid);
|
||||
sqlite_exec_void_retry(&retry,
|
||||
"DELETE FROM FILES WHERE id='%s'",
|
||||
slot->manifest->fileHexHash);
|
||||
sqlite_exec_void_retry(&retry, "DELETE FROM FILEBLOBS WHERE rowid=%lld",slot->rowid);
|
||||
sqlite_exec_void_retry(&retry, "DELETE FROM FILES WHERE id='%s'", slot->manifest->fileHexHash);
|
||||
rhizome_fetch_close(slot);
|
||||
RETURN(-1);
|
||||
} else {
|
||||
int ret = sqlite_exec_void_retry(&retry,
|
||||
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
|
||||
gettime_ms(), slot->manifest->fileHexHash);
|
||||
if (ret!=SQLITE_OK)
|
||||
if (config.debug.rhizome_rx)
|
||||
if (ret == -1 && config.debug.rhizome_rx)
|
||||
DEBUGF("error marking row valid: %s",sqlite3_errmsg(rhizome_db));
|
||||
}
|
||||
|
||||
|
||||
122
rhizome_store.c
122
rhizome_store.c
@ -29,7 +29,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK)
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
|
||||
return WHY("Failed to begin transaction");
|
||||
|
||||
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
|
||||
@ -45,10 +45,8 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
|
||||
int ret = sqlite_exec_void_retry(&retry,
|
||||
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
|
||||
write->id, (long long)file_length, priority, (long long)gettime_ms());
|
||||
if (ret!=SQLITE_OK) {
|
||||
WHYF("Failed to insert into files: %s", sqlite3_errmsg(rhizome_db));
|
||||
if (ret == -1)
|
||||
goto insert_row_fail;
|
||||
}
|
||||
|
||||
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%s',?)",write->id);
|
||||
if (!statement) {
|
||||
@ -85,9 +83,8 @@ insert_row_fail:
|
||||
write->blob_rowid = sqlite3_last_insert_rowid(rhizome_db);
|
||||
DEBUGF("Got rowid %lld for %s", write->blob_rowid, write->id);
|
||||
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){
|
||||
return WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
|
||||
}
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
|
||||
return -1;
|
||||
|
||||
write->file_length = file_length;
|
||||
write->file_offset = 0;
|
||||
@ -201,14 +198,9 @@ int rhizome_fail_write(struct rhizome_write *write){
|
||||
if (write->buffer)
|
||||
free(write->buffer);
|
||||
write->buffer=NULL;
|
||||
|
||||
// don't worry too much about sql failures.
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
sqlite_exec_void_retry(&retry,
|
||||
"DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
|
||||
sqlite_exec_void_retry(&retry,
|
||||
"DELETE FROM FILES WHERE id='%s'",
|
||||
write->id);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id='%s'", write->id);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -225,22 +217,17 @@ int rhizome_finish_write(struct rhizome_write *write){
|
||||
SHA512_End(&write->sha512_context, hash_out);
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK){
|
||||
WHY("Failed to begin transaction");
|
||||
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
|
||||
goto failure;
|
||||
}
|
||||
|
||||
if (write->id_known){
|
||||
if (strcasecmp(write->id, hash_out)){
|
||||
WHYF("Expected hash=%s, got %s", write->id, hash_out);
|
||||
goto failure;
|
||||
}
|
||||
if (sqlite_exec_void_retry(&retry,
|
||||
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
|
||||
gettime_ms(), write->id)!=SQLITE_OK){
|
||||
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
|
||||
if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
|
||||
gettime_ms(), write->id) == -1)
|
||||
goto failure;
|
||||
}
|
||||
}else{
|
||||
str_toupper_inplace(hash_out);
|
||||
|
||||
@ -249,28 +236,22 @@ int rhizome_finish_write(struct rhizome_write *write){
|
||||
rhizome_fail_write(write);
|
||||
}else{
|
||||
// delete any half finished records
|
||||
sqlite_exec_void_retry(&retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
|
||||
sqlite_exec_void_retry(&retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
|
||||
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
|
||||
|
||||
if (sqlite_exec_void_retry(&retry,
|
||||
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'",
|
||||
hash_out, gettime_ms(), write->id)!=SQLITE_OK){
|
||||
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
|
||||
hash_out, gettime_ms(), write->id) == -1)
|
||||
goto failure;
|
||||
}
|
||||
if (sqlite_exec_void_retry(&retry,
|
||||
"UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld",
|
||||
hash_out, write->blob_rowid)!=SQLITE_OK){
|
||||
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
|
||||
hash_out, write->blob_rowid) == -1)
|
||||
goto failure;
|
||||
}
|
||||
}
|
||||
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH);
|
||||
}
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){
|
||||
WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
|
||||
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
|
||||
goto failure;
|
||||
}
|
||||
return 0;
|
||||
|
||||
failure:
|
||||
@ -375,43 +356,24 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
|
||||
|
||||
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
||||
|
||||
/* Return -1 on error, 0 if file blob found, 1 if not found.
|
||||
*/
|
||||
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
|
||||
{
|
||||
strncpy(read->id, fileid, sizeof read->id);
|
||||
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
|
||||
str_toupper_inplace(read->id);
|
||||
|
||||
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = ? AND FILES.datavalid != 0");
|
||||
if (!statement)
|
||||
return WHYF("Failed to prepare statement: %s", sqlite3_errmsg(rhizome_db));
|
||||
|
||||
sqlite3_bind_text(statement, 1, read->id, -1, SQLITE_STATIC);
|
||||
|
||||
int ret = sqlite_step_retry(&retry, statement);
|
||||
if (ret != SQLITE_ROW){
|
||||
WHYF("Failed to open file blob: %s", sqlite3_errmsg(rhizome_db));
|
||||
sqlite3_finalize(statement);
|
||||
int64_t rowid = -1;
|
||||
if (sqlite_exec_int64(&rowid, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = '%s' AND FILES.datavalid != 0", read->id) == -1)
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!(sqlite3_column_count(statement) == 1
|
||||
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER)) {
|
||||
sqlite3_finalize(statement);
|
||||
return WHY("Incorrect statement column");
|
||||
}
|
||||
|
||||
read->blob_rowid = sqlite3_column_int64(statement, 0);
|
||||
if (rowid == -1)
|
||||
return 1;
|
||||
read->blob_rowid = rowid;
|
||||
read->hash=hash;
|
||||
read->offset=0;
|
||||
read->length=-1;
|
||||
|
||||
sqlite3_finalize(statement);
|
||||
|
||||
if (hash)
|
||||
SHA512_Init(&read->sha512_context);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -485,6 +447,8 @@ int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_le
|
||||
}while (1);
|
||||
}
|
||||
|
||||
/* Returns -1 on error, 0 on success.
|
||||
*/
|
||||
static int write_file(struct rhizome_read *read, const char *filepath){
|
||||
int fd=-1, ret=0;
|
||||
|
||||
@ -515,44 +479,44 @@ static int write_file(struct rhizome_read *read, const char *filepath){
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Extract the file related to a manifest to the file system.
|
||||
* The file will be de-crypted and verified while reading.
|
||||
* If filepath is not supplied, the file will still be checked.
|
||||
/* Extract the file related to a manifest to the file system. The file will be de-crypted and
|
||||
* verified while reading. If filepath is not supplied, the file will still be checked.
|
||||
*
|
||||
* Returns -1 on error, 0 if extracted successfully, 1 if not found.
|
||||
*/
|
||||
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){
|
||||
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk)
|
||||
{
|
||||
struct rhizome_read read_state;
|
||||
bzero(&read_state, sizeof read_state);
|
||||
|
||||
// for now, always hash the file
|
||||
if (rhizome_open_read(&read_state, m->fileHexHash, 1))
|
||||
return -1;
|
||||
|
||||
int ret = rhizome_open_read(&read_state, m->fileHexHash, 1);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
read_state.crypt=m->payloadEncryption;
|
||||
if (read_state.crypt){
|
||||
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
|
||||
if (rhizome_derive_key(m, bsk))
|
||||
return -1;
|
||||
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Decrypting file contents");
|
||||
|
||||
bcopy(m->payloadKey, read_state.key, sizeof(read_state.key));
|
||||
bcopy(m->payloadNonce, read_state.nonce, sizeof(read_state.nonce));
|
||||
}
|
||||
|
||||
return write_file(&read_state, filepath);
|
||||
}
|
||||
|
||||
/* dump the raw contents of a file */
|
||||
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length){
|
||||
/* dump the raw contents of a file
|
||||
*
|
||||
* Returns -1 on error, 0 if dumped successfully, 1 if not found.
|
||||
*/
|
||||
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
|
||||
{
|
||||
struct rhizome_read read_state;
|
||||
bzero(&read_state, sizeof read_state);
|
||||
|
||||
if (rhizome_open_read(&read_state, id, 1))
|
||||
return -1;
|
||||
|
||||
int ret = rhizome_open_read(&read_state, id, 1);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
if (length)
|
||||
*length = read_state.length;
|
||||
|
||||
return write_file(&read_state, filepath);
|
||||
}
|
||||
|
||||
1
serval.h
1
serval.h
@ -538,7 +538,6 @@ int serval_packetvisualise(XPRINTF xpf, const char *message, const unsigned char
|
||||
|
||||
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
|
||||
int rhizome_opendb();
|
||||
void rhizome_cleanup();
|
||||
|
||||
int parseCommandLine(const char *argv0, int argc, const char *const *argv);
|
||||
|
||||
|
||||
@ -386,14 +386,30 @@ extract_manifest_vars() {
|
||||
rhizome_add_file() {
|
||||
local name="$1"
|
||||
local size="${2:-64}"
|
||||
[ -e "$name" ] || create_file "$name" $size
|
||||
local sidvar="SID$instance_name"
|
||||
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "$name" --and-others
|
||||
rhizome_add_files --size="$size" "$name"
|
||||
extract_manifest_vars "$name.manifest"
|
||||
}
|
||||
|
||||
rhizome_add_files() {
|
||||
local size=64
|
||||
local sidvar="SID$instance_name"
|
||||
local -a names=()
|
||||
for arg; do
|
||||
case "$arg" in
|
||||
--size=*)
|
||||
size="${arg##*=}"
|
||||
;;
|
||||
*)
|
||||
local name="$arg"
|
||||
[ -e "$name" ] || create_file "$name" $size
|
||||
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
|
||||
names+=("$name")
|
||||
esac
|
||||
done
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "${names[@]}" --and-others
|
||||
}
|
||||
|
||||
rhizome_update_file() {
|
||||
local orig_name="$1"
|
||||
local new_name="$2"
|
||||
|
||||
@ -27,7 +27,7 @@ shopt -s extglob
|
||||
setup_rhizome() {
|
||||
set_instance +A
|
||||
executeOk_servald config set debug.rhizome on
|
||||
create_identities 1
|
||||
create_single_identity
|
||||
set_instance +B
|
||||
executeOk_servald config set debug.rhizome on
|
||||
create_identities 4
|
||||
@ -833,7 +833,7 @@ test_ImportOwnBundle() {
|
||||
assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
|
||||
}
|
||||
|
||||
doc_ImportCombinedBundle="Can generate a combined bundle, import into another instance and export again"
|
||||
doc_ImportCombinedBundle="Create and import combined bundle"
|
||||
setup_ImportCombinedBundle() {
|
||||
setup_servald
|
||||
setup_rhizome
|
||||
@ -859,4 +859,69 @@ test_ImportCombinedBundle() {
|
||||
assert diff fileA fileAx
|
||||
}
|
||||
|
||||
setup_delete() {
|
||||
setup_servald
|
||||
setup_rhizome
|
||||
set_instance +A
|
||||
executeOk_servald config set rhizome.clean_on_open off
|
||||
rhizome_add_files file{1..4}
|
||||
for i in {1..4}; do
|
||||
extract_manifest_id BID$i file$i.manifest
|
||||
extract_manifest_filehash HASH$i file$i.manifest
|
||||
done
|
||||
}
|
||||
|
||||
doc_DeleteManifest="Delete a manifest from store"
|
||||
setup_DeleteManifest() {
|
||||
setup_delete
|
||||
}
|
||||
test_DeleteManifest() {
|
||||
executeOk_servald rhizome delete manifest "$BID2"
|
||||
tfw_cat --stderr
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list file{1,3,4}
|
||||
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID2"
|
||||
executeOk_servald rhizome dump file "$HASH2" file2x
|
||||
assert diff file2 file2x
|
||||
}
|
||||
|
||||
doc_DeletePayload="Delete a payload from store"
|
||||
setup_DeletePayload() {
|
||||
setup_delete
|
||||
}
|
||||
test_DeletePayload() {
|
||||
executeOk_servald rhizome delete payload "$BID3"
|
||||
tfw_cat --stderr
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list file{1..4}
|
||||
executeOk_servald rhizome extract manifest "$BID3"
|
||||
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH3" file3x
|
||||
}
|
||||
|
||||
doc_DeleteBundle="Delete a bundle from store"
|
||||
setup_DeleteBundle() {
|
||||
setup_delete
|
||||
}
|
||||
test_DeleteBundle() {
|
||||
executeOk_servald rhizome delete bundle "$BID4"
|
||||
tfw_cat --stderr
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list file{1..3}
|
||||
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID4"
|
||||
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH4" file4x
|
||||
}
|
||||
|
||||
doc_DeleteFile="Delete a file from store"
|
||||
setup_DeleteFile() {
|
||||
setup_delete
|
||||
}
|
||||
test_DeleteFile() {
|
||||
executeOk_servald rhizome delete file "$HASH1"
|
||||
tfw_cat --stderr
|
||||
executeOk_servald rhizome list
|
||||
assert_rhizome_list file{1..4}
|
||||
executeOk_servald rhizome extract manifest "$BID1"
|
||||
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH1" file1x
|
||||
}
|
||||
|
||||
runTests "$@"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user