ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-06 11:09:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/development' into serial

Browse Source 
Conflicts:
	rhizome.h
	rhizome_database.c
	rhizome_fetch.c
	rhizome_store.c
This commit is contained in:
Jeremy Lakeman 2013-02-20 17:22:53 +10:30
commit 90386ce1b1
13 changed files with 522 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
commandline.c
View File

@ -1497,7 +1497,69 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, void *context)
return ret; return ret;
} }
int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context) int app_rhizome_delete(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *manifestid, *fileid;
if (cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, NULL) == -1)
return -1;
if (cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1;
/* Ensure the Rhizome database exists and is open */
if (create_serval_instance_dir() == -1)
return -1;
if (rhizome_opendb() == -1)
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
int ret=0;
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
if (!fileid)
return WHY("missing <fileid> argument");
unsigned char filehash[RHIZOME_FILEHASH_BYTES];
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1)
return WHY("Invalid file ID");
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1];
tohex(fileIDUpper, filehash, RHIZOME_FILEHASH_BYTES);
ret = rhizome_delete_file(fileIDUpper);
} else {
if (!manifestid)
return WHY("missing <manifestid> argument");
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
return WHY("Invalid manifest ID");
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0)
ret = rhizome_delete_bundle(manifestIdUpper);
else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0)
ret = rhizome_delete_manifest(manifestIdUpper);
else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0)
ret = rhizome_delete_payload(manifestIdUpper);
else
return WHY("unrecognised command");
}
return ret;
}
int app_rhizome_clean(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
struct rhizome_cleanup_report report;
if (rhizome_cleanup(&report) == -1)
return -1;
cli_field_name("deleted_stale_incoming_files", ":");
cli_put_long(report.deleted_stale_incoming_files, "\n");
cli_field_name("deleted_orphan_files", ":");
cli_put_long(report.deleted_orphan_files, "\n");
cli_field_name("deleted_orphan_fileblobs", ":");
cli_put_long(report.deleted_orphan_fileblobs, "\n");
return 0;
}
int app_rhizome_extract(const struct cli_parsed *parsed, void *context)
{ {
if (config.debug.verbose) if (config.debug.verbose)
DEBUG_cli_parsed(parsed); DEBUG_cli_parsed(parsed);
@ -1586,13 +1648,10 @@ int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
} }
} }
} }
if (retfile) if (retfile)
ret=retfile; ret = retfile == -1 ? -1 : 1;
if (m) if (m)
rhizome_manifest_free(m); rhizome_manifest_free(m);
return ret; return ret;
} }
@ -1604,24 +1663,20 @@ int app_rhizome_dump_file(const struct cli_parsed *parsed, void *context)
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1 if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1) || cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1; return -1;
if (create_serval_instance_dir() == -1) if (create_serval_instance_dir() == -1)
return -1; return -1;
if (rhizome_opendb() == -1) if (rhizome_opendb() == -1)
return -1; return -1;
if (!rhizome_exists(fileid)) if (!rhizome_exists(fileid))
return 1; return 1;
int64_t length; int64_t length;
if (rhizome_dump_file(fileid, filepath, &length)) int ret = rhizome_dump_file(fileid, filepath, &length);
return -1; if (ret)
return ret == -1 ? -1 : 1;
cli_puts("filehash"); cli_delim(":"); cli_puts("filehash"); cli_delim(":");
cli_puts(fileid); cli_delim("\n"); cli_puts(fileid); cli_delim("\n");
cli_puts("filesize"); cli_delim(":"); cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", length); cli_delim("\n"); cli_printf("%lld", length); cli_delim("\n");
return 0; return 0;
} }
@ -2301,28 +2356,40 @@ struct cli_schema command_line_options[]={
"Add a file to Rhizome and optionally write its manifest to the given path"}, "Add a file to Rhizome and optionally write its manifest to the given path"},
{app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE, {app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE,
"Import a payload/manifest pair into Rhizome"}, "Import a payload/manifest pair into Rhizome"},
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE, {app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,
"List all manifests and files in Rhizome"}, "[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
{app_rhizome_extract_bundle,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS, "List all manifests and files in Rhizome"},
{app_rhizome_extract,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest and decrypted file to the given paths."}, "Extract a manifest and decrypted file to the given paths."},
{app_rhizome_extract_bundle,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS, {app_rhizome_extract,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest from Rhizome and write it to the given path"}, "Extract a manifest from Rhizome and write it to the given path"},
{app_rhizome_extract_bundle,{"rhizome","extract","file" KEYRING_PIN_OPTIONS, {app_rhizome_extract,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
"<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE, "<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path"}, "Extract a file from Rhizome and write it to the given path"},
{app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE, {app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path without attempting decryption"}, "Extract a file from Rhizome and write it to the given path without attempting decryption"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\manifest",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."}, "Remove the manifest for the given bundle from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\payload",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."}, "Remove the payload for the given bundle from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, {app_rhizome_delete,{"rhizome","delete","\\bundle",
CLIFLAG_STANDALONE, "<manifestid>",NULL},CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."}, "Remove the manifest and payload for the given bundle from the Rhizome store"},
{app_rhizome_delete,{"rhizome","delete","\\file",
"<fileid>",NULL},CLIFLAG_STANDALONE,
"Remove the file with the given hash from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."},
{app_rhizome_clean,{"rhizome","clean",NULL},CLIFLAG_STANDALONE,
"Remove stale and orphaned content from the Rhizome store"},
{app_keyring_create,{"keyring","create",NULL},0, {app_keyring_create,{"keyring","create",NULL},0,
"Create a new keyring file."}, "Create a new keyring file."},
{app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE, {app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE,

2
conf_schema.c
View File

@ -425,6 +425,7 @@ static int cf_opt_network_interface_legacy(struct config_network_interface *nifp
return CFSTRINGOVERFLOW; return CFSTRINGOVERFLOW;
strncpy(nif.file, &name[1], len - 1)[len - 1] = '\0'; strncpy(nif.file, &name[1], len - 1)[len - 1] = '\0';
nif.match.patc = 0; nif.match.patc = 0;
nif.socket_type = SOCK_FILE;
} else { } else {
int star = (strchr(name, '*') != NULL) ? 1 : 0; int star = (strchr(name, '*') != NULL) ? 1 : 0;
if (len + star >= sizeof(nif.match.patv[0])) if (len + star >= sizeof(nif.match.patv[0]))
@ -433,6 +434,7 @@ static int cf_opt_network_interface_legacy(struct config_network_interface *nifp
if (star) if (star)
nif.match.patv[0][len] = '*'; nif.match.patv[0][len] = '*';
nif.match.patc = 1; nif.match.patc = 1;
nif.socket_type = SOCK_DGRAM;
} }
if (*p == '=') { if (*p == '=') {
const char *const type = p + 1; const char *const type = p + 1;

1
conf_schema.h
View File

@ -310,6 +310,7 @@ END_STRUCT
STRUCT(rhizome) STRUCT(rhizome)
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting") ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
ATOM(int, clean_on_open, 1, cf_opt_int_boolean,, "If true, Rhizome database is cleaned at start of every command")
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory") STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes") ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.") ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.")

7
crypto.c
View File

@ -28,12 +28,9 @@ int crypto_verify_signature(unsigned char *sas_key,
reassembled,sizeof(reassembled), reassembled,sizeof(reassembled),
sas_key); sas_key);
if (result) { if (result)
WHY("Signature verification failed"); RETURN(WHY("Signature verification failed"));
RETURN(-1);
}
RETURN(0); RETURN(0);
OUT();
} }
// verify the signature at the end of a message, on return message_len will be reduced by the length of the signature. // verify the signature at the end of a message, on return message_len will be reduced by the length of the signature.

3
overlay.c
View File

@ -130,7 +130,8 @@ schedule(&_sched_##X); }
/* Get rhizome server started BEFORE populating fd list so that /* Get rhizome server started BEFORE populating fd list so that
the server's listen socket is in the list for poll() */ the server's listen socket is in the list for poll() */
if (is_rhizome_enabled()) rhizome_opendb(); if (is_rhizome_enabled())
rhizome_opendb();
/* Rhizome http server needs to know which callback to attach /* Rhizome http server needs to know which callback to attach
to client sockets, so provide it here, along with the name to to client sockets, so provide it here, along with the name to

35
rhizome.h
View File

@ -197,8 +197,16 @@ extern sqlite3 *rhizome_db;
int rhizome_opendb(); int rhizome_opendb();
int rhizome_close_db(); int rhizome_close_db();
int rhizome_manifest_createid(rhizome_manifest *m);
struct rhizome_cleanup_report {
int deleted_stale_incoming_files;
int deleted_orphan_files;
int deleted_orphan_fileblobs;
};
int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_strn_is_manifest_id(const char *text); int rhizome_strn_is_manifest_id(const char *text);
int rhizome_str_is_manifest_id(const char *text); int rhizome_str_is_manifest_id(const char *text);
int rhizome_strn_is_bundle_key(const char *text); int rhizome_strn_is_bundle_key(const char *text);
@ -276,17 +284,20 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))();
int is_debug_rhizome(); int is_debug_rhizome();
int is_debug_rhizome_ads(); int is_debug_rhizome_ads();
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...); sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, strbuf stmt); sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt);
int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action); int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action); void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement); int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...); int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...);
int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...); int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...);
int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...); int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat,...); int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat,...); int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...); int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap);
#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) #define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb)) #define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb))
@ -297,9 +308,11 @@ int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...
#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__) #define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__) #define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__) #define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__) #define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__) #define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__) #define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__)
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
@ -314,6 +327,10 @@ int rhizome_list_manifests(const char *service, const char *name,
int limit, int offset, char count_rows); int limit, int offset, char count_rows);
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m); int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
int rhizome_advertise_manifest(rhizome_manifest *m); int rhizome_advertise_manifest(rhizome_manifest *m);
int rhizome_delete_bundle(const char *manifestid);
int rhizome_delete_manifest(const char *manifestid);
int rhizome_delete_payload(const char *manifestid);
int rhizome_delete_file(const char *fileid);
#define RHIZOME_DONTVERIFY 0 #define RHIZOME_DONTVERIFY 0
#define RHIZOME_VERIFY 1 #define RHIZOME_VERIFY 1

337
rhizome_database.c
View File

@ -255,8 +255,8 @@ int rhizome_opendb()
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
long long version; long long version;
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;")<0) if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;") == -1)
RETURN(WHY("Failed to check schema version")); RETURN(-1);
if (version<1){ if (version<1){
/* Create tables as required */ /* Create tables as required */
@ -300,6 +300,8 @@ int rhizome_opendb()
All changes should attempt to preserve any existing data */ All changes should attempt to preserve any existing data */
// We can't delete a file that is being transferred in another process at this very moment... // We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open)
rhizome_cleanup(NULL);
RETURN(0); RETURN(0);
OUT(); OUT();
} }
@ -496,13 +498,21 @@ int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_
} }
/* /*
Convenience wrapper for executing a prepared SQL statement that returns no value. If an error * Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
occurs then logs it at the given level and returns -1. If 'retry' is non-NULL and the BUSY error * Always finalises the statement before returning.
occurs (indicating the database is locked, ie, currently in use by another process), then resets *
the statement and retries while sqlite_retry() returns true. If sqlite_retry() returns false * If an error occurs then logs it at the given level and returns -1.
then returns -1. Otherwise returns zero. Always finalises the statement before returning. *
* If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
* currently in use by another process), then resets the statement and retries while sqlite_retry()
* returns true. If sqlite_retry() returns false then returns -1.
*
* Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
* was SQLITE_OK or SQLITE_DONE.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement) static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{ {
if (!statement) if (!statement)
return -1; return -1;
@ -510,22 +520,29 @@ static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level
int stepcode; int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW) while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW)
++rowcount; ++rowcount;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
sqlite3_finalize(statement); sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) ? 0 : -1; if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return sqlite_code_ok(stepcode) ? rowcount : -1;
} }
static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap) static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap)
{ {
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap); strbuf_vsprintf(stmt, sqlformat, ap);
return _sqlite_exec_void_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt)); int rowcount = _sqlite_exec_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
if (rowcount == -1)
return -1;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
return sqlite3_changes(rhizome_db);
} }
/* Convenience wrapper for executing an SQL command that returns no value. /* Convenience wrapper for executing an SQL command that returns no value.
If an error occurs then logs it at ERROR level and returns -1. Otherwise returns zero. * If an error occurs then logs it at ERROR level and returns -1. Otherwise returns the number of
@author Andrew Bettison <andrew@servalproject.com> * rows changed by the command.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...) int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
{ {
@ -538,7 +555,8 @@ int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
} }
/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR. /* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...) int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...)
{ {
@ -551,10 +569,11 @@ int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const
} }
/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is /* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at * instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately. * all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...) int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...)
{ {
@ -565,6 +584,19 @@ int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *ret
return ret; return ret;
} }
/* Same as sqlite_exec_void_retry(), but logs any error at the given level instead of ERROR.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void_retry_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_void(__whence, log_level, retry, sqlformat, ap);
va_end(ap);
return ret;
}
static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap) static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap)
{ {
strbuf stmt = strbuf_alloca(8192); strbuf stmt = strbuf_alloca(8192);
@ -585,16 +617,22 @@ static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *
if (rowcount > 1) if (rowcount > 1)
WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount); WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
sqlite3_finalize(statement); sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1; if (!sqlite_code_ok(stepcode) || ret == -1)
return -1;
if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return rowcount;
} }
/* /*
Convenience wrapper for executing an SQL command that returns a single int64 value. * Convenience wrapper for executing an SQL command that returns a single int64 value.
Logs an error and returns -1 if an error occurs. * Logs an error and returns -1 if an error occurs.
If no row is found, then returns 0 and does not alter *result. * If no row is found, then returns 0 and does not alter *result.
If exactly one row is found, the assigns its value to *result and returns 1. * If exactly one row is found, the assigns its value to *result and returns 1.
If more than one row is found, then logs a warning, assigns the value of the first row to *result * If more than one row is found, then logs a warning, assigns the value of the first row to *result
and returns the number of rows. * and returns the number of rows.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...) int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...)
{ {
@ -607,10 +645,11 @@ int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const cha
} }
/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is /* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at * instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately. * all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com> *
* @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...) int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...)
{ {
@ -621,26 +660,44 @@ int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *re
return ret; return ret;
} }
/* /* Convenience wrapper for executing an SQL command that returns a single text value.
Convenience wrapper for executing an SQL command that returns a single text value. * Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found: * 0 means no rows, nothing is appended to the strbuf
0 means no rows, nothing is appended to the strbuf * 1 means exactly one row, appends its column to the strbuf
1 means exactly one row, appends its column to the strbuf * 2 more than one row, logs a warning and appends the first row's column to the strbuf
2 more than one row, logs a warning and appends the first row's column to the strbuf *
@author Andrew Bettison <andrew@servalproject.com> * @author Andrew Bettison <andrew@servalproject.com>
*/ */
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...) int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
{ {
strbuf stmt = strbuf_alloca(8192);
strbuf_va_printf(stmt, sqlformat);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, &retry, stmt); va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap);
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt);
if (!statement) if (!statement)
return -1; return -1;
int ret = 0; int ret = 0;
int rowcount = 0; int rowcount = 0;
int stepcode; int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) { while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
int columncount = sqlite3_column_count(statement); int columncount = sqlite3_column_count(statement);
if (columncount != 1) if (columncount != 1)
ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement)); ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
@ -676,41 +733,44 @@ int rhizome_database_filehash_from_id(const char *id, uint64_t version, char has
OUT(); OUT();
} }
void rhizome_cleanup() static int rhizome_cleanup_external(sqlite_retry_state *retry, sqlite3_stmt *statement){
int ret=0;
while (sqlite_step_retry(retry, statement) == SQLITE_ROW) {
const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_store_delete(id)==0)
ret++;
}
return ret;
}
int rhizome_cleanup(struct rhizome_cleanup_report *report)
{ {
IN(); IN();
/* FIXME sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// cleanup external blobs for unreferenced files
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
int externals_removed=rhizome_cleanup_external(&retry, statement);
sqlite3_finalize(statement);
statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
externals_removed+=rhizome_cleanup_external(&retry, statement);
sqlite3_finalize(statement);
// clean out unreferenced files // clean out unreferenced files
// TODO keep updating inserttime for *very* long transfers? int ret;
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000)) { ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db)); if (report)
} report->deleted_stale_incoming_files = ret;
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000)) { ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db)); if (report)
} report->deleted_orphan_files = ret;
// Clean up unreferenced blobs and files
{ ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
char sqlcmd[1024]; if (report)
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd); report->deleted_orphan_fileblobs = ret + externals_removed;
strbuf_puts(b, "SELECT rowid FROM fileblobs WHERE TYPEOF(data)<>\"blob\" AND NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
if (!strbuf_overrun(b)) { RETURN(0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, "%s", strbuf_str(b));
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
int64_t rowid = sqlite3_column_int64(statement, 0);
char *blobfile=rhizome_database_get_blob_filename(rowid);
if (blobfile) {
INFOF("Cleaning up orphaned blob file %s",blobfile);
unlink(blobfile);
}
}
sqlite3_finalize(statement);
}
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
}
*/
OUT(); OUT();
} }
@ -724,7 +784,7 @@ int rhizome_make_space(int group_priority, long long bytes)
if (db_used == -1) if (db_used == -1)
return -1; return -1;
rhizome_cleanup(); rhizome_cleanup(NULL);
/* If there is already enough space now, then do nothing more */ /* If there is already enough space now, then do nothing more */
if (db_used<=(config.rhizome.database_size-bytes-65536)) if (db_used<=(config.rhizome.database_size-bytes-65536))
@ -878,7 +938,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction"); return WHY("Failed to begin transaction");
sqlite3_stmt *stmt; sqlite3_stmt *stmt;
@ -946,7 +1006,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt); sqlite3_finalize(stmt);
stmt = NULL; stmt = NULL;
} }
if (sqlite_exec_void_retry(&retry, "COMMIT;") == SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1){
// This message used in tests; do not modify or remove. // This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m, "service", NULL, 0); const char *service = rhizome_manifest_get(m, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld", INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
@ -1086,7 +1146,6 @@ int rhizome_list_manifests(const char *service, const char *name,
const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0); const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
long long blob_date = rhizome_manifest_get_ll(m, "date"); long long blob_date = rhizome_manifest_get_ll(m, "date");
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0); const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
int from_here = 0; int from_here = 0;
unsigned char senderSid[SID_SIZE]; unsigned char senderSid[SID_SIZE];
unsigned char recipientSid[SID_SIZE]; unsigned char recipientSid[SID_SIZE];
@ -1107,7 +1166,6 @@ int rhizome_list_manifests(const char *service, const char *name,
int cn = 0, in = 0, kp = 0; int cn = 0, in = 0, kp = 0;
from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid); from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid);
} }
if (config.debug.rhizome) DEBUGF("manifest payload size = %lld", blob_filesize);
cli_put_long(rowid, ":"); cli_put_long(rowid, ":");
cli_put_string(blob_service, ":"); cli_put_string(blob_service, ":");
@ -1162,8 +1220,8 @@ int rhizome_update_file_priority(const char *fileid)
" AND groupmemberships.groupid=grouplist.id;", " AND groupmemberships.groupid=grouplist.id;",
fileid) == -1) fileid) == -1)
return -1; return -1;
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) != 0) if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) == -1)
WHYF("cannot update priority for fileid=%s", fileid); return WHYF("cannot update priority for fileid=%s", fileid);
return 0; return 0;
} }
@ -1329,7 +1387,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
if (!inconsistent) { if (!inconsistent) {
*found = blob_m; *found = blob_m;
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s", if (config.debug.rhizome)
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : "" blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : ""
); );
ret = 1; ret = 1;
@ -1350,7 +1409,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
* Returns -1 on error * Returns -1 on error
* Caller is responsible for allocating and freeing rhizome_manifest * Caller is responsible for allocating and freeing rhizome_manifest
*/ */
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m){ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m)
{
int ret=0; int ret=0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
@ -1391,8 +1451,113 @@ done:
return ret; return ret;
} }
long long lookup_time=0; int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid)
long long last_bar_lookup=0; {
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?");
if (!statement)
return -1;
sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
return -1;
return sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
{
int ret = 0;
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}
int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid)
{
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = '%s'", manifestid);
if (rows == -1)
return -1;
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
return -1;
return 0;
}
/* Remove a manifest and its bundle from the database, given its manifest ID.
*
* Returns 0 if manifest is found and removed and bundle was either absent or removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_bundle(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_delete_payload_retry(&retry, manifestid) == -1)
return -1;
if (rhizome_delete_manifest_retry(&retry, manifestid) == -1)
return -1;
return 0;
}
/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
* untouched if present.
*
* Returns 0 if manifest is found and removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_manifest(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_manifest_retry(&retry, manifestid);
}
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
* untouched if present.
*
* Returns 0 if manifest is found, its payload is found and removed
* Returns 1 if manifest or payload is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_payload(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_payload_retry(&retry, manifestid);
}
/* Remove a file from the database, given its file hash.
*
* Returns 0 if file is found and removed
* Returns 1 if file is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_file(const char *fileid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_retry(&retry, fileid);
}
time_ms_t lookup_time=0;
time_ms_t last_bar_lookup=0;
int rhizome_is_bar_interesting(unsigned char *bar){ int rhizome_is_bar_interesting(unsigned char *bar){
IN(); IN();
@ -1409,7 +1574,7 @@ int rhizome_is_bar_interesting(unsigned char *bar){
} }
} }
long long start_time=gettime_ms(); time_ms_t start_time=gettime_ms();
int64_t version = rhizome_bar_version(bar); int64_t version = rhizome_bar_version(bar);
int ret=1; int ret=1;
@ -1446,7 +1611,7 @@ int rhizome_is_bar_interesting(unsigned char *bar){
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
long long end_time=gettime_ms(); time_ms_t end_time=gettime_ms();
lookup_time=end_time-start_time; lookup_time=end_time-start_time;
last_bar_lookup=end_time; last_bar_lookup=end_time;
if (lookup_time>50) WARNF("Looking up a BAR took %lldms",lookup_time); if (lookup_time>50) WARNF("Looking up a BAR took %lldms",lookup_time);

101
rhizome_store.c
View File

@ -29,7 +29,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK) if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction"); return WHY("Failed to begin transaction");
/* /*
@ -45,10 +45,8 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
int ret=sqlite_exec_void_retry(&retry, int ret=sqlite_exec_void_retry(&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);", "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
write->id, (long long)file_length, priority, (long long)gettime_ms()); write->id, (long long)file_length, priority, (long long)gettime_ms());
if (ret!=SQLITE_OK) { if (ret==-1)
WHYF("Failed to insert into files: %s", sqlite3_errmsg(rhizome_db));
goto insert_row_fail; goto insert_row_fail;
}
char blob_path[1024]; char blob_path[1024];
@ -108,12 +106,12 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
} }
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1){
if (write->blob_fd>0){ if (write->blob_fd>0){
close(write->blob_fd); close(write->blob_fd);
unlink(blob_path); unlink(blob_path);
} }
return WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db)); return -1;
} }
write->file_length = file_length; write->file_length = file_length;
@ -196,7 +194,6 @@ int rhizome_flush(struct rhizome_write *write_state){
if (sqlite_code_busy(ret)) if (sqlite_code_busy(ret))
goto again; goto again;
else if (ret==SQLITE_OK){ else if (ret==SQLITE_OK){
DEBUGF("Success");
break; break;
} }
@ -210,7 +207,6 @@ int rhizome_flush(struct rhizome_write *write_state){
}while(1); }while(1);
} }
DEBUGF("Wrote %d bytes", write_state->data_size);
SHA512_Update(&write_state->sha512_context, write_state->buffer, write_state->data_size); SHA512_Update(&write_state->sha512_context, write_state->buffer, write_state->data_size);
write_state->file_offset+=write_state->data_size; write_state->file_offset+=write_state->data_size;
if (config.debug.rhizome) if (config.debug.rhizome)
@ -254,7 +250,12 @@ int rhizome_store_delete(const char *id){
char blob_path[1024]; char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id)) if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id))
return -1; return -1;
return unlink(blob_path)?-1:0; if (unlink(blob_path)){
if (config.debug.externalblobs)
DEBUG_perror("unlink");
return -1;
}
return 0;
} }
int rhizome_fail_write(struct rhizome_write *write){ int rhizome_fail_write(struct rhizome_write *write){
@ -270,9 +271,9 @@ int rhizome_fail_write(struct rhizome_write *write){
// don't worry too much about sql failures. // don't worry too much about sql failures.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (!config.rhizome.external_blobs) if (!config.rhizome.external_blobs)
sqlite_exec_void_retry(&retry, sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid); "DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
sqlite_exec_void_retry(&retry, sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILES WHERE id='%s'", "DELETE FROM FILES WHERE id='%s'",
write->id); write->id);
return 0; return 0;
@ -293,22 +294,17 @@ int rhizome_finish_write(struct rhizome_write *write){
SHA512_End(&write->sha512_context, hash_out); SHA512_End(&write->sha512_context, hash_out);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
WHY("Failed to begin transaction");
goto failure; goto failure;
}
if (write->id_known){ if (write->id_known){
if (strcasecmp(write->id, hash_out)){ if (strcasecmp(write->id, hash_out)){
WHYF("Expected hash=%s, got %s", write->id, hash_out); WHYF("Expected hash=%s, got %s", write->id, hash_out);
goto failure; goto failure;
} }
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'", gettime_ms(), write->id) == -1)
gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
}else{ }else{
str_toupper_inplace(hash_out); str_toupper_inplace(hash_out);
@ -317,15 +313,13 @@ int rhizome_finish_write(struct rhizome_write *write){
rhizome_fail_write(write); rhizome_fail_write(write);
}else{ }else{
// delete any half finished records // delete any half finished records
sqlite_exec_void_retry(&retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
sqlite_exec_void_retry(&retry,"DELETE FROM FILES WHERE id='%s';",hash_out); sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'", "UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'",
hash_out, gettime_ms(), write->id)!=SQLITE_OK){ hash_out, gettime_ms(), write->id) == -1)
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
if (config.rhizome.external_blobs){ if (config.rhizome.external_blobs){
char blob_path[1024]; char blob_path[1024];
@ -349,18 +343,15 @@ int rhizome_finish_write(struct rhizome_write *write){
}else{ }else{
if (sqlite_exec_void_retry(&retry, if (sqlite_exec_void_retry(&retry,
"UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld", "UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld",
hash_out, write->blob_rowid)!=SQLITE_OK){ hash_out, write->blob_rowid) == -1){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
} }
} }
} }
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH); strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH);
} }
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){ if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
goto failure; goto failure;
}
return 0; return 0;
failure: failure:
@ -465,10 +456,10 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
return 0; return 0;
} }
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){ /* Return -1 on error, 0 if file blob found, 1 if not found.
*/
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
{
strncpy(read->id, fileid, sizeof read->id); strncpy(read->id, fileid, sizeof read->id);
read->id[RHIZOME_FILEHASH_STRLEN] = '\0'; read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(read->id); str_toupper_inplace(read->id);
@ -485,27 +476,10 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
read->length=lseek(read->blob_fd,0,SEEK_END); read->length=lseek(read->blob_fd,0,SEEK_END);
}else{ }else{
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = ? AND FILES.datavalid != 0"); if (sqlite_exec_int64(&read->blob_rowid, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = '%s' AND FILES.datavalid != 0", read->id) == -1)
if (!statement)
return WHYF("Failed to prepare statement: %s", sqlite3_errmsg(rhizome_db));
sqlite3_bind_text(statement, 1, read->id, -1, SQLITE_STATIC);
int ret = sqlite_step_retry(&retry, statement);
if (ret != SQLITE_ROW){
WHYF("Failed to open file blob: %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
return -1; return -1;
} if (read->blob_rowid == -1)
return 1;
if (!(sqlite3_column_count(statement) == 1
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER)) {
sqlite3_finalize(statement);
return WHY("Incorrect statement column");
}
read->blob_rowid = sqlite3_column_int64(statement, 0);
sqlite3_finalize(statement);
read->length=-1; read->length=-1;
} }
read->hash=hash; read->hash=hash;
@ -559,7 +533,7 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
else if(ret!=SQLITE_OK){ else if(ret!=SQLITE_OK){
WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db)); WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob); sqlite3_blob_close(blob);
return -1; RETURN(-1);
} }
} }
@ -570,7 +544,7 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
again: again:
if (blob) sqlite3_blob_close(blob); if (blob) sqlite3_blob_close(blob);
if (sqlite_retry(&retry, "sqlite3_blob_open")==0) if (sqlite_retry(&retry, "sqlite3_blob_open")==0)
return -1; RETURN(-1);
}while (1); }while (1);
} }
@ -607,6 +581,8 @@ int rhizome_read_close(struct rhizome_read *read){
return 0; return 0;
} }
/* Returns -1 on error, 0 on success.
*/
static int write_file(struct rhizome_read *read, const char *filepath){ static int write_file(struct rhizome_read *read, const char *filepath){
int fd=-1, ret=0; int fd=-1, ret=0;
@ -658,9 +634,10 @@ int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhi
return 0; return 0;
} }
/* Extract the file related to a manifest to the file system. /* Extract the file related to a manifest to the file system. The file will be de-crypted and
* The file will be de-crypted and verified while reading. * verified while reading. If filepath is not supplied, the file will still be checked.
* If filepath is not supplied, the file will still be checked. *
* Returns -1 on error, 0 if extracted successfully, 1 if not found.
*/ */
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){
struct rhizome_read read_state; struct rhizome_read read_state;
@ -672,8 +649,12 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
return ret; return ret;
} }
/* dump the raw contents of a file */ /* dump the raw contents of a file
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length){ *
* Returns -1 on error, 0 if dumped successfully, 1 if not found.
*/
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
{
struct rhizome_read read_state; struct rhizome_read read_state;
bzero(&read_state, sizeof read_state); bzero(&read_state, sizeof read_state);

1
serval.h
View File

@ -586,7 +586,6 @@ int serval_packetvisualise(XPRINTF xpf, const char *message, const unsigned char
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax); int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int rhizome_opendb(); int rhizome_opendb();
void rhizome_cleanup();
int parseCommandLine(const char *argv0, int argc, const char *const *argv); int parseCommandLine(const char *argv0, int argc, const char *const *argv);

4
server.c
View File

@ -132,10 +132,6 @@ int server(char *backing_file)
fprintf(f,"%d\n", server_getpid); fprintf(f,"%d\n", server_getpid);
fclose(f); fclose(f);
/* Open Rhizome database and clean out any cruft */
rhizome_opendb();
rhizome_cleanup();
overlayServerMode(); overlayServerMode();
RETURN(0); RETURN(0);

18
strbuf.c
View File

@ -54,11 +54,19 @@ strbuf strbuf_ncat(strbuf sb, const char *text, size_t len)
strbuf strbuf_puts(strbuf sb, const char *text) strbuf strbuf_puts(strbuf sb, const char *text)
{ {
if (sb->start && (!sb->end || sb->current < sb->end)) { if (sb->start) {
register size_t n = sb->end ? sb->end - sb->current : -1; if (!sb->end) {
while (n-- && (*sb->current = *text)) { while ((*sb->current = *text)) {
++sb->current; ++sb->current;
++text; ++text;
}
} else if (sb->current < sb->end) {
register size_t n = sb->end - sb->current;
while (n-- && (*sb->current = *text)) {
++sb->current;
++text;
}
*sb->current = '\0';
} }
} }
while (*text++) while (*text++)

26
testdefs_rhizome.sh
View File

@ -386,14 +386,30 @@ extract_manifest_vars() {
rhizome_add_file() { rhizome_add_file() {
local name="$1" local name="$1"
local size="${2:-64}" local size="${2:-64}"
[ -e "$name" ] || create_file "$name" $size rhizome_add_files --size="$size" "$name"
local sidvar="SID$instance_name"
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "$name" --and-others
extract_manifest_vars "$name.manifest" extract_manifest_vars "$name.manifest"
} }
rhizome_add_files() {
local size=64
local sidvar="SID$instance_name"
local -a names=()
for arg; do
case "$arg" in
--size=*)
size="${arg##*=}"
;;
*)
local name="$arg"
[ -e "$name" ] || create_file "$name" $size
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
names+=("$name")
esac
done
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "${names[@]}" --and-others
}
rhizome_update_file() { rhizome_update_file() {
local orig_name="$1" local orig_name="$1"
local new_name="$2" local new_name="$2"

69
tests/rhizomeops
View File

@ -27,7 +27,7 @@ shopt -s extglob
setup_rhizome() { setup_rhizome() {
set_instance +A set_instance +A
executeOk_servald config set debug.rhizome on executeOk_servald config set debug.rhizome on
create_identities 1 create_single_identity
set_instance +B set_instance +B
executeOk_servald config set debug.rhizome on executeOk_servald config set debug.rhizome on
create_identities 4 create_identities 4
@ -865,7 +865,7 @@ test_ImportOwnBundle() {
assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
} }
doc_ImportCombinedBundle="Can generate a combined bundle, import into another instance and export again" doc_ImportCombinedBundle="Create and import combined bundle"
setup_ImportCombinedBundle() { setup_ImportCombinedBundle() {
setup_servald setup_servald
setup_rhizome setup_rhizome
@ -891,4 +891,69 @@ test_ImportCombinedBundle() {
assert diff fileA fileAx assert diff fileA fileAx
} }
setup_delete() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.clean_on_open off
rhizome_add_files file{1..4}
for i in {1..4}; do
extract_manifest_id BID$i file$i.manifest
extract_manifest_filehash HASH$i file$i.manifest
done
}
doc_DeleteManifest="Delete a manifest from store"
setup_DeleteManifest() {
setup_delete
}
test_DeleteManifest() {
executeOk_servald rhizome delete manifest "$BID2"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1,3,4}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID2"
executeOk_servald rhizome dump file "$HASH2" file2x
assert diff file2 file2x
}
doc_DeletePayload="Delete a payload from store"
setup_DeletePayload() {
setup_delete
}
test_DeletePayload() {
executeOk_servald rhizome delete payload "$BID3"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID3"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH3" file3x
}
doc_DeleteBundle="Delete a bundle from store"
setup_DeleteBundle() {
setup_delete
}
test_DeleteBundle() {
executeOk_servald rhizome delete bundle "$BID4"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..3}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID4"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH4" file4x
}
doc_DeleteFile="Delete a file from store"
setup_DeleteFile() {
setup_delete
}
test_DeleteFile() {
executeOk_servald rhizome delete file "$HASH1"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID1"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH1" file1x
}
runTests "$@" runTests "$@"