ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-01-29 15:43:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/development' into serial

Browse Source 
Conflicts:
	rhizome.h
	rhizome_database.c
	rhizome_fetch.c
	rhizome_store.c
This commit is contained in:
Jeremy Lakeman 2013-02-20 17:22:53 +10:30
commit 90386ce1b1
13 changed files with 522 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
commandline.c
View File

@ -1497,7 +1497,69 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, void *context)
return ret;
}
int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
int app_rhizome_delete(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *manifestid, *fileid;
if (cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, NULL) == -1)
return -1;
if (cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1;
/* Ensure the Rhizome database exists and is open */
if (create_serval_instance_dir() == -1)
return -1;
if (rhizome_opendb() == -1)
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
int ret=0;
if (cli_arg(parsed, "file", NULL, NULL, NULL) == 0) {
if (!fileid)
return WHY("missing <fileid> argument");
unsigned char filehash[RHIZOME_FILEHASH_BYTES];
if (fromhexstr(filehash, fileid, RHIZOME_FILEHASH_BYTES) == -1)
return WHY("Invalid file ID");
char fileIDUpper[RHIZOME_FILEHASH_STRLEN + 1];
tohex(fileIDUpper, filehash, RHIZOME_FILEHASH_BYTES);
ret = rhizome_delete_file(fileIDUpper);
} else {
if (!manifestid)
return WHY("missing <manifestid> argument");
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
return WHY("Invalid manifest ID");
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0)
ret = rhizome_delete_bundle(manifestIdUpper);
else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0)
ret = rhizome_delete_manifest(manifestIdUpper);
else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0)
ret = rhizome_delete_payload(manifestIdUpper);
else
return WHY("unrecognised command");
}
return ret;
}
int app_rhizome_clean(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
struct rhizome_cleanup_report report;
if (rhizome_cleanup(&report) == -1)
return -1;
cli_field_name("deleted_stale_incoming_files", ":");
cli_put_long(report.deleted_stale_incoming_files, "\n");
cli_field_name("deleted_orphan_files", ":");
cli_put_long(report.deleted_orphan_files, "\n");
cli_field_name("deleted_orphan_fileblobs", ":");
cli_put_long(report.deleted_orphan_fileblobs, "\n");
return 0;
}
int app_rhizome_extract(const struct cli_parsed *parsed, void *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
@ -1586,13 +1648,10 @@ int app_rhizome_extract_bundle(const struct cli_parsed *parsed, void *context)
}
}
}
if (retfile)
ret=retfile;
ret = retfile == -1 ? -1 : 1;
if (m)
rhizome_manifest_free(m);
return ret;
}
@ -1604,24 +1663,20 @@ int app_rhizome_dump_file(const struct cli_parsed *parsed, void *context)
if ( cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|| cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1;
if (create_serval_instance_dir() == -1)
return -1;
if (rhizome_opendb() == -1)
return -1;
if (!rhizome_exists(fileid))
return 1;
int64_t length;
if (rhizome_dump_file(fileid, filepath, &length))
return -1;
int ret = rhizome_dump_file(fileid, filepath, &length);
if (ret)
return ret == -1 ? -1 : 1;
cli_puts("filehash"); cli_delim(":");
cli_puts(fileid); cli_delim("\n");
cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", length); cli_delim("\n");
return 0;
}
@ -2301,28 +2356,40 @@ struct cli_schema command_line_options[]={
"Add a file to Rhizome and optionally write its manifest to the given path"},
{app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL},CLIFLAG_STANDALONE,
"Import a payload/manifest pair into Rhizome"},
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
"List all manifests and files in Rhizome"},
{app_rhizome_extract_bundle,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,
"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
"List all manifests and files in Rhizome"},
{app_rhizome_extract,{"rhizome","extract","bundle" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest and decrypted file to the given paths."},
{app_rhizome_extract_bundle,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
{app_rhizome_extract,{"rhizome","extract","manifest" KEYRING_PIN_OPTIONS,
"<manifestid>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE,
"Extract a manifest from Rhizome and write it to the given path"},
{app_rhizome_extract_bundle,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
{app_rhizome_extract,{"rhizome","extract","file" KEYRING_PIN_OPTIONS,
"<manifestid>","[<filepath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path"},
{app_rhizome_dump_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE,
"Extract a file from Rhizome and write it to the given path without attempting decryption"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL},
CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL},
CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL},
CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."},
"Extract a file from Rhizome and write it to the given path without attempting decryption"},
{app_rhizome_delete,{"rhizome","delete","\\manifest",
"<manifestid>",NULL},CLIFLAG_STANDALONE,
"Remove the manifest for the given bundle from the Rhizome store"},
{app_rhizome_delete,{"rhizome","delete","\\payload",
"<manifestid>",NULL},CLIFLAG_STANDALONE,
"Remove the payload for the given bundle from the Rhizome store"},
{app_rhizome_delete,{"rhizome","delete","\\bundle",
"<manifestid>",NULL},CLIFLAG_STANDALONE,
"Remove the manifest and payload for the given bundle from the Rhizome store"},
{app_rhizome_delete,{"rhizome","delete","\\file",
"<fileid>",NULL},CLIFLAG_STANDALONE,
"Remove the file with the given hash from the Rhizome store"},
{app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Synchronise with the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","push","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Deliver all new content to the specified Rhizome Direct server. Return when done."},
{app_rhizome_direct_sync,{"rhizome","direct","pull","[peer url]",NULL}, CLIFLAG_STANDALONE,
"Fetch all new content from the specified Rhizome Direct server. Return when done."},
{app_rhizome_clean,{"rhizome","clean",NULL},CLIFLAG_STANDALONE,
"Remove stale and orphaned content from the Rhizome store"},
{app_keyring_create,{"keyring","create",NULL},0,
"Create a new keyring file."},
{app_keyring_list,{"keyring","list" KEYRING_PIN_OPTIONS,NULL},CLIFLAG_STANDALONE,

2
conf_schema.c
View File

@ -425,6 +425,7 @@ static int cf_opt_network_interface_legacy(struct config_network_interface *nifp
return CFSTRINGOVERFLOW;
strncpy(nif.file, &name[1], len - 1)[len - 1] = '\0';
nif.match.patc = 0;
nif.socket_type = SOCK_FILE;
} else {
int star = (strchr(name, '*') != NULL) ? 1 : 0;
if (len + star >= sizeof(nif.match.patv[0]))
@ -433,6 +434,7 @@ static int cf_opt_network_interface_legacy(struct config_network_interface *nifp
if (star)
nif.match.patv[0][len] = '*';
nif.match.patc = 1;
nif.socket_type = SOCK_DGRAM;
}
if (*p == '=') {
const char *const type = p + 1;

1
conf_schema.h
View File

@ -310,6 +310,7 @@ END_STRUCT
STRUCT(rhizome)
ATOM(int, enable, 1, cf_opt_int_boolean,, "If true, server opens Rhizome database when starting")
ATOM(int, clean_on_open, 1, cf_opt_int_boolean,, "If true, Rhizome database is cleaned at start of every command")
STRING(256, datastore_path, "", cf_opt_absolute_path,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, 1000000, cf_opt_uint64_scaled,, "Size of database in bytes")
ATOM(char, external_blobs, 0, cf_opt_char_boolean,, "Store rhizome bundles as separate files.")

7
crypto.c
View File

@ -28,12 +28,9 @@ int crypto_verify_signature(unsigned char *sas_key,
reassembled,sizeof(reassembled),
sas_key);
if (result) {
WHY("Signature verification failed");
RETURN(-1);
}
if (result)
RETURN(WHY("Signature verification failed"));
RETURN(0);
OUT();
}
// verify the signature at the end of a message, on return message_len will be reduced by the length of the signature.

3
overlay.c
View File

@ -130,7 +130,8 @@ schedule(&_sched_##X); }
/* Get rhizome server started BEFORE populating fd list so that
the server's listen socket is in the list for poll() */
if (is_rhizome_enabled()) rhizome_opendb();
if (is_rhizome_enabled())
rhizome_opendb();
/* Rhizome http server needs to know which callback to attach
to client sockets, so provide it here, along with the name to

35
rhizome.h
View File

@ -197,8 +197,16 @@ extern sqlite3 *rhizome_db;
int rhizome_opendb();
int rhizome_close_db();
int rhizome_manifest_createid(rhizome_manifest *m);
struct rhizome_cleanup_report {
int deleted_stale_incoming_files;
int deleted_orphan_files;
int deleted_orphan_fileblobs;
};
int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_strn_is_manifest_id(const char *text);
int rhizome_str_is_manifest_id(const char *text);
int rhizome_strn_is_bundle_key(const char *text);
@ -276,17 +284,20 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))();
int is_debug_rhizome();
int is_debug_rhizome_ads();
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...);
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, strbuf stmt);
int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action);
int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt);
int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...);
int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...);
int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat,...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat,...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...);
int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64(struct __sourceloc, long long *result, const char *sqlformat, ...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, long long *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap);
#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb))
@ -297,9 +308,11 @@ int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat,...
#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__)
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
@ -314,6 +327,10 @@ int rhizome_list_manifests(const char *service, const char *name,
int limit, int offset, char count_rows);
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
int rhizome_advertise_manifest(rhizome_manifest *m);
int rhizome_delete_bundle(const char *manifestid);
int rhizome_delete_manifest(const char *manifestid);
int rhizome_delete_payload(const char *manifestid);
int rhizome_delete_file(const char *fileid);
#define RHIZOME_DONTVERIFY 0
#define RHIZOME_VERIFY 1

339
rhizome_database.c
View File

@ -255,8 +255,8 @@ int rhizome_opendb()
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
long long version;
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;")<0)
RETURN(WHY("Failed to check schema version"));
if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;") == -1)
RETURN(-1);
if (version<1){
/* Create tables as required */
@ -300,6 +300,8 @@ int rhizome_opendb()
All changes should attempt to preserve any existing data */
// We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open)
rhizome_cleanup(NULL);
RETURN(0);
OUT();
}
@ -496,13 +498,21 @@ int _sqlite_step_retry(struct __sourceloc __whence, int log_level, sqlite_retry_
}
/*
Convenience wrapper for executing a prepared SQL statement that returns no value. If an error
occurs then logs it at the given level and returns -1. If 'retry' is non-NULL and the BUSY error
occurs (indicating the database is locked, ie, currently in use by another process), then resets
the statement and retries while sqlite_retry() returns true. If sqlite_retry() returns false
then returns -1. Otherwise returns zero. Always finalises the statement before returning.
* Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
* Always finalises the statement before returning.
*
* If an error occurs then logs it at the given level and returns -1.
*
* If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
* currently in use by another process), then resets the statement and retries while sqlite_retry()
* returns true. If sqlite_retry() returns false then returns -1.
*
* Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
* was SQLITE_OK or SQLITE_DONE.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
static int _sqlite_exec_prepared(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
if (!statement)
return -1;
@ -510,22 +520,29 @@ static int _sqlite_exec_void_prepared(struct __sourceloc __whence, int log_level
int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, log_level, retry, statement)) == SQLITE_ROW)
++rowcount;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) ? 0 : -1;
if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return sqlite_code_ok(stepcode) ? rowcount : -1;
}
static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap);
return _sqlite_exec_void_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
int rowcount = _sqlite_exec_prepared(__whence, log_level, retry, _sqlite_prepare_loglevel(__whence, log_level, retry, stmt));
if (rowcount == -1)
return -1;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
return sqlite3_changes(rhizome_db);
}
/* Convenience wrapper for executing an SQL command that returns no value.
If an error occurs then logs it at ERROR level and returns -1. Otherwise returns zero.
@author Andrew Bettison <andrew@servalproject.com>
* If an error occurs then logs it at ERROR level and returns -1. Otherwise returns the number of
* rows changed by the command.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
{
@ -538,7 +555,8 @@ int _sqlite_exec_void(struct __sourceloc __whence, const char *sqlformat, ...)
}
/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR.
@author Andrew Bettison <andrew@servalproject.com>
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const char *sqlformat, ...)
{
@ -551,10 +569,11 @@ int _sqlite_exec_void_loglevel(struct __sourceloc __whence, int log_level, const
}
/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com>
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
* all in the event of a busy condition, but will log it as an error and return immediately.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *sqlformat, ...)
{
@ -565,6 +584,19 @@ int _sqlite_exec_void_retry(struct __sourceloc __whence, sqlite_retry_state *ret
return ret;
}
/* Same as sqlite_exec_void_retry(), but logs any error at the given level instead of ERROR.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_void_retry_loglevel(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_void(__whence, log_level, retry, sqlformat, ap);
va_end(ap);
return ret;
}
static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap)
{
strbuf stmt = strbuf_alloca(8192);
@ -585,16 +617,22 @@ static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *
if (rowcount > 1)
WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
sqlite3_finalize(statement);
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
if (!sqlite_code_ok(stepcode) || ret == -1)
return -1;
if (sqlite_trace_func())
DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
return rowcount;
}
/*
Convenience wrapper for executing an SQL command that returns a single int64 value.
Logs an error and returns -1 if an error occurs.
If no row is found, then returns 0 and does not alter *result.
If exactly one row is found, the assigns its value to *result and returns 1.
If more than one row is found, then logs a warning, assigns the value of the first row to *result
and returns the number of rows.
* Convenience wrapper for executing an SQL command that returns a single int64 value.
* Logs an error and returns -1 if an error occurs.
* If no row is found, then returns 0 and does not alter *result.
* If exactly one row is found, the assigns its value to *result and returns 1.
* If more than one row is found, then logs a warning, assigns the value of the first row to *result
* and returns the number of rows.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const char *sqlformat,...)
{
@ -607,10 +645,11 @@ int _sqlite_exec_int64(struct __sourceloc __whence, long long *result, const cha
}
/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
all in the event of a busy condition, but will log it as an error and return immediately.
@author Andrew Bettison <andrew@servalproject.com>
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
* all in the event of a busy condition, but will log it as an error and return immediately.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, long long *result, const char *sqlformat,...)
{
@ -621,26 +660,44 @@ int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *re
return ret;
}
/*
Convenience wrapper for executing an SQL command that returns a single text value.
Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
0 means no rows, nothing is appended to the strbuf
1 means exactly one row, appends its column to the strbuf
2 more than one row, logs a warning and appends the first row's column to the strbuf
@author Andrew Bettison <andrew@servalproject.com>
/* Convenience wrapper for executing an SQL command that returns a single text value.
* Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
* 0 means no rows, nothing is appended to the strbuf
* 1 means exactly one row, appends its column to the strbuf
* 2 more than one row, logs a warning and appends the first row's column to the strbuf
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_va_printf(stmt, sqlformat);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, &retry, stmt);
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
{
va_list ap;
va_start(ap, sqlformat);
int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
va_end(ap);
return ret;
}
int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap)
{
strbuf stmt = strbuf_alloca(8192);
strbuf_vsprintf(stmt, sqlformat, ap);
sqlite3_stmt *statement = _sqlite_prepare_loglevel(__whence, LOG_LEVEL_ERROR, retry, stmt);
if (!statement)
return -1;
int ret = 0;
int rowcount = 0;
int stepcode;
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) {
while ((stepcode = _sqlite_step_retry(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
int columncount = sqlite3_column_count(statement);
if (columncount != 1)
ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
@ -676,41 +733,44 @@ int rhizome_database_filehash_from_id(const char *id, uint64_t version, char has
OUT();
}
void rhizome_cleanup()
static int rhizome_cleanup_external(sqlite_retry_state *retry, sqlite3_stmt *statement){
int ret=0;
while (sqlite_step_retry(retry, statement) == SQLITE_ROW) {
const char *id = (const char *) sqlite3_column_text(statement, 0);
if (rhizome_store_delete(id)==0)
ret++;
}
return ret;
}
int rhizome_cleanup(struct rhizome_cleanup_report *report)
{
IN();
/* FIXME
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// cleanup external blobs for unreferenced files
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
int externals_removed=rhizome_cleanup_external(&retry, statement);
sqlite3_finalize(statement);
statement = sqlite_prepare(&retry, "SELECT id FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
externals_removed+=rhizome_cleanup_external(&retry, statement);
sqlite3_finalize(statement);
// clean out unreferenced files
// TODO keep updating inserttime for *very* long transfers?
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000)) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000)) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
// Clean up unreferenced blobs and files
{
char sqlcmd[1024];
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
strbuf_puts(b, "SELECT rowid FROM fileblobs WHERE TYPEOF(data)<>\"blob\" AND NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
if (!strbuf_overrun(b)) {
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, "%s", strbuf_str(b));
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
int64_t rowid = sqlite3_column_int64(statement, 0);
char *blobfile=rhizome_database_get_blob_filename(rowid);
if (blobfile) {
INFOF("Cleaning up orphaned blob file %s",blobfile);
unlink(blobfile);
}
}
sqlite3_finalize(statement);
}
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
}
*/
int ret;
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000);
if (report)
report->deleted_stale_incoming_files = ret;
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000);
if (report)
report->deleted_orphan_files = ret;
ret = sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );");
if (report)
report->deleted_orphan_fileblobs = ret + externals_removed;
RETURN(0);
OUT();
}
@ -724,7 +784,7 @@ int rhizome_make_space(int group_priority, long long bytes)
if (db_used == -1)
return -1;
rhizome_cleanup();
rhizome_cleanup(NULL);
/* If there is already enough space now, then do nothing more */
if (db_used<=(config.rhizome.database_size-bytes-65536))
@ -878,7 +938,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK)
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction");
sqlite3_stmt *stmt;
@ -946,7 +1006,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt);
stmt = NULL;
}
if (sqlite_exec_void_retry(&retry, "COMMIT;") == SQLITE_OK){
if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1){
// This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
@ -1086,7 +1146,6 @@ int rhizome_list_manifests(const char *service, const char *name,
const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
long long blob_date = rhizome_manifest_get_ll(m, "date");
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
int from_here = 0;
unsigned char senderSid[SID_SIZE];
unsigned char recipientSid[SID_SIZE];
@ -1107,7 +1166,6 @@ int rhizome_list_manifests(const char *service, const char *name,
int cn = 0, in = 0, kp = 0;
from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid);
}
if (config.debug.rhizome) DEBUGF("manifest payload size = %lld", blob_filesize);
cli_put_long(rowid, ":");
cli_put_string(blob_service, ":");
@ -1162,8 +1220,8 @@ int rhizome_update_file_priority(const char *fileid)
" AND groupmemberships.groupid=grouplist.id;",
fileid) == -1)
return -1;
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) != 0)
WHYF("cannot update priority for fileid=%s", fileid);
if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) == -1)
return WHYF("cannot update priority for fileid=%s", fileid);
return 0;
}
@ -1329,7 +1387,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
if (!inconsistent) {
*found = blob_m;
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
if (config.debug.rhizome)
DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash, q_author ? q_author : ""
);
ret = 1;
@ -1350,7 +1409,8 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
* Returns -1 on error
* Caller is responsible for allocating and freeing rhizome_manifest
*/
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m){
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m)
{
int ret=0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
@ -1391,8 +1451,113 @@ done:
return ret;
}
long long lookup_time=0;
long long last_bar_lookup=0;
int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid)
{
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?");
if (!statement)
return -1;
sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
return -1;
return sqlite3_changes(rhizome_db) ? 0 : 1;
}
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
{
int ret = 0;
sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?");
if (!statement)
ret = -1;
else {
sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
if (_sqlite_exec_prepared(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
ret = -1;
}
return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}
int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid)
{
strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = '%s'", manifestid);
if (rows == -1)
return -1;
if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
return -1;
return 0;
}
/* Remove a manifest and its bundle from the database, given its manifest ID.
*
* Returns 0 if manifest is found and removed and bundle was either absent or removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_bundle(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (rhizome_delete_payload_retry(&retry, manifestid) == -1)
return -1;
if (rhizome_delete_manifest_retry(&retry, manifestid) == -1)
return -1;
return 0;
}
/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
* untouched if present.
*
* Returns 0 if manifest is found and removed
* Returns 1 if manifest is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_manifest(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_manifest_retry(&retry, manifestid);
}
/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
* untouched if present.
*
* Returns 0 if manifest is found, its payload is found and removed
* Returns 1 if manifest or payload is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_payload(const char *manifestid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_payload_retry(&retry, manifestid);
}
/* Remove a file from the database, given its file hash.
*
* Returns 0 if file is found and removed
* Returns 1 if file is not found
* Returns -1 on error
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_delete_file(const char *fileid)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
return rhizome_delete_file_retry(&retry, fileid);
}
time_ms_t lookup_time=0;
time_ms_t last_bar_lookup=0;
int rhizome_is_bar_interesting(unsigned char *bar){
IN();
@ -1409,7 +1574,7 @@ int rhizome_is_bar_interesting(unsigned char *bar){
}
}
long long start_time=gettime_ms();
time_ms_t start_time=gettime_ms();
int64_t version = rhizome_bar_version(bar);
int ret=1;
@ -1446,11 +1611,11 @@ int rhizome_is_bar_interesting(unsigned char *bar){
}
sqlite3_finalize(statement);
long long end_time=gettime_ms();
time_ms_t end_time=gettime_ms();
lookup_time=end_time-start_time;
last_bar_lookup=end_time;
if (lookup_time>50) WARNF("Looking up a BAR took %lldms",lookup_time);
RETURN(ret);
OUT();
}
}

103
rhizome_store.c
View File

@ -29,7 +29,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK)
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
return WHY("Failed to begin transaction");
/*
@ -45,10 +45,8 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
int ret=sqlite_exec_void_retry(&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES('%s',%lld,%d,0,%lld);",
write->id, (long long)file_length, priority, (long long)gettime_ms());
if (ret!=SQLITE_OK) {
WHYF("Failed to insert into files: %s", sqlite3_errmsg(rhizome_db));
if (ret==-1)
goto insert_row_fail;
}
char blob_path[1024];
@ -108,12 +106,12 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
}
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1){
if (write->blob_fd>0){
close(write->blob_fd);
unlink(blob_path);
}
return WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
return -1;
}
write->file_length = file_length;
@ -196,7 +194,6 @@ int rhizome_flush(struct rhizome_write *write_state){
if (sqlite_code_busy(ret))
goto again;
else if (ret==SQLITE_OK){
DEBUGF("Success");
break;
}
@ -210,7 +207,6 @@ int rhizome_flush(struct rhizome_write *write_state){
}while(1);
}
DEBUGF("Wrote %d bytes", write_state->data_size);
SHA512_Update(&write_state->sha512_context, write_state->buffer, write_state->data_size);
write_state->file_offset+=write_state->data_size;
if (config.debug.rhizome)
@ -254,7 +250,12 @@ int rhizome_store_delete(const char *id){
char blob_path[1024];
if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, id))
return -1;
return unlink(blob_path)?-1:0;
if (unlink(blob_path)){
if (config.debug.externalblobs)
DEBUG_perror("unlink");
return -1;
}
return 0;
}
int rhizome_fail_write(struct rhizome_write *write){
@ -270,9 +271,9 @@ int rhizome_fail_write(struct rhizome_write *write){
// don't worry too much about sql failures.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (!config.rhizome.external_blobs)
sqlite_exec_void_retry(&retry,
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILEBLOBS WHERE rowid=%lld",write->blob_rowid);
sqlite_exec_void_retry(&retry,
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
"DELETE FROM FILES WHERE id='%s'",
write->id);
return 0;
@ -293,22 +294,17 @@ int rhizome_finish_write(struct rhizome_write *write){
SHA512_End(&write->sha512_context, hash_out);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") != SQLITE_OK){
WHY("Failed to begin transaction");
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
goto failure;
}
if (write->id_known){
if (strcasecmp(write->id, hash_out)){
WHYF("Expected hash=%s, got %s", write->id, hash_out);
goto failure;
}
if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
gettime_ms(), write->id) == -1)
goto failure;
}
}else{
str_toupper_inplace(hash_out);
@ -317,15 +313,13 @@ int rhizome_finish_write(struct rhizome_write *write){
rhizome_fail_write(write);
}else{
// delete any half finished records
sqlite_exec_void_retry(&retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
sqlite_exec_void_retry(&retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'",
hash_out, gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
hash_out, gettime_ms(), write->id) == -1)
goto failure;
}
if (config.rhizome.external_blobs){
char blob_path[1024];
@ -349,18 +343,15 @@ int rhizome_finish_write(struct rhizome_write *write){
}else{
if (sqlite_exec_void_retry(&retry,
"UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld",
hash_out, write->blob_rowid)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
hash_out, write->blob_rowid) == -1){
goto failure;
}
}
}
strlcpy(write->id, hash_out, SHA512_DIGEST_STRING_LENGTH);
}
if (sqlite_exec_void_retry(&retry, "COMMIT;")!=SQLITE_OK){
WHYF("Failed to commit transaction: %s", sqlite3_errmsg(rhizome_db));
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
goto failure;
}
return 0;
failure:
@ -465,10 +456,10 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
return 0;
}
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
/* Return -1 on error, 0 if file blob found, 1 if not found.
*/
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash)
{
strncpy(read->id, fileid, sizeof read->id);
read->id[RHIZOME_FILEHASH_STRLEN] = '\0';
str_toupper_inplace(read->id);
@ -485,27 +476,10 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
read->length=lseek(read->blob_fd,0,SEEK_END);
}else{
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = ? AND FILES.datavalid != 0");
if (!statement)
return WHYF("Failed to prepare statement: %s", sqlite3_errmsg(rhizome_db));
sqlite3_bind_text(statement, 1, read->id, -1, SQLITE_STATIC);
int ret = sqlite_step_retry(&retry, statement);
if (ret != SQLITE_ROW){
WHYF("Failed to open file blob: %s", sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
if (sqlite_exec_int64(&read->blob_rowid, "SELECT FILEBLOBS.rowid FROM FILEBLOBS, FILES WHERE FILEBLOBS.id = FILES.id AND FILES.id = '%s' AND FILES.datavalid != 0", read->id) == -1)
return -1;
}
if (!(sqlite3_column_count(statement) == 1
&& sqlite3_column_type(statement, 0) == SQLITE_INTEGER)) {
sqlite3_finalize(statement);
return WHY("Incorrect statement column");
}
read->blob_rowid = sqlite3_column_int64(statement, 0);
sqlite3_finalize(statement);
if (read->blob_rowid == -1)
return 1;
read->length=-1;
}
read->hash=hash;
@ -513,7 +487,7 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash){
if (hash)
SHA512_Init(&read->sha512_context);
return 0;
}
@ -559,7 +533,7 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
else if(ret!=SQLITE_OK){
WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
return -1;
RETURN(-1);
}
}
@ -570,7 +544,7 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
again:
if (blob) sqlite3_blob_close(blob);
if (sqlite_retry(&retry, "sqlite3_blob_open")==0)
return -1;
RETURN(-1);
}while (1);
}
@ -607,6 +581,8 @@ int rhizome_read_close(struct rhizome_read *read){
return 0;
}
/* Returns -1 on error, 0 on success.
*/
static int write_file(struct rhizome_read *read, const char *filepath){
int fd=-1, ret=0;
@ -658,9 +634,10 @@ int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhi
return 0;
}
/* Extract the file related to a manifest to the file system.
* The file will be de-crypted and verified while reading.
* If filepath is not supplied, the file will still be checked.
/* Extract the file related to a manifest to the file system. The file will be de-crypted and
* verified while reading. If filepath is not supplied, the file will still be checked.
*
* Returns -1 on error, 0 if extracted successfully, 1 if not found.
*/
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){
struct rhizome_read read_state;
@ -672,8 +649,12 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t
return ret;
}
/* dump the raw contents of a file */
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length){
/* dump the raw contents of a file
*
* Returns -1 on error, 0 if dumped successfully, 1 if not found.
*/
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
{
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);

1
serval.h
View File

@ -586,7 +586,6 @@ int serval_packetvisualise(XPRINTF xpf, const char *message, const unsigned char
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int rhizome_opendb();
void rhizome_cleanup();
int parseCommandLine(const char *argv0, int argc, const char *const *argv);

4
server.c
View File

@ -132,10 +132,6 @@ int server(char *backing_file)
fprintf(f,"%d\n", server_getpid);
fclose(f);
/* Open Rhizome database and clean out any cruft */
rhizome_opendb();
rhizome_cleanup();
overlayServerMode();
RETURN(0);

18
strbuf.c
View File

@ -54,11 +54,19 @@ strbuf strbuf_ncat(strbuf sb, const char *text, size_t len)
strbuf strbuf_puts(strbuf sb, const char *text)
{
if (sb->start && (!sb->end || sb->current < sb->end)) {
register size_t n = sb->end ? sb->end - sb->current : -1;
while (n-- && (*sb->current = *text)) {
++sb->current;
++text;
if (sb->start) {
if (!sb->end) {
while ((*sb->current = *text)) {
++sb->current;
++text;
}
} else if (sb->current < sb->end) {
register size_t n = sb->end - sb->current;
while (n-- && (*sb->current = *text)) {
++sb->current;
++text;
}
*sb->current = '\0';
}
}
while (*text++)

26
testdefs_rhizome.sh
View File

@ -386,14 +386,30 @@ extract_manifest_vars() {
rhizome_add_file() {
local name="$1"
local size="${2:-64}"
[ -e "$name" ] || create_file "$name" $size
local sidvar="SID$instance_name"
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "$name" --and-others
rhizome_add_files --size="$size" "$name"
extract_manifest_vars "$name.manifest"
}
rhizome_add_files() {
local size=64
local sidvar="SID$instance_name"
local -a names=()
for arg; do
case "$arg" in
--size=*)
size="${arg##*=}"
;;
*)
local name="$arg"
[ -e "$name" ] || create_file "$name" $size
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
names+=("$name")
esac
done
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "${names[@]}" --and-others
}
rhizome_update_file() {
local orig_name="$1"
local new_name="$2"

69
tests/rhizomeops
View File

@ -27,7 +27,7 @@ shopt -s extglob
setup_rhizome() {
set_instance +A
executeOk_servald config set debug.rhizome on
create_identities 1
create_single_identity
set_instance +B
executeOk_servald config set debug.rhizome on
create_identities 4
@ -865,7 +865,7 @@ test_ImportOwnBundle() {
assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
}
doc_ImportCombinedBundle="Can generate a combined bundle, import into another instance and export again"
doc_ImportCombinedBundle="Create and import combined bundle"
setup_ImportCombinedBundle() {
setup_servald
setup_rhizome
@ -891,4 +891,69 @@ test_ImportCombinedBundle() {
assert diff fileA fileAx
}
setup_delete() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.clean_on_open off
rhizome_add_files file{1..4}
for i in {1..4}; do
extract_manifest_id BID$i file$i.manifest
extract_manifest_filehash HASH$i file$i.manifest
done
}
doc_DeleteManifest="Delete a manifest from store"
setup_DeleteManifest() {
setup_delete
}
test_DeleteManifest() {
executeOk_servald rhizome delete manifest "$BID2"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1,3,4}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID2"
executeOk_servald rhizome dump file "$HASH2" file2x
assert diff file2 file2x
}
doc_DeletePayload="Delete a payload from store"
setup_DeletePayload() {
setup_delete
}
test_DeletePayload() {
executeOk_servald rhizome delete payload "$BID3"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID3"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH3" file3x
}
doc_DeleteBundle="Delete a bundle from store"
setup_DeleteBundle() {
setup_delete
}
test_DeleteBundle() {
executeOk_servald rhizome delete bundle "$BID4"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..3}
execute --exit-status=1 --stderr $servald rhizome extract manifest "$BID4"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH4" file4x
}
doc_DeleteFile="Delete a file from store"
setup_DeleteFile() {
setup_delete
}
test_DeleteFile() {
executeOk_servald rhizome delete file "$HASH1"
tfw_cat --stderr
executeOk_servald rhizome list
assert_rhizome_list file{1..4}
executeOk_servald rhizome extract manifest "$BID1"
execute --exit-status=1 --stderr $servald rhizome dump file "$HASH1" file1x
}
runTests "$@"