2012-01-12 03:38:24 +00:00
|
|
|
/*
|
2014-11-19 13:31:12 +00:00
|
|
|
Serval DNA - Rhizome database operations
|
|
|
|
Copyright (C) 2012-2014 Serval Project Inc.
|
2018-04-17 02:31:02 +00:00
|
|
|
Copyright (C) 2016-2018 Flinders University
|
2012-08-23 08:13:35 +00:00
|
|
|
|
2012-01-12 03:38:24 +00:00
|
|
|
This program is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU General Public License
|
|
|
|
as published by the Free Software Foundation; either version 2
|
|
|
|
of the License, or (at your option) any later version.
|
2012-08-23 08:13:35 +00:00
|
|
|
|
2012-01-12 03:38:24 +00:00
|
|
|
This program is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
2012-08-23 08:13:35 +00:00
|
|
|
|
2012-01-12 03:38:24 +00:00
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with this program; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
*/
|
|
|
|
|
2012-08-24 05:56:25 +00:00
|
|
|
#define __RHIZOME_INLINE
|
2012-09-25 04:01:34 +00:00
|
|
|
#include <stdlib.h>
|
|
|
|
#include <time.h>
|
2013-10-02 08:49:20 +00:00
|
|
|
#include <ctype.h>
|
2013-10-03 13:46:45 +00:00
|
|
|
#include <assert.h>
|
2017-09-11 03:13:38 +00:00
|
|
|
#include "lang.h" // for FALLTHROUGH
|
2012-02-23 02:15:42 +00:00
|
|
|
#include "serval.h"
|
2012-12-04 03:42:28 +00:00
|
|
|
#include "conf.h"
|
2012-01-12 03:38:24 +00:00
|
|
|
#include "rhizome.h"
|
2012-05-20 03:32:41 +00:00
|
|
|
#include "strbuf.h"
|
2012-12-04 03:42:28 +00:00
|
|
|
#include "strbuf_helpers.h"
|
2012-09-25 04:01:34 +00:00
|
|
|
#include "str.h"
|
2013-10-16 03:00:00 +00:00
|
|
|
#include "keyring.h"
|
2014-05-29 07:11:27 +00:00
|
|
|
#include "server.h"
|
2016-11-11 00:09:50 +00:00
|
|
|
#include "commandline.h"
|
2017-03-06 03:31:14 +00:00
|
|
|
#include "mdp_client.h"
|
2017-11-24 00:03:39 +00:00
|
|
|
#include "debug.h"
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2013-10-03 13:46:45 +00:00
|
|
|
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
|
2013-08-21 06:15:18 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
__thread struct rhizome_database rhizome_database = {
|
|
|
|
.dir_path = "",
|
|
|
|
.db = NULL,
|
|
|
|
.uuid = SERVAL_UUID_INVALID
|
|
|
|
};
|
|
|
|
|
2014-11-11 06:41:33 +00:00
|
|
|
static time_ms_t rhizomeRetryLimit = -1;
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2012-12-11 05:29:46 +00:00
|
|
|
int is_debug_rhizome()
|
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
return IF_DEBUG(rhizome);
|
2012-12-11 05:29:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int is_debug_rhizome_ads()
|
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
return IF_DEBUG(rhizome_ads);
|
2012-12-11 05:29:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int (*sqlite_trace_func)() = is_debug_rhizome;
|
2012-10-16 06:16:52 +00:00
|
|
|
const struct __sourceloc *sqlite_trace_whence = NULL;
|
2013-11-13 02:15:32 +00:00
|
|
|
static int sqlite_trace_done;
|
2017-03-06 03:31:14 +00:00
|
|
|
static uint64_t max_rowid=0;
|
2012-10-10 02:52:30 +00:00
|
|
|
|
2016-09-19 02:51:57 +00:00
|
|
|
/* This callback conditionally logs all rendered SQL statements. This function is registered with
|
|
|
|
* SQLite as the 'trace callback'. SQLite invokes it with mask == SQLITE_TRACE_STMT when about to
|
|
|
|
* execute a statement using sqlite3_step(), and with mask == SQLITE_TRACE_PROFILE when the
|
|
|
|
* statement finishes executing.
|
2013-11-13 02:15:32 +00:00
|
|
|
*
|
2016-09-19 02:51:57 +00:00
|
|
|
* SQLite neglects to call its trace callback with mask==SQLITE_TRACE_STMT on some kinds of
|
|
|
|
* statements, eg, PRAGMA. However, it does appear to call its trace callback with
|
|
|
|
* mask==SQLITE_TRACE_PROFILE after every kind of statement. Hence this function uses both
|
|
|
|
* callbacks, and uses the 'sqlite_trace_done' flag, which gets reset to zero whenever Rhizome
|
|
|
|
* prepares a new SQLite statement, to avoid duplicate logging.
|
2013-11-13 02:15:32 +00:00
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2016-09-19 02:51:57 +00:00
|
|
|
static int sqlite_trace_callback(unsigned UNUSED(mask), void *UNUSED(context), void * p, void * x)
|
2013-11-13 02:15:32 +00:00
|
|
|
{
|
2016-09-19 02:51:57 +00:00
|
|
|
if (sqlite_trace_func()) {
|
2017-09-11 04:39:05 +00:00
|
|
|
const char * expanded_sql = NULL;
|
2016-09-19 02:51:57 +00:00
|
|
|
const char * rendered_sql = NULL;
|
|
|
|
switch (mask) {
|
|
|
|
case SQLITE_TRACE_STMT:
|
|
|
|
if (!sqlite_trace_done) {
|
|
|
|
sqlite3_stmt * stmt = p;
|
|
|
|
const char * unexpanded_sql = x;
|
2017-09-11 04:39:05 +00:00
|
|
|
rendered_sql = (unexpanded_sql[0] == '-' && unexpanded_sql[1] == '-') ?
|
|
|
|
&unexpanded_sql[2] : (expanded_sql = sqlite3_expanded_sql(stmt));
|
2016-09-19 02:51:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
case SQLITE_TRACE_PROFILE:
|
|
|
|
if (!sqlite_trace_done) {
|
|
|
|
sqlite3_stmt * stmt = p;
|
2017-09-11 04:39:05 +00:00
|
|
|
rendered_sql = (expanded_sql = sqlite3_expanded_sql(stmt));
|
2016-09-19 02:51:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (rendered_sql) {
|
Rewrite logging system
Rename the logging primitive functions and utility functions, prefixing
all with 'serval_log', eg: logMessage() -> serval_logf() etc.
Add an XPRINTF xhexdump() function and use it to implement the
serval_log_hexdump() utility, renamed from dump(). Add macros
WHY_dump(), WARN_dump(), HINT_dump() and DEBUG_dump(), and use them
everywhere.
Remove the 'log.console.dump_config' and 'log.file.dump_config'
configuration options; configuration is now dumped in every log prolog.
The logging system now constructs the log prolog by invoking the new
'log_prolog' trigger, so that it no longer depends on the version string
and configuration system. Any system that wants to present a message in
the log prolog can define its own trigger, which calls standard log
primitives to print the message.
Split the logging system into a front-end (log.c) that provides the
logging primitives and is independent of the configuration system, and a
set of back-end "outputters" (log_output_console.c, log_output_file.c,
log_output_android.c) that may depend on the configuration system and
are decoupled from the front-end using the 'logoutput' link section.
These log outputters are explicitly linked into executables by the
Makefile rules, but could also be linked in using USE_FEATURE(). The
USE_FEATURE() calls have _not_ been added to servald_features.c, so that
different daemon executables can be built with the same feature set but
different log outputs.
2017-11-29 13:34:54 +00:00
|
|
|
serval_logf(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql);
|
2016-09-19 02:51:57 +00:00
|
|
|
++sqlite_trace_done;
|
|
|
|
}
|
2017-09-11 04:39:05 +00:00
|
|
|
if (expanded_sql)
|
|
|
|
sqlite3_free((void*)expanded_sql);
|
2016-09-19 02:51:57 +00:00
|
|
|
}
|
|
|
|
return 0; // ignored by SQLite
|
2012-10-10 02:52:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* This function allows code like:
|
|
|
|
*
|
|
|
|
* debugflags_t oldmask = sqlite_set_debugmask(DEBUG_SOMETHING_ELSE);
|
|
|
|
* ...
|
|
|
|
* sqlite3_stmt *statement = sqlite_prepare(&retry, "select blah blah blah");
|
|
|
|
* while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
|
|
|
* // do blah blah blah
|
|
|
|
* }
|
|
|
|
* ...
|
|
|
|
* sqlite_set_debugmask(oldmask);
|
|
|
|
* return result;
|
|
|
|
*
|
|
|
|
* so that code can choose which DEBUG_ flags control the logging of rendered SQL queries.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2012-12-11 05:29:46 +00:00
|
|
|
int (*sqlite_set_tracefunc(int (*newfunc)()))()
|
2012-10-10 02:52:30 +00:00
|
|
|
{
|
2012-12-11 05:29:46 +00:00
|
|
|
int (*oldfunc)() = sqlite_trace_func;
|
|
|
|
sqlite_trace_func = newfunc;
|
|
|
|
return oldfunc;
|
2012-10-09 07:13:34 +00:00
|
|
|
}
|
|
|
|
|
2013-12-09 07:52:18 +00:00
|
|
|
void sqlite_log(void *UNUSED(ignored), int result, const char *msg)
|
2013-10-02 08:49:20 +00:00
|
|
|
{
|
2013-01-06 03:48:36 +00:00
|
|
|
WARNF("Sqlite: %d %s", result, msg);
|
|
|
|
}
|
|
|
|
|
2013-11-28 07:14:37 +00:00
|
|
|
void verify_bundles()
|
|
|
|
{
|
2013-01-16 00:26:09 +00:00
|
|
|
// assume that only the manifest itself can be trusted
|
2017-03-06 05:26:58 +00:00
|
|
|
// fetch all manifests, parse and update or delete them.
|
2013-01-16 00:26:09 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;");
|
2013-11-28 07:14:37 +00:00
|
|
|
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
2013-01-16 00:26:09 +00:00
|
|
|
sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
|
2013-12-19 08:37:14 +00:00
|
|
|
const void *blob = sqlite3_column_blob(statement, 1);
|
|
|
|
size_t blob_length = sqlite3_column_bytes(statement, 1);
|
|
|
|
rhizome_manifest *m = rhizome_new_manifest();
|
|
|
|
if (m) {
|
|
|
|
memcpy(m->manifestdata, blob, blob_length);
|
|
|
|
m->manifest_all_bytes = blob_length;
|
|
|
|
int ret = -1;
|
|
|
|
if ( rhizome_manifest_parse(m) != -1
|
|
|
|
&& rhizome_manifest_validate(m)
|
|
|
|
&& rhizome_manifest_verify(m)
|
|
|
|
) {
|
|
|
|
assert(m->finalised);
|
2017-03-06 05:26:58 +00:00
|
|
|
|
|
|
|
if (m->filesize == 0 || rhizome_exists(&m->filehash) == RHIZOME_PAYLOAD_STATUS_STORED){
|
|
|
|
// Attempt to update the manifest
|
|
|
|
rhizome_bar_t bar;
|
|
|
|
rhizome_manifest_to_bar(m, &bar);
|
|
|
|
rhizome_authenticate_author(m);
|
|
|
|
|
2017-04-19 05:16:11 +00:00
|
|
|
if (sqlite_exec_void("UPDATE MANIFESTS SET "
|
|
|
|
"id = ?, "
|
|
|
|
"version = ?, "
|
|
|
|
"bar = ?, "
|
|
|
|
"filesize = ?, "
|
|
|
|
"filehash = ?, "
|
|
|
|
"author = ?, "
|
|
|
|
"service = ?, "
|
|
|
|
"name = ?, "
|
|
|
|
"sender = ?, "
|
|
|
|
"recipient = ?, "
|
|
|
|
"tail = ?, "
|
|
|
|
"manifest_hash = ? "
|
2017-03-06 05:26:58 +00:00
|
|
|
"WHERE ROWID = ?;",
|
|
|
|
RHIZOME_BID_T, &m->keypair.public_key,
|
|
|
|
INT64, m->version,
|
|
|
|
RHIZOME_BAR_T, &bar,
|
|
|
|
INT64, m->filesize,
|
|
|
|
RHIZOME_FILEHASH_T|NUL, m->filesize > 0 ? &m->filehash : NULL,
|
|
|
|
SID_T|NUL, m->authorship == AUTHOR_AUTHENTIC ? &m->author : NULL,
|
|
|
|
STATIC_TEXT, m->service,
|
|
|
|
STATIC_TEXT|NUL, m->name,
|
|
|
|
SID_T|NUL, m->has_sender ? &m->sender : NULL,
|
|
|
|
SID_T|NUL, m->has_recipient ? &m->recipient : NULL,
|
|
|
|
INT64, m->tail,
|
|
|
|
RHIZOME_FILEHASH_T, &m->manifesthash,
|
|
|
|
INT64, rowid,
|
|
|
|
END
|
|
|
|
)!=-1)
|
|
|
|
ret = 0;
|
|
|
|
}
|
2013-12-19 08:37:14 +00:00
|
|
|
}
|
2017-03-06 05:26:58 +00:00
|
|
|
|
2013-12-19 08:37:14 +00:00
|
|
|
if (ret) {
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "Removing invalid manifest entry @%lld", rowid);
|
2013-12-19 08:37:14 +00:00
|
|
|
sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
|
|
|
|
}
|
|
|
|
rhizome_manifest_free(m);
|
2013-01-16 00:26:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
sqlite3_finalize(statement);
|
|
|
|
}
|
|
|
|
|
2012-10-15 07:38:31 +00:00
|
|
|
/*
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
* The MANIFESTS table 'author' column records the cryptographically verified SID of the author that
|
|
|
|
* has write permission on the bundle, ie, possesses the Rhizome secret key that generated the BID,
|
|
|
|
* and hence can derive the Bundle Secret from the bundle's BK field:
|
|
|
|
*
|
|
|
|
* - The MANIFESTS table 'author' column is set to the author SID when a bundle is created locally
|
|
|
|
* by a non-secret identity, so no verification need be performed for one's own bundles while they
|
|
|
|
* remain in the local Rhizome store.
|
|
|
|
*
|
|
|
|
* - When a bundle is imported, the 'author' column is set to NULL to indicate that no verification
|
|
|
|
* has passed yet. This includes one's own bundles that have been purged from the local Rhizome
|
|
|
|
* store then recovered from a remote Rhizome node.
|
|
|
|
*
|
|
|
|
* - When a manifest with NULL 'author' is examined closely, ie extracted, not merely listed, the
|
|
|
|
* keyring is searched for an identity that is the author. If the identity is found and its
|
|
|
|
* Rhizome Secret unlocks the Bundle Key (ie, reveals a Bundle Secret that yields the Bundle's ID
|
|
|
|
* as its public key), the MANIFESTS table 'author' column is updated. This allows one to regain
|
|
|
|
* the ability to overwrite one's own bundles that have been lost but
|
|
|
|
* recovered from an exterior Rhizome node.
|
|
|
|
*
|
2012-10-15 07:38:31 +00:00
|
|
|
* - The above check automates the "own bundle recovery" mechanism at the expense of a CPU-heavy
|
|
|
|
* cryptographic check every time a foreign bundle is examined, but at least listing is fast.
|
|
|
|
* This will not scale as many identities are added to the keyring. It will eventually have to be
|
|
|
|
* replaced with a means to cache positive and negative verifications in the Rhizome db for local,
|
|
|
|
* non-secret identities.
|
|
|
|
*
|
|
|
|
* -- Andrew Bettison <andrew@servalproject.com>, October 2012
|
|
|
|
*/
|
|
|
|
|
2012-01-12 03:38:24 +00:00
|
|
|
int rhizome_opendb()
|
|
|
|
{
|
2018-04-10 01:20:06 +00:00
|
|
|
if (rhizome_database.db) {
|
|
|
|
assert(serval_uuid_is_valid(&rhizome_database.uuid));
|
2013-11-12 07:44:14 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2012-06-27 07:24:42 +00:00
|
|
|
IN();
|
2016-04-12 06:13:56 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
if (sodium_init() == -1)
|
2016-04-12 06:13:56 +00:00
|
|
|
RETURN(WHY("Failed to initialise libsodium"));
|
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// Work out the absolute path of the directory that contains all the database
|
|
|
|
// files/subdirectories.
|
|
|
|
if (!FORMF_RHIZOME_STORE_PATH(rhizome_database.dir_path, "%s", ""))
|
2014-06-18 07:55:15 +00:00
|
|
|
RETURN(-1);
|
2018-04-17 02:31:02 +00:00
|
|
|
DEBUGF(rhizome, "Rhizome store directory path = %s", alloca_str_toprint(rhizome_database.dir_path));
|
|
|
|
|
|
|
|
// Work out the absolute paths of the database file and subdirectories.
|
|
|
|
char dbpath[sizeof rhizome_database.dir_path];
|
|
|
|
char blobpath[sizeof rhizome_database.dir_path];
|
|
|
|
char hashpath[sizeof rhizome_database.dir_path];
|
|
|
|
char temppath[sizeof rhizome_database.dir_path];
|
|
|
|
int db_exists = 0;
|
|
|
|
{
|
|
|
|
struct file_meta dbmeta;
|
|
|
|
if ( !FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db")
|
|
|
|
|| !FORMF_RHIZOME_STORE_PATH(blobpath, RHIZOME_BLOB_SUBDIR)
|
|
|
|
|| !FORMF_RHIZOME_STORE_PATH(hashpath, RHIZOME_HASH_SUBDIR)
|
|
|
|
|| !FORMF_RHIZOME_STORE_PATH(temppath, "sqlite3tmp")
|
|
|
|
|| get_file_meta(dbpath, &dbmeta) == -1)
|
|
|
|
RETURN(-1);
|
|
|
|
db_exists = is_file_meta_exists(&dbmeta);
|
|
|
|
}
|
2018-04-10 07:18:23 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// Create missing store directory.
|
|
|
|
if (emkdirs_info(rhizome_database.dir_path, 0700) == -1)
|
|
|
|
RETURN(-1);
|
2018-04-10 07:18:23 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// If the database file does not exist, then SQLite will create it. However, in 2014 a bug was
|
|
|
|
// introduced that always created the database file in the instance directory, regardless of the
|
|
|
|
// setting of the 'rhizome.datastore_path' config option. This bug has now been fixed, so in
|
|
|
|
// order to preserve the Rhizome data store when upgrading from an earlier (buggy) version, if
|
|
|
|
// there is a database file at this "legacy" location, then move it into the correct, configured
|
|
|
|
// location, instead of creating a fresh (empty) database file.
|
|
|
|
if (!db_exists) {
|
|
|
|
char legacy_dbpath[sizeof rhizome_database.dir_path];
|
|
|
|
if (FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dbpath, "rhizome.db")
|
|
|
|
&& strcmp(legacy_dbpath, dbpath) != 0
|
|
|
|
&& file_exists(legacy_dbpath)
|
|
|
|
) {
|
|
|
|
INFOF("Recover legacy Rhizome SQLite database");
|
|
|
|
// Move the legacy database file to its correct location.
|
|
|
|
if (erename_info(legacy_dbpath, dbpath) == -1)
|
|
|
|
RETURN(-1);
|
|
|
|
// Move any legacy "blob" and "hash" subdirectories too, if they are present.
|
|
|
|
char legacy_dirpath[sizeof rhizome_database.dir_path];
|
|
|
|
if ( !file_exists(blobpath)
|
|
|
|
&& FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_BLOB_SUBDIR)
|
|
|
|
&& file_exists(legacy_dirpath))
|
|
|
|
erename_info(legacy_dirpath, blobpath);
|
|
|
|
if ( !file_exists(hashpath)
|
|
|
|
&& FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_HASH_SUBDIR)
|
|
|
|
&& file_exists(legacy_dirpath))
|
|
|
|
erename_info(legacy_dirpath, hashpath);
|
2018-04-10 07:18:23 +00:00
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
else
|
|
|
|
INFOF("Creating Rhizome SQLite database: %s", dbpath);
|
2018-04-10 07:18:23 +00:00
|
|
|
}
|
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// Create missing sub-directories.
|
|
|
|
if ( emkdirs_info(blobpath, 0700) == -1
|
|
|
|
|| emkdirs_info(hashpath, 0700) == -1
|
|
|
|
|| emkdirs_info(temppath, 0700) == -1)
|
|
|
|
RETURN(-1);
|
2018-04-10 07:18:23 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// Inform SQLite of its temporary directory.
|
|
|
|
assert(!sqlite3_temp_directory);
|
|
|
|
sqlite3_temp_directory = sqlite3_mprintf("%s", temppath);
|
2018-04-10 07:18:23 +00:00
|
|
|
|
2018-04-17 02:31:02 +00:00
|
|
|
// Set up SQLite logging.
|
|
|
|
sqlite3_config(SQLITE_CONFIG_LOG, sqlite_log, NULL);
|
|
|
|
|
|
|
|
// Open the SQLite database file, creating it if necessary.
|
|
|
|
if (sqlite3_open(dbpath, &rhizome_database.db)){
|
2018-04-10 01:20:06 +00:00
|
|
|
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_database.db)));
|
2012-06-27 07:24:42 +00:00
|
|
|
}
|
2018-04-10 01:20:06 +00:00
|
|
|
sqlite3_trace_v2(rhizome_database.db, SQLITE_TRACE_STMT, sqlite_trace_callback, NULL);
|
2015-07-06 08:19:49 +00:00
|
|
|
int loglevel = IF_DEBUG(rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2018-04-10 01:20:06 +00:00
|
|
|
const char *env = getenv("SERVALD_rhizome_database.db_RETRY_LIMIT_MS");
|
2014-11-11 06:41:33 +00:00
|
|
|
rhizomeRetryLimit = env ? atoi(env) : -1;
|
|
|
|
|
2012-03-04 23:05:12 +00:00
|
|
|
/* Read Rhizome configuration */
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "Rhizome will use %"PRIu64"B of storage for its database.", (uint64_t) config.rhizome.database_size);
|
2018-04-17 02:31:02 +00:00
|
|
|
|
2013-01-06 02:34:49 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2012-10-03 15:54:39 +00:00
|
|
|
|
2013-12-11 00:41:34 +00:00
|
|
|
uint64_t version;
|
2017-03-06 05:26:58 +00:00
|
|
|
if (sqlite_exec_uint64_retry(&retry, &version, "PRAGMA user_version;", END) != SQLITE_ROW)
|
2013-02-20 04:14:29 +00:00
|
|
|
RETURN(-1);
|
2018-04-10 07:18:23 +00:00
|
|
|
|
2013-01-06 02:34:49 +00:00
|
|
|
if (version<1){
|
|
|
|
/* Create tables as required */
|
2016-03-07 04:04:53 +00:00
|
|
|
// Note that this will create the current schema
|
|
|
|
// further additional columns should be skipped.
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;", END);
|
2014-06-18 07:55:15 +00:00
|
|
|
if ( sqlite_exec_void_retry(&retry,
|
|
|
|
"CREATE TABLE IF NOT EXISTS MANIFESTS("
|
|
|
|
"id text not null primary key, "
|
|
|
|
"version integer, "
|
|
|
|
"inserttime integer, "
|
|
|
|
"filesize integer, "
|
|
|
|
"filehash text, "
|
|
|
|
"author text, "
|
|
|
|
"bar blob, "
|
|
|
|
"manifest blob, "
|
|
|
|
"service text, "
|
|
|
|
"name text, "
|
|
|
|
"sender text collate nocase, "
|
|
|
|
"recipient text collate nocase, "
|
2016-03-07 04:04:53 +00:00
|
|
|
"tail integer, "
|
|
|
|
"manifest_hash text collate nocase"
|
2014-06-18 07:55:15 +00:00
|
|
|
");", END) == -1
|
|
|
|
|| sqlite_exec_void_retry(&retry,
|
|
|
|
"CREATE TABLE IF NOT EXISTS FILES("
|
|
|
|
"id text not null primary key, "
|
|
|
|
"length integer, "
|
|
|
|
"datavalid integer, "
|
2014-07-04 04:55:28 +00:00
|
|
|
"inserttime integer, "
|
|
|
|
"last_verified integer"
|
2014-06-18 07:55:15 +00:00
|
|
|
");", END) == -1
|
|
|
|
|| sqlite_exec_void_retry(&retry,
|
|
|
|
"CREATE TABLE IF NOT EXISTS FILEBLOBS("
|
|
|
|
"id text not null primary key, "
|
|
|
|
"data blob"
|
|
|
|
");", END) == -1
|
|
|
|
|| sqlite_exec_void_retry(&retry,
|
|
|
|
"CREATE TABLE IF NOT EXISTS IDENTITY("
|
|
|
|
"uuid text not null"
|
|
|
|
"); ", END) == -1
|
2013-01-06 02:34:49 +00:00
|
|
|
) {
|
|
|
|
RETURN(WHY("Failed to create schema"));
|
|
|
|
}
|
|
|
|
/* Create indexes if they don't already exist */
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
|
2013-01-06 02:34:49 +00:00
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
if (version<2 && db_exists){
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
|
2013-01-16 00:26:09 +00:00
|
|
|
}
|
2013-02-15 06:38:51 +00:00
|
|
|
if (version<3){
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
|
2013-02-15 06:38:51 +00:00
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
if (version<4 && db_exists){
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
|
2013-07-22 05:34:02 +00:00
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
if (version<5 && db_exists){
|
2013-11-12 07:44:14 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
|
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
if (version<6 && db_exists){
|
2014-07-04 04:55:28 +00:00
|
|
|
// we've always been at war with eurasia
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPLIST; ", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPMEMBERSHIPS; ", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS VERIFICATIONS; ", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS FILEMANIFESTS;", END);
|
|
|
|
}
|
2018-04-17 02:31:02 +00:00
|
|
|
if (version<7 && db_exists){
|
2016-03-07 04:04:53 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE FILES ADD COLUMN last_verified integer;", END);
|
2014-07-04 04:55:28 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=7;", END);
|
2014-06-18 07:55:15 +00:00
|
|
|
}
|
|
|
|
|
2016-03-07 04:04:53 +00:00
|
|
|
if (version<8){
|
2018-04-17 02:31:02 +00:00
|
|
|
if (db_exists)
|
2016-03-07 04:04:53 +00:00
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN manifest_hash text collate nocase;", END);
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFEST_HASH ON MANIFESTS(manifest_hash);", END);
|
|
|
|
|
|
|
|
// we need to populate fields on upgrade from older versions, we can simply re-insert all old manifests
|
|
|
|
// if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
|
|
|
|
verify_bundles();
|
|
|
|
|
|
|
|
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=8;", END);
|
|
|
|
}
|
|
|
|
|
2014-06-18 07:55:15 +00:00
|
|
|
// TODO recreate tables with collate nocase on all hex columns
|
|
|
|
|
|
|
|
/* Future schema updates should be performed here.
|
|
|
|
The above schema can be assumed to exist, no matter which version we upgraded from.
|
|
|
|
All changes should attempt to preserve all existing interesting data */
|
2013-11-12 07:44:14 +00:00
|
|
|
|
2017-10-09 01:11:44 +00:00
|
|
|
char buf[SERVAL_UUID_STRLEN + 1];
|
2015-10-13 08:12:22 +00:00
|
|
|
int r = sqlite_exec_strbuf_retry(&retry, strbuf_local_buf(buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
|
2013-11-12 07:44:14 +00:00
|
|
|
if (r == -1)
|
|
|
|
RETURN(-1);
|
|
|
|
if (r) {
|
2018-04-10 01:20:06 +00:00
|
|
|
if (!str_to_serval_uuid(buf, &rhizome_database.uuid, NULL)) {
|
2013-11-12 07:44:14 +00:00
|
|
|
WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf));
|
2018-04-10 01:20:06 +00:00
|
|
|
if (serval_uuid_generate_random(&rhizome_database.uuid) == -1)
|
2013-11-12 07:44:14 +00:00
|
|
|
RETURN(WHY("Cannot generate new UUID for Rhizome database"));
|
2018-04-10 01:20:06 +00:00
|
|
|
if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", SERVAL_UUID_T, &rhizome_database.uuid, END) == -1)
|
2013-11-12 07:44:14 +00:00
|
|
|
RETURN(WHY("Failed to update new UUID in Rhizome database"));
|
2018-04-10 01:20:06 +00:00
|
|
|
DEBUGF(rhizome, "Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_database.uuid));
|
2013-11-12 07:44:14 +00:00
|
|
|
}
|
|
|
|
} else if (r == 0) {
|
2018-04-10 01:20:06 +00:00
|
|
|
if (serval_uuid_generate_random(&rhizome_database.uuid) == -1)
|
2013-11-12 07:44:14 +00:00
|
|
|
RETURN(WHY("Cannot generate UUID for Rhizome database"));
|
2018-04-10 01:20:06 +00:00
|
|
|
if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", SERVAL_UUID_T, &rhizome_database.uuid, END) == -1)
|
2013-11-12 07:44:14 +00:00
|
|
|
RETURN(WHY("Failed to insert UUID into Rhizome database"));
|
2018-04-10 01:20:06 +00:00
|
|
|
DEBUGF(rhizome, "Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_database.uuid));
|
2013-11-12 07:44:14 +00:00
|
|
|
}
|
|
|
|
|
2012-12-28 02:47:04 +00:00
|
|
|
// We can't delete a file that is being transferred in another process at this very moment...
|
2013-02-20 04:13:20 +00:00
|
|
|
if (config.rhizome.clean_on_open)
|
2013-02-20 04:14:29 +00:00
|
|
|
rhizome_cleanup(NULL);
|
2017-03-06 03:31:14 +00:00
|
|
|
|
2018-06-01 08:10:09 +00:00
|
|
|
if (serverMode != SERVER_NOT_RUNNING) {
|
2017-03-06 03:31:14 +00:00
|
|
|
sqlite_exec_uint64_retry(&retry, &max_rowid,
|
|
|
|
"SELECT max(rowid) "
|
|
|
|
"FROM manifests", END);
|
|
|
|
}
|
|
|
|
|
2018-04-10 01:20:06 +00:00
|
|
|
INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_database.uuid));
|
2012-06-27 07:24:42 +00:00
|
|
|
RETURN(0);
|
2013-02-16 17:47:24 +00:00
|
|
|
OUT();
|
2012-01-12 03:38:24 +00:00
|
|
|
}
|
|
|
|
|
2012-10-30 05:42:40 +00:00
|
|
|
int rhizome_close_db()
|
|
|
|
{
|
2013-01-03 03:46:33 +00:00
|
|
|
IN();
|
2018-04-10 01:20:06 +00:00
|
|
|
if (rhizome_database.db) {
|
2013-08-16 05:27:28 +00:00
|
|
|
rhizome_cache_close();
|
2018-04-17 02:31:02 +00:00
|
|
|
|
2018-04-10 01:20:06 +00:00
|
|
|
if (!sqlite3_get_autocommit(rhizome_database.db)){
|
2013-01-06 09:13:14 +00:00
|
|
|
WHY("Uncommitted transaction!");
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void("ROLLBACK;", END);
|
2013-01-06 09:13:14 +00:00
|
|
|
}
|
2012-10-30 05:42:40 +00:00
|
|
|
sqlite3_stmt *stmt = NULL;
|
2018-04-10 01:20:06 +00:00
|
|
|
while ((stmt = sqlite3_next_stmt(rhizome_database.db, stmt))) {
|
2012-10-30 05:42:40 +00:00
|
|
|
const char *sql = sqlite3_sql(stmt);
|
|
|
|
WARNF("closing Rhizome db with unfinalised statement: %s", sql ? sql : "BLOB");
|
|
|
|
}
|
2018-04-10 01:20:06 +00:00
|
|
|
int r = sqlite3_close(rhizome_database.db);
|
2012-10-29 01:25:14 +00:00
|
|
|
if (r != SQLITE_OK)
|
2018-04-10 01:20:06 +00:00
|
|
|
RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_database.db)));
|
2018-04-17 02:31:02 +00:00
|
|
|
rhizome_database.db = NULL;
|
|
|
|
assert(sqlite3_temp_directory);
|
2018-04-10 07:18:23 +00:00
|
|
|
sqlite3_free(sqlite3_temp_directory);
|
2018-04-17 02:31:02 +00:00
|
|
|
sqlite3_temp_directory = NULL;
|
|
|
|
}
|
2013-01-03 03:46:33 +00:00
|
|
|
RETURN(0);
|
2013-02-16 17:47:24 +00:00
|
|
|
OUT();
|
2012-10-29 01:25:14 +00:00
|
|
|
}
|
|
|
|
|
2016-11-11 00:09:50 +00:00
|
|
|
/* Close the Rhizome database after every CLI command.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static void rhizome_on_cmd_cleanup();
|
|
|
|
DEFINE_TRIGGER(cmd_cleanup, rhizome_on_cmd_cleanup);
|
|
|
|
static void rhizome_on_cmd_cleanup()
|
|
|
|
{
|
|
|
|
rhizome_close_db();
|
|
|
|
}
|
|
|
|
|
2012-08-22 09:39:30 +00:00
|
|
|
/* SQL query retry logic.
|
|
|
|
|
|
|
|
The common retry-on-busy logic is factored into this function. This logic encapsulates the
|
|
|
|
maximum time (timeout) that the caller may wait for a lock to be released and the sleep interval
|
|
|
|
while waiting. The way to use it is this:
|
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2012-08-22 09:39:30 +00:00
|
|
|
do ret = some_sqlite_operation(...);
|
2012-08-24 05:56:25 +00:00
|
|
|
while (is_busy(ret) && sqlite_retry(&retry, "some_sqlite_operation"));
|
2012-08-22 09:39:30 +00:00
|
|
|
if (is_error(ret) || is_busy(ret))
|
|
|
|
return -1; // an error has already been logged
|
2012-08-24 05:56:25 +00:00
|
|
|
sqlite_retry_done(&retry, "some_sqlite_operation");
|
2012-08-22 09:39:30 +00:00
|
|
|
...
|
|
|
|
|
|
|
|
If the database is currently locked for updates, then some_sqlite_operation() will return a code
|
|
|
|
indicating busy (which is distinguishable from the codes for success or any other error).
|
|
|
|
sqlite_retry() will then log a DEBUG or INFO message, sleep for a short period and return true if
|
|
|
|
the timeout has not been reached. It keeps this information in the 'retry' variable, which must
|
|
|
|
be initialised as shown. As long as the timeout has not been reached, sqlite_retry() will keep
|
|
|
|
sleeping and returning true. If the timeout is reached, then sqlite_retry() will log an error
|
|
|
|
and return false. If the operation is successful, sqlite_retry_done() must be called to log the
|
|
|
|
success as a DEBUG or INFO message to provide closure to the prior messages already logged by
|
|
|
|
sqlite_retry() and to reset the 'retry' variable for re-use.
|
|
|
|
|
|
|
|
The timeout and sleep interval depend on whether the caller is the servald server process or not.
|
2012-08-23 08:13:35 +00:00
|
|
|
See the definition of the SQLITE_RETRY_STATE_DEFAULT macro for the default settings.
|
2012-08-22 09:39:30 +00:00
|
|
|
|
|
|
|
A single 'retry' variable may be initialised once then used for a succession of database
|
|
|
|
operations. If invoked by the server process, then the timeout timer will not be reset by
|
|
|
|
sqlite_retry() or sqlite_retry_done(), so that the timeout limit will apply to the cumulative
|
|
|
|
latency, not just to each individual query, which could potentially add up to much greater
|
|
|
|
latency than desired. However, in non-server processes, each query may be allowed its own
|
|
|
|
timeout, giving a greater chance of success at the expense of potentially greater latency.
|
|
|
|
*/
|
|
|
|
|
2012-08-24 05:56:25 +00:00
|
|
|
/* In the servald server process, by default we retry every 10 ms for up to 50 ms, so as to not
|
2012-08-23 08:13:35 +00:00
|
|
|
introduce too much latency into server responsiveness. In other processes (eg, Batphone MeshMS
|
|
|
|
thread), by default we allow busy retries to go for over a second, waiting 100 ms between each
|
|
|
|
retry.
|
|
|
|
*/
|
|
|
|
sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int otherLimit, int otherSleep)
|
|
|
|
{
|
|
|
|
return (sqlite_retry_state){
|
2018-06-01 08:10:09 +00:00
|
|
|
.limit = rhizomeRetryLimit >= 0 ? rhizomeRetryLimit :
|
|
|
|
serverMode != SERVER_NOT_RUNNING ? (serverLimit < 0 ? 50 : serverLimit)
|
|
|
|
: (otherLimit < 0 ? 5000 : otherLimit),
|
|
|
|
.sleep = serverMode != SERVER_NOT_RUNNING ? (serverSleep < 0 ? 10 : serverSleep)
|
|
|
|
: (otherSleep < 0 ? 100 : otherSleep),
|
2012-08-23 08:13:35 +00:00
|
|
|
.elapsed = 0,
|
|
|
|
.start = -1,
|
2012-08-24 05:56:25 +00:00
|
|
|
.busytries = 0
|
2012-08-23 08:13:35 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2012-10-16 06:16:52 +00:00
|
|
|
int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action)
|
2012-08-22 09:39:30 +00:00
|
|
|
{
|
|
|
|
time_ms_t now = gettime_ms();
|
2012-08-24 05:56:25 +00:00
|
|
|
++retry->busytries;
|
2012-08-22 09:39:30 +00:00
|
|
|
if (retry->start == -1)
|
|
|
|
retry->start = now;
|
2012-12-27 04:45:23 +00:00
|
|
|
retry->elapsed = now - retry->start;
|
|
|
|
|
|
|
|
INFOF("%s on try %u after %.3f seconds (limit %.3f): %s",
|
2018-04-10 01:20:06 +00:00
|
|
|
sqlite3_errmsg(rhizome_database.db),
|
2012-08-24 05:56:25 +00:00
|
|
|
retry->busytries,
|
2012-12-27 04:45:23 +00:00
|
|
|
(retry->elapsed) / 1e3,
|
|
|
|
(retry->limit) / 1e3,
|
2012-08-24 05:56:25 +00:00
|
|
|
action
|
|
|
|
);
|
2012-12-27 04:45:23 +00:00
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
if (retry->elapsed >= retry->limit) {
|
|
|
|
// reset ready for next query
|
2012-08-24 05:56:25 +00:00
|
|
|
retry->busytries = 0;
|
2018-06-01 08:10:09 +00:00
|
|
|
if (serverMode == SERVER_NOT_RUNNING)
|
2012-08-22 09:39:30 +00:00
|
|
|
retry->start = -1;
|
|
|
|
return 0; // tell caller to stop trying
|
|
|
|
}
|
2012-12-27 04:45:23 +00:00
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
if (retry->sleep)
|
|
|
|
sleep_ms(retry->sleep);
|
2012-08-22 09:39:30 +00:00
|
|
|
return 1; // tell caller to try again
|
|
|
|
}
|
|
|
|
|
2012-10-16 06:16:52 +00:00
|
|
|
void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action)
|
2012-08-22 09:39:30 +00:00
|
|
|
{
|
2012-08-24 05:56:25 +00:00
|
|
|
if (retry->busytries) {
|
2012-08-22 09:39:30 +00:00
|
|
|
time_ms_t now = gettime_ms();
|
2012-12-27 04:45:23 +00:00
|
|
|
INFOF("succeeded on try %u after %.3f seconds (limit %.3f): %s",
|
2012-08-24 05:56:25 +00:00
|
|
|
retry->busytries + 1,
|
2012-08-22 09:39:30 +00:00
|
|
|
(now - retry->start) / 1e3,
|
2012-12-27 04:45:23 +00:00
|
|
|
(retry->limit) / 1e3,
|
2012-08-22 09:39:30 +00:00
|
|
|
action
|
|
|
|
);
|
|
|
|
}
|
2012-08-23 08:13:35 +00:00
|
|
|
// reset ready for next query
|
2012-08-24 05:56:25 +00:00
|
|
|
retry->busytries = 0;
|
2018-06-01 08:10:09 +00:00
|
|
|
if (serverMode == SERVER_NOT_RUNNING)
|
2012-08-22 09:39:30 +00:00
|
|
|
retry->start = -1;
|
|
|
|
}
|
|
|
|
|
2013-10-03 05:45:30 +00:00
|
|
|
/* Prepare an SQL command from a simple string. Returns NULL if an error occurs (logged as an
|
|
|
|
* error), otherwise returns a pointer to the prepared SQLite statement.
|
2013-10-02 08:49:20 +00:00
|
|
|
*
|
|
|
|
* IMPORTANT! Do not form statement strings using sprintf(3) or strbuf_sprintf() or similar
|
|
|
|
* methods, because those are susceptible to SQL injection attacks. Instead, use bound parameters
|
2013-10-03 05:45:30 +00:00
|
|
|
* and bind them using the _sqlite_bind() function below.
|
2013-10-02 08:49:20 +00:00
|
|
|
*
|
2013-10-03 05:45:30 +00:00
|
|
|
* IMPORTANT! Do not add sprintf(3)-like functionality to this method. It used to take
|
|
|
|
* sprintf(3)-style varargs and these were deliberately removed. It is vital to discourage bad
|
|
|
|
* practice, and adding sprintf(3)-style args to this function would be a step in the wrong
|
|
|
|
* direction.
|
2013-10-02 15:46:10 +00:00
|
|
|
*
|
|
|
|
* See GitHub issue #69.
|
2013-10-02 08:49:20 +00:00
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-05-25 10:12:45 +00:00
|
|
|
*/
|
2013-10-03 05:45:30 +00:00
|
|
|
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext)
|
2012-05-25 10:12:45 +00:00
|
|
|
{
|
2013-06-14 04:39:42 +00:00
|
|
|
IN();
|
2012-08-20 09:04:35 +00:00
|
|
|
sqlite3_stmt *statement = NULL;
|
2018-04-10 01:20:06 +00:00
|
|
|
assert(rhizome_database.db);
|
2012-10-04 05:00:20 +00:00
|
|
|
while (1) {
|
2018-04-10 01:20:06 +00:00
|
|
|
switch (sqlite3_prepare_v2(rhizome_database.db, sqltext, -1, &statement, NULL)) {
|
2012-10-04 05:00:20 +00:00
|
|
|
case SQLITE_OK:
|
2013-11-18 05:54:03 +00:00
|
|
|
sqlite_trace_done = 0;
|
2013-06-14 04:39:42 +00:00
|
|
|
RETURN(statement);
|
2012-10-04 05:00:20 +00:00
|
|
|
case SQLITE_BUSY:
|
|
|
|
case SQLITE_LOCKED:
|
2013-10-02 08:49:20 +00:00
|
|
|
if (retry && _sqlite_retry(__whence, retry, sqltext)) {
|
2012-10-04 05:00:20 +00:00
|
|
|
break; // back to sqlite3_prepare_v2()
|
|
|
|
}
|
|
|
|
// fall through...
|
|
|
|
default:
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_database.db), sqltext);
|
2012-10-04 05:00:20 +00:00
|
|
|
sqlite3_finalize(statement);
|
2013-06-14 04:39:42 +00:00
|
|
|
RETURN(NULL);
|
2012-10-04 05:00:20 +00:00
|
|
|
}
|
2012-05-25 10:12:45 +00:00
|
|
|
}
|
2012-08-20 09:04:35 +00:00
|
|
|
}
|
|
|
|
|
2013-10-03 05:45:30 +00:00
|
|
|
/* Bind some parameters to a prepared SQL statement. Returns -1 if an error occurs (logged as an
|
|
|
|
* error), otherwise zero with the prepared statement in *statement.
|
|
|
|
*
|
|
|
|
* Developed as part of GitHub issue #69.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2013-10-02 15:46:10 +00:00
|
|
|
int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap)
|
2013-10-02 08:49:20 +00:00
|
|
|
{
|
2018-04-10 01:20:06 +00:00
|
|
|
const int index_limit = sqlite3_limit(rhizome_database.db, SQLITE_LIMIT_VARIABLE_NUMBER, -1);
|
2013-10-10 06:45:52 +00:00
|
|
|
unsigned argnum = 0;
|
2013-10-02 15:46:10 +00:00
|
|
|
int index_counter = 0;
|
2013-10-10 06:45:52 +00:00
|
|
|
enum sqlbind_type typ;
|
|
|
|
while ((typ = va_arg(ap, int)) != END) {
|
|
|
|
++argnum;
|
2013-10-02 08:49:20 +00:00
|
|
|
int index;
|
2013-10-03 05:45:30 +00:00
|
|
|
const char *name = NULL;
|
2013-10-10 06:45:52 +00:00
|
|
|
strbuf ext = NULL;
|
2016-11-07 03:46:51 +00:00
|
|
|
if (typ & INDEX) {
|
|
|
|
typ &= ~INDEX;
|
2013-10-02 15:46:10 +00:00
|
|
|
index = va_arg(ap, int);
|
2013-10-10 06:45:52 +00:00
|
|
|
++argnum;
|
2013-10-02 15:46:10 +00:00
|
|
|
if (index < 1 || index > index_limit) {
|
2013-10-10 06:45:52 +00:00
|
|
|
LOGF(log_level, "at bind arg %u, illegal index=%d: %s", argnum, index, sqlite3_sql(statement));
|
2013-10-02 15:46:10 +00:00
|
|
|
return -1;
|
|
|
|
}
|
2015-07-06 08:19:49 +00:00
|
|
|
if (IF_DEBUG(rhizome))
|
2013-10-10 06:45:52 +00:00
|
|
|
strbuf_sprintf((ext = strbuf_alloca(35)), "|INDEX(%d)", index);
|
2016-11-07 03:46:51 +00:00
|
|
|
} else if (typ & NAMED) {
|
|
|
|
typ &= ~NAMED;
|
2013-10-03 05:45:30 +00:00
|
|
|
name = va_arg(ap, const char *);
|
2013-10-10 06:45:52 +00:00
|
|
|
++argnum;
|
2013-10-02 08:49:20 +00:00
|
|
|
index = sqlite3_bind_parameter_index(statement, name);
|
|
|
|
if (index == 0) {
|
2013-10-10 06:45:52 +00:00
|
|
|
LOGF(log_level, "at bind arg %u, no parameter named %s in query: %s", argnum, alloca_str_toprint(name), sqlite3_sql(statement));
|
2013-10-02 15:46:10 +00:00
|
|
|
return -1;
|
2013-10-02 08:49:20 +00:00
|
|
|
}
|
2015-07-06 08:19:49 +00:00
|
|
|
if (IF_DEBUG(rhizome)) {
|
2013-10-10 06:45:52 +00:00
|
|
|
ext = strbuf_alloca(30 + toprint_str_len(name, "\"\""));
|
|
|
|
strbuf_puts(ext, "|NAMED(");
|
2013-10-03 05:45:30 +00:00
|
|
|
strbuf_toprint_quoted(ext, "\"\"", name);
|
2013-10-10 06:45:52 +00:00
|
|
|
strbuf_puts(ext, ")");
|
2013-10-03 05:45:30 +00:00
|
|
|
}
|
2016-11-07 03:46:51 +00:00
|
|
|
} else {
|
2013-10-02 15:46:10 +00:00
|
|
|
index = ++index_counter;
|
2015-07-06 08:19:49 +00:00
|
|
|
if (IF_DEBUG(rhizome))
|
2013-10-10 06:45:52 +00:00
|
|
|
ext = strbuf_alloca(10);
|
2013-10-03 05:45:30 +00:00
|
|
|
}
|
|
|
|
#define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome_sql_bind, "%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement))
|
2013-10-02 15:46:10 +00:00
|
|
|
#define BIND_RETRY(FUNC, ...) \
|
|
|
|
do { \
|
|
|
|
switch (FUNC(statement, index, ##__VA_ARGS__)) { \
|
|
|
|
case SQLITE_OK: \
|
|
|
|
break; \
|
|
|
|
case SQLITE_BUSY: \
|
|
|
|
case SQLITE_LOCKED: \
|
|
|
|
if (retry && _sqlite_retry(__whence, retry, #FUNC "()")) \
|
|
|
|
continue; \
|
2020-07-09 23:58:02 +00:00
|
|
|
FALLTHROUGH; /* fall through */ \
|
2013-10-02 15:46:10 +00:00
|
|
|
default: \
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, #FUNC "(%d) failed, %s: %s", index, sqlite3_errmsg(rhizome_database.db), sqlite3_sql(statement)); \
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite3_finalize(statement); \
|
|
|
|
return -1; \
|
|
|
|
} \
|
|
|
|
break; \
|
|
|
|
} while (1)
|
2013-10-10 06:45:52 +00:00
|
|
|
#define BIND_NULL(TYP) \
|
|
|
|
if (typ & NUL) { \
|
|
|
|
BIND_DEBUG(TYP, sqlite3_bind_null, ""); \
|
|
|
|
BIND_RETRY(sqlite3_bind_null); \
|
|
|
|
} else { \
|
|
|
|
LOGF(log_level, "at bind arg %u, %s%s parameter is NULL: %s", argnum, #TYP, strbuf_str(ext), sqlite3_sql(statement)); \
|
|
|
|
sqlite3_finalize(statement); \
|
|
|
|
return -1; \
|
|
|
|
}
|
2013-10-02 08:49:20 +00:00
|
|
|
switch (typ) {
|
|
|
|
case NUL:
|
2013-10-03 05:45:30 +00:00
|
|
|
BIND_DEBUG(NUL, sqlite3_bind_null, "");
|
2013-10-02 15:46:10 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_null);
|
2013-10-02 08:49:20 +00:00
|
|
|
break;
|
2013-10-10 06:45:52 +00:00
|
|
|
default:
|
2015-07-06 08:19:49 +00:00
|
|
|
if ((typ & NUL) && IF_DEBUG(rhizome))
|
2013-10-10 06:45:52 +00:00
|
|
|
strbuf_puts(ext, "|NUL");
|
|
|
|
switch (typ & ~NUL) {
|
|
|
|
case INT: {
|
|
|
|
int value = va_arg(ap, int);
|
|
|
|
++argnum;
|
|
|
|
BIND_DEBUG(INT, sqlite3_bind_int, "%d", value);
|
|
|
|
BIND_RETRY(sqlite3_bind_int, value);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case INT_TOSTR: {
|
|
|
|
int value = va_arg(ap, int);
|
|
|
|
++argnum;
|
|
|
|
char str[25];
|
|
|
|
sprintf(str, "%d", value);
|
|
|
|
BIND_DEBUG(INT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case UINT_TOSTR: {
|
|
|
|
unsigned value = va_arg(ap, unsigned);
|
|
|
|
++argnum;
|
|
|
|
char str[25];
|
|
|
|
sprintf(str, "%u", value);
|
|
|
|
BIND_DEBUG(UINT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case INT64: {
|
|
|
|
sqlite3_int64 value = va_arg(ap, int64_t);
|
|
|
|
BIND_DEBUG(INT64, sqlite3_bind_int64, "%"PRId64, (int64_t)value);
|
|
|
|
BIND_RETRY(sqlite3_bind_int64, value);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case INT64_TOSTR: {
|
|
|
|
int64_t value = va_arg(ap, int64_t);
|
|
|
|
++argnum;
|
|
|
|
char str[35];
|
|
|
|
sprintf(str, "%"PRId64, value);
|
|
|
|
BIND_DEBUG(INT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case UINT64_TOSTR: {
|
|
|
|
uint64_t value = va_arg(ap, uint64_t);
|
|
|
|
++argnum;
|
|
|
|
char str[35];
|
|
|
|
sprintf(str, "%"PRIu64, value);
|
|
|
|
BIND_DEBUG(UINT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TEXT: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
++argnum;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(TEXT);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(TEXT, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(text));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, text, -1, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TEXT_LEN: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
int bytes = va_arg(ap, int);
|
|
|
|
argnum += 2;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(TEXT_LEN);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(TEXT_LEN, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_str_toprint(text), bytes);
|
|
|
|
BIND_RETRY(sqlite3_bind_text, text, bytes, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case STATIC_TEXT: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
++argnum;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(STATIC_TEXT);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(STATIC_TEXT, sqlite3_bind_text, "%s,-1,SQLITE_STATIC", alloca_str_toprint(text));
|
|
|
|
BIND_RETRY(sqlite3_bind_text, text, -1, SQLITE_STATIC);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case STATIC_TEXT_LEN: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
int bytes = va_arg(ap, int);
|
|
|
|
argnum += 2;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(STATIC_TEXT_LEN);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(STATIC_TEXT_LEN, sqlite3_bind_text, "%s,%d,SQLITE_STATIC", alloca_str_toprint(text), bytes);
|
|
|
|
BIND_RETRY(sqlite3_bind_text, text, bytes, SQLITE_STATIC);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case STATIC_BLOB: {
|
|
|
|
const void *blob = va_arg(ap, const void *);
|
|
|
|
int bytes = va_arg(ap, int);
|
|
|
|
argnum += 2;
|
|
|
|
if (blob == NULL) {
|
|
|
|
BIND_NULL(STATIC_BLOB);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(STATIC_BLOB, sqlite3_bind_blob, "%s,%d,SQLITE_STATIC", alloca_toprint(20, blob, bytes), bytes);
|
|
|
|
BIND_RETRY(sqlite3_bind_blob, blob, bytes, SQLITE_STATIC);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
break;
|
|
|
|
case ZEROBLOB: {
|
|
|
|
int bytes = va_arg(ap, int);
|
|
|
|
++argnum;
|
|
|
|
BIND_DEBUG(ZEROBLOB, sqlite3_bind_zeroblob, "%d,SQLITE_STATIC", bytes);
|
|
|
|
BIND_RETRY(sqlite3_bind_zeroblob, bytes);
|
|
|
|
};
|
|
|
|
break;
|
|
|
|
case SID_T: {
|
|
|
|
const sid_t *sidp = va_arg(ap, const sid_t *);
|
|
|
|
++argnum;
|
|
|
|
if (sidp == NULL) {
|
|
|
|
BIND_NULL(SID_T);
|
|
|
|
} else {
|
|
|
|
const char *sid_hex = alloca_tohex_sid_t(*sidp);
|
2013-10-15 00:45:31 +00:00
|
|
|
BIND_DEBUG(SID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", sid_hex, SID_STRLEN);
|
2013-10-10 06:45:52 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, sid_hex, SID_STRLEN, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case RHIZOME_BID_T: {
|
|
|
|
const rhizome_bid_t *bidp = va_arg(ap, const rhizome_bid_t *);
|
|
|
|
++argnum;
|
|
|
|
if (bidp == NULL) {
|
|
|
|
BIND_NULL(RHIZOME_BID_T);
|
|
|
|
} else {
|
|
|
|
const char *bid_hex = alloca_tohex_rhizome_bid_t(*bidp);
|
2013-10-15 00:45:31 +00:00
|
|
|
BIND_DEBUG(RHIZOME_BID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", bid_hex, RHIZOME_MANIFEST_ID_STRLEN);
|
2013-10-10 06:45:52 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, bid_hex, RHIZOME_MANIFEST_ID_STRLEN, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
2014-06-16 04:51:42 +00:00
|
|
|
case RHIZOME_BAR_T: {
|
|
|
|
const rhizome_bar_t *barp = va_arg(ap, const rhizome_bar_t *);
|
|
|
|
++argnum;
|
|
|
|
if (barp == NULL) {
|
|
|
|
BIND_NULL(RHIZOME_BAR_T);
|
|
|
|
} else {
|
|
|
|
BIND_DEBUG(STATIC_BLOB, sqlite3_bind_blob, "%s,%d,SQLITE_STATIC", alloca_toprint(20, barp->binary, RHIZOME_BAR_BYTES), RHIZOME_BAR_BYTES);
|
|
|
|
BIND_RETRY(sqlite3_bind_blob, barp->binary, RHIZOME_BAR_BYTES, SQLITE_STATIC);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
2013-10-10 07:53:06 +00:00
|
|
|
case RHIZOME_FILEHASH_T: {
|
|
|
|
const rhizome_filehash_t *hashp = va_arg(ap, const rhizome_filehash_t *);
|
2013-10-10 06:45:52 +00:00
|
|
|
++argnum;
|
2013-10-10 07:53:06 +00:00
|
|
|
if (hashp == NULL) {
|
|
|
|
BIND_NULL(RHIZOME_FILEHASH_T);
|
2013-10-10 06:45:52 +00:00
|
|
|
} else {
|
2013-11-15 06:40:33 +00:00
|
|
|
char hash_hex[RHIZOME_FILEHASH_STRLEN + 1];
|
|
|
|
tohex(hash_hex, RHIZOME_FILEHASH_STRLEN, hashp->binary);
|
2013-11-21 05:49:14 +00:00
|
|
|
BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
|
2013-11-15 06:40:33 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT);
|
2013-10-10 06:45:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TOHEX: {
|
|
|
|
const unsigned char *binary = va_arg(ap, const unsigned char *);
|
|
|
|
unsigned bytes = va_arg(ap, unsigned);
|
|
|
|
argnum += 2;
|
|
|
|
if (binary == NULL) {
|
|
|
|
BIND_NULL(TOHEX);
|
|
|
|
} else {
|
|
|
|
const char *hex = alloca_tohex(binary, bytes);
|
2013-10-15 00:45:31 +00:00
|
|
|
BIND_DEBUG(TOHEX, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", hex, bytes * 2);
|
2013-10-10 06:45:52 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, hex, bytes * 2, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TEXT_TOUPPER: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
++argnum;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(TEXT_TOUPPER);
|
|
|
|
} else {
|
|
|
|
unsigned bytes = strlen(text);
|
|
|
|
char upper[bytes + 1];
|
|
|
|
str_toupper_inplace(strcpy(upper, text));
|
2013-10-15 00:45:31 +00:00
|
|
|
BIND_DEBUG(TEXT_TOUPPER, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes);
|
2013-10-10 06:45:52 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TEXT_LEN_TOUPPER: {
|
|
|
|
const char *text = va_arg(ap, const char *);
|
|
|
|
unsigned bytes = va_arg(ap, unsigned);
|
|
|
|
argnum += 2;
|
|
|
|
if (text == NULL) {
|
|
|
|
BIND_NULL(TEXT);
|
|
|
|
} else {
|
|
|
|
char upper[bytes];
|
|
|
|
unsigned i;
|
|
|
|
for (i = 0; i != bytes; ++i)
|
|
|
|
upper[i] = toupper(text[i]);
|
2013-10-15 00:45:31 +00:00
|
|
|
BIND_DEBUG(TEXT_LEN_TOUPPER, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes);
|
2013-10-10 06:45:52 +00:00
|
|
|
BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
2014-02-17 02:56:03 +00:00
|
|
|
case SERVAL_UUID_T: {
|
|
|
|
const serval_uuid_t *uuidp = va_arg(ap, const serval_uuid_t *);
|
2013-11-12 07:44:14 +00:00
|
|
|
++argnum;
|
|
|
|
if (uuidp == NULL) {
|
2014-02-17 02:56:03 +00:00
|
|
|
BIND_NULL(SERVAL_UUID_T);
|
2013-11-12 07:44:14 +00:00
|
|
|
} else {
|
2017-10-09 01:11:44 +00:00
|
|
|
char uuid_str[SERVAL_UUID_STRLEN + 1];
|
|
|
|
serval_uuid_to_str(uuidp, uuid_str);
|
|
|
|
BIND_DEBUG(SERVAL_UUID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", uuid_str, SERVAL_UUID_STRLEN);
|
|
|
|
BIND_RETRY(sqlite3_bind_text, uuid_str, SERVAL_UUID_STRLEN, SQLITE_TRANSIENT);
|
2013-11-12 07:44:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
2013-10-10 06:45:52 +00:00
|
|
|
#undef BIND_RETRY
|
|
|
|
default:
|
|
|
|
FATALF("at bind arg %u, unsupported bind code typ=0x%08x: %s", argnum, typ, sqlite3_sql(statement));
|
2013-10-02 08:49:20 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2013-10-10 06:45:52 +00:00
|
|
|
}
|
2013-10-02 15:46:10 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...)
|
|
|
|
{
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, statement);
|
|
|
|
int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap);
|
2013-10-02 08:49:20 +00:00
|
|
|
va_end(ap);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2013-10-03 05:45:30 +00:00
|
|
|
/* Prepare an SQL statement and bind some parameters. Returns a pointer to the SQLite statement if
|
|
|
|
* successful or NULL if an error occurs (which is logged at the given log level).
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
|
|
|
sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...)
|
|
|
|
{
|
|
|
|
sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext);
|
|
|
|
if (statement != NULL) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, sqltext);
|
|
|
|
int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap);
|
|
|
|
va_end(ap);
|
|
|
|
if (ret == -1) {
|
|
|
|
sqlite3_finalize(statement);
|
|
|
|
statement = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return statement;
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
|
2012-08-20 09:04:35 +00:00
|
|
|
{
|
2013-06-14 04:39:42 +00:00
|
|
|
IN();
|
2012-10-10 02:52:30 +00:00
|
|
|
int ret = -1;
|
2012-10-16 06:16:52 +00:00
|
|
|
sqlite_trace_whence = &__whence;
|
2012-10-10 02:52:30 +00:00
|
|
|
while (statement) {
|
2016-05-23 03:24:27 +00:00
|
|
|
ret = sqlite3_step(statement);
|
|
|
|
switch (ret) {
|
2012-08-22 09:39:30 +00:00
|
|
|
case SQLITE_OK:
|
|
|
|
case SQLITE_DONE:
|
|
|
|
case SQLITE_ROW:
|
|
|
|
if (retry)
|
2012-10-16 06:16:52 +00:00
|
|
|
_sqlite_retry_done(__whence, retry, sqlite3_sql(statement));
|
2012-10-10 02:52:30 +00:00
|
|
|
statement = NULL;
|
|
|
|
break;
|
2012-08-22 09:39:30 +00:00
|
|
|
case SQLITE_BUSY:
|
|
|
|
case SQLITE_LOCKED:
|
2012-10-16 06:16:52 +00:00
|
|
|
if (retry && _sqlite_retry(__whence, retry, sqlite3_sql(statement))) {
|
2012-08-24 05:56:25 +00:00
|
|
|
sqlite3_reset(statement);
|
2012-08-23 08:13:35 +00:00
|
|
|
break; // back to sqlite3_step()
|
2012-08-24 05:56:25 +00:00
|
|
|
}
|
2012-08-22 09:39:30 +00:00
|
|
|
// fall through...
|
|
|
|
default:
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, "query failed (%d), %s: %s", ret, sqlite3_errmsg(rhizome_database.db), sqlite3_sql(statement));
|
2012-10-10 02:52:30 +00:00
|
|
|
statement = NULL;
|
|
|
|
break;
|
2012-08-22 09:39:30 +00:00
|
|
|
}
|
|
|
|
}
|
2012-10-16 06:16:52 +00:00
|
|
|
sqlite_trace_whence = NULL;
|
2013-06-14 04:39:42 +00:00
|
|
|
OUT();
|
2012-10-10 02:52:30 +00:00
|
|
|
return ret;
|
2012-08-23 08:13:35 +00:00
|
|
|
}
|
|
|
|
|
2016-05-23 03:24:27 +00:00
|
|
|
int _sqlite_exec_code(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, int *rowcount)
|
|
|
|
{
|
|
|
|
*rowcount = 0;
|
|
|
|
if (!statement)
|
|
|
|
return SQLITE_ERROR;
|
|
|
|
int stepcode;
|
|
|
|
while ((stepcode = _sqlite_step(__whence, log_level, retry, statement)) == SQLITE_ROW)
|
|
|
|
++(*rowcount);
|
|
|
|
sqlite3_finalize(statement);
|
|
|
|
if (sqlite_trace_func())
|
2018-04-10 01:20:06 +00:00
|
|
|
_DEBUGF("rowcount=%d changes=%d", *rowcount, sqlite3_changes(rhizome_database.db));
|
2016-05-23 03:24:27 +00:00
|
|
|
return stepcode;
|
|
|
|
}
|
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
/*
|
2013-02-20 04:14:29 +00:00
|
|
|
* Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
|
|
|
|
* Always finalises the statement before returning.
|
|
|
|
*
|
|
|
|
* If an error occurs then logs it at the given level and returns -1.
|
|
|
|
*
|
|
|
|
* If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
|
|
|
|
* currently in use by another process), then resets the statement and retries while sqlite_retry()
|
|
|
|
* returns true. If sqlite_retry() returns false then returns -1.
|
|
|
|
*
|
|
|
|
* Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
|
|
|
|
* was SQLITE_OK or SQLITE_DONE.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-08-23 08:13:35 +00:00
|
|
|
*/
|
2014-06-18 07:55:15 +00:00
|
|
|
int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
|
2012-08-23 08:13:35 +00:00
|
|
|
{
|
2016-05-23 03:24:27 +00:00
|
|
|
int rowcount;
|
|
|
|
int stepcode = _sqlite_exec_code(__whence, log_level, retry, statement, &rowcount);
|
2013-02-20 04:14:29 +00:00
|
|
|
return sqlite_code_ok(stepcode) ? rowcount : -1;
|
2012-05-25 10:12:45 +00:00
|
|
|
}
|
|
|
|
|
2016-05-23 03:24:27 +00:00
|
|
|
static int _sqlite_vexec_void_code(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, int *rowcount, int *changes, const char *sqltext, va_list ap)
|
2012-08-22 09:39:30 +00:00
|
|
|
{
|
2016-05-23 03:24:27 +00:00
|
|
|
*changes=0;
|
|
|
|
*rowcount=0;
|
2013-10-03 05:45:30 +00:00
|
|
|
sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext);
|
2013-10-02 15:46:10 +00:00
|
|
|
if (!statement)
|
2016-05-23 03:24:27 +00:00
|
|
|
return SQLITE_ERROR;
|
2013-10-02 15:46:10 +00:00
|
|
|
if (_sqlite_vbind(__whence, log_level, retry, statement, ap) == -1)
|
2016-05-23 03:24:27 +00:00
|
|
|
return SQLITE_ERROR;
|
|
|
|
int stepcode = _sqlite_exec_code(__whence, log_level, retry, statement, rowcount);
|
|
|
|
if (sqlite_code_ok(stepcode)){
|
2018-04-10 01:20:06 +00:00
|
|
|
*changes = sqlite3_changes(rhizome_database.db);
|
2016-05-23 03:24:27 +00:00
|
|
|
}
|
|
|
|
return stepcode;
|
2012-08-22 09:39:30 +00:00
|
|
|
}
|
|
|
|
|
2013-09-30 06:45:01 +00:00
|
|
|
/* Convenience wrapper for executing an SQL command that returns no value. If an error occurs then
|
|
|
|
* logs it at ERROR level and returns -1. Otherwise returns the number of rows changed by the
|
|
|
|
* command.
|
2013-02-20 04:14:29 +00:00
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-06-12 08:42:36 +00:00
|
|
|
*/
|
2013-10-03 05:45:30 +00:00
|
|
|
int _sqlite_exec_void(struct __sourceloc __whence, int log_level, const char *sqltext, ...)
|
2012-06-12 08:42:36 +00:00
|
|
|
{
|
2016-05-23 03:24:27 +00:00
|
|
|
int rowcount, changes;
|
2012-08-22 09:39:30 +00:00
|
|
|
va_list ap;
|
2013-10-02 15:46:10 +00:00
|
|
|
va_start(ap, sqltext);
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2016-05-23 03:24:27 +00:00
|
|
|
int stepcode = _sqlite_vexec_void_code(__whence, log_level, &retry, &rowcount, &changes, sqltext, ap);
|
2012-08-22 09:39:30 +00:00
|
|
|
va_end(ap);
|
2016-05-23 03:24:27 +00:00
|
|
|
if (!sqlite_code_ok(stepcode))
|
|
|
|
return -1;
|
|
|
|
if (rowcount)
|
|
|
|
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
|
|
|
|
return changes;
|
2012-06-12 08:42:36 +00:00
|
|
|
}
|
|
|
|
|
2012-08-22 09:39:30 +00:00
|
|
|
/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
|
2013-02-20 04:14:29 +00:00
|
|
|
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
|
|
|
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
|
|
|
* all in the event of a busy condition, but will log it as an error and return immediately.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-06-12 08:42:36 +00:00
|
|
|
*/
|
2013-10-03 05:45:30 +00:00
|
|
|
int _sqlite_exec_void_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
2016-05-23 03:24:27 +00:00
|
|
|
int rowcount, changes;
|
2013-02-20 04:14:29 +00:00
|
|
|
va_list ap;
|
2013-10-02 15:46:10 +00:00
|
|
|
va_start(ap, sqltext);
|
2016-05-23 03:24:27 +00:00
|
|
|
int stepcode = _sqlite_vexec_void_code(__whence, log_level, retry, &rowcount, &changes, sqltext, ap);
|
2013-02-20 04:14:29 +00:00
|
|
|
va_end(ap);
|
2016-05-23 03:24:27 +00:00
|
|
|
if (!sqlite_code_ok(stepcode))
|
|
|
|
return -1;
|
|
|
|
if (rowcount)
|
|
|
|
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
|
|
|
|
return changes;
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_exec_changes_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, int *rowcount, int *changes, const char *sqltext, ...)
|
|
|
|
{
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, sqltext);
|
|
|
|
int stepcode = _sqlite_vexec_void_code(__whence, log_level, retry, rowcount, changes, sqltext, ap);
|
|
|
|
va_end(ap);
|
|
|
|
return stepcode;
|
2013-02-20 04:14:29 +00:00
|
|
|
}
|
|
|
|
|
2013-12-11 00:41:34 +00:00
|
|
|
static int _sqlite_vexec_uint64(struct __sourceloc __whence, sqlite_retry_state *retry, uint64_t *result, const char *sqltext, va_list ap)
|
2012-01-12 03:38:24 +00:00
|
|
|
{
|
2013-10-03 05:45:30 +00:00
|
|
|
sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext);
|
2012-08-23 08:13:35 +00:00
|
|
|
if (!statement)
|
|
|
|
return -1;
|
2013-10-02 15:46:10 +00:00
|
|
|
if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1)
|
|
|
|
return -1;
|
2017-03-06 05:26:58 +00:00
|
|
|
int rows = 0;
|
2012-08-23 08:13:35 +00:00
|
|
|
int stepcode;
|
2013-10-03 05:45:30 +00:00
|
|
|
while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
|
2012-08-23 08:13:35 +00:00
|
|
|
int columncount = sqlite3_column_count(statement);
|
2012-10-16 06:16:52 +00:00
|
|
|
if (columncount != 1)
|
2017-03-06 05:26:58 +00:00
|
|
|
FATALF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
|
|
|
|
else if (++rows == 1)
|
2012-08-23 08:13:35 +00:00
|
|
|
*result = sqlite3_column_int64(statement, 0);
|
2012-05-25 04:59:55 +00:00
|
|
|
}
|
2017-03-06 05:26:58 +00:00
|
|
|
if (rows > 1)
|
|
|
|
FATALF("query unexpectedly returned %d rows", rows);
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite3_finalize(statement);
|
2013-02-20 04:14:29 +00:00
|
|
|
if (sqlite_trace_func())
|
2018-04-10 01:20:06 +00:00
|
|
|
_DEBUGF("rowcount=%d changes=%d result=%"PRIu64, rows, sqlite3_changes(rhizome_database.db), *result);
|
2017-03-06 05:26:58 +00:00
|
|
|
if (sqlite_code_ok(stepcode) && rows>0)
|
|
|
|
return SQLITE_ROW;
|
|
|
|
return stepcode;
|
2012-05-25 04:59:55 +00:00
|
|
|
}
|
|
|
|
|
2012-08-22 09:39:30 +00:00
|
|
|
/*
|
2013-02-20 04:14:29 +00:00
|
|
|
* Convenience wrapper for executing an SQL command that returns a single int64 value.
|
|
|
|
* Logs an error and returns -1 if an error occurs.
|
2017-03-06 05:26:58 +00:00
|
|
|
* If no row is found, then returns SQLITE_OK / SQLITE_DONE and does not alter *result.
|
|
|
|
* If exactly one row is found, the assigns its value to *result and returns SQLITE_ROW.
|
|
|
|
* Otherwise an SQLITE_ stepcode will be returned
|
|
|
|
* If more than one row is found, or the query returns more that one column then this function asserts
|
2013-02-20 04:14:29 +00:00
|
|
|
*
|
2017-03-06 05:26:58 +00:00
|
|
|
* if the statement cannot be executed because the database is
|
2013-02-20 04:14:29 +00:00
|
|
|
* currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
|
|
|
|
* instead of its own, internal one. If 'retry' is passed as NULL, then will not sleep and retry at
|
|
|
|
* all in the event of a busy condition, but will log it as an error and return immediately.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-08-22 09:39:30 +00:00
|
|
|
*/
|
2013-12-11 00:41:34 +00:00
|
|
|
int _sqlite_exec_uint64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, uint64_t *result, const char *sqlformat,...)
|
2012-08-22 09:39:30 +00:00
|
|
|
{
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, sqlformat);
|
2013-12-11 00:41:34 +00:00
|
|
|
int ret = _sqlite_vexec_uint64(__whence, retry, result, sqlformat, ap);
|
2012-08-22 09:39:30 +00:00
|
|
|
va_end(ap);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2013-02-20 04:14:29 +00:00
|
|
|
/* Convenience wrapper for executing an SQL command that returns a single text value.
|
|
|
|
* Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
|
|
|
|
* 0 means no rows, nothing is appended to the strbuf
|
|
|
|
* 1 means exactly one row, appends its column to the strbuf
|
|
|
|
* 2 more than one row, logs a warning and appends the first row's column to the strbuf
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-05-25 04:59:55 +00:00
|
|
|
*/
|
2012-10-16 06:16:52 +00:00
|
|
|
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
|
2012-05-25 04:59:55 +00:00
|
|
|
{
|
2012-10-04 05:00:20 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-02-20 04:14:29 +00:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, sqlformat);
|
|
|
|
int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
|
|
|
|
va_end(ap);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
|
|
|
|
{
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, sqlformat);
|
|
|
|
int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
|
|
|
|
va_end(ap);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2013-10-02 15:46:10 +00:00
|
|
|
int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
2013-10-03 05:45:30 +00:00
|
|
|
sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext);
|
2012-08-23 08:13:35 +00:00
|
|
|
if (!statement)
|
|
|
|
return -1;
|
2013-10-02 15:46:10 +00:00
|
|
|
if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1)
|
|
|
|
return -1;
|
2012-08-23 08:13:35 +00:00
|
|
|
int ret = 0;
|
|
|
|
int rowcount = 0;
|
|
|
|
int stepcode;
|
2013-10-03 05:45:30 +00:00
|
|
|
while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
|
2012-08-23 08:13:35 +00:00
|
|
|
int columncount = sqlite3_column_count(statement);
|
2012-10-16 06:16:52 +00:00
|
|
|
if (columncount != 1)
|
2014-08-14 08:09:44 +00:00
|
|
|
ret = WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
|
2012-08-23 08:13:35 +00:00
|
|
|
else if (++rowcount == 1)
|
|
|
|
strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
|
2012-05-25 04:59:55 +00:00
|
|
|
}
|
2012-08-23 08:13:35 +00:00
|
|
|
if (rowcount > 1)
|
2012-10-16 06:16:52 +00:00
|
|
|
WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite3_finalize(statement);
|
|
|
|
return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
|
2012-01-12 03:38:24 +00:00
|
|
|
}
|
|
|
|
|
2013-12-30 04:26:08 +00:00
|
|
|
int _sqlite_blob_open_retry(
|
|
|
|
struct __sourceloc __whence,
|
|
|
|
int log_level,
|
|
|
|
sqlite_retry_state *retry,
|
|
|
|
const char *dbname,
|
|
|
|
const char *tablename,
|
|
|
|
const char *colname,
|
|
|
|
sqlite3_int64 rowid,
|
|
|
|
int flags,
|
|
|
|
sqlite3_blob **blobp
|
|
|
|
)
|
|
|
|
{
|
|
|
|
IN();
|
|
|
|
while (1) {
|
2018-04-10 01:20:06 +00:00
|
|
|
int code = sqlite3_blob_open(rhizome_database.db, dbname, tablename, colname, rowid, flags, blobp);
|
2013-12-30 04:26:08 +00:00
|
|
|
switch (code) {
|
|
|
|
case SQLITE_OK:
|
|
|
|
if (retry)
|
|
|
|
_sqlite_retry_done(__whence, retry, "sqlite3_blob_open()");
|
|
|
|
RETURN(code);
|
|
|
|
case SQLITE_DONE:
|
|
|
|
case SQLITE_ROW:
|
|
|
|
LOGF(log_level, "sqlite3_blob_open() returned unexpected code (%d)", code);
|
|
|
|
RETURN(-1);
|
|
|
|
case SQLITE_BUSY:
|
|
|
|
case SQLITE_LOCKED:
|
|
|
|
if (retry && _sqlite_retry(__whence, retry, "sqlite3_blob_open()"))
|
|
|
|
break; // back to sqlite3_blob_open()
|
|
|
|
// fall through...
|
|
|
|
default:
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, "sqlite3_blob_open() failed (%d), %s", code, sqlite3_errmsg(rhizome_database.db));
|
2013-12-30 04:26:08 +00:00
|
|
|
RETURN(-1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
FATAL("not reached");
|
|
|
|
OUT();
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_blob_write_retry(
|
|
|
|
struct __sourceloc __whence,
|
|
|
|
int log_level,
|
|
|
|
sqlite_retry_state *retry,
|
|
|
|
sqlite3_blob *blob,
|
|
|
|
const void *buf,
|
|
|
|
int len,
|
|
|
|
int offset
|
|
|
|
)
|
|
|
|
{
|
|
|
|
IN();
|
|
|
|
while (1) {
|
|
|
|
int code = sqlite3_blob_write(blob, buf, len, offset);
|
|
|
|
switch (code) {
|
|
|
|
case SQLITE_OK:
|
|
|
|
if (retry)
|
|
|
|
_sqlite_retry_done(__whence, retry, "sqlite3_blob_write()");
|
|
|
|
RETURN(code);
|
|
|
|
case SQLITE_DONE:
|
|
|
|
case SQLITE_ROW:
|
|
|
|
LOGF(log_level, "sqlite3_blob_write() returned unexpected code (%d)", code);
|
|
|
|
RETURN(-1);
|
|
|
|
case SQLITE_BUSY:
|
|
|
|
case SQLITE_LOCKED:
|
|
|
|
if (retry && _sqlite_retry(__whence, retry, "sqlite3_blob_write()"))
|
|
|
|
break; // back to sqlite3_blob_open()
|
|
|
|
// fall through...
|
|
|
|
default:
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, "sqlite3_blob_write() failed (%d), %s", code, sqlite3_errmsg(rhizome_database.db));
|
2013-12-30 04:26:08 +00:00
|
|
|
RETURN(-1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
FATAL("not reached");
|
|
|
|
OUT();
|
|
|
|
}
|
|
|
|
|
|
|
|
int _sqlite_blob_close(struct __sourceloc __whence, int log_level, sqlite3_blob *blob)
|
|
|
|
{
|
|
|
|
int code = sqlite3_blob_close(blob);
|
|
|
|
if (code != SQLITE_OK)
|
2018-04-10 01:20:06 +00:00
|
|
|
LOGF(log_level, "sqlite3_blob_close() failed: %s", sqlite3_errmsg(rhizome_database.db));
|
2013-12-30 04:26:08 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-10-10 07:53:06 +00:00
|
|
|
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp)
|
2013-02-16 09:55:26 +00:00
|
|
|
{
|
|
|
|
IN();
|
2013-10-10 07:53:06 +00:00
|
|
|
strbuf hash_sb = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
|
2014-07-07 02:01:09 +00:00
|
|
|
int r = sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE version = ? AND id = ?;",
|
|
|
|
INT64, version, RHIZOME_BID_T, bidp, END);
|
|
|
|
if (r == -1)
|
2013-10-10 07:53:06 +00:00
|
|
|
RETURN(-1);
|
2014-07-07 02:01:09 +00:00
|
|
|
// this bundle / version was not found
|
|
|
|
if (r != 1)
|
|
|
|
RETURN(1);
|
2013-10-10 07:53:06 +00:00
|
|
|
if (strbuf_overrun(hash_sb) || str_to_rhizome_filehash_t(hashp, strbuf_str(hash_sb)) == -1)
|
2014-07-07 02:01:09 +00:00
|
|
|
RETURN(WHYF("malformed file hash (%s) for bid=%s version=%"PRIu64,
|
|
|
|
strbuf_str(hash_sb), alloca_tohex_rhizome_bid_t(*bidp), version));
|
2013-10-10 07:53:06 +00:00
|
|
|
RETURN(0);
|
2013-02-16 17:47:24 +00:00
|
|
|
OUT();
|
2013-02-16 09:55:26 +00:00
|
|
|
}
|
|
|
|
|
2014-06-19 04:28:01 +00:00
|
|
|
void rhizome_vacuum_db(sqlite_retry_state *retry){
|
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(retry, "PRAGMA incremental_vacuum;", END);
|
|
|
|
if (!statement)
|
|
|
|
return;
|
|
|
|
sqlite_exec_retry(retry, statement);
|
|
|
|
}
|
|
|
|
|
2013-02-20 04:14:29 +00:00
|
|
|
int rhizome_cleanup(struct rhizome_cleanup_report *report)
|
2012-12-31 04:39:12 +00:00
|
|
|
{
|
2013-01-03 03:46:33 +00:00
|
|
|
IN();
|
2015-07-06 08:19:49 +00:00
|
|
|
if (IF_DEBUG(rhizome) && report == NULL)
|
2013-10-06 17:12:17 +00:00
|
|
|
report = alloca(sizeof *report);
|
|
|
|
if (report)
|
|
|
|
bzero(report, sizeof *report);
|
2013-02-20 06:52:53 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
|
2014-06-19 04:28:01 +00:00
|
|
|
// make sure we are under our database size limit
|
|
|
|
rhizome_store_cleanup(report);
|
|
|
|
|
2013-10-06 17:12:17 +00:00
|
|
|
/* For testing, it helps to speed up the cleanup process. */
|
|
|
|
const char *orphan_payload_persist_ms = getenv("SERVALD_ORPHAN_PAYLOAD_PERSIST_MS");
|
2013-09-30 06:50:50 +00:00
|
|
|
time_ms_t now = gettime_ms();
|
2013-10-06 17:12:17 +00:00
|
|
|
time_ms_t insert_horizon_no_manifest = now - (orphan_payload_persist_ms ? atoi(orphan_payload_persist_ms) : 1000); // 1 second ago
|
2013-09-30 06:50:50 +00:00
|
|
|
|
2014-03-07 06:25:38 +00:00
|
|
|
// Remove external payload files for stale, incomplete payloads.
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
2014-06-18 07:55:15 +00:00
|
|
|
"SELECT id FROM FILES WHERE datavalid = 0;", END);
|
2013-08-27 04:12:56 +00:00
|
|
|
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
|
|
|
const char *id = (const char *) sqlite3_column_text(statement, 0);
|
2018-04-09 23:21:45 +00:00
|
|
|
rhizome_filehash_t filehash;
|
|
|
|
if (str_to_rhizome_filehash_t(&filehash, id) != -1
|
|
|
|
&& rhizome_delete_file(&filehash)==0 && report)
|
2014-03-18 05:33:26 +00:00
|
|
|
++report->deleted_stale_incoming_files;
|
2013-08-27 04:12:56 +00:00
|
|
|
}
|
2013-02-20 06:52:53 +00:00
|
|
|
sqlite3_finalize(statement);
|
2013-10-10 07:53:06 +00:00
|
|
|
|
2014-03-07 06:25:38 +00:00
|
|
|
// Remove external payload files for old, unreferenced payloads.
|
2013-10-02 15:46:10 +00:00
|
|
|
statement = sqlite_prepare_bind(&retry,
|
2014-06-16 06:26:31 +00:00
|
|
|
"SELECT id FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
|
2013-10-02 15:46:10 +00:00
|
|
|
INT64, insert_horizon_no_manifest, END);
|
2013-08-27 04:12:56 +00:00
|
|
|
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
|
|
|
const char *id = (const char *) sqlite3_column_text(statement, 0);
|
2018-04-09 23:21:45 +00:00
|
|
|
rhizome_filehash_t filehash;
|
|
|
|
if (str_to_rhizome_filehash_t(&filehash, id) != -1
|
|
|
|
&& rhizome_delete_file(&filehash)==0 && report)
|
2013-10-06 17:12:17 +00:00
|
|
|
++report->deleted_orphan_files;
|
2013-08-27 04:12:56 +00:00
|
|
|
}
|
2013-02-20 06:52:53 +00:00
|
|
|
sqlite3_finalize(statement);
|
2014-03-07 06:25:38 +00:00
|
|
|
|
|
|
|
// TODO Iterate through all files in RHIZOME_BLOB_SUBDIR and delete any which are no longer
|
|
|
|
// referenced or are stale. This could take a long time, so for scalability should be done
|
|
|
|
// in an incremental background task. See GitHub issue #50.
|
|
|
|
|
|
|
|
// Remove payload blobs that are no longer referenced.
|
2014-06-19 04:28:01 +00:00
|
|
|
int ret = sqlite_exec_void_retry(&retry,
|
2014-06-18 07:55:15 +00:00
|
|
|
"DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
|
|
|
|
END);
|
|
|
|
if (ret > 0 && report)
|
2013-09-30 06:50:50 +00:00
|
|
|
report->deleted_orphan_fileblobs += ret;
|
2014-03-07 06:25:38 +00:00
|
|
|
|
2014-06-16 06:26:31 +00:00
|
|
|
// delete manifests that no longer have payload files
|
2014-06-19 04:28:01 +00:00
|
|
|
ret = sqlite_exec_void_retry(&retry,
|
2014-06-18 07:55:15 +00:00
|
|
|
"DELETE FROM MANIFESTS WHERE filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);", END);
|
2014-06-16 06:26:31 +00:00
|
|
|
if (report && ret > 0)
|
|
|
|
report->deleted_orphan_manifests += ret;
|
|
|
|
|
2014-06-19 04:28:01 +00:00
|
|
|
rhizome_vacuum_db(&retry);
|
|
|
|
|
2015-07-06 08:19:49 +00:00
|
|
|
if (report)
|
|
|
|
DEBUGF(rhizome, "report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u deleted_orphan_manifests=%u",
|
|
|
|
report->deleted_stale_incoming_files,
|
|
|
|
report->deleted_orphan_files,
|
|
|
|
report->deleted_orphan_fileblobs,
|
|
|
|
report->deleted_orphan_manifests
|
|
|
|
);
|
2013-02-20 04:14:29 +00:00
|
|
|
RETURN(0);
|
2013-01-03 03:46:33 +00:00
|
|
|
OUT();
|
2012-12-31 04:39:12 +00:00
|
|
|
}
|
|
|
|
|
2017-03-06 03:31:14 +00:00
|
|
|
static void sync_rhizome(){
|
|
|
|
if (server_pid()<=0)
|
|
|
|
return;
|
|
|
|
/* Bind to MDP socket and await confirmation */
|
|
|
|
struct mdp_header mdp_header = {
|
|
|
|
.remote.port = MDP_SYNC_RHIZOME,
|
|
|
|
};
|
|
|
|
int mdpsock = mdp_socket();
|
|
|
|
if (mdpsock == -1)
|
|
|
|
WARN("cannot create MDP socket");
|
|
|
|
set_nonblock(mdpsock);
|
|
|
|
int r = mdp_send(mdpsock, &mdp_header, NULL, 0);
|
|
|
|
if (r == -1)
|
|
|
|
goto end;
|
|
|
|
time_ms_t deadline = gettime_ms() + 10000; // TODO add --timeout option?
|
|
|
|
struct mdp_header rev_header;
|
|
|
|
do {
|
|
|
|
ssize_t len = mdp_poll_recv(mdpsock, deadline, &rev_header, NULL, 0);
|
|
|
|
if (len == -1){
|
|
|
|
r = -1;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
if (len == -2) {
|
|
|
|
WHYF("timeout while synchronising daemon bundle list");
|
|
|
|
r = -1;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
} while (!(rev_header.flags & MDP_FLAG_CLOSE));
|
|
|
|
r = 0;
|
|
|
|
|
|
|
|
end:
|
|
|
|
mdp_close(mdpsock);
|
|
|
|
}
|
|
|
|
|
2017-03-06 05:26:58 +00:00
|
|
|
/* Insert the manifest 'm' into the Rhizome store. This function encapsulates all the invariants
|
|
|
|
* that a manifest must satisfy before it is allowed into the store, so it is used by both the sync
|
|
|
|
* protocol and the application layer.
|
|
|
|
*
|
|
|
|
* - If the manifest is not valid then returns RHIZOME_BUNDLE_STATUS_INVALID. A valid manifest is
|
|
|
|
* one with all the core (transport) fields present and consistent ('id', 'version', 'filesize',
|
|
|
|
* 'filehash', 'tail'), all mandatory application fields present and consistent ('service',
|
|
|
|
* 'date') and any other service-dependent mandatory fields present (eg, 'sender', 'recipient').
|
|
|
|
*
|
|
|
|
* - If the manifest's signature does not verify, then returns RHIZOME_BUNDLE_STATUS_FAKE.
|
|
|
|
*
|
|
|
|
* - If the manifest has a payload (filesize != 0) but the payload is not present in the store
|
|
|
|
* (filehash), then returns an internal error RHIZOME_BUNDLE_STATUS_ERROR (-1).
|
|
|
|
*
|
|
|
|
* - If the store will not accept the manifest because there is already the same or a newer
|
|
|
|
* manifest in the store, then returns RHIZOME_BUNDLE_STATUS_SAME or RHIZOME_BUNDLE_STATUS_OLD.
|
|
|
|
*
|
|
|
|
* This function then attempts to store the manifest. If this fails due to an internal error,
|
|
|
|
* then returns RHIZOME_BUNDLE_STATUS_ERROR (-1), otherwise returns RHIZOME_BUNDLE_STATUS_NEW to
|
|
|
|
* indicate that the manifest was successfully stored.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-01-12 03:38:24 +00:00
|
|
|
*/
|
2017-03-06 05:26:58 +00:00
|
|
|
enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rhizome_manifest **mout)
|
2012-01-12 03:38:24 +00:00
|
|
|
{
|
2017-03-06 05:26:58 +00:00
|
|
|
if (mout == NULL)
|
|
|
|
DEBUGF(rhizome, "%s(m=manifest %p, mout=NULL)", __func__, m);
|
|
|
|
else
|
|
|
|
DEBUGF(rhizome, "%s(m=manifest %p, *mout=manifest %p)", __func__, m, *mout);
|
|
|
|
if (!m->finalised && !rhizome_manifest_validate(m))
|
|
|
|
return RHIZOME_BUNDLE_STATUS_INVALID;
|
2013-12-30 07:39:56 +00:00
|
|
|
assert(m->finalised);
|
2017-03-06 05:26:58 +00:00
|
|
|
if (!m->selfSigned && !rhizome_manifest_verify(m))
|
|
|
|
return RHIZOME_BUNDLE_STATUS_FAKE;
|
|
|
|
|
|
|
|
assert(m->filesize != RHIZOME_SIZE_UNSET);
|
|
|
|
if (m->filesize > 0){
|
|
|
|
switch (rhizome_exists(&m->filehash)){
|
|
|
|
case RHIZOME_PAYLOAD_STATUS_BUSY:
|
|
|
|
return RHIZOME_BUNDLE_STATUS_BUSY;
|
|
|
|
case RHIZOME_PAYLOAD_STATUS_STORED:
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return WHY("Payload has not been stored");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
enum rhizome_bundle_status status = rhizome_manifest_check_stored(m, mout);
|
|
|
|
if (status != RHIZOME_BUNDLE_STATUS_NEW)
|
|
|
|
return status;
|
|
|
|
|
2017-05-17 00:46:46 +00:00
|
|
|
if (mout && *mout){
|
|
|
|
if (m != *mout)
|
|
|
|
rhizome_manifest_free(*mout);
|
|
|
|
*mout = NULL;
|
|
|
|
}
|
|
|
|
|
2017-03-06 05:26:58 +00:00
|
|
|
// manifest is complete, and not already stored
|
2012-08-23 08:13:35 +00:00
|
|
|
|
2012-01-12 03:38:24 +00:00
|
|
|
/* Bind BAR to data field */
|
2014-06-16 04:51:42 +00:00
|
|
|
rhizome_bar_t bar;
|
|
|
|
rhizome_manifest_to_bar(m, &bar);
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-10-02 15:46:10 +00:00
|
|
|
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
|
2012-12-28 02:16:07 +00:00
|
|
|
return WHY("Failed to begin transaction");
|
2013-10-10 06:45:52 +00:00
|
|
|
|
2013-11-05 07:28:03 +00:00
|
|
|
time_ms_t now = gettime_ms();
|
|
|
|
|
2013-11-07 06:52:06 +00:00
|
|
|
// The INSERT OR REPLACE statement will delete a row with the same ID (primary key) if it exists,
|
|
|
|
// so a new autoincremented ROWID will be allocated whether or not the manifest with this ID is
|
|
|
|
// already in the table. Other code depends on this property: that ROWID is monotonically
|
|
|
|
// increasing with time and unique.
|
2012-08-23 08:13:35 +00:00
|
|
|
sqlite3_stmt *stmt;
|
2013-10-10 06:45:52 +00:00
|
|
|
if ((stmt = sqlite_prepare_bind(&retry,
|
2013-07-22 05:34:02 +00:00
|
|
|
"INSERT OR REPLACE INTO MANIFESTS("
|
|
|
|
"id,"
|
|
|
|
"manifest,"
|
|
|
|
"version,"
|
|
|
|
"inserttime,"
|
|
|
|
"bar,"
|
|
|
|
"filesize,"
|
|
|
|
"filehash,"
|
|
|
|
"author,"
|
|
|
|
"service,"
|
|
|
|
"name,"
|
|
|
|
"sender,"
|
|
|
|
"recipient,"
|
2016-03-07 04:04:53 +00:00
|
|
|
"tail,"
|
|
|
|
"manifest_hash"
|
2013-07-22 05:34:02 +00:00
|
|
|
") VALUES("
|
2016-03-07 04:04:53 +00:00
|
|
|
"?,?,?,?,?,?,?,?,?,?,?,?,?,?"
|
2013-10-10 06:45:52 +00:00
|
|
|
");",
|
2016-09-27 00:58:46 +00:00
|
|
|
RHIZOME_BID_T, &m->keypair.public_key,
|
2013-11-28 07:14:37 +00:00
|
|
|
STATIC_BLOB, m->manifestdata, m->manifest_all_bytes,
|
2013-10-10 06:45:52 +00:00
|
|
|
INT64, m->version,
|
2013-11-05 07:28:03 +00:00
|
|
|
INT64, (int64_t) now,
|
2014-06-16 04:51:42 +00:00
|
|
|
RHIZOME_BAR_T, &bar,
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
INT64, m->filesize,
|
|
|
|
RHIZOME_FILEHASH_T|NUL, m->filesize > 0 ? &m->filehash : NULL,
|
2013-11-05 07:28:03 +00:00
|
|
|
// Only store the author if it is known to be authentic.
|
|
|
|
SID_T|NUL, m->authorship == AUTHOR_AUTHENTIC ? &m->author : NULL,
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
STATIC_TEXT, m->service,
|
|
|
|
STATIC_TEXT|NUL, m->name,
|
|
|
|
SID_T|NUL, m->has_sender ? &m->sender : NULL,
|
|
|
|
SID_T|NUL, m->has_recipient ? &m->recipient : NULL,
|
|
|
|
INT64, m->tail,
|
2016-03-07 04:04:53 +00:00
|
|
|
RHIZOME_FILEHASH_T, &m->manifesthash,
|
2013-10-10 06:45:52 +00:00
|
|
|
END
|
|
|
|
)
|
|
|
|
) == NULL)
|
2012-08-23 08:13:35 +00:00
|
|
|
goto rollback;
|
2014-09-08 07:44:55 +00:00
|
|
|
if (!sqlite_code_ok(sqlite_step_retry(&retry, stmt)))
|
2012-08-23 08:13:35 +00:00
|
|
|
goto rollback;
|
|
|
|
sqlite3_finalize(stmt);
|
|
|
|
stmt = NULL;
|
2018-04-10 01:20:06 +00:00
|
|
|
rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_database.db));
|
2013-11-05 07:28:03 +00:00
|
|
|
rhizome_manifest_set_inserttime(m, now);
|
2012-08-23 08:13:35 +00:00
|
|
|
|
2013-10-02 15:46:10 +00:00
|
|
|
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
|
2013-01-15 00:02:48 +00:00
|
|
|
// This message used in tests; do not modify or remove.
|
2013-12-11 00:41:34 +00:00
|
|
|
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRIu64,
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
m->service ? m->service : "NULL",
|
2016-09-27 00:58:46 +00:00
|
|
|
alloca_tohex_rhizome_bid_t(m->keypair.public_key),
|
2013-01-15 00:02:48 +00:00
|
|
|
m->version
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
);
|
2018-06-01 08:10:09 +00:00
|
|
|
if (serverMode != SERVER_NOT_RUNNING) {
|
2017-03-06 03:31:14 +00:00
|
|
|
assert(max_rowid < m->rowid);
|
|
|
|
// detect any bundles added by the CLI
|
|
|
|
// due to potential race conditions, we have to do this here
|
|
|
|
// even though the CLI will try to send us a MDP_SYNC_RHIZOME message
|
|
|
|
if (m->rowid > max_rowid+1)
|
2018-05-31 15:41:44 +00:00
|
|
|
rhizome_process_added_bundles(m->rowid);
|
2017-03-06 03:31:14 +00:00
|
|
|
max_rowid = m->rowid;
|
2016-03-30 06:48:34 +00:00
|
|
|
CALL_TRIGGER(bundle_add, m);
|
2017-03-06 03:31:14 +00:00
|
|
|
}else{
|
|
|
|
sync_rhizome();
|
|
|
|
}
|
2017-05-17 00:46:46 +00:00
|
|
|
if (mout)
|
|
|
|
*mout = m;
|
2017-03-06 05:26:58 +00:00
|
|
|
return RHIZOME_BUNDLE_STATUS_NEW;
|
2013-01-15 00:02:48 +00:00
|
|
|
}
|
2017-03-06 05:26:58 +00:00
|
|
|
|
2012-08-23 08:13:35 +00:00
|
|
|
rollback:
|
|
|
|
if (stmt)
|
|
|
|
sqlite3_finalize(stmt);
|
2016-09-27 00:58:46 +00:00
|
|
|
WHYF("Failed to store bundle bid=%s", alloca_tohex_rhizome_bid_t(m->keypair.public_key));
|
2013-10-02 15:46:10 +00:00
|
|
|
sqlite_exec_void_retry(&retry, "ROLLBACK;", END);
|
2017-03-06 05:26:58 +00:00
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
2012-01-12 03:38:24 +00:00
|
|
|
}
|
|
|
|
|
2015-05-25 09:05:10 +00:00
|
|
|
static void trigger_rhizome_bundle_added_debug(rhizome_manifest *m)
|
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "TRIGGER rhizome_bundle_added service=%s bid=%s version=%"PRIu64,
|
|
|
|
m->service ? m->service : "NULL",
|
2016-09-27 00:58:46 +00:00
|
|
|
alloca_tohex_rhizome_bid_t(m->keypair.public_key),
|
2015-07-06 08:19:49 +00:00
|
|
|
m->version
|
2015-05-25 09:05:10 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2015-10-19 11:46:15 +00:00
|
|
|
DEFINE_TRIGGER(bundle_add, trigger_rhizome_bundle_added_debug);
|
2015-05-25 09:05:10 +00:00
|
|
|
|
2013-11-05 07:28:03 +00:00
|
|
|
/* The cursor struct must be zerofilled and the query parameters optionally filled in prior to
|
|
|
|
* calling this function.
|
2013-11-04 13:17:09 +00:00
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2014-01-22 06:50:06 +00:00
|
|
|
int rhizome_list_open(struct rhizome_list_cursor *c)
|
2013-11-04 13:17:09 +00:00
|
|
|
{
|
2016-07-18 06:49:16 +00:00
|
|
|
DEBUGF(rhizome, "c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->oldest_first=%d c->_rowid_last=%"PRIu64,
|
2015-07-06 08:19:49 +00:00
|
|
|
c,
|
|
|
|
alloca_str_toprint(c->service),
|
|
|
|
alloca_str_toprint(c->name),
|
|
|
|
c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET",
|
|
|
|
c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET",
|
|
|
|
c->rowid_since,
|
2016-07-18 06:49:16 +00:00
|
|
|
c->oldest_first,
|
2015-07-06 08:19:49 +00:00
|
|
|
c->_rowid_last
|
|
|
|
);
|
2013-11-04 13:17:09 +00:00
|
|
|
IN();
|
2012-05-20 14:39:14 +00:00
|
|
|
strbuf b = strbuf_alloca(1024);
|
2013-01-22 04:56:40 +00:00
|
|
|
strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->service)
|
2013-11-04 13:17:09 +00:00
|
|
|
strbuf_puts(b, " AND service = @service");
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->name)
|
2013-11-04 13:17:09 +00:00
|
|
|
strbuf_puts(b, " AND name like @name");
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->is_sender_set)
|
2013-11-04 13:17:09 +00:00
|
|
|
strbuf_puts(b, " AND sender = @sender");
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->is_recipient_set)
|
2013-11-04 13:17:09 +00:00
|
|
|
strbuf_puts(b, " AND recipient = @recipient");
|
2016-07-18 06:49:16 +00:00
|
|
|
if (c->rowid_since)
|
|
|
|
strbuf_puts(b, " AND rowid > @since");
|
|
|
|
if (c->oldest_first){
|
|
|
|
if (c->_rowid_last)
|
|
|
|
strbuf_puts(b, " AND rowid > @last");
|
|
|
|
strbuf_puts(b, " ORDER BY rowid ASC");
|
|
|
|
}else{
|
2013-11-19 05:11:12 +00:00
|
|
|
if (c->_rowid_last)
|
|
|
|
strbuf_puts(b, " AND rowid < @last");
|
2016-07-18 06:49:16 +00:00
|
|
|
strbuf_puts(b, " ORDER BY rowid DESC");
|
2013-11-19 05:11:12 +00:00
|
|
|
}
|
2012-05-20 14:39:14 +00:00
|
|
|
if (strbuf_overrun(b))
|
2013-07-15 00:29:24 +00:00
|
|
|
RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
|
2014-01-22 06:50:06 +00:00
|
|
|
c->_retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
c->_statement = sqlite_prepare(&c->_retry, strbuf_str(b));
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->_statement == NULL)
|
2012-11-12 04:07:58 +00:00
|
|
|
RETURN(-1);
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->service && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
goto failure;
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->name && sqlite_bind(&c->_retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
goto failure;
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->is_sender_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
goto failure;
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->is_recipient_set && sqlite_bind(&c->_retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
goto failure;
|
2016-07-18 06:49:16 +00:00
|
|
|
if (c->rowid_since && sqlite_bind(&c->_retry, c->_statement, NAMED|INT64, "@since", c->rowid_since, END) == -1)
|
|
|
|
goto failure;
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->_rowid_last && sqlite_bind(&c->_retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
goto failure;
|
2013-11-07 06:52:06 +00:00
|
|
|
c->manifest = NULL;
|
2013-11-13 06:28:28 +00:00
|
|
|
c->_rowid_current = 0;
|
2013-11-04 13:17:09 +00:00
|
|
|
RETURN(0);
|
|
|
|
OUT();
|
|
|
|
failure:
|
2013-11-07 06:52:06 +00:00
|
|
|
sqlite3_finalize(c->_statement);
|
|
|
|
c->_statement = NULL;
|
2013-11-04 13:17:09 +00:00
|
|
|
RETURN(-1);
|
|
|
|
OUT();
|
|
|
|
}
|
|
|
|
|
2013-11-13 06:28:28 +00:00
|
|
|
/* Guaranteed to return manifests with monotonically descending rowid. The first manifest will have
|
|
|
|
* the greatest rowid.
|
|
|
|
*
|
|
|
|
* Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest'
|
|
|
|
* field points to the fetched manifest. Returns 0 if there are no more manifests in the list.
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2014-01-22 06:50:06 +00:00
|
|
|
int rhizome_list_next(struct rhizome_list_cursor *c)
|
2013-11-04 13:17:09 +00:00
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64,
|
|
|
|
c,
|
|
|
|
alloca_str_toprint(c->service),
|
|
|
|
alloca_str_toprint(c->name),
|
|
|
|
c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET",
|
|
|
|
c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET",
|
|
|
|
c->rowid_since,
|
|
|
|
c->_rowid_last
|
|
|
|
);
|
2013-11-04 13:17:09 +00:00
|
|
|
IN();
|
2014-01-22 06:50:06 +00:00
|
|
|
if (c->_statement == NULL && rhizome_list_open(c) == -1)
|
2013-11-04 13:17:09 +00:00
|
|
|
RETURN(-1);
|
2014-09-08 07:44:55 +00:00
|
|
|
int r=0;
|
2013-11-13 06:28:28 +00:00
|
|
|
while (1) {
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->manifest) {
|
|
|
|
rhizome_manifest_free(c->manifest);
|
2013-11-13 06:28:28 +00:00
|
|
|
c->_rowid_current = 0;
|
2013-11-07 06:52:06 +00:00
|
|
|
c->manifest = NULL;
|
2013-11-04 13:17:09 +00:00
|
|
|
}
|
2014-09-08 07:44:55 +00:00
|
|
|
if ((r=sqlite_step_retry(&c->_retry, c->_statement)) != SQLITE_ROW)
|
2013-11-13 06:28:28 +00:00
|
|
|
break;
|
2013-11-07 06:52:06 +00:00
|
|
|
assert(sqlite3_column_count(c->_statement) == 6);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 2) == SQLITE_INTEGER);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
|
|
|
|
assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
|
2016-07-18 06:49:16 +00:00
|
|
|
|
|
|
|
uint64_t q_rowid = c->_rowid_current = sqlite3_column_int64(c->_statement, 5);
|
2013-11-07 06:52:06 +00:00
|
|
|
const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
|
|
|
|
const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
|
|
|
|
size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
|
2013-12-11 00:41:34 +00:00
|
|
|
uint64_t q_version = sqlite3_column_int64(c->_statement, 2);
|
2013-11-07 06:52:06 +00:00
|
|
|
int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
|
|
|
|
const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
|
2016-01-25 03:39:44 +00:00
|
|
|
sid_t author;
|
2013-11-04 13:17:09 +00:00
|
|
|
if (q_author) {
|
2016-01-25 03:39:44 +00:00
|
|
|
if (str_to_sid_t(&author, q_author) == -1) {
|
2013-11-04 13:17:09 +00:00
|
|
|
WHYF("MANIFESTS row id=%s has invalid author column %s -- skipped", q_manifestid, alloca_str_toprint(q_author));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2013-11-07 06:52:06 +00:00
|
|
|
rhizome_manifest *m = c->manifest = rhizome_new_manifest();
|
2013-11-05 07:28:03 +00:00
|
|
|
if (m == NULL)
|
|
|
|
RETURN(-1);
|
2013-12-19 08:37:14 +00:00
|
|
|
memcpy(m->manifestdata, manifestblob, manifestblobsize);
|
|
|
|
m->manifest_all_bytes = manifestblobsize;
|
|
|
|
if ( rhizome_manifest_parse(m) == -1
|
2013-11-28 07:14:37 +00:00
|
|
|
|| !rhizome_manifest_validate(m)
|
|
|
|
) {
|
2013-11-05 07:28:03 +00:00
|
|
|
WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
|
2013-11-04 13:17:09 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (m->version != q_version) {
|
2013-12-11 00:41:34 +00:00
|
|
|
WHYF("MANIFESTS row id=%s version=%"PRIu64" does not match manifest blob version=%"PRIu64" -- skipped",
|
2013-11-04 13:17:09 +00:00
|
|
|
q_manifestid, q_version, m->version);
|
|
|
|
continue;
|
|
|
|
}
|
2016-01-25 03:39:44 +00:00
|
|
|
if (q_author)
|
|
|
|
rhizome_manifest_set_author(m, &author);
|
2013-11-11 07:43:38 +00:00
|
|
|
rhizome_manifest_set_rowid(m, q_rowid);
|
2013-11-05 07:28:03 +00:00
|
|
|
rhizome_manifest_set_inserttime(m, q_inserttime);
|
2013-11-13 06:28:28 +00:00
|
|
|
assert(c->_rowid_current != 0);
|
2013-11-04 13:17:09 +00:00
|
|
|
// Don't do rhizome_verify_author(m); too CPU expensive for a listing. Save that for when
|
|
|
|
// the bundle is extracted or exported.
|
|
|
|
RETURN(1);
|
2013-01-17 01:13:31 +00:00
|
|
|
}
|
2013-11-13 06:28:28 +00:00
|
|
|
assert(c->_rowid_current == 0);
|
2014-09-08 07:44:55 +00:00
|
|
|
RETURN(sqlite_code_ok(r)?0:-1);
|
2013-11-04 13:17:09 +00:00
|
|
|
OUT();
|
|
|
|
}
|
|
|
|
|
2013-11-08 07:59:49 +00:00
|
|
|
void rhizome_list_commit(struct rhizome_list_cursor *c)
|
2013-11-04 13:17:09 +00:00
|
|
|
{
|
2016-07-18 06:49:16 +00:00
|
|
|
DEBUGF(rhizome, "c=%p c->oldest_first=%d c->_rowid_current=%"PRIu64" c->_rowid_last=%"PRIu64,
|
|
|
|
c, c->oldest_first, c->_rowid_current, c->_rowid_last);
|
2013-11-13 06:28:28 +00:00
|
|
|
assert(c->_rowid_current != 0);
|
2016-07-18 06:49:16 +00:00
|
|
|
if (c->_rowid_last == 0 || (c->oldest_first ? c->_rowid_current > c->_rowid_last : c->_rowid_current < c->_rowid_last))
|
2013-11-13 06:28:28 +00:00
|
|
|
c->_rowid_last = c->_rowid_current;
|
2013-11-04 13:17:09 +00:00
|
|
|
}
|
|
|
|
|
2013-11-07 06:52:06 +00:00
|
|
|
void rhizome_list_release(struct rhizome_list_cursor *c)
|
2013-11-04 13:17:09 +00:00
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "c=%p", c);
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->manifest) {
|
|
|
|
rhizome_manifest_free(c->manifest);
|
2013-11-13 06:28:28 +00:00
|
|
|
c->_rowid_current = 0;
|
2013-11-07 06:52:06 +00:00
|
|
|
c->manifest = NULL;
|
2013-11-04 13:17:09 +00:00
|
|
|
}
|
2013-11-07 06:52:06 +00:00
|
|
|
if (c->_statement) {
|
|
|
|
sqlite3_finalize(c->_statement);
|
|
|
|
c->_statement = NULL;
|
2013-01-22 04:56:40 +00:00
|
|
|
}
|
2012-04-02 08:12:40 +00:00
|
|
|
}
|
2012-01-12 03:38:24 +00:00
|
|
|
|
2012-05-23 06:34:00 +00:00
|
|
|
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
|
2012-01-12 03:38:24 +00:00
|
|
|
{
|
2013-10-09 08:24:21 +00:00
|
|
|
(void) tohex(out, byteCount * 2, in);
|
2012-01-12 03:38:24 +00:00
|
|
|
}
|
|
|
|
|
2013-09-30 06:45:01 +00:00
|
|
|
/* Search the database for a manifest having the same name and payload content, and if the version
|
2013-12-19 08:43:39 +00:00
|
|
|
* is known, having the same version. Returns RHIZOME_BUNDLE_STATUS_DUPLICATE if a duplicate is found
|
|
|
|
* (setting *found to point to the duplicate's manifest), returns RHIZOME_BUNDLE_STATUS_NEW if no
|
|
|
|
* duplicate is found (leaving *found unchanged). Returns -1 on error (leaving *found undefined).
|
2013-09-30 06:45:01 +00:00
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
2012-04-12 09:00:52 +00:00
|
|
|
*/
|
2013-12-19 08:43:39 +00:00
|
|
|
enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
2012-04-12 09:00:52 +00:00
|
|
|
{
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->service == NULL)
|
2012-05-20 14:39:14 +00:00
|
|
|
return WHY("Manifest has no service");
|
2012-04-12 09:00:52 +00:00
|
|
|
char sqlcmd[1024];
|
2015-10-13 08:12:22 +00:00
|
|
|
strbuf b = strbuf_local_buf(sqlcmd);
|
2013-08-01 02:07:35 +00:00
|
|
|
strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
assert(m->filesize != RHIZOME_SIZE_UNSET);
|
|
|
|
if (m->filesize > 0)
|
2013-08-01 02:07:35 +00:00
|
|
|
strbuf_puts(b, " AND filehash = ?");
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->name)
|
2013-08-01 02:07:35 +00:00
|
|
|
strbuf_puts(b, " AND name = ?");
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->has_sender)
|
2013-08-01 02:07:35 +00:00
|
|
|
strbuf_puts(b, " AND sender = ?");
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->has_recipient)
|
2013-08-01 02:07:35 +00:00
|
|
|
strbuf_puts(b, " AND recipient = ?");
|
2012-07-16 09:01:00 +00:00
|
|
|
if (strbuf_overrun(b))
|
|
|
|
return WHYF("SQL command too long: %s", strbuf_str(b));
|
2013-12-19 08:43:39 +00:00
|
|
|
int ret = RHIZOME_BUNDLE_STATUS_NEW;
|
2012-10-04 05:00:20 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END);
|
2012-08-23 08:13:35 +00:00
|
|
|
if (!statement)
|
|
|
|
return -1;
|
2013-10-10 07:53:06 +00:00
|
|
|
int field = 2;
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->filesize > 0)
|
2013-10-10 07:53:06 +00:00
|
|
|
sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END);
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
if (m->name)
|
|
|
|
sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, m->name, END);
|
|
|
|
if (m->has_sender)
|
|
|
|
sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->sender, END);
|
|
|
|
if (m->has_recipient)
|
|
|
|
sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->recipient, END);
|
2013-10-10 07:53:06 +00:00
|
|
|
|
2013-07-15 00:29:24 +00:00
|
|
|
int rows = 0;
|
2014-09-08 07:44:55 +00:00
|
|
|
int r=0;
|
|
|
|
while ((r=sqlite_step_retry(&retry, statement)) == SQLITE_ROW) {
|
2012-08-23 08:13:35 +00:00
|
|
|
++rows;
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "Row %d", rows);
|
2012-08-23 08:13:35 +00:00
|
|
|
rhizome_manifest *blob_m = rhizome_new_manifest();
|
|
|
|
if (blob_m == NULL) {
|
|
|
|
ret = WHY("Out of manifests");
|
|
|
|
break;
|
|
|
|
}
|
2013-08-01 02:07:35 +00:00
|
|
|
const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
|
|
|
|
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
|
|
|
|
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
2013-12-19 08:37:14 +00:00
|
|
|
memcpy(blob_m->manifestdata, manifestblob, manifestblobsize);
|
|
|
|
blob_m->manifest_all_bytes = manifestblobsize;
|
|
|
|
if ( rhizome_manifest_parse(blob_m) == -1
|
2013-11-28 07:14:37 +00:00
|
|
|
|| !rhizome_manifest_validate(blob_m)
|
2013-12-19 08:43:39 +00:00
|
|
|
) {
|
2012-08-23 08:13:35 +00:00
|
|
|
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
|
2013-08-01 02:07:35 +00:00
|
|
|
goto next;
|
|
|
|
}
|
2013-11-28 07:14:37 +00:00
|
|
|
if (!rhizome_manifest_verify(blob_m)) {
|
2012-08-23 08:13:35 +00:00
|
|
|
WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
|
2013-08-01 02:07:35 +00:00
|
|
|
goto next;
|
|
|
|
}
|
|
|
|
const char *q_author = (const char *) sqlite3_column_text(statement, 2);
|
|
|
|
if (q_author) {
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
sid_t author;
|
|
|
|
if (str_to_sid_t(&author, q_author) == -1)
|
|
|
|
WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_manifestid, alloca_str_toprint(q_author));
|
|
|
|
else
|
|
|
|
rhizome_manifest_set_author(blob_m, &author);
|
2013-08-01 02:07:35 +00:00
|
|
|
}
|
|
|
|
// check that we can re-author this manifest
|
2013-11-05 07:28:03 +00:00
|
|
|
rhizome_authenticate_author(blob_m);
|
|
|
|
if (m->authorship != AUTHOR_AUTHENTIC)
|
2013-08-01 02:07:35 +00:00
|
|
|
goto next;
|
|
|
|
*found = blob_m;
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "Found duplicate payload, %s", q_manifestid);
|
2013-12-19 08:43:39 +00:00
|
|
|
ret = RHIZOME_BUNDLE_STATUS_DUPLICATE;
|
2013-08-01 02:07:35 +00:00
|
|
|
break;
|
|
|
|
next:
|
2012-08-23 08:13:35 +00:00
|
|
|
if (blob_m)
|
|
|
|
rhizome_manifest_free(blob_m);
|
2012-04-12 09:00:52 +00:00
|
|
|
}
|
|
|
|
sqlite3_finalize(statement);
|
2014-09-08 07:44:55 +00:00
|
|
|
if (!sqlite_code_ok(r))
|
|
|
|
ret=-1;
|
2012-04-12 09:00:52 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2012-05-02 06:33:09 +00:00
|
|
|
|
2015-09-21 09:12:13 +00:00
|
|
|
/* Unpacks a database MANIFESTS table row into a manifest structure.
|
|
|
|
*
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_SAME if unpack succeeds
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_NEW if manifest is not found
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_ERROR on error
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_BUSY if the database is locked
|
|
|
|
* Caller is responsible for allocating and freeing rhizome_manifest
|
|
|
|
*/
|
2017-03-06 03:31:14 +00:00
|
|
|
static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement){
|
2013-10-03 13:46:45 +00:00
|
|
|
const char *q_id = (const char *) sqlite3_column_text(statement, 0);
|
|
|
|
const char *q_blob = (char *) sqlite3_column_blob(statement, 1);
|
2013-12-11 00:41:34 +00:00
|
|
|
uint64_t q_version = sqlite3_column_int64(statement, 2);
|
2013-10-03 13:46:45 +00:00
|
|
|
int64_t q_inserttime = sqlite3_column_int64(statement, 3);
|
|
|
|
const char *q_author = (const char *) sqlite3_column_text(statement, 4);
|
|
|
|
size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
2013-11-11 07:43:38 +00:00
|
|
|
uint64_t q_rowid = sqlite3_column_int64(statement, 5);
|
2013-12-19 08:37:14 +00:00
|
|
|
memcpy(m->manifestdata, q_blob, q_blobsize);
|
|
|
|
m->manifest_all_bytes = q_blobsize;
|
|
|
|
if (rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m))
|
2013-11-28 07:14:37 +00:00
|
|
|
return WHYF("Manifest bid=%s in database but invalid", q_id);
|
2013-10-03 13:46:45 +00:00
|
|
|
if (q_author) {
|
Refactor manifest: specific setter functions
Replace generic rhizome_manifest_set() and rhizome_manifest_set_ll()
with per-field setter functions, eg, rhizome_manifest_set_filesize().
Struct rhizome_manifest elements for all known fields, to replace the
use of rhizome_manifest_get() and rhizome_manifest_get_ll() everywhere:
sender, recipient, service, name, date, bundle_key.
Add boolean validity flags for binary blob types, to avoid having to compare
with many bytes of all-zero to detect presence, eg, has_sender, has_recipient,
has_author, has_bundle_key. These maintained by the setter functions.
Rename existing manifest struct elements to be the same as their field
names: fileLength -> filesize, journalTail -> tail.
More use of unsigned int, size_t and uint64_t for payload sizes, offsets, byte
counts, etc. especially in rhizome_store.c and meshms.c. More uniform use of
size_t to dimension memory buffers. Fix some printf(3) style format strings
for 64-bit correctness on 32-bit systems. Use new constant RHIZOME_SIZE_UNSET
instead of -1 to indicate unknown dimension, and explicitly assert its absence
before comparisons and arithmetic, for safety.
Replace some 'int' loop variables with 'unsigned' where appropriate.
Fix bugs discovered in MeshMS bundle private/public key generation and
bundle secret key handling for export/extract commands.
Instrument the first MeshMS test case to aid debugging.
New debug config flag: debug.manifest logs all modifications to all manifest
fields by setter functions.
Rename debug config flag: debug.rhizome_bind -> debug.rhizome_sql_bind.
2013-10-30 12:52:19 +00:00
|
|
|
sid_t author;
|
|
|
|
if (str_to_sid_t(&author, q_author) == -1)
|
|
|
|
WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author));
|
|
|
|
else
|
|
|
|
rhizome_manifest_set_author(m, &author);
|
2013-10-03 13:46:45 +00:00
|
|
|
}
|
|
|
|
if (m->version != q_version)
|
2013-12-11 00:41:34 +00:00
|
|
|
WARNF("Version mismatch, manifest is %"PRIu64", database is %"PRIu64, m->version, q_version);
|
2013-11-11 07:43:38 +00:00
|
|
|
rhizome_manifest_set_rowid(m, q_rowid);
|
2013-11-05 07:28:03 +00:00
|
|
|
rhizome_manifest_set_inserttime(m, q_inserttime);
|
2017-03-06 03:31:14 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static enum rhizome_bundle_status step_unpack_manifest_row(sqlite_retry_state *retry, rhizome_manifest *m, sqlite3_stmt *statement)
|
|
|
|
{
|
|
|
|
int r=sqlite_step_retry(retry, statement);
|
|
|
|
if (sqlite_code_busy(r))
|
|
|
|
return RHIZOME_BUNDLE_STATUS_BUSY;
|
|
|
|
if (!sqlite_code_ok(r))
|
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
if (r!=SQLITE_ROW)
|
|
|
|
return RHIZOME_BUNDLE_STATUS_NEW;
|
|
|
|
|
|
|
|
if (unpack_manifest_row(m, statement)==-1)
|
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
2014-09-08 07:44:55 +00:00
|
|
|
return RHIZOME_BUNDLE_STATUS_SAME;
|
2013-10-03 13:46:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Retrieve a manifest from the database, given its Bundle ID.
|
2012-05-02 08:27:35 +00:00
|
|
|
*
|
2014-09-08 07:44:55 +00:00
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_SAME if manifest is found
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_NEW if manifest is not found
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_ERROR on error
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_BUSY if the database is locked
|
2013-01-02 00:42:15 +00:00
|
|
|
* Caller is responsible for allocating and freeing rhizome_manifest
|
2012-05-02 06:33:09 +00:00
|
|
|
*/
|
2014-09-08 07:44:55 +00:00
|
|
|
enum rhizome_bundle_status rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
2015-07-06 08:19:49 +00:00
|
|
|
DEBUGF(rhizome, "retrieve manifest bid=%s", bidp ? alloca_tohex_rhizome_bid_t(*bidp) : "<NULL>");
|
2012-10-04 05:00:20 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-10-03 07:14:06 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
2013-11-11 07:43:38 +00:00
|
|
|
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id = ?",
|
2013-10-03 17:42:52 +00:00
|
|
|
RHIZOME_BID_T, bidp,
|
2013-10-03 07:14:06 +00:00
|
|
|
END);
|
2012-08-23 08:13:35 +00:00
|
|
|
if (!statement)
|
2014-09-08 07:44:55 +00:00
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
2017-03-06 03:31:14 +00:00
|
|
|
enum rhizome_bundle_status ret = step_unpack_manifest_row(&retry, m, statement);
|
2013-10-03 13:46:45 +00:00
|
|
|
sqlite3_finalize(statement);
|
|
|
|
return ret;
|
|
|
|
}
|
2013-01-02 00:42:15 +00:00
|
|
|
|
2013-10-03 13:46:45 +00:00
|
|
|
/* Retrieve any manifest from the database whose Bundle ID starts with the given prefix.
|
|
|
|
*
|
2014-09-08 07:44:55 +00:00
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_SAME if manifest is found
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_NEW if manifest is not found
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_ERROR on error
|
|
|
|
* Returns RHIZOME_BUNDLE_STATUS_BUSY if the database is locked
|
2013-10-03 13:46:45 +00:00
|
|
|
* Caller is responsible for allocating and freeing rhizome_manifest
|
|
|
|
*/
|
2014-09-08 07:44:55 +00:00
|
|
|
enum rhizome_bundle_status rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m)
|
2013-10-03 13:46:45 +00:00
|
|
|
{
|
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-10-09 08:24:21 +00:00
|
|
|
const unsigned prefix_strlen = prefix_len * 2;
|
|
|
|
char like[prefix_strlen + 2];
|
|
|
|
tohex(like, prefix_strlen, prefix);
|
|
|
|
like[prefix_strlen] = '%';
|
|
|
|
like[prefix_strlen + 1] = '\0';
|
2013-10-03 13:46:45 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
2013-11-11 07:43:38 +00:00
|
|
|
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id like ?",
|
2013-10-03 13:46:45 +00:00
|
|
|
TEXT, like,
|
|
|
|
END);
|
|
|
|
if (!statement)
|
2014-09-08 07:44:55 +00:00
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
2017-03-06 03:31:14 +00:00
|
|
|
enum rhizome_bundle_status ret = step_unpack_manifest_row(&retry, m, statement);
|
2012-05-02 06:33:09 +00:00
|
|
|
sqlite3_finalize(statement);
|
2013-10-03 13:46:45 +00:00
|
|
|
return ret;
|
2012-05-02 06:33:09 +00:00
|
|
|
}
|
2013-02-12 00:04:04 +00:00
|
|
|
|
2016-03-09 02:02:23 +00:00
|
|
|
enum rhizome_bundle_status rhizome_retrieve_manifest_by_hash_prefix(const uint8_t *prefix, unsigned prefix_len, rhizome_manifest *m)
|
|
|
|
{
|
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
const unsigned prefix_strlen = prefix_len * 2;
|
|
|
|
char like[prefix_strlen + 2];
|
|
|
|
tohex(like, prefix_strlen, prefix);
|
|
|
|
like[prefix_strlen] = '%';
|
|
|
|
like[prefix_strlen + 1] = '\0';
|
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
|
|
|
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE manifest_hash like ?",
|
|
|
|
TEXT, like,
|
|
|
|
END);
|
|
|
|
if (!statement)
|
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
2017-03-06 03:31:14 +00:00
|
|
|
enum rhizome_bundle_status ret = step_unpack_manifest_row(&retry, m, statement);
|
2016-03-09 02:02:23 +00:00
|
|
|
sqlite3_finalize(statement);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-03-22 05:58:44 +00:00
|
|
|
enum rhizome_bundle_status rhizome_retrieve_bar_by_hash_prefix(const uint8_t *prefix, unsigned prefix_len, rhizome_bar_t *bar)
|
|
|
|
{
|
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
const unsigned prefix_strlen = prefix_len * 2;
|
|
|
|
char like[prefix_strlen + 2];
|
|
|
|
tohex(like, prefix_strlen, prefix);
|
|
|
|
like[prefix_strlen] = '%';
|
|
|
|
like[prefix_strlen + 1] = '\0';
|
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
|
|
|
"SELECT bar FROM manifests WHERE manifest_hash like ?",
|
|
|
|
TEXT, like,
|
|
|
|
END);
|
|
|
|
if (!statement)
|
|
|
|
return RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
enum rhizome_bundle_status ret;
|
|
|
|
|
|
|
|
int r=sqlite_step_retry(&retry, statement);
|
|
|
|
if (sqlite_code_busy(r)){
|
|
|
|
ret = RHIZOME_BUNDLE_STATUS_BUSY;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
if (!sqlite_code_ok(r)){
|
|
|
|
ret = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
if (r!=SQLITE_ROW){
|
|
|
|
ret = RHIZOME_BUNDLE_STATUS_NEW;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
const uint8_t *db_bar = sqlite3_column_blob(statement, 0);
|
|
|
|
size_t bar_size = sqlite3_column_bytes(statement, 0);
|
|
|
|
|
|
|
|
if (bar_size != RHIZOME_BAR_BYTES){
|
|
|
|
ret = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
bcopy(db_bar, bar, RHIZOME_BAR_BYTES);
|
|
|
|
ret = RHIZOME_BUNDLE_STATUS_SAME;
|
|
|
|
|
|
|
|
end:
|
|
|
|
sqlite3_finalize(statement);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2018-05-31 15:41:44 +00:00
|
|
|
// Detect bundles added from the cmdline, and call trigger functions.
|
|
|
|
void rhizome_process_added_bundles(uint64_t up_to_rowid) {
|
2018-06-01 08:10:09 +00:00
|
|
|
assert(serverMode != SERVER_NOT_RUNNING);
|
2017-03-06 03:31:14 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
|
|
|
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE rowid > ? AND rowid < ?"
|
|
|
|
"ORDER BY rowid",
|
2018-05-31 15:41:44 +00:00
|
|
|
INT64, max_rowid, INT64, up_to_rowid, END);
|
2017-03-06 03:31:14 +00:00
|
|
|
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
|
|
|
|
rhizome_manifest *m = rhizome_new_manifest();
|
|
|
|
if (!m)
|
|
|
|
break;
|
|
|
|
if (unpack_manifest_row(m, statement)!=-1){
|
|
|
|
if (rhizome_manifest_verify(m)){
|
2017-03-21 02:37:33 +00:00
|
|
|
if (max_rowid < m->rowid)
|
|
|
|
max_rowid = m->rowid;
|
2017-03-06 03:31:14 +00:00
|
|
|
CALL_TRIGGER(bundle_add, m);
|
2017-03-21 02:37:33 +00:00
|
|
|
// Note that a trigger might cause a new bundle to be added, and max_rowid to jump
|
2017-03-06 03:31:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
rhizome_manifest_free(m);
|
|
|
|
}
|
|
|
|
sqlite3_finalize(statement);
|
|
|
|
}
|
|
|
|
|
2013-10-03 13:46:45 +00:00
|
|
|
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
2013-10-03 07:14:06 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(retry,
|
|
|
|
"DELETE FROM manifests WHERE id = ?",
|
2013-10-03 13:46:45 +00:00
|
|
|
RHIZOME_BID_T, bidp,
|
2013-10-03 07:14:06 +00:00
|
|
|
END);
|
2013-02-20 04:14:29 +00:00
|
|
|
if (!statement)
|
|
|
|
return -1;
|
2013-10-03 05:45:30 +00:00
|
|
|
if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
|
2013-02-20 04:14:29 +00:00
|
|
|
return -1;
|
2018-04-10 01:20:06 +00:00
|
|
|
return sqlite3_changes(rhizome_database.db) ? 0 : 1;
|
2013-02-20 04:14:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove a manifest and its bundle from the database, given its manifest ID.
|
|
|
|
*
|
|
|
|
* Returns 0 if manifest is found and removed and bundle was either absent or removed
|
|
|
|
* Returns 1 if manifest is not found
|
|
|
|
* Returns -1 on error
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2013-10-03 13:46:45 +00:00
|
|
|
int rhizome_delete_bundle(const rhizome_bid_t *bidp)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
2014-06-18 07:55:15 +00:00
|
|
|
if (rhizome_delete_payload(bidp) == -1)
|
2013-02-20 04:14:29 +00:00
|
|
|
return -1;
|
2014-06-18 07:55:15 +00:00
|
|
|
if (rhizome_delete_manifest(bidp) == -1)
|
2013-02-20 04:14:29 +00:00
|
|
|
return -1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
|
|
|
|
* untouched if present.
|
|
|
|
*
|
|
|
|
* Returns 0 if manifest is found and removed
|
|
|
|
* Returns 1 if manifest is not found
|
|
|
|
* Returns -1 on error
|
|
|
|
*
|
|
|
|
* @author Andrew Bettison <andrew@servalproject.com>
|
|
|
|
*/
|
2013-10-03 13:46:45 +00:00
|
|
|
int rhizome_delete_manifest(const rhizome_bid_t *bidp)
|
2013-02-20 04:14:29 +00:00
|
|
|
{
|
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-10-03 13:46:45 +00:00
|
|
|
return rhizome_delete_manifest_retry(&retry, bidp);
|
2013-02-20 04:14:29 +00:00
|
|
|
}
|
|
|
|
|
2017-10-19 11:43:30 +00:00
|
|
|
static enum rhizome_bundle_status is_interesting(const char *id_hex, uint64_t version, uint64_t *filesizep)
|
2013-08-22 05:44:21 +00:00
|
|
|
{
|
2013-02-15 06:37:45 +00:00
|
|
|
IN();
|
2013-04-11 05:54:13 +00:00
|
|
|
|
2013-02-14 22:42:01 +00:00
|
|
|
// do we have this bundle [or later]?
|
2013-02-12 00:04:04 +00:00
|
|
|
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
|
2013-10-03 19:20:37 +00:00
|
|
|
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
|
2017-10-19 11:43:30 +00:00
|
|
|
"SELECT version, filesize, filehash FROM MANIFESTS WHERE id LIKE ? AND version >= ?",
|
2013-10-03 07:14:06 +00:00
|
|
|
TEXT_TOUPPER, id_hex,
|
|
|
|
INT64, version,
|
|
|
|
END);
|
2013-06-18 03:57:26 +00:00
|
|
|
if (!statement)
|
2017-03-06 05:26:58 +00:00
|
|
|
RETURN(RHIZOME_BUNDLE_STATUS_ERROR);
|
|
|
|
|
|
|
|
enum rhizome_bundle_status status = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
int stepcode;
|
|
|
|
if ((stepcode = sqlite_step_retry(&retry, statement)) == SQLITE_ROW){
|
2017-05-17 04:38:01 +00:00
|
|
|
uint64_t q_version = sqlite3_column_int64(statement, 0);
|
2017-10-19 11:43:30 +00:00
|
|
|
uint64_t q_filesize = sqlite3_column_int64(statement, 1);
|
|
|
|
const char *q_filehash = (const char *) sqlite3_column_text(statement, 2);
|
|
|
|
|
|
|
|
if (filesizep)
|
|
|
|
*filesizep = q_filesize;
|
2017-05-17 04:38:01 +00:00
|
|
|
|
|
|
|
if (q_version > version){
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_OLD;
|
2017-03-06 05:26:58 +00:00
|
|
|
}else{
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_SAME;
|
2017-10-19 11:43:30 +00:00
|
|
|
if (q_filesize) {
|
|
|
|
if (q_filehash && *q_filehash) {
|
|
|
|
rhizome_filehash_t hash;
|
|
|
|
if (str_to_rhizome_filehash_t(&hash, q_filehash) == -1) {
|
|
|
|
WHYF("Malformed filehash %s", q_filehash);
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
}else{
|
|
|
|
// unless we are missing the payload...
|
|
|
|
enum rhizome_payload_status pstatus;
|
|
|
|
switch((pstatus = rhizome_exists(&hash))){
|
|
|
|
case RHIZOME_PAYLOAD_STATUS_NEW:
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_NEW;
|
|
|
|
break;
|
|
|
|
case RHIZOME_PAYLOAD_STATUS_STORED:
|
|
|
|
break;
|
|
|
|
case RHIZOME_PAYLOAD_STATUS_BUSY:
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_BUSY;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
break;
|
|
|
|
}
|
2017-05-17 04:38:01 +00:00
|
|
|
}
|
2017-10-19 11:43:30 +00:00
|
|
|
} else {
|
|
|
|
WHYF("Missing filehash");
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_ERROR;
|
2017-05-17 04:38:01 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-06 05:26:58 +00:00
|
|
|
}
|
|
|
|
}else if (sqlite_code_busy(stepcode)){
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_BUSY;
|
|
|
|
}else if (!sqlite_code_ok(stepcode)){
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_ERROR;
|
|
|
|
}else{
|
|
|
|
status = RHIZOME_BUNDLE_STATUS_NEW;
|
|
|
|
}
|
2013-02-12 00:04:04 +00:00
|
|
|
sqlite3_finalize(statement);
|
2017-03-06 05:26:58 +00:00
|
|
|
RETURN(status);
|
2013-02-16 17:47:24 +00:00
|
|
|
OUT();
|
2013-04-04 07:07:49 +00:00
|
|
|
}
|
2013-08-22 05:44:21 +00:00
|
|
|
|
2017-03-06 05:26:58 +00:00
|
|
|
enum rhizome_bundle_status rhizome_is_bar_interesting(const rhizome_bar_t *bar)
|
2013-08-22 05:44:21 +00:00
|
|
|
{
|
2014-06-28 10:41:43 +00:00
|
|
|
char id_hex[RHIZOME_BAR_PREFIX_BYTES *2 + 2];
|
|
|
|
tohex(id_hex, RHIZOME_BAR_PREFIX_BYTES * 2, rhizome_bar_prefix(bar));
|
|
|
|
strcat(id_hex, "%");
|
2017-10-19 11:43:30 +00:00
|
|
|
return is_interesting(id_hex, rhizome_bar_version(bar), NULL);
|
2013-08-22 05:44:21 +00:00
|
|
|
}
|
|
|
|
|
2017-10-19 11:43:30 +00:00
|
|
|
enum rhizome_bundle_status rhizome_is_interesting(const rhizome_bid_t *bid, uint64_t version, uint64_t *filesizep)
|
2013-08-22 05:44:21 +00:00
|
|
|
{
|
2017-10-19 11:43:30 +00:00
|
|
|
return is_interesting(alloca_tohex_rhizome_bid_t(*bid), version, filesizep);
|
2013-09-30 06:50:50 +00:00
|
|
|
}
|