serval-dna/rhizome_database.c

/*
Serval Rhizome file sharing
Copyright (C) 2012 The Serval Project, Inc.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#define __RHIZOME_INLINE
#include <stdlib.h>
#include <time.h>
#include <ctype.h>
#include "serval.h"
#include "conf.h"
#include "rhizome.h"
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "str.h"

static char rhizome_thisdatastore_path[256];

static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid);
static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid);
static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid);

const char *rhizome_datastore_path()
{
  if (!rhizome_thisdatastore_path[0])
    rhizome_set_datastore_path(NULL);
  return rhizome_thisdatastore_path;
}

int rhizome_set_datastore_path(const char *path)
{
  strbuf b = strbuf_local(rhizome_thisdatastore_path, sizeof rhizome_thisdatastore_path);
  strbuf_path_join(b, serval_instancepath(), config.rhizome.datastore_path, path, NULL);
  INFOF("Rhizome datastore path = %s", alloca_str_toprint(rhizome_thisdatastore_path));
  return 0;
}

int form_rhizome_datastore_path(char * buf, size_t bufsiz, const char *fmt, ...)
{
  va_list ap;
  strbuf b = strbuf_local(buf, bufsiz);
  strbuf_puts(b, rhizome_datastore_path());
  if (fmt) {
    va_start(ap, fmt);
    if (*strbuf_substr(b, -1) != '/')
      strbuf_putc(b, '/');
    strbuf_vsprintf(b, fmt, ap);
    va_end(ap);
  }
  if (strbuf_overrun(b)) {
      WHY("Path buffer overrun");
      return 0;
  }
  return 1;
}

int create_rhizome_datastore_dir()
{
  if (config.debug.rhizome) DEBUGF("mkdirs(%s, 0700)", rhizome_datastore_path());
  return emkdirs(rhizome_datastore_path(), 0700);
}

sqlite3 *rhizome_db=NULL;

/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id)
{
  int64_t result = 0;
  if (sqlite_exec_int64_retry(retry, &result,
	"SELECT max(grouplist.priorty) FROM GROUPLIST,MANIFESTS,GROUPMEMBERSHIPS"
	" WHERE MANIFESTS.id = ?"
	"   AND GROUPLIST.id = GROUPMEMBERSHIPS.groupid"
	"   AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id;",
	TEXT_TOUPPER, id, END
      ) == -1
  )
    return -1;
  return (int) result;
}

int is_debug_rhizome()
{
  return config.debug.rhizome;
}

int is_debug_rhizome_ads()
{
  return config.debug.rhizome_ads;
}

static int (*sqlite_trace_func)() = is_debug_rhizome;
const struct __sourceloc *sqlite_trace_whence = NULL;

static void sqlite_trace_callback(void *context, const char *rendered_sql)
{
  if (sqlite_trace_func())
    logMessage(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql);
}

/* This function allows code like:
 *
 *    debugflags_t oldmask = sqlite_set_debugmask(DEBUG_SOMETHING_ELSE);
 *    ...
 *    sqlite3_stmt *statement = sqlite_prepare(&retry, "select blah blah blah");
 *    while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
 *	// do blah blah blah
 *    }
 *    ...
 *    sqlite_set_debugmask(oldmask);
 *    return result;
 *
 * so that code can choose which DEBUG_ flags control the logging of rendered SQL queries.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int (*sqlite_set_tracefunc(int (*newfunc)()))()
{
  int (*oldfunc)() = sqlite_trace_func;
  sqlite_trace_func = newfunc;
  return oldfunc;
}

void sqlite_log(void *ignored, int result, const char *msg)
{
  WARNF("Sqlite: %d %s", result, msg);
}

void verify_bundles(){
  // assume that only the manifest itself can be trusted
  // fetch all manifests and reinsert them.
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  // This cursor must be ordered descending as re-inserting the manifests will give them a new higher manifest id.
  // If we didn't, we'd get stuck in an infinite loop.
  sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;");
  while(sqlite_step_retry(&retry, statement)==SQLITE_ROW){
    sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
    const void *manifest = sqlite3_column_blob(statement, 1);
    int manifest_length = sqlite3_column_bytes(statement, 1);
    
    rhizome_manifest *m=rhizome_new_manifest();
    int ret=0;
    ret = rhizome_read_manifest_file(m, manifest, manifest_length);
    if (ret==0 && m->errors)
      ret=-1;
    if (ret==0)
      ret=rhizome_manifest_verify(m);
    if (ret==0){
      m->finalised=1;
      m->manifest_bytes=m->manifest_all_bytes;
      // store it again, to ensure it is valid and stored correctly with matching file content.
      ret=rhizome_store_bundle(m);
    }
    if (ret!=0){
      DEBUGF("Removing invalid manifest entry @%lld", rowid);
      sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
    }
    rhizome_manifest_free(m);
  }
  sqlite3_finalize(statement);
}

/*
 * The MANIFESTS table 'author' column records the cryptographically verified SID of the author
 * that has write permission on the bundle, ie, possesses the Rhizome secret key that generated the
 * BID, and hence can derive the Bundle Secret from the bundle's BK field:
 * - The MANIFESTS table 'author' column is set to the author SID when a bundle is created
 *   locally by a non-secret identity, so no verification need be performed for one's own
 *   bundles while they remain in the Rhizome store.
 * - When a bundle is imported, the 'author' column is set to NULL to indicate that no
 *   verification has passed yet.  This includes one's own bundles that have been purged from
 *   the local Rhizome store then recovered from a remote Rhizome node.
 * - When a manifest with NULL 'author' is examined closely, ie extracted, not merely
 *   listed, the keyring is searched for an identity that is the author.  If an author is
 *   found, the MANIFESTS table 'author' column is updated.  This allows one to regain the
 *   ability to overwrite one's own bundles that have been lost but recovered from an exterior
 *   Rhizome node.
 * - The above check automates the "own bundle recovery" mechanism at the expense of a CPU-heavy
 *   cryptographic check every time a foreign bundle is examined, but at least listing is fast.
 *   This will not scale as many identities are added to the keyring.  It will eventually have to be
 *   replaced with a means to cache positive and negative verifications in the Rhizome db for local,
 *   non-secret identities.
 *
 * -- Andrew Bettison <andrew@servalproject.com>, October 2012
 */

int rhizome_opendb()
{
  if (rhizome_db) return 0;

  IN();
  
  if (create_rhizome_datastore_dir() == -1){
    RETURN(WHY("No Directory"));
  }
  char dbpath[1024];
  if (!sqlite3_temp_directory){
    if (!FORM_RHIZOME_DATASTORE_PATH(dbpath, "")){
      RETURN(WHY("Invalid path"));
    }
    sqlite3_temp_directory = sqlite3_mprintf("%s", dbpath);
  }
  
  if (!FORM_RHIZOME_DATASTORE_PATH(dbpath, "rhizome.db")){
    RETURN(WHY("Invalid path"));
  }

  sqlite3_config(SQLITE_CONFIG_LOG,sqlite_log,NULL);
  
  if (sqlite3_open(dbpath,&rhizome_db)){
    RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
  }
  sqlite3_trace(rhizome_db, sqlite_trace_callback, NULL);
  int loglevel = (config.debug.rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;

  /* Read Rhizome configuration */
  if (config.debug.rhizome) {
    DEBUGF("Rhizome will use %lluB of storage for its database.", (unsigned long long) config.rhizome.database_size);
  }
  
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;

  int64_t version;
  if (sqlite_exec_int64_retry(&retry, &version, "PRAGMA user_version;", END) == -1)
    RETURN(-1);
  
  if (version<1){
    /* Create tables as required */
    sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;", END);
    if (	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);", END) == -1
      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, version integer,inserttime integer, filesize integer, filehash text, author text, bar blob, manifest blob);", END) == -1
      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, length integer, highestpriority integer, datavalid integer, inserttime integer);", END) == -1
      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILEBLOBS(id text not null primary key, data blob);", END) == -1
      ||	sqlite_exec_void_retry(&retry, "DROP TABLE IF EXISTS FILEMANIFESTS;", END) == -1
      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);", END) == -1
      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);", END) == -1
    ) {
      RETURN(WHY("Failed to create schema"));
    }

    /* Create indexes if they don't already exist */
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
    
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
  }
  if (version<2){
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;", END);
    // if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
    verify_bundles();
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
  }
  
  if (version<3){
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
  }
  
  if (version<4){
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
  }
  
  // TODO recreate tables with collate nocase on hex columns
  
  /* Future schema updates should be performed here. 
   The above schema can be assumed to exist.
   All changes should attempt to preserve any existing data */
  
  // We can't delete a file that is being transferred in another process at this very moment...
  if (config.rhizome.clean_on_open)
    rhizome_cleanup(NULL);
  RETURN(0);
  OUT();
}

int rhizome_close_db()
{
  IN();
  if (rhizome_db) {
    rhizome_cache_close();
    
    if (!sqlite3_get_autocommit(rhizome_db)){
      WHY("Uncommitted transaction!");
      sqlite_exec_void("ROLLBACK;", END);
    }
    sqlite3_stmt *stmt = NULL;
    while ((stmt = sqlite3_next_stmt(rhizome_db, stmt))) {
      const char *sql = sqlite3_sql(stmt);
      WARNF("closing Rhizome db with unfinalised statement: %s", sql ? sql : "BLOB");
    }
    int r = sqlite3_close(rhizome_db);
    if (r != SQLITE_OK)
      RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_db)));
  }
  rhizome_db=NULL;
  RETURN(0);
  OUT();
}

/* SQL query retry logic.

   The common retry-on-busy logic is factored into this function.  This logic encapsulates the
   maximum time (timeout) that the caller may wait for a lock to be released and the sleep interval
   while waiting.  The way to use it is this:

      sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
      do ret = some_sqlite_operation(...);
        while (is_busy(ret) && sqlite_retry(&retry, "some_sqlite_operation"));
      if (is_error(ret) || is_busy(ret))
	return -1; // an error has already been logged
      sqlite_retry_done(&retry, "some_sqlite_operation");
      ...

   If the database is currently locked for updates, then some_sqlite_operation() will return a code
   indicating busy (which is distinguishable from the codes for success or any other error).
   sqlite_retry() will then log a DEBUG or INFO message, sleep for a short period and return true if
   the timeout has not been reached.  It keeps this information in the 'retry' variable, which must
   be initialised as shown.  As long as the timeout has not been reached, sqlite_retry() will keep
   sleeping and returning true.  If the timeout is reached, then sqlite_retry() will log an error
   and return false.  If the operation is successful, sqlite_retry_done() must be called to log the
   success as a DEBUG or INFO message to provide closure to the prior messages already logged by
   sqlite_retry() and to reset the 'retry' variable for re-use.

   The timeout and sleep interval depend on whether the caller is the servald server process or not.
   See the definition of the SQLITE_RETRY_STATE_DEFAULT macro for the default settings.

   A single 'retry' variable may be initialised once then used for a succession of database
   operations.  If invoked by the server process, then the timeout timer will not be reset by
   sqlite_retry() or sqlite_retry_done(), so that the timeout limit will apply to the cumulative
   latency, not just to each individual query, which could potentially add up to much greater
   latency than desired.  However, in non-server processes, each query may be allowed its own
   timeout, giving a greater chance of success at the expense of potentially greater latency.
 */

/* In the servald server process, by default we retry every 10 ms for up to 50 ms, so as to not
   introduce too much latency into server responsiveness.  In other processes (eg, Batphone MeshMS
   thread), by default we allow busy retries to go for over a second, waiting 100 ms between each
   retry.
 */
sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int otherLimit, int otherSleep)
{
  return (sqlite_retry_state){
      .limit = serverMode ? (serverLimit < 0 ? 50 : serverLimit) : (otherLimit < 0 ? 5000 : otherLimit),
      .sleep = serverMode ? (serverSleep < 0 ? 10 : serverSleep) : (otherSleep < 0 ? 100 : otherSleep),
      .elapsed = 0,
      .start = -1,
      .busytries = 0
    };
}

int _sqlite_retry(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action)
{
  time_ms_t now = gettime_ms();
  ++retry->busytries;
  if (retry->start == -1)
    retry->start = now;
  retry->elapsed = now - retry->start;
  
  INFOF("%s on try %u after %.3f seconds (limit %.3f): %s",
      sqlite3_errmsg(rhizome_db),
      retry->busytries,
      (retry->elapsed) / 1e3,
      (retry->limit) / 1e3,
      action
    );
  
  if (retry->elapsed >= retry->limit) {
    // reset ready for next query
    retry->busytries = 0;
    if (!serverMode)
      retry->start = -1;
    return 0; // tell caller to stop trying
  }
  
  if (retry->sleep)
    sleep_ms(retry->sleep);
  return 1; // tell caller to try again
}

void _sqlite_retry_done(struct __sourceloc __whence, sqlite_retry_state *retry, const char *action)
{
  if (retry->busytries) {
    time_ms_t now = gettime_ms();
    INFOF("succeeded on try %u after %.3f seconds (limit %.3f): %s",
	retry->busytries + 1,
	(now - retry->start) / 1e3,
	(retry->limit) / 1e3,
	action
      );
  }
  // reset ready for next query
  retry->busytries = 0;
  if (!serverMode)
    retry->start = -1;
}

/* Prepare an SQL command from a simple string.  Returns NULL if an error occurs (logged as an
 * error), otherwise returns a pointer to the prepared SQLite statement.
 *
 * IMPORTANT!  Do not form statement strings using sprintf(3) or strbuf_sprintf() or similar
 * methods, because those are susceptible to SQL injection attacks.  Instead, use bound parameters
 * and bind them using the _sqlite_bind() function below.
 *
 * IMPORTANT!  Do not add sprintf(3)-like functionality to this method.  It used to take
 * sprintf(3)-style varargs and these were deliberately removed.  It is vital to discourage bad
 * practice, and adding sprintf(3)-style args to this function would be a step in the wrong
 * direction.
 *
 * See GitHub issue #69.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext)
{
  IN();
  sqlite3_stmt *statement = NULL;
  if (!rhizome_db && rhizome_opendb() == -1)
    RETURN(NULL);
  while (1) {
    switch (sqlite3_prepare_v2(rhizome_db, sqltext, -1, &statement, NULL)) {
      case SQLITE_OK:
	RETURN(statement);
      case SQLITE_BUSY:
      case SQLITE_LOCKED:
	if (retry && _sqlite_retry(__whence, retry, sqltext)) {
	  break; // back to sqlite3_prepare_v2()
	}
	// fall through...
      default:
	LOGF(log_level, "query invalid, %s: %s", sqlite3_errmsg(rhizome_db), sqltext);
	sqlite3_finalize(statement);
	RETURN(NULL);
    }
  }
}

/* Bind some parameters to a prepared SQL statement.  Returns -1 if an error occurs (logged as an
 * error), otherwise zero with the prepared statement in *statement.
 *
 * Developed as part of GitHub issue #69.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap)
{
  const int index_limit = sqlite3_limit(rhizome_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1);
  enum sqlbind_type typ;
  int index_counter = 0;
  strbuf ext = NULL;
  do {
    typ = va_arg(ap, int);
    int index;
    const char *name = NULL;
    if ((typ & 0xffff0000) == INDEX) {
      typ &= 0xffff;
      index = va_arg(ap, int);
      if (index < 1 || index > index_limit) {
	LOGF(log_level, "illegal index %d: %s", index, sqlite3_sql(statement));
	return -1;
      }
      if (config.debug.rhizome)
	strbuf_sprintf((ext = strbuf_alloca(25)), "|INDEX index=%d", index);
    } else if ((typ & 0xffff0000) == NAMED) {
      typ &= 0xffff;
      name = va_arg(ap, const char *);
      index = sqlite3_bind_parameter_index(statement, name);
      if (index == 0) {
	LOGF(log_level, "no parameter %s in query: %s", alloca_str_toprint(name), sqlite3_sql(statement));
	return -1;
      }
      if (config.debug.rhizome) {
	ext = strbuf_alloca(20 + toprint_str_len(name, "\"\""));
	strbuf_puts(ext, "|NAMED name=");
	strbuf_toprint_quoted(ext, "\"\"", name);
      }
    } else {
      index = ++index_counter;
      if (config.debug.rhizome)
	ext = strbuf_alloca(1);
    }
#define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \
	  if (config.debug.rhizome) \
	    DEBUGF("%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement))
#define BIND_RETRY(FUNC, ...) \
	do { \
	  switch (FUNC(statement, index, ##__VA_ARGS__)) { \
	    case SQLITE_OK: \
	      break; \
	    case SQLITE_BUSY: \
	    case SQLITE_LOCKED: \
	      if (retry && _sqlite_retry(__whence, retry, #FUNC "()")) \
		continue; \
	    default: \
	      LOGF(log_level, #FUNC "(%d) failed, %s: %s", index, sqlite3_errmsg(rhizome_db), sqlite3_sql(statement)); \
	      sqlite3_finalize(statement); \
	      return -1; \
	  } \
	  break; \
	} while (1)
    switch (typ) {
      case END:
	break;
      case NUL:
	BIND_DEBUG(NUL, sqlite3_bind_null, "");
	BIND_RETRY(sqlite3_bind_null);
	break;
      case INT: {
	  int value = va_arg(ap, int);
	  BIND_DEBUG(INT, sqlite3_bind_int, "%d", value);
	  BIND_RETRY(sqlite3_bind_int, value);
	}
	break;
      case INT_TOSTR: {
	  int value = va_arg(ap, int);
	  char str[25];
	  sprintf(str, "%d", value);
	  BIND_DEBUG(INT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
	  BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
	}
	break;
      case UINT_TOSTR: {
	  unsigned value = va_arg(ap, unsigned);
	  char str[25];
	  sprintf(str, "%u", value);
	  BIND_DEBUG(UINT_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
	  BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
	}
	break;
      case INT64: {
	  int64_t value = va_arg(ap, int64_t);
	  BIND_DEBUG(INT64, sqlite3_bind_int64, "%"PRId64, value);
	  BIND_RETRY(sqlite3_bind_int64, value);
	}
	break;
      case INT64_TOSTR: {
	  int64_t value = va_arg(ap, int64_t);
	  char str[35];
	  sprintf(str, "%"PRId64, value);
	  BIND_DEBUG(INT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
	  BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
	}
	break;
      case UINT64_TOSTR: {
	  uint64_t value = va_arg(ap, uint64_t);
	  char str[35];
	  sprintf(str, "%"PRIu64, value);
	  BIND_DEBUG(UINT64_TOSTR, sqlite3_bind_text, "%s,-1,SQLITE_TRANSIENT", alloca_str_toprint(str));
	  BIND_RETRY(sqlite3_bind_text, str, -1, SQLITE_TRANSIENT);
	}
	break;
      case STATIC_TEXT: {
	  const char *text = va_arg(ap, const char *);
	  BIND_DEBUG(STATIC_TEXT, sqlite3_bind_text, "%s,-1,SQLITE_STATIC", alloca_str_toprint(text));
	  BIND_RETRY(sqlite3_bind_text, text, -1, SQLITE_STATIC);
	}
	break;
      case STATIC_TEXT_LEN: {
	  const char *text = va_arg(ap, const char *);
	  int bytes = va_arg(ap, int);
	  BIND_DEBUG(STATIC_TEXT_LEN, sqlite3_bind_text, "%s,%d,SQLITE_STATIC", alloca_str_toprint(text), bytes);
	  BIND_RETRY(sqlite3_bind_text, text, bytes, SQLITE_STATIC);
	}
	break;
      case STATIC_BLOB: {
	  const void *blob = va_arg(ap, const void *);
	  int bytes = va_arg(ap, int);
	  BIND_DEBUG(STATIC_BLOB, sqlite3_bind_blob, "%s,%d,SQLITE_STATIC", alloca_toprint(20, blob, bytes), bytes);
	  BIND_RETRY(sqlite3_bind_blob, blob, bytes, SQLITE_STATIC);
	};
	break;
      case ZEROBLOB: {
	  int bytes = va_arg(ap, int);
	  BIND_DEBUG(ZEROBLOB, sqlite3_bind_zeroblob, "%d,SQLITE_STATIC", bytes);
	  BIND_RETRY(sqlite3_bind_zeroblob, bytes);
	};
	break;
      case SID_T: {
	  const sid_t *sidp = va_arg(ap, const sid_t *);
	  const char *sid_hex = alloca_tohex_sid_t(*sidp);
	  BIND_DEBUG(SID_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", sid_hex, SID_STRLEN);
	  BIND_RETRY(sqlite3_bind_text, sid_hex, SID_STRLEN, SQLITE_TRANSIENT);
	}
	break;
      case BUNDLE_ID_T: {
	  const char *bid_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_MANIFEST_ID_BYTES);
	  BIND_DEBUG(BUNDLE_ID_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", bid_hex, RHIZOME_MANIFEST_ID_STRLEN);
	  BIND_RETRY(sqlite3_bind_text, bid_hex, RHIZOME_MANIFEST_ID_STRLEN, SQLITE_TRANSIENT);
	}
	break;
      case FILEHASH_T: {
	  const char *hash_hex = alloca_tohex(va_arg(ap, const unsigned char *), RHIZOME_FILEHASH_BYTES);
	  BIND_DEBUG(FILEHASH_T, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
	  BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT);
	}
	break;
      case TOHEX: {
	  const unsigned char *binary = va_arg(ap, const unsigned char *);
	  unsigned bytes = va_arg(ap, unsigned);
	  const char *hex = alloca_tohex(binary, bytes);
	  BIND_DEBUG(TOHEX, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", hex, bytes * 2);
	  BIND_RETRY(sqlite3_bind_text, hex, bytes * 2, SQLITE_TRANSIENT);
	}
	break;
      case TEXT_TOUPPER: {
	  const char *text = va_arg(ap, const char *);
	  unsigned bytes = strlen(text);
	  char upper[bytes + 1];
	  str_toupper_inplace(strcpy(upper, text));
	  BIND_DEBUG(TEXT_TOUPPER, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes);
	  BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT);
	}
	break;
      case TEXT_LEN_TOUPPER: {
	  const char *text = va_arg(ap, const char *);
	  unsigned bytes = va_arg(ap, unsigned);
	  char upper[bytes];
	  unsigned i;
	  for (i = 0; i != bytes; ++i)
	    upper[i] = toupper(text[i]);
	  BIND_DEBUG(TEXT_LEN_TOUPPER, sqlite3_bind_text, "%s,%d,SQLITE_TRANSIENT", alloca_toprint(-1, upper, bytes), bytes);
	  BIND_RETRY(sqlite3_bind_text, upper, bytes, SQLITE_TRANSIENT);
	}
	break;
#undef BIND_RETRY
      default:
	FATALF("unsupported bind code, index=%d typ=0x%08x: %s", index, typ, sqlite3_sql(statement));
    }
  } while (typ != END);
  return 0;
}

int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...)
{
  va_list ap;
  va_start(ap, statement);
  int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap);
  va_end(ap);
  return ret;
}

/* Prepare an SQL statement and bind some parameters.  Returns a pointer to the SQLite statement if
 * successful or NULL if an error occurs (which is logged at the given log level).
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...)
{
  sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext);
  if (statement != NULL) {
    va_list ap;
    va_start(ap, sqltext);
    int ret = _sqlite_vbind(__whence, log_level, retry, statement, ap);
    va_end(ap);
    if (ret == -1) {
      sqlite3_finalize(statement);
      statement = NULL;
    }
  }
  return statement;
}

int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
  IN();
  int ret = -1;
  sqlite_trace_whence = &__whence;
  while (statement) {
    int stepcode = sqlite3_step(statement);
    switch (stepcode) {
      case SQLITE_OK:
      case SQLITE_DONE:
      case SQLITE_ROW:
	if (retry)
	  _sqlite_retry_done(__whence, retry, sqlite3_sql(statement));
	ret = stepcode;
	statement = NULL;
	break;
      case SQLITE_BUSY:
      case SQLITE_LOCKED:
	if (retry && _sqlite_retry(__whence, retry, sqlite3_sql(statement))) {
	  sqlite3_reset(statement);
	  break; // back to sqlite3_step()
	}
	// fall through...
      default:
	LOGF(log_level, "query failed (%d), %s: %s", stepcode, sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
	ret = -1;
	statement = NULL;
	break;
    }
  }
  sqlite_trace_whence = NULL;
  OUT();
  return ret;
}

/*
 * Convenience wrapper for executing a prepared SQL statement where the row outputs are not wanted.
 * Always finalises the statement before returning.
 *
 * If an error occurs then logs it at the given level and returns -1.
 *
 * If 'retry' is non-NULL and the BUSY error occurs (indicating the database is locked, ie,
 * currently in use by another process), then resets the statement and retries while sqlite_retry()
 * returns true.  If sqlite_retry() returns false then returns -1.
 *
 * Otherwise returns the number of rows (SQLITE_ROW) results, which will be zero if the first result
 * was SQLITE_OK or SQLITE_DONE.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
static int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
  if (!statement)
    return -1;
  int rowcount = 0;
  int stepcode;
  while ((stepcode = _sqlite_step(__whence, log_level, retry, statement)) == SQLITE_ROW)
    ++rowcount;
  sqlite3_finalize(statement);
  if (sqlite_trace_func())
    DEBUGF("rowcount=%d changes=%d", rowcount, sqlite3_changes(rhizome_db));
  return sqlite_code_ok(stepcode) ? rowcount : -1;
}

/* Execute an SQL command that returns no value.  If an error occurs then logs it at ERROR level and
 * returns -1.  Otherwise returns the number of rows changed by the command.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
static int _sqlite_vexec_void(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, va_list ap)
{
  sqlite3_stmt *statement = _sqlite_prepare(__whence, log_level, retry, sqltext);
  if (!statement)
    return -1;
  if (_sqlite_vbind(__whence, log_level, retry, statement, ap) == -1)
    return -1;
  int rowcount = _sqlite_exec(__whence, log_level, retry, statement);
  if (rowcount == -1)
    return -1;
  if (rowcount)
    WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
  return sqlite3_changes(rhizome_db);
}

/* Convenience wrapper for executing an SQL command that returns no value.  If an error occurs then
 * logs it at ERROR level and returns -1.  Otherwise returns the number of rows changed by the
 * command.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_void(struct __sourceloc __whence, int log_level, const char *sqltext, ...)
{
  va_list ap;
  va_start(ap, sqltext);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = _sqlite_vexec_void(__whence, log_level, &retry, sqltext, ap);
  va_end(ap);
  return ret;
}

/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
 * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
 * instead of its own, internal one.  If 'retry' is passed as NULL, then will not sleep and retry at
 * all in the event of a busy condition, but will log it as an error and return immediately.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_void_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, const char *sqltext, ...)
{
  va_list ap;
  va_start(ap, sqltext);
  int ret = _sqlite_vexec_void(__whence, log_level, retry, sqltext, ap);
  va_end(ap);
  return ret;
}

static int _sqlite_vexec_int64(struct __sourceloc __whence, sqlite_retry_state *retry, int64_t *result, const char *sqltext, va_list ap)
{
  sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext);
  if (!statement)
    return -1;
  if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1)
    return -1;
  int ret = 0;
  int rowcount = 0;
  int stepcode;
  while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
    int columncount = sqlite3_column_count(statement);
    if (columncount != 1)
      ret = WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
    else if (++rowcount == 1)
      *result = sqlite3_column_int64(statement, 0);
  }
  if (rowcount > 1)
    WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
  sqlite3_finalize(statement);
  if (!sqlite_code_ok(stepcode) || ret == -1)
    return -1;
  if (sqlite_trace_func())
    DEBUGF("rowcount=%d changes=%d result=%lld", rowcount, sqlite3_changes(rhizome_db), (long long)*result);
  return rowcount;
}

/*
 * Convenience wrapper for executing an SQL command that returns a single int64 value.
 * Logs an error and returns -1 if an error occurs.
 * If no row is found, then returns 0 and does not alter *result.
 * If exactly one row is found, the assigns its value to *result and returns 1.
 * If more than one row is found, then logs a warning, assigns the value of the first row to *result
 * and returns the number of rows.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_int64(struct __sourceloc __whence, int64_t *result, const char *sqlformat,...)
{
  va_list ap;
  va_start(ap, sqlformat);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = _sqlite_vexec_int64(__whence, &retry, result, sqlformat, ap);
  va_end(ap);
  return ret;
}

/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
 * currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
 * instead of its own, internal one.  If 'retry' is passed as NULL, then will not sleep and retry at
 * all in the event of a busy condition, but will log it as an error and return immediately.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_int64_retry(struct __sourceloc __whence, sqlite_retry_state *retry, int64_t *result, const char *sqlformat,...)
{
  va_list ap;
  va_start(ap, sqlformat);
  int ret = _sqlite_vexec_int64(__whence, retry, result, sqlformat, ap);
  va_end(ap);
  return ret;
}

/* Convenience wrapper for executing an SQL command that returns a single text value.
 * Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
 *  0 means no rows, nothing is appended to the strbuf
 *  1 means exactly one row, appends its column to the strbuf
 *  2 more than one row, logs a warning and appends the first row's column to the strbuf
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_strbuf(struct __sourceloc __whence, strbuf sb, const char *sqlformat,...)
{
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  va_list ap;
  va_start(ap, sqlformat);
  int ret = _sqlite_vexec_strbuf_retry(__whence, &retry, sb, sqlformat, ap);
  va_end(ap);
  return ret;
}

int _sqlite_exec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...)
{
  va_list ap;
  va_start(ap, sqlformat);
  int ret = _sqlite_vexec_strbuf_retry(__whence, retry, sb, sqlformat, ap);
  va_end(ap);
  return ret;
}

int _sqlite_vexec_strbuf_retry(struct __sourceloc __whence, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap)
{
  sqlite3_stmt *statement = _sqlite_prepare(__whence, LOG_LEVEL_ERROR, retry, sqltext);
  if (!statement)
    return -1;
  if (_sqlite_vbind(__whence, LOG_LEVEL_ERROR, retry, statement, ap) == -1)
    return -1;
  int ret = 0;
  int rowcount = 0;
  int stepcode;
  while ((stepcode = _sqlite_step(__whence, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
    int columncount = sqlite3_column_count(statement);
    if (columncount != 1)
      ret - WHYF("incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
    else if (++rowcount == 1)
      strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
  }
  if (rowcount > 1)
    WARNF("query unexpectedly returned %d rows, ignored all but first", rowcount);
  sqlite3_finalize(statement);
  return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
}

int64_t rhizome_database_used_bytes()
{
  int64_t db_page_size;
  int64_t db_page_count;
  int64_t db_free_page_count;
  if (	sqlite_exec_int64(&db_page_size, "PRAGMA page_size;", END) == -1LL
    ||  sqlite_exec_int64(&db_page_count, "PRAGMA page_count;", END) == -1LL
    ||	sqlite_exec_int64(&db_free_page_count, "PRAGMA free_count;", END) == -1LL
  )
    return WHY("Cannot measure database used bytes");
  return db_page_size * (db_page_count - db_free_page_count);
}

int rhizome_database_filehash_from_id(const char *id, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH])
{
  IN();
  strbuf hash_sb = strbuf_local(hash, SHA512_DIGEST_STRING_LENGTH);
  RETURN(sqlite_exec_strbuf(hash_sb, "SELECT filehash FROM MANIFESTS WHERE manifests.version = ? AND manifests.id = ?;",
			    INT64, version, TEXT_TOUPPER, id, END));
  OUT();
}

static int rhizome_delete_external(const char *fileid)
{
  // attempt to remove any external blob
  char blob_path[1024];
  if (!FORM_RHIZOME_DATASTORE_PATH(blob_path, fileid))
    return -1;
  return unlink(blob_path);
}

static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
{
  return sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
      "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
      END);
}

int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const char *fileid)
{
  int ret = 0;
  if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
	  "DELETE FROM FILES WHERE id = ? and datavalid = 0;",
	  TEXT_TOUPPER, fileid, END
	) == -1
  )
    ret = -1;
  if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
	  "DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
	  TEXT_TOUPPER, fileid, END
	) == -1
  )
    ret = -1;
  return ret;
}

int rhizome_cleanup(struct rhizome_cleanup_report *report)
{
  IN();
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;

  time_ms_t now = gettime_ms();
  time_ms_t insert_horizon_no_manifest = now - 1000; // 1 second ago
  time_ms_t insert_horizon = now - 300000; // 5 minutes ago

  // cleanup external blobs for unreferenced files
  int candidates=0;
  if (report)
    report->deleted_orphan_fileblobs = 0;
  sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
      "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 0;",
      INT64, insert_horizon, END);
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    candidates++;
    const char *id = (const char *) sqlite3_column_text(statement, 0);
    if (rhizome_delete_external(id) == 0 && report)
      ++report->deleted_orphan_fileblobs;
  }
  sqlite3_finalize(statement);
  
  statement = sqlite_prepare_bind(&retry,
      "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
      INT64, insert_horizon_no_manifest, END);
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    candidates++;
    const char *id = (const char *) sqlite3_column_text(statement, 0);
    if (rhizome_delete_external(id) == 0 && report)
      ++report->deleted_orphan_fileblobs;
  }
  sqlite3_finalize(statement);
  
  int ret;
  if (candidates) {
    // clean out unreferenced files
    ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
	"DELETE FROM FILES WHERE inserttime < ? AND datavalid = 0;",
	INT64, insert_horizon, END);
    if (report)
      report->deleted_stale_incoming_files = ret;
    ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
	"DELETE FROM FILES WHERE inserttime < ? AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
	INT64, insert_horizon_no_manifest, END);
    if (report)
      report->deleted_orphan_files = ret;
  }
  
  if ((ret = rhizome_delete_orphan_fileblobs_retry(&retry)) > 0 && report)
    report->deleted_orphan_fileblobs += ret;
   
  RETURN(0);
  OUT();
}

int rhizome_make_space(int group_priority, long long bytes)
{
  /* Asked for impossibly large amount */
  if (bytes>=(config.rhizome.database_size-65536))
    return WHYF("bytes=%lld is too large", bytes);

  long long db_used = rhizome_database_used_bytes();
  if (db_used == -1)
    return -1;
  
  rhizome_cleanup(NULL);
  
  /* If there is already enough space now, then do nothing more */
  if (db_used<=(config.rhizome.database_size-bytes-65536))
    return 0;

  /* Okay, not enough space, so free up some. */
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
      "SELECT id,length FROM FILES WHERE highestpriority < ? ORDER BY DESCENDING LENGTH",
      INT, group_priority, END);
  if (!statement)
    return -1;
  while (bytes > (config.rhizome.database_size - 65536 - rhizome_database_used_bytes())
      && sqlite_step_retry(&retry, statement) == SQLITE_ROW
  ) {
    /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
    const unsigned char *id;

    /* Get values */
    if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
      id = sqlite3_column_text(statement, 0);
    else {
      WHY("Incorrect type in id column of files table");
      break;
    }
    if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER)
      ; //length=sqlite3_column_int(statement, 1);
    else {
      WHY("Incorrect type in length column of files table");
      break;
    }
    /* Try to drop this file from storage, discarding any references that do not trump the priority
       of this request.  The query done earlier should ensure this, but it doesn't hurt to be
       paranoid, and it also protects against inconsistency in the database. */
    rhizome_drop_stored_file((char *)id, group_priority + 1);
  }
  sqlite3_finalize(statement);

  //long long equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM FILES WHERE highestpriority=%d and length>%lld",group_priority,bytes);
  /* XXX Get rid of any equal priority files that are larger than this one */

  /* XXX Get rid of any higher priority files that are not relevant in this time or location */

  /* Couldn't make space */
  return WHY("Incomplete");
}

/* Drop the specified file from storage, and any manifests that reference it, 
   provided that none of those manifests are being retained at a higher priority
   than the maximum specified here. */
int rhizome_drop_stored_file(const char *id,int maximum_priority)
{
  if (!rhizome_str_is_file_hash(id))
    return WHYF("invalid file hash id=%s", alloca_str_toprint(id));
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", TEXT_TOUPPER, id, END);
  if (!statement)
    return WHYF("Could not drop stored file id=%s", id);
  int can_drop = 1;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    /* Find manifests for this file */
    if (sqlite3_column_type(statement, 0) != SQLITE_TEXT) {
      WHYF("Incorrect type in id column of manifests table");
      break;
    }
    const char *manifestId = (char *) sqlite3_column_text(statement, 0);
    /* Check that manifest is not part of a higher priority group.
	If so, we cannot drop the manifest or the file.
	However, we will keep iterating, as we can still drop any other manifests pointing to this file
	that are lower priority, and thus free up a little space. */
    int priority = rhizome_manifest_priority(&retry, manifestId);
    if (priority == -1)
      WHYF("Cannot drop fileid=%s due to error, manifestId=%s", id, manifestId);
    else if (priority > maximum_priority) {
      WHYF("Cannot drop fileid=%s due to manifest priority, manifestId=%s", id, manifestId);
      can_drop = 0;
    } else {
      if (config.debug.rhizome)
	DEBUGF("removing stale manifests, groupmemberships");
      sqlite_exec_void_retry(&retry, "delete from manifests where id = ?;", TEXT_TOUPPER, manifestId, END);
      sqlite_exec_void_retry(&retry, "delete from keypairs where public = ?;", TEXT_TOUPPER, manifestId, END);
      sqlite_exec_void_retry(&retry, "delete from groupmemberships where manifestid = ?;", TEXT_TOUPPER, manifestId, END);
    }
  }
  sqlite3_finalize(statement);
  if (can_drop)
    rhizome_delete_file_retry(&retry, id);
  return 0;
}

/*
  Store the specified manifest into the sqlite database.
  We assume that sufficient space has been made for us.
  The manifest should be finalised, and so we don't need to
  look at the underlying manifest file, but can just write m->manifest_data
  as a blob.

  associated_filename needs to be read in and stored as a blob.  Hopefully that
  can be done in pieces so that we don't have memory exhaustion issues on small
  architectures.  However, we do know it's hash apriori from m, and so we can
  skip loading the file in if it is already stored.  mmap() apparently works on
  Linux FAT file systems, and is probably the best choice since it doesn't need
  all pages to be in RAM at the same time.

  SQLite does allow modifying of blobs once stored in the database.
  The trick is to insert the blob as all zeroes using a special function, and then
  substitute bytes in the blog progressively.

  We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, 
   and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
 */
int rhizome_store_bundle(rhizome_manifest *m)
{
  if (!m->finalised) return WHY("Manifest was not finalised");

  if (m->haveSecret) {
    /* We used to store the secret in the database, but we don't anymore, as we use 
     the BK field in the manifest. So nothing to do here. */
  } else {
    /* We don't have the secret for this manifest, so only allow updates if 
     the self-signature is valid */
    if (!m->selfSigned)
      return WHY("Manifest is not signed, and I don't have the key.  Manifest might be forged or corrupt.");
  }

  char manifestid[RHIZOME_MANIFEST_ID_STRLEN + 1];
  rhizome_manifest_get(m, "id", manifestid, sizeof manifestid);
  str_toupper_inplace(manifestid);

  /* Bind BAR to data field */
  unsigned char bar[RHIZOME_BAR_BYTES];
  rhizome_manifest_to_bar(m,bar);

  /* Store the file (but not if it is already in the database) */
  char filehash[RHIZOME_FILEHASH_STRLEN + 1];
  if (m->fileLength > 0) {
    strncpy(filehash, m->fileHexHash, sizeof filehash);
    str_toupper_inplace(filehash);

    if (!rhizome_exists(filehash))
      return WHY("File should already be stored by now");
  } else {
    filehash[0] = '\0';
  }

  const char *author = is_sid_any(m->author) ? NULL : alloca_tohex_sid(m->author);
  const char *name = rhizome_manifest_get(m, "name", NULL, 0);
  const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
  const char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
  
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
    return WHY("Failed to begin transaction");
  
  sqlite3_stmt *stmt;
  if ((stmt = sqlite_prepare(&retry, 
	"INSERT OR REPLACE INTO MANIFESTS("
	  "id,"
	  "manifest,"
	  "version,"
	  "inserttime,"
	  "bar,"
	  "filesize,"
	  "filehash,"
	  "author,"
	  "service,"
	  "name,"
	  "sender,"
	  "recipient,"
	  "tail"
	") VALUES("
	  "?,?,?,?,?,?,?,?,?,?,?,?,?"
	");")) == NULL)
    goto rollback;
  if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_STATIC))
        && sqlite_code_ok(sqlite3_bind_blob(stmt, 2, m->manifestdata, m->manifest_bytes, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 3, m->version))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 4, (long long) gettime_ms()))
	&& sqlite_code_ok(sqlite3_bind_blob(stmt, 5, bar, RHIZOME_BAR_BYTES, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 6, m->fileLength))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 7, filehash, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 8, author, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 9, service, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 10, name, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 11, sender, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 12, recipient, -1, SQLITE_STATIC))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 13, m->journalTail))
  )) {
    WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
    goto rollback;
  }
  if (sqlite_step_retry(&retry, stmt) == -1)
    goto rollback;
  sqlite3_finalize(stmt);
  stmt = NULL;

//  if (serverMode)
//    rhizome_sync_bundle_inserted(bar);

  // TODO remove old payload?
  
  if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
    int closed=rhizome_manifest_get_ll(m,"closedgroup");
    if (closed<1) closed=0;
    int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
    if (ciphered<1) ciphered=0;
    if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPLIST(id,closed,ciphered,priority) VALUES (?,?,?,?);")) == NULL)
      goto rollback;
    if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_TRANSIENT))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 2, closed))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 3, ciphered))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 4, RHIZOME_PRIORITY_DEFAULT))
    )) {
      WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
      goto rollback;
    }
    if (sqlite_step_retry(&retry, stmt) == -1)
      goto rollback;
    sqlite3_finalize(stmt);
    stmt = NULL;
  }

  if (m->group_count > 0) {
    if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS(manifestid,groupid) VALUES(?, ?);")) == NULL)
      goto rollback;
    int i;
    for (i=0;i<m->group_count;i++){
      if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_TRANSIENT))
	    && sqlite_code_ok(sqlite3_bind_text(stmt, 2, m->groups[i], -1, SQLITE_TRANSIENT))
      )) {
	WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
	goto rollback;
      }
      if (sqlite_step_retry(&retry, stmt) == -1)
	goto rollback;
      sqlite3_reset(stmt);
    }
    sqlite3_finalize(stmt);
    stmt = NULL;
  }
  if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
    // This message used in tests; do not modify or remove.
    const char *service = rhizome_manifest_get(m, "service", NULL, 0);
    INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRId64,
	  service ? service : "NULL",
	  alloca_tohex_sid(m->cryptoSignPublic),
	  m->version
	  );
    monitor_announce_bundle(m);
    if (serverMode)
      rhizome_sync_announce();
    return 0;
  }
rollback:
  if (stmt)
    sqlite3_finalize(stmt);
  WHYF("Failed to store bundle bid=%s", manifestid);
  sqlite_exec_void_retry(&retry, "ROLLBACK;", END);
  return -1;
}

int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name, 
			   const char *sender_sid, const char *recipient_sid, 
			   int limit, int offset, char count_rows)
{
  IN();
  strbuf b = strbuf_alloca(1024);
  strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
  
  if (service && *service)
    strbuf_sprintf(b, " AND service = ?1");
  if (name && *name)
    strbuf_sprintf(b, " AND name like ?2");
  if (sender_sid && *sender_sid)
    strbuf_sprintf(b, " AND sender = ?3");
  if (recipient_sid && *recipient_sid)
    strbuf_sprintf(b, " AND recipient = ?4");
  
  strbuf_sprintf(b, " ORDER BY inserttime DESC");
  
  if (offset)
    strbuf_sprintf(b, " OFFSET %u", offset);
  
  if (strbuf_overrun(b))
    RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
  
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b));
  if (!statement)
    RETURN(-1);
  
  int ret = 0;
  
  if (service && *service)
    ret = sqlite3_bind_text(statement, 1, service, -1, SQLITE_STATIC);
  if (ret==SQLITE_OK && name && *name)
    ret = sqlite3_bind_text(statement, 2, name, -1, SQLITE_STATIC);
  if (ret==SQLITE_OK && sender_sid && *sender_sid)
    ret = sqlite3_bind_text(statement, 3, sender_sid, -1, SQLITE_STATIC);
  if (ret==SQLITE_OK && recipient_sid && *recipient_sid)
    ret = sqlite3_bind_text(statement, 4, recipient_sid, -1, SQLITE_STATIC);
  
  if (ret!=SQLITE_OK){
    ret = WHYF("Failed to bind parameters: %s", sqlite3_errmsg(rhizome_db));
    goto cleanup;
  }
  
  ret=0;
  size_t rows = 0;
  
  const char *names[]={
    "_id",
    "service",
    "id",
    "version",
    "date",
    ".inserttime",
    ".author",
    ".fromhere",
    "filesize",
    "filehash",
    "sender",
    "recipient",
    "name"
  };
  cli_columns(context, 13, names);
  
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    ++rows;
    if (limit>0 && rows>limit)
      break;
    if (!(   sqlite3_column_count(statement) == 6
	  && sqlite3_column_type(statement, 0) == SQLITE_TEXT
	  && sqlite3_column_type(statement, 1) == SQLITE_BLOB
	  && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
	  && sqlite3_column_type(statement, 3) == SQLITE_INTEGER
	  && (	sqlite3_column_type(statement, 4) == SQLITE_TEXT
	     || sqlite3_column_type(statement, 4) == SQLITE_NULL
	     )
    )) { 
      ret = WHY("Incorrect statement column");
      break;
    }
    rhizome_manifest *m = rhizome_new_manifest();
    if (m == NULL) {
      ret = WHY("Out of manifests");
      break;
    }
    const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
    long long q_version = sqlite3_column_int64(statement, 2);
    long long q_inserttime = sqlite3_column_int64(statement, 3);
    const char *q_author = (const char *) sqlite3_column_text(statement, 4);
    long long rowid = sqlite3_column_int64(statement, 5);
    
    if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
      WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
    } else {
      long long blob_version = rhizome_manifest_get_ll(m, "version");
      if (blob_version != q_version)
	WARNF("MANIFESTS row id=%s version=%lld does not match manifest blob.version=%lld", q_manifestid, q_version, blob_version);
      int match = 1;
      
      const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
      if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0))
	match = 0;
      const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0);
      const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
      if (match && sender_sid[0]) {
	if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0))
	  match = 0;
      }
      if (match && recipient_sid[0]) {
	if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0))
	  match = 0;
      }
      
      if (match) {
	const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
	long long blob_date = rhizome_manifest_get_ll(m, "date");
	const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
	int from_here = 0;
	unsigned char senderSid[SID_SIZE];
	unsigned char recipientSid[SID_SIZE];
	
	if (blob_sender)
	  stowSid(senderSid, 0, blob_sender);
	if (blob_recipient)
	  stowSid(recipientSid, 0, blob_recipient);
	
	if (q_author) {
	  if (config.debug.rhizome) DEBUGF("q_author=%s", alloca_str_toprint(q_author));
	  stowSid(m->author, 0, q_author);
	  int cn = 0, in = 0, kp = 0;
	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, m->author);
	}
	if (!from_here && blob_sender) {
	  if (config.debug.rhizome) DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender));
	  int cn = 0, in = 0, kp = 0;
	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, senderSid);
	}
	
	cli_put_long(context, rowid, ":");
	cli_put_string(context, blob_service, ":");
	cli_put_hexvalue(context, m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES, ":");
	cli_put_long(context, blob_version, ":");
	cli_put_long(context, blob_date, ":");
	cli_put_long(context, q_inserttime, ":");
	cli_put_hexvalue(context, q_author?m->author:NULL, SID_SIZE, ":");
	cli_put_long(context, from_here, ":");
	cli_put_long(context, m->fileLength, ":");
	
	unsigned char filehash[SHA512_DIGEST_LENGTH];
	if (m->fileLength)
	  fromhex(filehash, blob_filehash, SHA512_DIGEST_LENGTH);
	
	cli_put_hexvalue(context, m->fileLength?filehash:NULL, SHA512_DIGEST_LENGTH, ":");
	
	cli_put_hexvalue(context, blob_sender?senderSid:NULL, SID_SIZE, ":");
	cli_put_hexvalue(context, blob_recipient?recipientSid:NULL, SID_SIZE, ":");
	cli_put_string(context, blob_name, "\n");
      }
    }
    if (m) rhizome_manifest_free(m);
  }
  
  if (ret==0 && count_rows){
    while (sqlite_step_retry(&retry, statement) == SQLITE_ROW)
      ++rows;
  }
  cli_row_count(context, rows);
  
cleanup:
  sqlite3_finalize(statement);
  RETURN(ret);
  OUT();
}

void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
{
  (void) tohex(out, in, byteCount);
}

int rhizome_update_file_priority(const char *fileid)
{
  /* work out the highest priority of any referrer */
  int64_t highestPriority = -1;
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (sqlite_exec_int64_retry(&retry, &highestPriority,
	"SELECT max(grouplist.priority) FROM MANIFESTS, GROUPMEMBERSHIPS, GROUPLIST"
	" WHERE MANIFESTS.filehash = ?"
	"   AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id"
	"   AND GROUPMEMBERSHIPS.groupid = GROUPLIST.id;",
	TEXT_TOUPPER, fileid, END) == -1)
    return -1;
  if (   highestPriority >= 0
      && sqlite_exec_void_retry(&retry,
	      "UPDATE files SET highestPriority = ? WHERE id = ?;",
	      INT, highestPriority, TEXT_TOUPPER, fileid, END
	  ) == -1
  )
    return WHYF("cannot update priority for fileid=%s", fileid);
  return 0;
}

/* Search the database for a manifest having the same name and payload content, and if the version
 * is known, having the same version.  Returns 1 if a duplicate is found (setting *found to point to
 * the duplicate's manifest), returns 0 if no duplicate is found (leaving *found unchanged).
 * Returns -1 on error (leaving *found undefined).
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
{
  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
  if (service == NULL)
    return WHY("Manifest has no service");
  
  const char *name = rhizome_manifest_get(m, "name", NULL, 0);
  const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
  const char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
  
  char sqlcmd[1024];
  strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
  strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
  
  if (m->fileLength != 0)
    strbuf_puts(b, " AND filehash = ?");
  if (name)
    strbuf_puts(b, " AND name = ?");
  if (sender)
    strbuf_puts(b, " AND sender = ?");
  if (recipient)
    strbuf_puts(b, " AND recipient = ?");
  
  if (strbuf_overrun(b))
    return WHYF("SQL command too long: %s", strbuf_str(b));
  
  int ret = 0;
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b));
  if (!statement)
    return -1;
  
  int field = 1;
  sqlite3_bind_int(statement, field++, m->fileLength);
  sqlite3_bind_text(statement, field++, service, -1, SQLITE_STATIC);
  
  if (m->fileLength != 0)
    sqlite3_bind_text(statement, field++, m->fileHexHash, -1, SQLITE_STATIC);
  if (name)
    sqlite3_bind_text(statement, field++, name, -1, SQLITE_STATIC);
  if (sender)
    sqlite3_bind_text(statement, field++, sender, -1, SQLITE_STATIC);
  if (recipient)
    sqlite3_bind_text(statement, field++, recipient, -1, SQLITE_STATIC);
  
  int rows = 0;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    ++rows;
    if (config.debug.rhizome)
      DEBUGF("Row %d", rows);
      
    rhizome_manifest *blob_m = rhizome_new_manifest();
    if (blob_m == NULL) {
      ret = WHY("Out of manifests");
      break;
    }
    
    const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
    
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
    if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
      WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
      goto next;
    }
    
    if (rhizome_manifest_verify(blob_m)) {
      WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
      goto next;
    }
    
    const char *q_author = (const char *) sqlite3_column_text(statement, 2);
    if (q_author) {
      if (config.debug.rhizome)
	strbuf_sprintf(b, " .author=%s", q_author);
      stowSid(blob_m->author, 0, q_author);
    }
    
    // check that we can re-author this manifest
    if (rhizome_extract_privatekey(blob_m, NULL)){
      goto next;
    }
    
    *found = blob_m;
    if (config.debug.rhizome)
      DEBUGF("Found duplicate payload, %s", q_manifestid);
    ret = 1;
    break;
    
next:
    if (blob_m)
      rhizome_manifest_free(blob_m);
  }
  sqlite3_finalize(statement);
  return ret;
}

/* Retrieve a manifest from the database, given its manifest ID.
 *
 * Returns 0 if manifest is found
 * Returns 1 if manifest is not found
 * Returns -1 on error
 * Caller is responsible for allocating and freeing rhizome_manifest
 */
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m)
{
  int ret=0;
  
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  
  sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT manifest, version, inserttime, author FROM manifests WHERE id like ?");
  if (!statement)
    return -1;

  sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
  if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 0);
    int64_t q_version = sqlite3_column_int64(statement, 1);
    int64_t q_inserttime = sqlite3_column_int64(statement, 2);
    const char *q_author = (const char *) sqlite3_column_text(statement, 3);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 0); // must call after sqlite3_column_blob()
    
    if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize)){
      ret=WHYF("Manifest %s exists but is invalid", manifestid);
      goto done;
    }
    
    if (q_author){
      if (stowSid(m->author, 0, q_author) == -1)
	WARNF("Manifest %s contains invalid author=%s -- ignored", manifestid, alloca_str_toprint(q_author));
    }
    
    if (m->version!=q_version)
      WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version);
    
    m->inserttime = q_inserttime;
  }else{
    INFOF("Manifest %s was not found", manifestid);
    ret=1;
  }
  
done:
  sqlite3_finalize(statement);
  return ret;  
}

static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const char *manifestid)
{
  sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM manifests WHERE id = ?");
  if (!statement)
    return -1;
  sqlite3_bind_text(statement, 1, manifestid, -1, SQLITE_STATIC);
  if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
    return -1;
  return sqlite3_changes(rhizome_db) ? 0 : 1;
}

static int rhizome_delete_file_retry(sqlite_retry_state *retry, const char *fileid)
{
  int ret = 0;
  
  rhizome_delete_external(fileid);
  
  sqlite3_stmt *statement = sqlite_prepare(retry, "DELETE FROM files WHERE id = ?");
  if (!statement)
    ret = -1;
  else {
    sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
    if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
      ret = -1;
  }
  statement = sqlite_prepare(retry, "DELETE FROM fileblobs WHERE id = ?");
  if (!statement)
    ret = -1;
  else {
    sqlite3_bind_text(statement, 1, fileid, -1, SQLITE_STATIC);
    if (_sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, retry, statement) == -1)
      ret = -1;
  }
  return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
}

static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const char *manifestid)
{
  strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
  int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", TEXT_TOUPPER, manifestid, END);
  if (rows == -1)
    return -1;
  if (rows && rhizome_delete_file_retry(retry, strbuf_str(fh)) == -1)
    return -1;
  return 0;
}

/* Remove a manifest and its bundle from the database, given its manifest ID.
 *
 * Returns 0 if manifest is found and removed and bundle was either absent or removed
 * Returns 1 if manifest is not found
 * Returns -1 on error
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_delete_bundle(const char *manifestid)
{
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (rhizome_delete_payload_retry(&retry, manifestid) == -1)
    return -1;
  if (rhizome_delete_manifest_retry(&retry, manifestid) == -1)
    return -1;
  return 0;
}

/* Remove a manifest from the database, given its manifest ID, leaving its bundle (fileblob)
 * untouched if present.
 *
 * Returns 0 if manifest is found and removed
 * Returns 1 if manifest is not found
 * Returns -1 on error
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_delete_manifest(const char *manifestid)
{
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  return rhizome_delete_manifest_retry(&retry, manifestid);
}

/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
 * untouched if present.
 *
 * Returns 0 if manifest is found, its payload is found and removed
 * Returns 1 if manifest or payload is not found
 * Returns -1 on error
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_delete_payload(const char *manifestid)
{
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  return rhizome_delete_payload_retry(&retry, manifestid);
}

/* Remove a file from the database, given its file hash.
 *
 * Returns 0 if file is found and removed
 * Returns 1 if file is not found
 * Returns -1 on error
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_delete_file(const char *fileid)
{
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  return rhizome_delete_file_retry(&retry, fileid);
}

static int is_interesting(const char *id_hex, int64_t version)
{
  IN();
  int ret=1;

  // do we have this bundle [or later]?
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  sqlite3_stmt *statement = sqlite_prepare(&retry, 
    "SELECT filehash FROM manifests WHERE id like ? and version >= ?");

  if (!statement)
    RETURN(-1);

  sqlite3_bind_text(statement, 1, id_hex, -1, SQLITE_STATIC);
  sqlite3_bind_int64(statement, 2, version);
  
  if (sqlite_step_retry(&retry, statement) == SQLITE_ROW){
    const char *q_filehash = (const char *) sqlite3_column_text(statement, 0);
    ret=0;
    if (q_filehash && *q_filehash && !rhizome_exists(q_filehash))
      ret=1;
  }
  sqlite3_finalize(statement);

  RETURN(ret);
  OUT();
}

int rhizome_is_bar_interesting(unsigned char *bar)
{
  int64_t version = rhizome_bar_version(bar);
  char id_hex[RHIZOME_MANIFEST_ID_STRLEN];
  tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES);
  strcat(id_hex, "%");
  return is_interesting(id_hex, version);
}

int rhizome_is_manifest_interesting(rhizome_manifest *m)
{
  char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
  if (!rhizome_manifest_get(m, "id", id, sizeof id))
    // dodgy manifest, we don't want to receive it
    return WHY("Ignoring bad manifest (no ID field)");
  str_toupper_inplace(id);
  return is_interesting(id, m->version);
}