serval-dna/rhizome_database.c

/*
Serval Rhizome file sharing
Copyright (C) 2012 The Serval Project, Inc.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#define __RHIZOME_INLINE
#include "serval.h"
#include "rhizome.h"
#include "strbuf.h"
#include <stdlib.h>
#include <time.h>

long long rhizome_space=0;
static const char *rhizome_thisdatastore_path = NULL;

const char *rhizome_datastore_path()
{
  if (!rhizome_thisdatastore_path)
    rhizome_set_datastore_path(NULL);
  return rhizome_thisdatastore_path;
}

int rhizome_set_datastore_path(const char *path)
{
  if (!path)
    path = confValueGet("rhizome.datastore_path", NULL);
  if (path) {
    rhizome_thisdatastore_path = strdup(path);
    if (path[0] != '/')
      WARNF("Dangerous rhizome.datastore_path setting: '%s' -- should be absolute", rhizome_thisdatastore_path);
  } else {
    rhizome_thisdatastore_path = serval_instancepath();
    WARNF("Rhizome datastore path not configured -- using instance path '%s'", rhizome_thisdatastore_path);
  }
  return 0;
}

int form_rhizome_datastore_path(char * buf, size_t bufsiz, const char *fmt, ...)
{
  va_list ap;
  strbuf b = strbuf_local(buf, bufsiz);
  strbuf_puts(b, rhizome_datastore_path());
  if (fmt) {
    va_start(ap, fmt);
    if (*strbuf_substr(b, -1) != '/')
      strbuf_putc(b, '/');
    strbuf_vsprintf(b, fmt, ap);
    va_end(ap);
  }
  if (strbuf_overrun(b)) {
      WHY("Path buffer overrun");
      return 0;
  }
  return 1;
}

int form_rhizome_import_path(char * buf, size_t bufsiz, const char *fmt, ...)
{
  va_list ap;
  strbuf b = strbuf_local(buf, bufsiz);
  strbuf_sprintf(b, "%s/import", rhizome_datastore_path());
  if (fmt) {
    va_start(ap, fmt);
    strbuf_putc(b, '/');
    strbuf_vsprintf(b, fmt, ap);
    va_end(ap);
  }
  if (strbuf_overrun(b)) {
      WHY("Path buffer overrun");
      return 0;
  }
  return 1;
}

int create_rhizome_datastore_dir()
{
  if (debug & DEBUG_RHIZOME) DEBUGF("mkdirs(%s, 0700)", rhizome_datastore_path());
  return mkdirs(rhizome_datastore_path(), 0700);
}

int create_rhizome_import_dir()
{
  char dirname[1024];
  if (!form_rhizome_import_path(dirname, sizeof dirname, NULL))
    return -1;
  if (debug & DEBUG_RHIZOME) DEBUGF("mkdirs(%s, 0700)", dirname);
  return mkdirs(dirname, 0700);
}

sqlite3 *rhizome_db=NULL;

/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id)
{
  long long result = 0;
  if (sqlite_exec_int64_retry(retry, &result,
	"select max(grouplist.priorty) from grouplist,manifests,groupmemberships"
	" where manifests.id='%s'"
	"   and grouplist.id=groupmemberships.groupid"
	"   and groupmemberships.manifestid=manifests.id;",
	id
      ) == -1
  )
    return -1;
  return (int) result;
}

int rhizome_opendb()
{
  if (rhizome_db) return 0;

  IN();
  
  if (create_rhizome_datastore_dir() == -1){
    RETURN(WHY("No Directory"));
  }
  char dbpath[1024];
  if (!FORM_RHIZOME_DATASTORE_PATH(dbpath, "rhizome.db")){
    RETURN(WHY("Invalid path"));
  }

  if (sqlite3_open(dbpath,&rhizome_db)){
    RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
  }
  int loglevel = (debug & DEBUG_RHIZOME) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;

  /* Read Rhizome configuration */
  double rhizome_kb = atof(confValueGet("rhizome_kb", "1024"));
  rhizome_space = 1024LL * rhizome_kb;
  if (debug&DEBUG_RHIZOME) {
    DEBUGF("serval.conf:rhizome_kb=%.f", rhizome_kb);
    DEBUGF("Rhizome will use %lldB of storage for its database.", rhizome_space);
  }
  /* Create tables as required */
  sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;");
  if (	sqlite_exec_void("CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);") == -1
    ||	sqlite_exec_void("CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer,inserttime integer, bar blob, filesize integer, filehash text);") == -1
    ||	sqlite_exec_void("CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, data blob, length integer, highestpriority integer, datavalid integer, inserttime integer);") == -1
    ||	sqlite_exec_void("DROP TABLE IF EXISTS FILEMANIFESTS;") == -1
    ||	sqlite_exec_void("CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);") == -1
    ||	sqlite_exec_void("CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);") == -1
  ) {
    RETURN(WHY("Failed to create schema"));
  }

  /* Create indexes if they don't already exist */
  sqlite_exec_void_loglevel(LOG_LEVEL_WARN,"CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);");
  sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);");

  /* Clean out half-finished entries from the database */
  sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash IS NULL;");
  sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE NOT EXISTS( SELECT  1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);");
  sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash != '' AND NOT EXISTS( SELECT  1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);");
  RETURN(0);
}

/* SQL query retry logic.

   The common retry-on-busy logic is factored into this function.  This logic encapsulates the
   maximum time (timeout) that the caller may wait for a lock to be released and the sleep interval
   while waiting.  The way to use it is this:

      sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
      do ret = some_sqlite_operation(...);
        while (is_busy(ret) && sqlite_retry(&retry, "some_sqlite_operation"));
      if (is_error(ret) || is_busy(ret))
	return -1; // an error has already been logged
      sqlite_retry_done(&retry, "some_sqlite_operation");
      ...

   If the database is currently locked for updates, then some_sqlite_operation() will return a code
   indicating busy (which is distinguishable from the codes for success or any other error).
   sqlite_retry() will then log a DEBUG or INFO message, sleep for a short period and return true if
   the timeout has not been reached.  It keeps this information in the 'retry' variable, which must
   be initialised as shown.  As long as the timeout has not been reached, sqlite_retry() will keep
   sleeping and returning true.  If the timeout is reached, then sqlite_retry() will log an error
   and return false.  If the operation is successful, sqlite_retry_done() must be called to log the
   success as a DEBUG or INFO message to provide closure to the prior messages already logged by
   sqlite_retry() and to reset the 'retry' variable for re-use.

   The timeout and sleep interval depend on whether the caller is the servald server process or not.
   See the definition of the SQLITE_RETRY_STATE_DEFAULT macro for the default settings.

   A single 'retry' variable may be initialised once then used for a succession of database
   operations.  If invoked by the server process, then the timeout timer will not be reset by
   sqlite_retry() or sqlite_retry_done(), so that the timeout limit will apply to the cumulative
   latency, not just to each individual query, which could potentially add up to much greater
   latency than desired.  However, in non-server processes, each query may be allowed its own
   timeout, giving a greater chance of success at the expense of potentially greater latency.
 */

/* In the servald server process, by default we retry every 10 ms for up to 50 ms, so as to not
   introduce too much latency into server responsiveness.  In other processes (eg, Batphone MeshMS
   thread), by default we allow busy retries to go for over a second, waiting 100 ms between each
   retry.
 */
sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int otherLimit, int otherSleep)
{
  return (sqlite_retry_state){
      .limit = serverMode ? (serverLimit < 0 ? 50 : serverLimit) : (otherLimit < 0 ? 1500 : otherLimit),
      .sleep = serverMode ? (serverSleep < 0 ? 10 : serverSleep) : (otherSleep < 0 ? 100 : otherSleep),
      .elapsed = 0,
      .start = -1,
      .busytries = 0
    };
}

int _sqlite_retry(struct __sourceloc where, sqlite_retry_state *retry, const char *action)
{
  time_ms_t now = gettime_ms();
  ++retry->busytries;
  if (retry->start == -1)
    retry->start = now;
  else
    retry->elapsed += now - retry->start;
  logMessage(LOG_LEVEL_INFO, where,
      "%s on try %u after %.3f seconds (%.3f elapsed): %s",
      sqlite3_errmsg(rhizome_db),
      retry->busytries,
      (now - retry->start) / 1e3,
      retry->elapsed / 1e3,
      action
    );
  if (retry->elapsed >= retry->limit) {
    // reset ready for next query
    retry->busytries = 0;
    if (!serverMode)
      retry->start = -1;
    return 0; // tell caller to stop trying
  }
  if (retry->sleep)
    sleep_ms(retry->sleep);
  return 1; // tell caller to try again
}

void _sqlite_retry_done(struct __sourceloc where, sqlite_retry_state *retry, const char *action)
{
  if (retry->busytries) {
    time_ms_t now = gettime_ms();
    logMessage(LOG_LEVEL_INFO, where,
	"succeeded on try %u after %.3f seconds (%.3f elapsed): %s",
	retry->busytries + 1,
	(now - retry->start) / 1e3,
	retry->elapsed / 1e3,
	action
      );
  }
  // reset ready for next query
  retry->busytries = 0;
  if (!serverMode)
    retry->start = -1;
}

/*
   Convenience wrapper for preparing an SQL command.
   Returns -1 if an error occurs (logged as an error), otherwise zero with the prepared
   statement in *statement.
 */
sqlite3_stmt *_sqlite_prepare(struct __sourceloc where, const char *sqlformat, ...)
{
  strbuf sql = strbuf_alloca(8192);
  strbuf_va_printf(sql, sqlformat);
  return _sqlite_prepare_loglevel(where, LOG_LEVEL_ERROR, sql);
}

sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc where, int log_level, strbuf stmt)
{
  sqlite3_stmt *statement = NULL;
  if (strbuf_overrun(stmt)) {
    logMessage(LOG_LEVEL_ERROR, where, "SQL overrun: %s", strbuf_str(stmt));
    return NULL;
  }
  if (!rhizome_db && rhizome_opendb() == -1)
    return NULL;
  switch (sqlite3_prepare_v2(rhizome_db, strbuf_str(stmt), -1, &statement, NULL)) {
    case SQLITE_OK:
    case SQLITE_DONE:
      break;
    default:
      logMessage(log_level, where, "query invalid, %s: %s", sqlite3_errmsg(rhizome_db), strbuf_str(stmt));
      sqlite3_finalize(statement);
      return NULL;
  }
  return statement;
}

int _sqlite_step_retry(struct __sourceloc where, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
  if (!statement)
    return -1;
  while (1) {
    int stepcode = sqlite3_step(statement);
    switch (stepcode) {
      case SQLITE_OK:
      case SQLITE_DONE:
      case SQLITE_ROW:
	if (retry)
	  _sqlite_retry_done(where, retry, sqlite3_sql(statement));
	return stepcode;
      case SQLITE_BUSY:
      case SQLITE_LOCKED:
	if (retry && _sqlite_retry(where, retry, sqlite3_sql(statement))) {
	  sqlite3_reset(statement);
	  break; // back to sqlite3_step()
	}
	// fall through...
      default:
	logMessage(log_level, where, "query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
	return -1;
    }
  }
}

/*
   Convenience wrapper for executing a prepared SQL statement that returns no value.  If an error
   occurs then logs it at the given level and returns -1.  If 'retry' is non-NULL and the BUSY error
   occurs (indicating the database is locked, ie, currently in use by another process), then resets
   the statement and retries while sqlite_retry() returns true.  If sqlite_retry() returns false
   then returns -1.  Otherwise returns zero.  Always finalises the statement before returning.
 */
static int _sqlite_exec_void_prepared(struct __sourceloc where, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
{
  if (!statement)
    return -1;
  int rowcount = 0;
  int stepcode;
  while ((stepcode = _sqlite_step_retry(where, log_level, retry, statement)) == SQLITE_ROW)
    ++rowcount;
  if (rowcount)
    logMessage(LOG_LEVEL_WARN, where, "void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
  sqlite3_finalize(statement);
  return sqlite_code_ok(stepcode) ? 0 : -1;
}

static int _sqlite_vexec_void(struct __sourceloc where, int log_level, sqlite_retry_state *retry, const char *sqlformat, va_list ap)
{
  strbuf stmt = strbuf_alloca(8192);
  strbuf_vsprintf(stmt, sqlformat, ap);
  return _sqlite_exec_void_prepared(where, log_level, retry, sqlite_prepare_loglevel(log_level, stmt));
}

/* Convenience wrapper for executing an SQL command that returns no value.
   If an error occurs then logs it at ERROR level and returns -1.  Otherwise returns zero.
   @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_void(struct __sourceloc where, const char *sqlformat, ...)
{
  va_list ap;
  va_start(ap, sqlformat);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = _sqlite_vexec_void(where, LOG_LEVEL_ERROR, &retry, sqlformat, ap);
  va_end(ap);
  return ret;
}

/* Same as sqlite_exec_void(), but logs any error at the given level instead of ERROR.
   @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_void_loglevel(struct __sourceloc where, int log_level, const char *sqlformat, ...)
{
  va_list ap;
  va_start(ap, sqlformat);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = _sqlite_vexec_void(where, log_level, &retry, sqlformat, ap);
  va_end(ap);
  return ret;
}

/* Same as sqlite_exec_void() but if the statement cannot be executed because the database is
   currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
   instead of its own, internal one.  If 'retry' is passed as NULL, then will not sleep and retry at
   all in the event of a busy condition, but will log it as an error and return immediately.
   @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_void_retry(struct __sourceloc where, sqlite_retry_state *retry, const char *sqlformat, ...)
{
  va_list ap;
  va_start(ap, sqlformat);
  int ret = _sqlite_vexec_void(where, LOG_LEVEL_ERROR, retry, sqlformat, ap);
  va_end(ap);
  return ret;
}

static int _sqlite_vexec_int64(struct __sourceloc where, sqlite_retry_state *retry, long long *result, const char *sqlformat, va_list ap)
{
  strbuf stmt = strbuf_alloca(8192);
  strbuf_vsprintf(stmt, sqlformat, ap);
  sqlite3_stmt *statement = _sqlite_prepare_loglevel(where, LOG_LEVEL_ERROR, stmt);
  if (!statement)
    return -1;
  int ret = 0;
  int rowcount = 0;
  int stepcode;
  while ((stepcode = _sqlite_step_retry(where, LOG_LEVEL_ERROR, retry, statement)) == SQLITE_ROW) {
    int columncount = sqlite3_column_count(statement);
    if (columncount != 1) {
      logMessage(LOG_LEVEL_ERROR, where, "incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
      ret = -1;
    }
    else if (++rowcount == 1)
      *result = sqlite3_column_int64(statement, 0);
  }
  if (rowcount > 1)
    logMessage(LOG_LEVEL_WARN, where, "query unexpectedly returned %d rows, ignored all but first", rowcount);
  sqlite3_finalize(statement);
  return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
}

/*
   Convenience wrapper for executing an SQL command that returns a single int64 value.
   Logs an error and returns -1 if an error occurs.
   If no row is found, then returns 0 and does not alter *result.
   If exactly one row is found, the assigns its value to *result and returns 1.
   If more than one row is found, then logs a warning, assigns the value of the first row to *result
   and returns the number of rows.
 */
int _sqlite_exec_int64(struct __sourceloc where, long long *result, const char *sqlformat,...)
{
  va_list ap;
  va_start(ap, sqlformat);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = _sqlite_vexec_int64(where, &retry, result, sqlformat, ap);
  va_end(ap);
  return ret;
}

/* Same as sqlite_exec_int64() but if the statement cannot be executed because the database is
   currently locked for updates, then will call sqlite_retry() on the supplied retry state variable
   instead of its own, internal one.  If 'retry' is passed as NULL, then will not sleep and retry at
   all in the event of a busy condition, but will log it as an error and return immediately.
   @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_int64_retry(struct __sourceloc where, sqlite_retry_state *retry, long long *result, const char *sqlformat,...)
{
  va_list ap;
  va_start(ap, sqlformat);
  int ret = _sqlite_vexec_int64(where, retry, result, sqlformat, ap);
  va_end(ap);
  return ret;
}

/*
   Convenience wrapper for executing an SQL command that returns a single text value.
   Logs an error and returns -1 if an error occurs, otherwise the number of rows that were found:
    0 means no rows, nothing is appended to the strbuf
    1 means exactly one row, appends its column to the strbuf
    2 more than one row, logs a warning and appends the first row's column to the strbuf
   @author Andrew Bettison <andrew@servalproject.com>
 */
int _sqlite_exec_strbuf(struct __sourceloc where, strbuf sb, const char *sqlformat,...)
{
  strbuf stmt = strbuf_alloca(8192);
  strbuf_va_printf(stmt, sqlformat);
  sqlite3_stmt *statement = _sqlite_prepare_loglevel(where, LOG_LEVEL_ERROR, stmt);
  if (!statement)
    return -1;
  int ret = 0;
  int rowcount = 0;
  int stepcode;
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  while ((stepcode = _sqlite_step_retry(where, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW) {
    int columncount = sqlite3_column_count(statement);
    if (columncount != 1) {
      logMessage(LOG_LEVEL_ERROR, where, "incorrect column count %d (should be 1): %s", columncount, sqlite3_sql(statement));
      ret = -1;
    }
    else if (++rowcount == 1)
      strbuf_puts(sb, (const char *)sqlite3_column_text(statement, 0));
  }
  if (rowcount > 1)
    logMessage(LOG_LEVEL_WARN, where, "query unexpectedly returned %d rows, ignored all but first", rowcount);
  sqlite3_finalize(statement);
  return sqlite_code_ok(stepcode) && ret != -1 ? rowcount : -1;
}

long long rhizome_database_used_bytes()
{
  long long db_page_size;
  long long db_page_count;
  long long db_free_page_count;
  if (	sqlite_exec_int64(&db_page_size, "PRAGMA page_size;") == -1LL
    ||  sqlite_exec_int64(&db_page_count, "PRAGMA page_count;") == -1LL
    ||	sqlite_exec_int64(&db_free_page_count, "PRAGMA free_count;") == -1LL
  )
    return WHY("Cannot measure database used bytes");
  return db_page_size * (db_page_count - db_free_page_count);
}

int rhizome_make_space(int group_priority, long long bytes)
{
  /* Asked for impossibly large amount */
  if (bytes>=(rhizome_space-65536))
    return WHYF("bytes=%lld is too large", bytes);

  long long db_used = rhizome_database_used_bytes();
  if (db_used == -1)
    return -1;
  
  /* If there is already enough space now, then do nothing more */
  if (db_used<=(rhizome_space-bytes-65536))
    return 0;

  /* Okay, not enough space, so free up some. */
  sqlite3_stmt *statement = sqlite_prepare("select id,length from files where highestpriority < %d order by descending length", group_priority);
  if (!statement)
    return -1;

  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  while (bytes > (rhizome_space - 65536 - rhizome_database_used_bytes())
      && sqlite_step_retry(&retry, statement) == SQLITE_ROW
  ) {
    /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
    const unsigned char *id;

    /* Get values */
    if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
      id = sqlite3_column_text(statement, 0);
    else {
      WHY("Incorrect type in id column of files table");
      break;
    }
    if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER)
      ; //length=sqlite3_column_int(statement, 1);
    else {
      WHY("Incorrect type in length column of files table");
      break;
    }
    /* Try to drop this file from storage, discarding any references that do not trump the priority
       of this request.  The query done earlier should ensure this, but it doesn't hurt to be
       paranoid, and it also protects against inconsistency in the database. */
    rhizome_drop_stored_file((char *)id, group_priority + 1);
  }
  sqlite3_finalize(statement);

  //long long equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM FILES WHERE highestpriority=%d and length>%lld",group_priority,bytes);
  /* XXX Get rid of any equal priority files that are larger than this one */

  /* XXX Get rid of any higher priority files that are not relevant in this time or location */

  /* Couldn't make space */
  return WHY("Incomplete");
}

/* Drop the specified file from storage, and any manifests that reference it, 
   provided that none of those manifests are being retained at a higher priority
   than the maximum specified here. */
int rhizome_drop_stored_file(const char *id,int maximum_priority)
{
  if (!rhizome_str_is_file_hash(id))
    return WHYF("invalid file hash id=%s", alloca_toprint(-1, id, strlen(id)));
  sqlite3_stmt *statement = sqlite_prepare("select id from manifests where filehash='%s'", id);
  if (!statement)
    return WHYF("Could not drop stored file id=%s", id);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int can_drop = 1;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    /* Find manifests for this file */
    if (sqlite3_column_type(statement, 0) != SQLITE_TEXT) {
      WHYF("Incorrect type in id column of manifests table");
      break;
    }
    const char *manifestId = (char *) sqlite3_column_text(statement, 0);
    /* Check that manifest is not part of a higher priority group.
	If so, we cannot drop the manifest or the file.
	However, we will keep iterating, as we can still drop any other manifests pointing to this file
	that are lower priority, and thus free up a little space. */
    int priority = rhizome_manifest_priority(&retry, manifestId);
    if (priority == -1)
      WHYF("Cannot drop fileid=%s due to error, manifestId=%s", id, manifestId);
    else if (priority > maximum_priority) {
      WHYF("Cannot drop fileid=%s due to manifest priority, manifestId=%s", id, manifestId);
      can_drop = 0;
    } else {
      if (debug & DEBUG_RHIZOME)
	DEBUGF("removing stale manifests, groupmemberships");
      sqlite_exec_void_retry(&retry, "delete from manifests where id='%s';", manifestId);
      sqlite_exec_void_retry(&retry, "delete from keypairs where public='%s';", manifestId);
      sqlite_exec_void_retry(&retry, "delete from groupmemberships where manifestid='%s';", manifestId);
    }
  }
  sqlite3_finalize(statement);
  if (can_drop)
    sqlite_exec_void_retry(&retry, "delete from files where id='%s';",id);
  return 0;
}


/*
  Store the specified manifest into the sqlite database.
  We assume that sufficient space has been made for us.
  The manifest should be finalised, and so we don't need to
  look at the underlying manifest file, but can just write m->manifest_data
  as a blob.

  associated_filename needs to be read in and stored as a blob.  Hopefully that
  can be done in pieces so that we don't have memory exhaustion issues on small
  architectures.  However, we do know it's hash apriori from m, and so we can
  skip loading the file in if it is already stored.  mmap() apparently works on
  Linux FAT file systems, and is probably the best choice since it doesn't need
  all pages to be in RAM at the same time.

  SQLite does allow modifying of blobs once stored in the database.
  The trick is to insert the blob as all zeroes using a special function, and then
  substitute bytes in the blog progressively.

  We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, 
   and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
 */
int rhizome_store_bundle(rhizome_manifest *m)
{
  if (!m->finalised) return WHY("Manifest was not finalised");

  if (m->haveSecret) {
    /* We used to store the secret in the database, but we don't anymore, as we use 
     the BK field in the manifest. So nothing to do here. */
  } else {
    /* We don't have the secret for this manifest, so only allow updates if 
     the self-signature is valid */
    if (!m->selfSigned)
      return WHY("Manifest is not signed, and I don't have the key.  Manifest might be forged or corrupt.");
  }

  char manifestid[RHIZOME_MANIFEST_ID_STRLEN + 1];
  rhizome_manifest_get(m, "id", manifestid, sizeof manifestid);
  str_toupper_inplace(manifestid);

  /* Bind BAR to data field */
  unsigned char bar[RHIZOME_BAR_BYTES];
  rhizome_manifest_to_bar(m,bar);

  /* Store the file (but not if it is already in the database) */
  // TODO encrypted payloads - pass encryption key here. Filehash should be of
  // encrypted data.
  // We should add the file in the same transaction, but closing the blob seems
  // to cause some issues.
  char filehash[RHIZOME_FILEHASH_STRLEN + 1];
  if (m->fileLength > 0) {
    if (!m->fileHashedP)
      return WHY("Manifest payload hash unknown");
    strncpy(filehash, m->fileHexHash, sizeof filehash);
    str_toupper_inplace(filehash);

    /* rhizome_store_file() checks if it is already in the database, so we just
       call it normally. */
    if (rhizome_store_file(m, NULL))
      return WHY("Could not store file");
  } else {
    filehash[0] = '\0';
  }

  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
    return -1;

  sqlite3_stmt *stmt;
  if ((stmt = sqlite_prepare("INSERT OR REPLACE INTO MANIFESTS(id,manifest,version,inserttime,bar,filesize,filehash) VALUES(?,?,?,?,?,?,?);")) == NULL)
    goto rollback;
  if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_TRANSIENT))
        && sqlite_code_ok(sqlite3_bind_blob(stmt, 2, m->manifestdata, m->manifest_bytes, SQLITE_TRANSIENT))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 3, m->version))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 4, (long long) gettime_ms()))
	&& sqlite_code_ok(sqlite3_bind_blob(stmt, 5, bar, RHIZOME_BAR_BYTES, SQLITE_TRANSIENT))
	&& sqlite_code_ok(sqlite3_bind_int64(stmt, 6, m->fileLength))
	&& sqlite_code_ok(sqlite3_bind_text(stmt, 7, filehash, -1, SQLITE_TRANSIENT))
  )) {
    WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
    goto rollback;
  }
  if (sqlite_step_retry(&retry, stmt) == -1)
    goto rollback;
  sqlite3_finalize(stmt);
  stmt = NULL;

  // we might need to leave the old file around for a bit
  // clean out unreferenced files first
  if ((stmt = sqlite_prepare("DELETE FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT  1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);")) == NULL)
    goto rollback;
  if (!sqlite_code_ok(sqlite3_bind_int64(stmt, 1, (long long)(gettime_ms() - 60000)))) {
    WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
    goto rollback;
  }
  if (sqlite_step_retry(&retry, stmt) == -1)
    goto rollback;
  sqlite3_finalize(stmt);
  stmt = NULL;

  if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
    int closed=rhizome_manifest_get_ll(m,"closedgroup");
    if (closed<1) closed=0;
    int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
    if (ciphered<1) ciphered=0;
    if ((stmt = sqlite_prepare("INSERT OR REPLACE INTO GROUPLIST(id,closed,ciphered,priority) VALUES (?,?,?,?);")) == NULL)
      goto rollback;
    if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_TRANSIENT))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 2, closed))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 3, ciphered))
          && sqlite_code_ok(sqlite3_bind_int(stmt, 4, RHIZOME_PRIORITY_DEFAULT))
    )) {
      WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
      goto rollback;
    }
    if (sqlite_step_retry(&retry, stmt) == -1)
      goto rollback;
    sqlite3_finalize(stmt);
    stmt = NULL;
  }

  if (m->group_count > 0) {
    if ((stmt = sqlite_prepare("INSERT OR REPLACE INTO GROUPMEMBERSHIPS(manifestid,groupid) VALUES(?, ?);")) == NULL)
      goto rollback;
    int i;
    for (i=0;i<m->group_count;i++){
      if (!(   sqlite_code_ok(sqlite3_bind_text(stmt, 1, manifestid, -1, SQLITE_TRANSIENT))
	    && sqlite_code_ok(sqlite3_bind_text(stmt, 2, m->groups[i], -1, SQLITE_TRANSIENT))
      )) {
	WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
	goto rollback;
      }
      if (sqlite_step_retry(&retry, stmt) == -1)
	goto rollback;
      sqlite3_reset(stmt);
    }
    sqlite3_finalize(stmt);
    stmt = NULL;
  }
  if (sqlite_exec_void_retry(&retry, "COMMIT;") != -1)
    return 0;
rollback:
  if (stmt)
    sqlite3_finalize(stmt);
  WHYF("Failed to store bundle bid=%s", manifestid);
  sqlite_exec_void_retry(&retry, "ROLLBACK;");
  return -1;
}

int rhizome_list_manifests(const char *service, const char *sender_sid, const char *recipient_sid, int limit, int offset)
{
  IN();
  strbuf b = strbuf_alloca(1024);
  strbuf_sprintf(b, "SELECT id, manifest, version, inserttime FROM manifests ORDER BY inserttime DESC");
  if (limit)
    strbuf_sprintf(b, " LIMIT %u", limit);
  if (offset)
    strbuf_sprintf(b, " OFFSET %u", offset);
  if (strbuf_overrun(b))
    RETURN(WHYF("SQL command too long: ", strbuf_str(b)));
  sqlite3_stmt *statement = sqlite_prepare("%s", strbuf_str(b));
  if (!statement)
    return -1;
  int ret = 0;
  size_t rows = 0;
  cli_puts("11"); cli_delim("\n"); // number of columns
  cli_puts("service"); cli_delim(":");
  cli_puts("id"); cli_delim(":");
  cli_puts("version"); cli_delim(":");
  cli_puts("date"); cli_delim(":");
  cli_puts(".inserttime"); cli_delim(":");
  cli_puts(".selfsigned"); cli_delim(":");
  cli_puts("filesize"); cli_delim(":");
  cli_puts("filehash"); cli_delim(":");
  cli_puts("sender"); cli_delim(":");
  cli_puts("recipient"); cli_delim(":");
  cli_puts("name"); cli_delim("\n"); // should be last, because name may contain ':'
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    ++rows;
    if (!(   sqlite3_column_count(statement) == 4
	  && sqlite3_column_type(statement, 0) == SQLITE_TEXT
	  && sqlite3_column_type(statement, 1) == SQLITE_BLOB
	  && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
	  && sqlite3_column_type(statement, 3) == SQLITE_INTEGER
    )) { 
      ret = WHY("Incorrect statement column");
      break;
    }
    rhizome_manifest *m = rhizome_new_manifest();
    if (m == NULL) {
      ret = WHY("Out of manifests");
      break;
    }
    const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
    long long q_version = sqlite3_column_int64(statement, 2);
    long long q_inserttime = sqlite3_column_int64(statement, 3);
    if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
      WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
    } else {
      long long blob_version = rhizome_manifest_get_ll(m, "version");
      if (blob_version != q_version)
	WARNF("MANIFESTS row id=%s version=%lld does not match manifest blob.version=%lld", q_manifestid, q_version, blob_version);
      int match = 1;
      const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
      if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0))
	match = 0;
      const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0);
      const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
      if (match && sender_sid[0]) {
	if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0))
	  match = 0;
      }
      if (match && recipient_sid[0]) {
	if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0))
	  match = 0;
      }
      if (match) {
	const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
	long long blob_date = rhizome_manifest_get_ll(m, "date");
	const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
	long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
	int self_signed = rhizome_is_self_signed(m) ? 0 : 1;
	if (debug & DEBUG_RHIZOME) DEBUGF("manifest payload size = %lld", blob_filesize);
	cli_puts(blob_service ? blob_service : ""); cli_delim(":");
	cli_puts(q_manifestid); cli_delim(":");
	cli_printf("%lld", blob_version); cli_delim(":");
	cli_printf("%lld", blob_date); cli_delim(":");
	cli_printf("%lld", q_inserttime); cli_delim(":");
	cli_printf("%d", self_signed); cli_delim(":");
	cli_printf("%lld", blob_filesize); cli_delim(":");
	cli_puts(blob_filehash ? blob_filehash : ""); cli_delim(":");
	cli_puts(blob_sender ? blob_sender : ""); cli_delim(":");
	cli_puts(blob_recipient ? blob_recipient : ""); cli_delim(":");
	cli_puts(blob_name ? blob_name : ""); cli_delim("\n");
      }
    }
    if (m) rhizome_manifest_free(m);
  }
  sqlite3_finalize(statement);
  RETURN(ret);
}

/* The following function just stores the file (or silently returns if it already exists).
   The relationships of manifests to this file are the responsibility of the caller. */
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key)
{
  const char *file=m->dataFileName;
  const char *hash=m->fileHexHash;
  int priority=m->fileHighestPriority;
  if (m->payloadEncryption) 
    return WHY("Writing encrypted payloads not implemented");

  if (!m->fileHashedP)
    return WHY("Cannot store bundle file until it has been hashed");

  int fd = -1;

  /* See if the file is already stored, and if so, don't bother storing it again.
     Do this check BEFORE trying to open the associated file, because if the caller
     has received a manifest and checked that it exists in the database, it may 
     (sensibly) elect not supply the file. Rhizome Direct does this. */
  long long count = 0;
  if (sqlite_exec_int64(&count, "SELECT COUNT(*) FROM FILES WHERE id='%s' AND datavalid<>0;", hash) < 1) {
    WHY("Failed to count stored files");
    goto error;
  }
  if (count >= 1) {
    /* File is already stored, so just update the highestPriority field if required. */
    long long storedPriority = -1;
    if (sqlite_exec_int64(&storedPriority, "SELECT highestPriority FROM FILES WHERE id='%s' AND datavalid!=0", hash) == -1) {
      WHY("Failed to select highest priority");
      goto error;
    }
    if (storedPriority<priority) {
      if (sqlite_exec_void("UPDATE FILES SET highestPriority=%d WHERE id='%s';", priority, hash) == -1) {
	WHY("SQLite failed to update highestPriority field for stored file.");
	goto error;
      }
    }
    return 0;
  }

  fd = open(file, O_RDONLY);
  if (fd == -1) {
    WHYF_perror("open(%s)", alloca_str_toprint(file));
    WHY("Could not open associated file");
    goto error;
  }
  
  struct stat stat;
  if (fstat(fd, &stat)) {
    WHYF_perror("fstat(%d)", fd);
    WHY("Could not stat() associated file");
    goto error;
  }
  if (stat.st_size < m->fileLength) {
    WHYF("File has shrunk by %lld bytes from %lld to %lld, not stored",
	(long long)(m->fileLength - stat.st_size), (long long) m->fileLength, (long long) stat.st_size
      );
    goto error;
  } else if (stat.st_size > m->fileLength) {
    // If the file has grown, store the original , in the hope that it will match the hash.
    WARNF("File has grown by +%lld bytes to %lld, only storing %lld",
	(long long)(stat.st_size - m->fileLength), (long long) stat.st_size, (long long) m->fileLength
      );
  }

  unsigned char *addr = mmap(NULL, m->fileLength, PROT_READ, MAP_SHARED, fd, 0);
  if (addr==MAP_FAILED) {
    WHYF_perror("mmap(NULL, %lld, PROT_READ, MAP_SHARED, %d, 0)", (long long) m->fileLength, fd);
    WHY("mmap() of associated file failed.");
    goto error;
  }

  /* Okay, so there are no records that match, but we should delete any half-baked record (with datavalid=0) so that the insert below doesn't fail.
   Don't worry about the return result, since it might not delete any records. */
  sqlite_exec_void("DELETE FROM FILES WHERE datavalid=0;");

  /* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
   BUT, we have to do this incrementally so that we can handle blobs larger than available memory.
  This is possible using:
     int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
  That binds an all zeroes blob to a field.  We can then populate the data by
  opening a handle to the blob using:
     int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
  */

  sqlite3_stmt *statement = sqlite_prepare("INSERT OR REPLACE INTO FILES(id,data,length,highestpriority,datavalid,inserttime) VALUES('%s',?,%lld,%d,0,%lld);",
	  hash, (long long)m->fileLength, priority, (long long)gettime_ms()
	);
  if (!statement)
    goto insert_row_fail;
  /* Bind appropriate sized zero-filled blob to data field */
  if (sqlite3_bind_zeroblob(statement, 1, m->fileLength) != SQLITE_OK) {
    WHYF("sqlite3_bind_zeroblob() failed: %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
    sqlite3_finalize(statement);
    goto insert_row_fail;
  }
  /* Do actual insert, and abort if it fails */
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (_sqlite_exec_void_prepared(__HERE__, LOG_LEVEL_ERROR, &retry, statement) == -1) {
insert_row_fail:
    WHYF("Failed to insert row for fileid=%s", hash);
    goto error;
  }

  /* Get rowid for inserted row, so that we can modify the blob */
  int64_t rowid = sqlite3_last_insert_rowid(rhizome_db);
  if (rowid<1) {
    WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
    WHYF("Failed to get row ID of newly inserted row for fileid=%s", hash);
    goto error;
  }
  // We write the blob inside a transaction so that we can't get SQLITE_BUSY from
  // sqlite3_blob_close(), which cannot be retried.  Using an explicit transaction, defers BUSY
  // detection to the COMMIT, which can be retried.
  if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
    goto error;
  sqlite3_blob *blob;
  int ret;
  do ret = sqlite3_blob_open(rhizome_db, "main", "FILES", "data", rowid, 1 /* read/write */, &blob);
  while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
  if (ret != SQLITE_OK) {
    WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
    goto rollback_blob;
  }
  sqlite_retry_done(&retry, "sqlite3_blob_open");

  /* Calculate hash of file as we go, so that we can report if
     the contents have changed during import.  This is also why we
     use the m->fileLength instead of size returned by stat, in case
     the file has been appended, e.g., if a journal is being appended to
     by a separate process.  This has already been shown to happen with
     Serval Maps, and it is also quite possible with MeshMS and other
     services. */
  char hash_out[crypto_hash_sha512_BYTES*2+1];
  {
    unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
    unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
    bzero(nonce, sizeof nonce);
    SHA512_CTX context;
    SHA512_Init(&context);
    long long i;
    for (i = 0; i < m->fileLength; i += RHIZOME_CRYPT_PAGE_SIZE) {
      int n = RHIZOME_CRYPT_PAGE_SIZE;
      if (i + n > m->fileLength)
	n = m->fileLength - i;
      const unsigned char *writeable = &addr[i];
      SHA512_Update(&context, writeable, n);
      if (key) {
	/* calculate block nonce */
	int j;
	for (j=0;j<8;j++)
	  nonce[i]=(i>>(j*8))&0xff;
	crypto_stream_xsalsa20_xor(buffer, writeable, n, nonce, key);
	writeable = buffer;
      }
      do ret = sqlite3_blob_write(blob, writeable, n, i);
	while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_write"));
      if (ret != SQLITE_OK) {
	WHYF("sqlite3_blob_write() failed, %s", sqlite3_errmsg(rhizome_db));
	goto rollback_blob;
      }
      sqlite_retry_done(&retry, "sqlite3_blob_write");
    }
     SHA512_End(&context, (char *)hash_out);
     str_toupper_inplace(hash_out);
  }
  if (strcasecmp(hash_out, hash) != 0) {
    WHYF("File hash %s does not match computed hash %s -- has file been modified while being stored?",
	hash_out, hash
      );
    goto rollback_blob;
  }
  // sqlite3_blob_close() always closes the blob, regardless of return value, so it cannot be
  // retried on returning SQLITE_BUSY.
  ret = sqlite3_blob_close(blob);
  blob = NULL;
  if (!sqlite_code_ok(ret)) {
    WHYF("sqlite3_blob_close() failed, %s", sqlite3_errmsg(rhizome_db));
    goto rollback_blob;
  }

  if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
    goto rollback;

  /* Mark file as up-to-date */
  if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET datavalid=1 WHERE id='%s';", hash) != 0) {
    WHY("Failed to set datavalid");
    goto error;
  }

  close(fd);
  return 0;

rollback_blob:
  WHYF("Failed to write blob in newly inserted row for fileid=%s", hash);
  if (blob)
    sqlite3_blob_close(blob);
rollback:
  sqlite_exec_void_retry(&retry, "ROLLBACK;");
error:
  if (fd != -1)
    close(fd);
  return -1;
}


void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
{
  (void) tohex(out, in, byteCount);
}

int rhizome_update_file_priority(const char *fileid)
{
  /* work out the highest priority of any referrer */
  long long highestPriority = -1;
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  if (sqlite_exec_int64_retry(&retry, &highestPriority,
	"SELECT max(grouplist.priority) FROM MANIFESTS,GROUPMEMBERSHIPS,GROUPLIST"
	" where manifests.filehash='%s'"
	"   AND groupmemberships.manifestid=manifests.id"
	"   AND groupmemberships.groupid=grouplist.id;",
	fileid) == -1)
    return -1;
  if (highestPriority >= 0 && sqlite_exec_void_retry(&retry, "UPDATE files set highestPriority=%lld WHERE id='%s';", highestPriority, fileid) != 0)
    WHYF("cannot update priority for fileid=%s", fileid);
  return 0;
}

/* Search the database for a manifest having the same name and payload content,
   and if the version is known, having the same version.

   @author Andrew Bettison <andrew@servalproject.com>
 */
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found, int checkVersionP)
{
  const char *service = rhizome_manifest_get(m, "service", NULL, 0);
  const char *name = NULL;
  const char *sender = NULL;
  const char *recipient = NULL;
  if (service == NULL) {
    return WHY("Manifest has no service");
  } else if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
    name = rhizome_manifest_get(m, "name", NULL, 0);
    if (!name) return WHY("Manifest has no name");
  } else if (strcasecmp(service, RHIZOME_SERVICE_MESHMS) == 0) {
    sender = rhizome_manifest_get(m, "sender", NULL, 0);
    recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
    if (!sender) return WHY("Manifest has no sender");
    if (!recipient) return WHY("Manifest has no recipient");
  } else {
    return WHYF("Unsupported service '%s'", service);
  }
  char sqlcmd[1024];
  strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
  strbuf_puts(b, "SELECT id, manifest, version FROM manifests WHERE ");
  if (m->fileLength != 0) {
    if (!m->fileHashedP)
      return WHY("Manifest payload is not hashed");
    strbuf_puts(b, "filehash = ?");
  } else
    strbuf_puts(b, "filesize = 0");
  if (checkVersionP)
    strbuf_puts(b, " AND version = ?");
  if (strbuf_overrun(b))
    return WHYF("SQL command too long: %s", strbuf_str(b));
  if (debug & DEBUG_RHIZOME)
    DEBUGF("sql query: %s", sqlcmd);
  int ret = 0;
  sqlite3_stmt *statement = sqlite_prepare("%s", strbuf_str(b));
  if (!statement)
    return -1;
  int field = 1;
  char filehash[RHIZOME_FILEHASH_STRLEN + 1];
  if (m->fileLength != 0) {
    strncpy(filehash, m->fileHexHash, sizeof filehash);
    str_toupper_inplace(filehash);
    if (debug & DEBUG_RHIZOME)
      DEBUGF("filehash=\"%s\"", filehash);
    sqlite3_bind_text(statement, field++, filehash, -1, SQLITE_STATIC);
  }
  if (checkVersionP)
    sqlite3_bind_int64(statement, field++, m->version);
  size_t rows = 0;
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    ++rows;
    if (debug & DEBUG_RHIZOME) DEBUGF("Row %d", rows);
    if (!(   sqlite3_column_count(statement) == 3
	  && sqlite3_column_type(statement, 0) == SQLITE_TEXT
	  && sqlite3_column_type(statement, 1) == SQLITE_BLOB
	  && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
    )) {
      ret = WHY("Incorrect statement columns");
      break;
    }
    const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
    size_t manifestidsize = sqlite3_column_bytes(statement, 0); // must call after sqlite3_column_text()
    unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
    if (  manifestidsize != crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES * 2
      ||  fromhexstr(manifest_id, q_manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1
    ) {
      ret = WHYF("Malformed manifest.id from query: %s", q_manifestid);
      break;
    }
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
    long long q_version = sqlite3_column_int64(statement, 2);
    rhizome_manifest *blob_m = rhizome_new_manifest();
    if (blob_m == NULL) {
      ret = WHY("Out of manifests");
      break;
    }
    if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
      WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
    } else if (rhizome_manifest_verify(blob_m)) {
      WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
    } else {
      const char *blob_service = rhizome_manifest_get(blob_m, "service", NULL, 0);
      const char *blob_id = rhizome_manifest_get(blob_m, "id", NULL, 0);
      long long blob_version = rhizome_manifest_get_ll(blob_m, "version");
      const char *blob_filehash = rhizome_manifest_get(blob_m, "filehash", NULL, 0);
      long long blob_filesize = rhizome_manifest_get_ll(blob_m, "filesize");
      if (debug & DEBUG_RHIZOME)
	DEBUGF("Consider manifest.service=%s manifest.id=%s manifest.version=%lld", blob_service, q_manifestid, blob_version);
      /* Perform consistency checks, because we're paranoid. */
      int inconsistent = 0;
      if (blob_id && strcasecmp(blob_id, q_manifestid)) {
	WARNF("MANIFESTS row id=%s has inconsistent blob with id=%s -- skipped", q_manifestid, blob_id);
	++inconsistent;
      }
      if (checkVersionP && blob_version != q_version) {
	WARNF("MANIFESTS row id=%s has inconsistent blob: manifests.version=%lld, blob.version=%lld -- skipped",
	      q_manifestid, q_version, blob_version);
	++inconsistent;
      }
      if (blob_filesize != -1 && blob_filesize != m->fileLength) {
	WARNF("MANIFESTS row id=%s has inconsistent blob: known file size %lld, blob.filesize=%lld -- skipped",
	      q_manifestid, m->fileLength, blob_filesize);
	++inconsistent;
      }
      if (m->fileLength != 0) {
	if (!blob_filehash && strcasecmp(blob_filehash, m->fileHexHash)) {
	  WARNF("MANIFESTS row id=%s has inconsistent blob: manifests.filehash=%s, blob.filehash=%s -- skipped",
		q_manifestid, m->fileHexHash, blob_filehash);
	  ++inconsistent;
	}
      } else {
	if (blob_filehash) {
	  WARNF("MANIFESTS row id=%s has inconsistent blob: blob.filehash should be absent -- skipped",
		q_manifestid);
	  ++inconsistent;
	}
      }
      if (checkVersionP && q_version != m->version) {
	WARNF("SELECT query with version=%lld returned incorrect row: manifests.version=%lld -- skipped", m->version, q_version);
	++inconsistent;
      }
      if (blob_service == NULL) {
	WARNF("MANIFESTS row id=%s has blob with no 'service' -- skipped", q_manifestid, blob_id);
	++inconsistent;
      }
      if (!inconsistent) {
	strbuf b = strbuf_alloca(1024);
	if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
	  const char *blob_name = rhizome_manifest_get(blob_m, "name", NULL, 0);
	  if (blob_name && !strcmp(blob_name, name)) {
	    if (debug & DEBUG_RHIZOME)
	      strbuf_sprintf(b, " name=\"%s\"", blob_name);
	    ret = 1;
	  }
	} else if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
	  const char *blob_sender = rhizome_manifest_get(blob_m, "sender", NULL, 0);
	  const char *blob_recipient = rhizome_manifest_get(blob_m, "recipient", NULL, 0);
	  if (blob_sender && !strcasecmp(blob_sender, sender) && blob_recipient && !strcasecmp(blob_recipient, recipient)) {
	    if (debug & DEBUG_RHIZOME)
	      strbuf_sprintf(b, " sender=%s recipient=%s", blob_sender, blob_recipient);
	    ret = 1;
	  }
	}
	if (ret == 1) {
	  memcpy(blob_m->cryptoSignPublic, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
	  memcpy(blob_m->fileHexHash, m->fileHexHash, RHIZOME_FILEHASH_STRLEN + 1);
	  blob_m->fileHashedP = 1;
	  blob_m->fileLength = m->fileLength;
	  blob_m->version = q_version;
	  *found = blob_m;
	  DEBUGF("Found duplicate payload: service=%s%s version=%llu hexhash=%s",
		  blob_service, strbuf_str(b), blob_m->version, blob_m->fileHexHash
		);
	  break;
	}
      }
    }
    if (blob_m)
      rhizome_manifest_free(blob_m);
  }
  sqlite3_finalize(statement);
  return ret;
}

/* Retrieve a manifest from the database, given its manifest ID.
 *
 * Returns 1 if manifest is found (if mp != NULL then a new manifest struct is allocated, made
 * finalisable and * assigned to *mp, caller is responsible for freeing).
 * Returns 0 if manifest is not found (*mp is unchanged).
 * Returns -1 on error (*mp is unchanged).
 */
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
{
  unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
  if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
    return WHY("Invalid manifest ID");
  sqlite3_stmt *statement = sqlite_prepare("SELECT id, manifest, version, inserttime FROM manifests WHERE id = ?");
  if (!statement)
    return -1;
  char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
  tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
  sqlite3_bind_text(statement, 1, manifestIdUpper, -1, SQLITE_STATIC);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int ret = 0;
  rhizome_manifest *m = NULL;
  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
    if (!(   sqlite3_column_count(statement) == 4
	  && sqlite3_column_type(statement, 0) == SQLITE_TEXT
	  && sqlite3_column_type(statement, 1) == SQLITE_BLOB
	  && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
	  && sqlite3_column_type(statement, 3) == SQLITE_INTEGER
    )) {
      ret = WHY("Incorrect statement column");
      break;
    }
    const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
    const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
    size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
    if (mp) {
      m = rhizome_new_manifest();
      if (m == NULL) {
	WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
	ret = WHY("Out of manifests");
      } else if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
	WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
	ret = WHY("Invalid manifest blob from database");
      } else {
	ret = 1;
	memcpy(m->cryptoSignPublic, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
	const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
	if (blob_service == NULL)
	  ret = WHY("Manifest is missing 'service' field");
	long long filesizeq = rhizome_manifest_get_ll(m, "filesize");
	if (filesizeq == -1)
	  ret = WHY("Manifest is missing 'filesize' field");
	else
	  m->fileLength = filesizeq;
	const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
	if (m->fileLength != 0) {
	  if (blob_filehash == NULL)
	    ret = WHY("Manifest is missing 'filehash' field");
	  else {
	    memcpy(m->fileHexHash, blob_filehash, RHIZOME_FILEHASH_STRLEN + 1);
	    m->fileHashedP = 1;
	  }
	} else {
	  if (blob_filehash != NULL)
	    WARN("Manifest contains spurious 'filehash' field");
	  m->fileHexHash[0] = '\0';
	  m->fileHashedP = 0;
	}
	long long blob_version = rhizome_manifest_get_ll(m, "version");
	if (blob_version == -1)
	  ret = WHY("Manifest is missing 'version' field");
	else
	  m->version = blob_version;
	if (ret == 1) {
	  cli_puts("service"); cli_delim(":");
	  cli_puts(blob_service); cli_delim("\n");
	  cli_puts("manifestid"); cli_delim(":");
	  cli_puts(q_manifestid); cli_delim("\n");
	  cli_puts("version"); cli_delim(":");
	  cli_printf("%lld", (long long) sqlite3_column_int64(statement, 2)); cli_delim("\n");
	  cli_puts("inserttime"); cli_delim(":");
	  cli_printf("%lld", (long long) sqlite3_column_int64(statement, 3)); cli_delim("\n");
	  cli_puts("filesize"); cli_delim(":");
	  cli_printf("%lld", (long long) m->fileLength); cli_delim("\n");
	  if (m->fileLength != 0) {
	    cli_puts("filehash"); cli_delim(":");
	    cli_puts(m->fileHexHash); cli_delim("\n");
	  }
	  // Could write the manifest blob to the CLI output here, but that would require the output to
	  // support byte[] fields as well as String fields.
	}
      }
    }
    break;
  }
  sqlite3_finalize(statement);
  if (mp && ret == 1)
    *mp = m;
  return ret;
}

/* Retrieve a file from the database, given its file hash.
 *
 * Returns 1 if file is found (contents are written to filepath if given).
 * Returns 0 if file is not found.
 * Returns -1 on error.
 */
int rhizome_retrieve_file(const char *fileid, const char *filepath, const unsigned char *key)
{
  if (rhizome_update_file_priority(fileid) == -1) {
    WHY("Failed to update file priority");
    return 0;
  }
  sqlite3_stmt *statement = sqlite_prepare("SELECT id, rowid, length FROM files WHERE id = ? AND datavalid != 0");
  if (!statement)
    return -1;
  int ret = 0;
  char fileIdUpper[RHIZOME_FILEHASH_STRLEN + 1];
  strncpy(fileIdUpper, fileid, sizeof fileIdUpper);
  fileIdUpper[RHIZOME_FILEHASH_STRLEN] = '\0';
  str_toupper_inplace(fileIdUpper);
  sqlite3_bind_text(statement, 1, fileIdUpper, -1, SQLITE_STATIC);
  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
  int stepcode = sqlite_step_retry(&retry, statement);
  if (stepcode != SQLITE_ROW) {
    ret = 0; // no files found
  } else if (!(   sqlite3_column_count(statement) == 3
		  && sqlite3_column_type(statement, 0) == SQLITE_TEXT
		  && sqlite3_column_type(statement, 1) == SQLITE_INTEGER
		  && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
  )) { 
    ret = WHY("Incorrect statement column");
  } else {
    long long length = sqlite3_column_int64(statement, 2);
    int64_t rowid = sqlite3_column_int64(statement, 1);
    sqlite3_blob *blob = NULL;
    int code;
    do code = sqlite3_blob_open(rhizome_db, "main", "FILES", "data", rowid, 0 /* read only */, &blob);
    while (sqlite_code_busy(code) && sqlite_retry(&retry, "sqlite3_blob_open"));
    if (!sqlite_code_ok(code)) {
      ret = WHY("Could not open blob for reading");
    } else {
      cli_puts("filehash"); cli_delim(":");
      cli_puts((const char *)sqlite3_column_text(statement, 0)); cli_delim("\n");
      cli_puts("filesize"); cli_delim(":");
      cli_printf("%lld", length); cli_delim("\n");
      ret = 1;
      if (filepath&&filepath[0]) {
	int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
	if (fd == -1) {
	  WHY_perror("open");
	  ret = WHYF("Cannot open %s for write/create", filepath);
	} else {
	  /* read from blob and write to disk, decrypting if necessary as we go.  Each 4KB block of
	     data has a nonce which is fed with the key into crypto_stream_xsalsa20().  The nonce is
	     the file address divided by 4KB.  This approach is used as it allows us to append to
	     files easily, without having to get the XOR stream for the whole file, and without the
	     cipher on existing bytes having to change.  Both of these are important properties for
	     journal bundles, such as will be used by MeshMS.  For non-journal bundles where it is
	     important that changing the payload changes the encryption key (so that the XOR between
	     any two versions of the payload cannot be easily obtained).  We will do this by having
	     journal manifests identified, causing the key to be locked, rather than based on the
	     version number.  But anyway, we are supplied with the key here, so all we need to do is
	     do the block counting and call crypto_stream_xsalsa20().
	  */
	  long long offset;
	  unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
	  bzero(nonce,crypto_stream_xsalsa20_NONCEBYTES);
	  unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
	  for (offset = 0; offset < length; offset += RHIZOME_CRYPT_PAGE_SIZE) {
	    long long count=length-offset;
	    if (count>RHIZOME_CRYPT_PAGE_SIZE) count=RHIZOME_CRYPT_PAGE_SIZE;
	    if(sqlite3_blob_read(blob,&buffer[0],count,offset)!=SQLITE_OK) {
	      ret = 0;
	      WHYF("query failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
	      WHYF("Error reading %lld bytes of data from blob at offset 0x%llx", count, offset);
	    }
	    if (key) {
	      /* calculate block nonce */
	      int i; for(i=0;i<8;i++) nonce[i]=(offset>>(i*8))&0xff;
	      crypto_stream_xsalsa20_xor(&buffer[0],&buffer[0],count, nonce,key);
	    }
	    if (write(fd,buffer,count)!=count) {
	      ret =0;
	      WHY("Failed to write data to file");
	    }
	  }
	  sqlite3_blob_close(blob);
	  blob = NULL;
	}
	if (fd != -1 && close(fd) == -1) {
	  WHY_perror("close");
	  WHYF("Error flushing to %s ", filepath);
	  ret = 0;
	}
      }
      if (blob)
	sqlite3_blob_close(blob);
    }
  }
  sqlite3_finalize(statement);
  return ret;
}

int rhizome_import_from_files(const char *manifestpath,const char *filepath)
{
  rhizome_manifest *m = rhizome_new_manifest();
  if (!m)
    return WHY("Out of manifests.");
  int status = -1;
  if (rhizome_read_manifest_file(m, manifestpath, 0) == -1) {
    status = WHY("could not read manifest file");
  } else if (rhizome_manifest_verify(m) == -1) {
    status = WHY("Could not verify manifest file.");
  } else {
    /* Make sure we store signatures */
    m->manifest_bytes=m->manifest_all_bytes;
    DEBUGF("selfsignedP=%d",
	   rhizome_is_self_signed(m));

    /* Add the manifest and its associated file to the Rhizome database. */
    m->dataFileName = strdup(filepath);
    if (rhizome_manifest_check_file(m))
      status = WHY("file does not belong to manifest");
    else {
      int ret = rhizome_manifest_check_duplicate(m, NULL);
      if (ret == -1)
	status = WHY("rhizome_manifest_check_duplicate() failed");
      else if (ret) {
	INFO("Duplicate found in store");
	status = 1;
      } else if (rhizome_add_manifest(m, 1) == -1) { // ttl = 1
	status = WHY("rhizome_add_manifest() failed");
      } else {
	status = 0;
      }
      if (status != -1) {
	const char *service = rhizome_manifest_get(m, "service", NULL, 0);
	if (service) {
	  cli_puts("service");
	  cli_delim(":");
	  cli_puts(service);
	  cli_delim("\n");
	}
	{
	  cli_puts("manifestid");
	  cli_delim(":");
	  cli_puts(alloca_tohex(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES));
	  cli_delim("\n");
	}
	cli_puts("filesize");
	cli_delim(":");
	cli_printf("%lld", m->fileLength);
	cli_delim("\n");
	if (m->fileLength != 0) {
	  cli_puts("filehash");
	  cli_delim(":");
	  cli_puts(m->fileHexHash);
	  cli_delim("\n");
	}
	const char *name = rhizome_manifest_get(m, "name", NULL, 0);
	if (name) {
	  cli_puts("name");
	  cli_delim(":");
	  cli_puts(name);
	  cli_delim("\n");
	}
      }
    }
  }
  rhizome_manifest_free(m);
  return status;
}