serval-dna/rhizome.h

/*
Serval DNA Rhizome file distribution
Copyright (C) 2016-2018 Flinders University
Copyright (C) 2010-2014 Serval Project Inc.
Copyright (C) 2010 Paul Gardner-Stephen
 
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#ifndef __SERVAL_DNA__RHIZOME_H
#define __SERVAL_DNA__RHIZOME_H

#include "sqlite3.h"
#include "serval_types.h"
#include "rhizome_types.h"
#include "overlay_address.h"
#include "overlay_packet.h"
#include "fdqueue.h"
#include "os.h"
#include "serval_uuid.h"
#include "trigger.h"

#ifndef __RHIZOME_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
#  define __RHIZOME_INLINE extern inline
# else
#  define __RHIZOME_INLINE inline
# endif
#endif

// assumed to always be 2^n
#define RHIZOME_CRYPT_PAGE_SIZE         4096

extern time_ms_t rhizome_voice_timeout;

#define RHIZOME_IDLE_TIMEOUT 20000

#define RHIZOME_BAR_COMPARE_BYTES 31
#define RHIZOME_BAR_TTL_OFFSET 31

#define MAX_MANIFEST_VARS 256
#define MAX_MANIFEST_BYTES 8192
#define MAX_MANIFEST_FIELD_LABEL_LEN 80

typedef struct rhizome_manifest
{

  /* CryptoSign key pair for this manifest.  The public key is the Bundle ID
   * (aka Manifest ID).
   */
  sign_keypair_t keypair;

  /* What do we know about keypair.private_key? (ie, bundle secret is known)
   */
  enum { SECRET_UNKNOWN = 0, EXISTING_BUNDLE_ID, NEW_BUNDLE_ID } haveSecret;

  /* Version of the manifest.  Typically the number of milliseconds since 1970.
   * A value of zero (0) means it has not been set yet.
   */
  uint64_t version;

  /* Payload is described by the offset of its tail (number of missing bytes
   * before the first byte in the payload), its size (number of bytes) and the
   * hash of its content.  Bundle size = tail + filesize.
   */
  uint64_t tail;               // only valid if is_journal
  uint64_t filesize;
  rhizome_filehash_t filehash; // only valid if filesize != 0 and has_filehash

  /* All the manifest fields in original order (the order affects the manifest
   * hash which was used to sign the manifest, so the signature can only be
   * checked if order is preserved).
   *
   * TODO: reduce to only unknown fields.
   *
   * TODO: store all vars and values as NUL terminated strings within
   * manifestdata[], not malloc()/free() heap, to reduce memory fragmentation
   * and allow manifest struct copying without string lifetime issues.
   */
  unsigned short var_count;
  const char *vars[MAX_MANIFEST_VARS];
  const char *values[MAX_MANIFEST_VARS];

  /* Parties who have signed this manifest (binary format, malloc(3)).
   * Recognised signature types:
   *    0x17 = crypto_sign_edwards25519sha512batch()
   */
  unsigned short sig_count;
  unsigned char *signatories[MAX_MANIFEST_VARS];
  uint8_t signatureTypes[MAX_MANIFEST_VARS];

  /* Set to non-NULL if a manifest has been parsed that cannot be fully
   * understood by this version of Rhizome (probably from a future or a very
   * old past version of Rhizome).  During add (local injection), the manifest
   * should not be imported.  During extract (local decode) a warning or error
   * message should be logged.  Manifests marked as malformed are still
   * transported, imported and exported normally, as long as their signature is
   * valid.
   */
  const char *malformed;

  /* Set non-zero after variables have been packed and signature blocks
   * appended.  All fields below may not be valid until the manifest has been
   * finalised.
   */
  bool_t finalised:1;

  /* Whether the manifest contains a signature that corresponds to the manifest
   * id (ie public key).
   */
  bool_t selfSigned:1;

  /* Set if the ID field (keypair.public_key) contains a bundle ID.
   */
  bool_t has_id:1;

  /* Set if the filehash field contains a file hash.
   */
  bool_t has_filehash:1;

  /* Set if the tail field is valid, ie, the bundle is a journal.
   */
  bool_t is_journal:1;

  /* Set if the date field is valid, ie, the manifest contains a valid "date"
   * field.
   */
  bool_t has_date:1;

  /* Set if the bundle_key field is valid, ie, the manifest contains a valid
   * "BK" field.
   */
  bool_t has_bundle_key:1;

  /* Set if the sender and recipient fields are valid, ie, the manifest
   * contains a valid "sender"/"recipient" field.
   */
  bool_t has_sender:1;
  bool_t has_recipient:1;

  /* Local authorship.  Useful for dividing bundle lists between "sent" and
   * "inbox" views.
   */
  enum rhizome_bundle_authorship { 
    ANONYMOUS = 0, // 'author' element is not valid
    AUTHOR_NOT_CHECKED, // 'author' element is valid but not checked
    AUTHENTICATION_ERROR, // author check failed, don't try again
    AUTHOR_UNKNOWN, // author is not a local identity
    AUTHOR_LOCAL, // author is in keyring (unlocked) but not verified
    AUTHOR_IMPOSTOR, // author is a local identity but fails verification
    AUTHOR_AUTHENTIC, // a local identity is the verified author
    AUTHOR_REMOTE // the author of this bundle has signed it, but isn't in our keyring
  } authorship;

  /* Whether the paylaod is encrypted or not */
  enum rhizome_manifest_crypt {
        PAYLOAD_CRYPT_UNKNOWN = 0,
        PAYLOAD_CLEAR,
        PAYLOAD_ENCRYPTED
    } payloadEncryption;
  unsigned char payloadKey[RHIZOME_CRYPT_KEY_BYTES];
  unsigned char payloadNonce[crypto_box_NONCEBYTES];

  /* From the "date" field, if present.  The number of milliseconds since 1970
   * when the bundle was last modified.
   */
  time_ms_t date;

  /* From the "service" field, which should always be present.
   */
  const char *service;

  /* From the optional "name" field.  NULL if there is no "name" field in the
   * manifest.
   */
  const char *name;

  /* Bundle Key "BK" field from the manifest.
   */
  rhizome_bk_t bundle_key;

  /* Sender and recipient fields, if present in the manifest.
   */
  sid_t sender;
  sid_t recipient;

  /* Local data, not encapsulated in the bundle.  The ROWID of the SQLite
   * MANIFESTS table row in which this manifest is stored.  Zero if the
   * manifest has not been stored yet.
   */
  uint64_t rowid;

  /* Local data, not encapsulated in the bundle.  The system time of the most
   * recent INSERT or UPDATE of the manifest into the store.  Zero if the manifest
   * has not been stored yet.
   */
  time_ms_t inserttime;

  /* Local data, not encapsulated in the bundle.  The author of the manifest.
   * A reference to a local keyring entry.  Manifests not authored locally will
   * have an ANY author (all zeros).
   */
  sid_t author;
  const struct keyring_identity *author_identity;

  size_t manifest_body_bytes;
  size_t manifest_all_bytes;
  unsigned char manifestdata[MAX_MANIFEST_BYTES];
  rhizome_filehash_t manifesthash;

} rhizome_manifest;

/* These setter functions (methods) are needed because the relevant attributes
 * are stored in two places: in the vars[] array and in a dedicated struct
 * element.
 *
 * TODO: refactor to remove the redundancy, possibly removing these setter
 * functions as well.
 *
 * @author Andrew Bettison <andrew@servalproject.com>
 */
#define rhizome_manifest_set_id(m,v)            _rhizome_manifest_set_id(__WHENCE__,(m),(v))
#define rhizome_manifest_set_version(m,v)       _rhizome_manifest_set_version(__WHENCE__,(m),(v))
#define rhizome_manifest_del_version(m)         _rhizome_manifest_del_version(__WHENCE__,(m))
#define rhizome_manifest_set_filesize(m,v)      _rhizome_manifest_set_filesize(__WHENCE__,(m),(v))
#define rhizome_manifest_del_filesize(m)        _rhizome_manifest_del_filesize(__WHENCE__,(m))
#define rhizome_manifest_set_filehash(m,v)      _rhizome_manifest_set_filehash(__WHENCE__,(m),(v))
#define rhizome_manifest_del_filehash(m)        _rhizome_manifest_del_filehash(__WHENCE__,(m))
#define rhizome_manifest_set_tail(m,v)          _rhizome_manifest_set_tail(__WHENCE__,(m),(v))
#define rhizome_manifest_set_bundle_key(m,v)    _rhizome_manifest_set_bundle_key(__WHENCE__,(m),(v))
#define rhizome_manifest_del_bundle_key(m)      _rhizome_manifest_del_bundle_key(__WHENCE__,(m))
#define rhizome_manifest_set_service(m,v)       _rhizome_manifest_set_service(__WHENCE__,(m),(v))
#define rhizome_manifest_del_service(m)         _rhizome_manifest_del_service(__WHENCE__,(m))
#define rhizome_manifest_set_name(m,v)          _rhizome_manifest_set_name(__WHENCE__,(m),(v))
#define rhizome_manifest_del_name(m)            _rhizome_manifest_del_name(__WHENCE__,(m))
#define rhizome_manifest_set_date(m,v)          _rhizome_manifest_set_date(__WHENCE__,(m),(v))
#define rhizome_manifest_del_date(m)            _rhizome_manifest_del_date(__WHENCE__,(m))
#define rhizome_manifest_set_sender(m,v)        _rhizome_manifest_set_sender(__WHENCE__,(m),(v))
#define rhizome_manifest_del_sender(m)          _rhizome_manifest_del_sender(__WHENCE__,(m))
#define rhizome_manifest_set_recipient(m,v)     _rhizome_manifest_set_recipient(__WHENCE__,(m),(v))
#define rhizome_manifest_del_recipient(m)       _rhizome_manifest_del_recipient(__WHENCE__,(m))
#define rhizome_manifest_set_crypt(m,v)         _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
#define rhizome_manifest_set_rowid(m,v)         _rhizome_manifest_set_rowid(__WHENCE__,(m),(v))
#define rhizome_manifest_set_inserttime(m,v)    _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v))
#define rhizome_manifest_set_author(m,v)        _rhizome_manifest_set_author(__WHENCE__,(m),NULL,(v))
#define rhizome_manifest_set_author_identity(m,v) _rhizome_manifest_set_author(__WHENCE__,(m),(v),NULL)
#define rhizome_manifest_del_author(m)          _rhizome_manifest_del_author(__WHENCE__,(m))

void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *);
void _rhizome_manifest_set_version(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_del_version(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_filesize(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_del_filesize(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_filehash(struct __sourceloc, rhizome_manifest *, const rhizome_filehash_t *);
void _rhizome_manifest_del_filehash(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_tail(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_bundle_key(struct __sourceloc, rhizome_manifest *, const rhizome_bk_t *);
void _rhizome_manifest_del_bundle_key(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_service(struct __sourceloc, rhizome_manifest *, const char *);
void _rhizome_manifest_del_service(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_name(struct __sourceloc, rhizome_manifest *, const char *);
void _rhizome_manifest_del_name(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_date(struct __sourceloc, rhizome_manifest *, time_ms_t);
void _rhizome_manifest_del_date(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_sender(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_sender(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t);
void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const struct keyring_identity *, const sid_t *);
void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);

#define rhizome_manifest_overwrite(dstm,srcm)   _rhizome_manifest_overwrite(__WHENCE__,(dstm),(srcm))

int _rhizome_manifest_overwrite(struct __sourceloc, rhizome_manifest *m, const rhizome_manifest *srcm);

enum rhizome_manifest_parse_status {
    RHIZOME_MANIFEST_ERROR = -1,          // unrecoverable error while constructing manifest
    RHIZOME_MANIFEST_OK = 0,              // field parsed ok; manifest updated
    RHIZOME_MANIFEST_SYNTAX_ERROR = 1,    // field label violates syntax
    RHIZOME_MANIFEST_DUPLICATE_FIELD = 2, // field is already set in manifest
    RHIZOME_MANIFEST_INVALID = 3,         // core field value does not parse
    RHIZOME_MANIFEST_MALFORMED = 4,       // non-core field value does not parse
    RHIZOME_MANIFEST_OVERFLOW = 5,        // maximum field count exceeded
};

/* This structure represents a manifest field assignment as received by the API
 * operations "add file" or "journal append" or any other operation that takes an
 * existing manifest and modifies it to produce a new one.
 *
 * The 'label' and 'value' strings are pointer-length instead of NUL terminated,
 * to allow them to refer directly to fragments of an existing, larger text
 * without requiring the caller to allocate new strings to hold them.
 */
struct rhizome_manifest_field_assignment {
    const char *label;
    size_t labellen;
    const char *value;
    size_t valuelen;
};

int rhizome_manifest_field_label_is_valid(const char *field_label, size_t field_label_len);
int rhizome_manifest_field_value_is_valid(const char *field_value, size_t field_value_len);
enum rhizome_manifest_parse_status
    rhizome_manifest_parse_field(rhizome_manifest *m,
                                 const char *field_label, size_t field_label_len,
                                 const char *field_value, size_t field_value_len);
int rhizome_manifest_remove_field(rhizome_manifest *, const char *field_label, size_t field_label_len);

/* Supported service identifiers.  These go in the 'service' field of every
 * manifest, and indicate which application must be used to process the bundle
 * after it is received by Rhizome.
 */
#define     RHIZOME_SERVICE_FILE    "file"
#define     RHIZOME_SERVICE_MESHMS  "MeshMS1"
#define     RHIZOME_SERVICE_MESHMS2  "MeshMS2"
#define     RHIZOME_SERVICE_MESHMB  "MeshMB1"
#define     RHIZOME_SERVICE_PRIVATE    "private"

extern int64_t rhizome_space;

int log2ll(uint64_t x);
int rhizome_configure();
int rhizome_enabled();
int rhizome_fetch_delay_ms();

#define RHIZOME_DEFAULT_PATH "rhizome"
#define RHIZOME_BLOB_SUBDIR "blob"
#define RHIZOME_HASH_SUBDIR "hash"

struct rhizome_database {
  char dir_path[1024];
  sqlite3 *db;
  serval_uuid_t uuid;
};

extern __thread struct rhizome_database rhizome_database;

int rhizome_opendb();
int rhizome_close_db();
void verify_bundles();

typedef struct sqlite_retry_state {
  unsigned int limit; // do not retry once elapsed >= limit
  unsigned int sleep; // number of milliseconds to sleep between retries
  unsigned int elapsed; // the total number of milliseconds elapsed doing retries
  time_ms_t start; // the gettime_ms() value just after the current SQL query first returned BUSY
  unsigned int busytries; // the number of times the current SQL query has returned BUSY
}
    sqlite_retry_state;

sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int otherLimit, int otherSleep);

#define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)

struct rhizome_cleanup_report {
    unsigned deleted_stale_incoming_files;
    unsigned deleted_expired_files;
    unsigned deleted_orphan_files;
    unsigned deleted_orphan_fileblobs;
    unsigned deleted_orphan_manifests;
};

int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_store_cleanup(struct rhizome_cleanup_report *report);
void rhizome_vacuum_db(sqlite_retry_state *retry);
int rhizome_manifest_createid(rhizome_manifest *m);
struct rhizome_bundle_result rhizome_private_bundle(rhizome_manifest *m, const sign_keypair_t *keypair);
void rhizome_new_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk);

struct rhizome_manifest_summary {
  rhizome_bid_t bid;
  uint64_t version;
  size_t body_len;
};

int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ);

enum rhizome_bundle_status {
    RHIZOME_BUNDLE_STATUS_ERROR = -1,
    RHIZOME_BUNDLE_STATUS_NEW = 0, // bundle is newer than store
    RHIZOME_BUNDLE_STATUS_SAME = 1, // same version already in store
    RHIZOME_BUNDLE_STATUS_DUPLICATE = 2, // equivalent bundle already in store
    RHIZOME_BUNDLE_STATUS_OLD = 3, // newer version already in store
    RHIZOME_BUNDLE_STATUS_INVALID = 4, // manifest is invalid
    RHIZOME_BUNDLE_STATUS_FAKE = 5, // manifest signature not valid
    RHIZOME_BUNDLE_STATUS_INCONSISTENT = 6, // manifest filesize/filehash does not match supplied payload
    RHIZOME_BUNDLE_STATUS_NO_ROOM = 7, // doesn't fit; store may contain more important bundles
    RHIZOME_BUNDLE_STATUS_READONLY = 8, // cannot modify manifest; secret unknown
    RHIZOME_BUNDLE_STATUS_BUSY = 9, // the database is currently busy
    RHIZOME_BUNDLE_STATUS_MANIFEST_TOO_BIG = 10, // manifest + signature exceeds size limit
};

// Useful for initialising a variable then checking later that it was set to a
// valid value (typically FATAL if not).
#define INVALID_RHIZOME_BUNDLE_STATUS ((enum rhizome_bundle_status)-2)

const char *rhizome_bundle_status_message(enum rhizome_bundle_status);
const char *rhizome_bundle_status_message_nonnull(enum rhizome_bundle_status);

// Encapsulate a status enum value and an optional text message to assist with
// diagnostics.  Useful as a return value from functions that can fail in all
// sorts of ways.
struct rhizome_bundle_result {
    enum rhizome_bundle_status status;
    const char *message;
    void (*release)(void *); // call r.release(r.message) before destroying r
};

#define INVALID_RHIZOME_BUNDLE_RESULT ((struct rhizome_bundle_result){ .status = INVALID_RHIZOME_BUNDLE_STATUS, .message = NULL, .release = NULL })

// Call this before discarding a struct rhizome_bundle_result.
void rhizome_bundle_result_free(struct rhizome_bundle_result *);

// Convenience functions for constructing a struct rhizome_bundle_result and
// logging errors and debug messages in the process.
struct rhizome_bundle_result _rhizome_bundle_result(struct __sourceloc, enum rhizome_bundle_status);
struct rhizome_bundle_result _rhizome_bundle_result_static(struct __sourceloc, enum rhizome_bundle_status, const char *);
struct rhizome_bundle_result _rhizome_bundle_result_strdup(struct __sourceloc, enum rhizome_bundle_status, const char *);
struct rhizome_bundle_result _rhizome_bundle_result_sprintf(struct __sourceloc, enum rhizome_bundle_status, const char *fmt, ...);

#define rhizome_bundle_result(status)                   _rhizome_bundle_result(__WHENCE__, status)
#define rhizome_bundle_result_static(status, str)       _rhizome_bundle_result_static(__WHENCE__, status, str)
#define rhizome_bundle_result_strdup(status, str)       _rhizome_bundle_result_strdup(__WHENCE__, status, str)
#define rhizome_bundle_result_sprintf(status, fmt, ...) _rhizome_bundle_result_sprintf(__WHENCE__, status, fmt, ## __VA_ARGS__);

// Functions for extracting information from a struct rhizome_bundle_result.
const char *rhizome_bundle_result_message(struct rhizome_bundle_result); // NULL only if invalid
const char *rhizome_bundle_result_message_nonnull(struct rhizome_bundle_result);

// Function to assist logging struct rhizome_bundle_result.
#define alloca_rhizome_bundle_result(result)            strbuf_str(strbuf_append_rhizome_bundle_result(strbuf_alloca(50 + (result.message ? strlen(result.message) : 0)), result))
strbuf strbuf_append_rhizome_bundle_result(strbuf, struct rhizome_bundle_result);

enum rhizome_payload_status {
    RHIZOME_PAYLOAD_STATUS_ERROR = -1,
    RHIZOME_PAYLOAD_STATUS_EMPTY = 0, // payload is empty (zero length)
    RHIZOME_PAYLOAD_STATUS_NEW = 1, // payload is not yet in store (added)
    RHIZOME_PAYLOAD_STATUS_STORED = 2, // payload is already in store
    RHIZOME_PAYLOAD_STATUS_WRONG_SIZE = 3, // payload's size does not match manifest
    RHIZOME_PAYLOAD_STATUS_WRONG_HASH = 4, // payload's hash does not match manifest
    RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL = 5, // cannot encrypt/decrypt (payload key unknown)
    RHIZOME_PAYLOAD_STATUS_TOO_BIG = 6, // payload will never fit in our store
    RHIZOME_PAYLOAD_STATUS_EVICTED = 7, // other payloads in our store are more important
    RHIZOME_PAYLOAD_STATUS_BUSY = 8, // storage is currently busy, you may try again at a later time
};

// Useful for initialising a variable then checking later that it was set to a
// valid value (typically FATAL if not).
#define INVALID_RHIZOME_PAYLOAD_STATUS ((enum rhizome_payload_status)-2)

const char *rhizome_payload_status_message(enum rhizome_payload_status);
const char *rhizome_payload_status_message_nonnull(enum rhizome_payload_status);

int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_validate(rhizome_manifest *m);
const char *rhizome_manifest_validate_reason(rhizome_manifest *m);
int rhizome_manifest_parse(rhizome_manifest *m);
int rhizome_manifest_verify(rhizome_manifest *m);

void _rhizome_manifest_free(struct __sourceloc, rhizome_manifest *m);
#define rhizome_manifest_free(m) _rhizome_manifest_free(__WHENCE__,m)
rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)

int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);

int rhizome_add_field_assignment(struct rhizome_manifest_field_assignment *field, const char *arg, size_t arglen);
int rhizome_parse_field_assignments(struct rhizome_manifest_field_assignment *fields, unsigned argc, const char *const *args);
struct rhizome_bundle_result rhizome_apply_assignments(rhizome_manifest *m,
  unsigned nassignments, const struct rhizome_manifest_field_assignment *assignments);

struct rhizome_bundle_result rhizome_manifest_add_file(int appending,
                                                       rhizome_manifest *m,
                                                       rhizome_manifest **mout,
                                                       const rhizome_bid_t *bid,
                                                       const rhizome_bk_t *bsk,
                                                       const sid_t *author,
                                                       const char *file_path,
                                                       unsigned nassignments,
                                                       const struct rhizome_manifest_field_assignment *assignments);
int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath, int zip_files);

int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath);
struct rhizome_bundle_result rhizome_fill_manifest(rhizome_manifest *m, const char *filepath);

int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *);
int rhizome_manifest_add_bundle_key(rhizome_manifest *);

int rhizome_lookup_author(rhizome_manifest *m);
void rhizome_authenticate_author(rhizome_manifest *m);

struct rhizome_bundle_result rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **m_out, int deduplicate);
enum rhizome_bundle_status rhizome_manifest_check_stored(rhizome_manifest *m, rhizome_manifest **m_out);
enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m_in, rhizome_manifest **m_out);

void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount);
int rhizome_find_privatekey(rhizome_manifest *m);

__RHIZOME_INLINE int sqlite_code_ok(int code)
{
  return code == SQLITE_OK || code == SQLITE_DONE || code == SQLITE_ROW;
}

__RHIZOME_INLINE int sqlite_code_busy(int code)
{
  return code == SQLITE_BUSY || code == SQLITE_LOCKED;
}

int (*sqlite_set_tracefunc(int (*newfunc)()))();
int is_debug_rhizome();
int is_debug_rhizome_ads();

enum sqlbind_type {
  INT = 1,	      // int value
  INT_TOSTR,	      // int value
  UINT_TOSTR,	      // unsigned value
  INT64,	      // int64_t value
  INT64_TOSTR,	      // int64_t value
  UINT64_TOSTR,	      // uint64_t value
  TEXT,	              // const char *text,
  TEXT_LEN,           // const char *text, int bytes
  STATIC_TEXT,	      // const char *text,
  STATIC_TEXT_LEN,    // const char *text, int bytes
  STATIC_BLOB,	      // const void *blob, int bytes
  ZEROBLOB,	      // int bytes
  SID_T,	      // const sid_t *sidp
  RHIZOME_BID_T,      // const rhizome_bid_t *bidp
  RHIZOME_BAR_T,      // const rhizome_bar_t *barp
  RHIZOME_FILEHASH_T, // const rhizome_filehash_t *hashp
  TOHEX,              // const unsigned char *binary, unsigned bytes
  TEXT_TOUPPER,       // const char *text,
  TEXT_LEN_TOUPPER,   // const char *text, unsigned bytes
  SERVAL_UUID_T,      // const serval_uuid_t *uuidp
  NUL = 1 << 8,      // NUL (no arg) ; NUL|INT, ...
  INDEX = 1 << 9, // INDEX|INT, int index, ...
  NAMED = 1 << 10,  // NAMED|INT, const char *label, ...
  END = 1 << 11,
};

sqlite3_stmt *_sqlite_prepare(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext);
int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...);
int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap);
sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_code(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, int *rowcount);
int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, int log_level, const char *sqltext, ...);
int _sqlite_exec_void_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
int _sqlite_exec_changes_retry(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, int *rowcount, int *changes, const char *sqltext, ...);
int _sqlite_exec_uint64_retry(struct __sourceloc, sqlite_retry_state *retry, uint64_t *result, const char *sqltext, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqltext, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap);
int _sqlite_blob_open_retry(
  struct __sourceloc,
  int log_level,
  sqlite_retry_state *retry,
  const char *dbname,
  const char *tablename,
  const char *colname,
  sqlite3_int64 rowid,
  int flags,
  sqlite3_blob **blobp
);
int _sqlite_blob_write_retry(
  struct __sourceloc,
  int log_level,
  sqlite_retry_state *retry,
  sqlite3_blob *blob,
  const void *buf,
  int len,
  int offset
);
int _sqlite_blob_close(struct __sourceloc, int log_level, sqlite3_blob *blob);

// The 'arg' arguments in the following macros appear to be unnecessary, but
// they serve a very useful purpose, so don't remove them!  They ensure that
// programmers do not forget the bind args, of which there must be at least
// one, even if it is only 'END' to make no bindings at all.
#define sqlite_prepare(rs,sql)                          _sqlite_prepare(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql))
#define sqlite_prepare_loglevel(ll,rs,sql)              _sqlite_prepare(__WHENCE__, (ll), (rs), (sql))
#define sqlite_prepare_bind(rs,sql,arg,...)             _sqlite_prepare_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_prepare_bind_loglevel(ll,rs,sql,arg,...) _sqlite_prepare_bind(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_bind(rs,stmt,arg,...)                    _sqlite_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt), arg, ##__VA_ARGS__)
#define sqlite_bind_loglevel(ll,rs,stmt,arg,...)        _sqlite_bind(__WHENCE__, (ll), (rs), (stmt), arg, ##__VA_ARGS__)
#define sqlite_retry(rs,action)                         _sqlite_retry(__WHENCE__, (rs), (action))
#define sqlite_retry_done(rs,action)                    _sqlite_retry_done(__WHENCE__, (rs), (action))
#define sqlite_exec(stmt)                               _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt))
#define sqlite_exec_retry(rs,stmt)                      _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt))
#define sqlite_exec_retry_loglevel(ll,rs,stmt)          _sqlite_exec(__WHENCE__, (ll), (rs), (stmt))
#define sqlite_exec_code(stmt, rowcount)                _sqlite_exec_code(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt), (rowcount))
#define sqlite_exec_code_retry(rs, stmt, rowcount)      _sqlite_exec_code(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt), (rowcount))
#define sqlite_step(stmt)                               _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt))
#define sqlite_step_retry(rs,stmt)                      _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt))
#define sqlite_exec_void(sql,arg,...)                   _sqlite_exec_void(__WHENCE__, LOG_LEVEL_ERROR, (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,sql,arg,...)       _sqlite_exec_void(__WHENCE__, (ll), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,sql,arg,...)          _sqlite_exec_void_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_changes_retry(rs,ROWS,CHANGES,sql,arg,...) _sqlite_exec_changes_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (ROWS), (CHANGES), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_changes_retry_loglevel(ll,rs,ROWS,CHANGES,sql,arg,...) _sqlite_exec_changes_retry(__WHENCE__, (ll), (rs), (ROWS), (CHANGES), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,sql,arg,...) _sqlite_exec_void_retry(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_uint64_retry(rs,res,sql,arg,...)    _sqlite_exec_uint64_retry(__WHENCE__, (rs), (res), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,sql,arg,...)              _sqlite_exec_strbuf(__WHENCE__, (sb), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...)     _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__)
#define sqlite_blob_open_retry(rs,db,table,col,row,flags,blobp) \
                                                        _sqlite_blob_open_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (db), (table), (col), (row), (flags), (blobp))
#define sqlite_blob_close(blob)                         _sqlite_blob_close(__WHENCE__, LOG_LEVEL_ERROR, (blob));
#define sqlite_blob_write_retry(rs,blob,buf,siz,off)    _sqlite_blob_write_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (blob), (buf), (siz), (off))

double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
int rhizome_manifest_to_bar(rhizome_manifest *m, rhizome_bar_t *bar);
enum rhizome_bundle_status rhizome_is_bar_interesting(const rhizome_bar_t *bar);
enum rhizome_bundle_status rhizome_is_interesting(const rhizome_bid_t *bid, uint64_t version, uint64_t *filesizep);
#define rhizome_is_manifest_interesting(M) rhizome_is_interesting(&(M)->keypair.public_key, (M)->version, NULL)
enum rhizome_bundle_status rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m);
enum rhizome_bundle_status rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m);
enum rhizome_bundle_status rhizome_retrieve_manifest_by_hash_prefix(const uint8_t *prefix, unsigned prefix_len, rhizome_manifest *m);
enum rhizome_bundle_status rhizome_retrieve_bar_by_hash_prefix(const uint8_t *prefix, unsigned prefix_len, rhizome_bar_t *bar);
int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
int rhizome_delete_bundle(const rhizome_bid_t *bidp);
int rhizome_delete_manifest(const rhizome_bid_t *bidp);
int rhizome_delete_payload(const rhizome_bid_t *bidp);
int rhizome_delete_file(const rhizome_filehash_t *hashp);

#define RHIZOME_DONTVERIFY 0
#define RHIZOME_VERIFY 1

int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);

int rhizome_queue_ignore_manifest(const unsigned char *bid_prefix, int prefix_len, int timeout);
int rhizome_ignore_manifest_check(const unsigned char *bid_prefix, int prefix_len);

/* Rhizome list cursor for iterating over all or a subset of manifests in the store.
 */
struct rhizome_list_cursor {
  // Query parameters that narrow the set of listed bundles.
  const char *service;
  const char *name;
  bool_t is_sender_set;
  bool_t is_recipient_set;
  bool_t oldest_first;
  sid_t sender;
  sid_t recipient;
  // If set, then the cursor moves in ascending (chronological) order starting
  // from the first bundle with rowid > rowid_since.  If zero, then the cursor
  // moves in descending (reverse chronological) order starting from the most
  // recent bundle.
  uint64_t rowid_since;
  // Set by calling the next() function.
  rhizome_manifest *manifest;
  // Private state - implementation that could change.
  sqlite_retry_state _retry;
  sqlite3_stmt *_statement;
  uint64_t _rowid_current;
  uint64_t _rowid_last; // for re-opening query
};

int rhizome_list_open(struct rhizome_list_cursor *);
int rhizome_list_next(struct rhizome_list_cursor *);
void rhizome_list_commit(struct rhizome_list_cursor *);
void rhizome_list_release(struct rhizome_list_cursor *);

#define MAX_CANDIDATES 32

int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct socket_address *addr, const struct subscriber *peer);
rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_length);
int rhizome_fetch_bar_queued(const rhizome_bar_t *bar);

/* Rhizome file storage api */
struct rhizome_write_buffer
{
  struct rhizome_write_buffer *_next;
  uint64_t offset;
  size_t buffer_size;
  size_t data_size;
  unsigned char data[0];
};

struct rhizome_write
{
  uint64_t temp_id;
  uint64_t tail;
  uint64_t file_offset;
  uint64_t written_offset;
  uint64_t file_length;
  struct rhizome_write_buffer *buffer_list;
  size_t buffer_size;
  
  struct crypto_hash_sha512_state sha512_context;
  uint64_t blob_rowid;
  int blob_fd;
  sqlite3_blob *sql_blob;
  
  rhizome_filehash_t id;
  uint8_t id_known:1;
  uint8_t crypt:1;
  uint8_t journal:1;

  unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
  unsigned char nonce[crypto_box_NONCEBYTES];
};

struct rhizome_read_buffer{
  uint64_t offset;
  unsigned char data[RHIZOME_CRYPT_PAGE_SIZE];
  size_t len;
};

struct rhizome_read
{
  uint64_t hash_offset;
  struct crypto_hash_sha512_state sha512_context;
  
  uint64_t blob_rowid;
  int blob_fd;
  
  uint64_t tail;
  uint64_t offset;
  uint64_t length;
  
  int8_t verified;
  uint8_t crypt;
  rhizome_filehash_t id;
  unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
  unsigned char nonce[crypto_box_NONCEBYTES];
};

int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, 
			     uint64_t offset, size_t count,unsigned char *bytes);

int is_rhizome_enabled();
int is_rhizome_mdp_enabled();
int is_rhizome_http_enabled();
int is_rhizome_advertise_enabled();
int is_rhizome_mdp_server_running();

typedef struct rhizome_direct_bundle_cursor {
  /* Where the current fill started */
  int64_t start_size_high;
  rhizome_bid_t start_bid_low;

  /* Limit of where this cursor may traverse */
  int64_t limit_size_high;
  rhizome_bid_t limit_bid_high;

  int64_t size_low;
  int64_t size_high;
  rhizome_bid_t bid_low;
  rhizome_bid_t bid_high;
  unsigned char *buffer;
  size_t buffer_size;
  size_t buffer_used;
  size_t buffer_offset_bytes;
} rhizome_direct_bundle_cursor;

rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size);
void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r);
int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r,
						unsigned char *pickled,
						int pickle_buffer_size);
int rhizome_direct_bundle_iterator_unpickle_range(rhizome_direct_bundle_cursor *r,
						  const unsigned char *pickled,
						  int pickle_buffer_size);
int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,
					int max_bars);
void rhizome_direct_bundle_iterator_free(rhizome_direct_bundle_cursor **c);
int rhizome_direct_get_bars(const rhizome_bid_t *bid_low,
			    rhizome_bid_t *bid_high,
			    int64_t size_low, int64_t size_high,
			    const rhizome_bid_t *bid_max,
			    unsigned char *bars_out,
			    int bars_requested);

typedef struct rhizome_direct_sync_request {
  struct sched_ent alarm;
  rhizome_direct_bundle_cursor *cursor;

  int pushP;
  int pullP;

  /* Sync interval in seconds.  zero = sync only once */
  int interval;

  /* The dispatch function will be called each time a sync request can
     be sent off, i.e., one cursor->buffer full of data.
     Will differ based on underlying transport. HTTP is the initial 
     supported transport, but deLorme inReach will likely follow soon after.
  */
  void (*dispatch_function)(struct rhizome_direct_sync_request *);

  /* General purpose pointer for transport-dependent state */
  void *transport_specific_state;

  /* Statistics.
     Each sync will consist of one or more "fills" of the cursor buffer, which 
     will then be dispatched by the transport-specific dispatch function.
     Each of those dispatches may then result in zero or 
   */
  int syncs_started;
  int syncs_completed;
  int fills_sent;
  int fill_responses_processed;
  int bundles_pushed;
  int bundles_pulled;
  int bundle_transfers_in_progress;

} rhizome_direct_sync_request;

#define RHIZOME_DIRECT_MAX_SYNC_HANDLES 16
extern rhizome_direct_sync_request *rd_sync_handles[RHIZOME_DIRECT_MAX_SYNC_HANDLES];
extern int rd_sync_handle_count;

rhizome_direct_sync_request
*rhizome_direct_new_sync_request(void (*transport_specific_dispatch_function)(struct rhizome_direct_sync_request *),
				 size_t buffer_size,
                                 int interval,
                                 int mode, 
				 void *transport_specific_state);
int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r);
int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r);
rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *buffer,int size,int max_response_bytes);

typedef struct rhizome_direct_transport_state_http {
  int port;
  char host[1024];  
} rhizome_direct_transport_state_http;

void rhizome_direct_http_dispatch(rhizome_direct_sync_request *);

extern unsigned char favicon_bytes[];
extern int favicon_len;

int rhizome_import_from_files(const char *manifestpath,const char *filepath);

enum rhizome_start_fetch_result {
  STARTED = 0,
  SAMEBUNDLE,
  SAMEPAYLOAD,
  SUPERSEDED,
  OLDERBUNDLE,
  NEWERBUNDLE,
  DONOTWANT,
  IMPORTED,
  SLOTBUSY
};

enum rhizome_start_fetch_result
rhizome_fetch_request_manifest_by_prefix(const struct socket_address *addr, 
					 const struct subscriber *peer,
					 const unsigned char *prefix, size_t prefix_length);
int rhizome_any_fetch_active();
int rhizome_any_fetch_queued();
int rhizome_fetch_status_html(struct strbuf *b);
int rhizome_fetch_has_queue_space(unsigned char log2_size);

/* Rhizome storage methods */

enum rhizome_payload_status rhizome_exists(const rhizome_filehash_t *hashp);
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length);
int rhizome_write_buffer(struct rhizome_write *write_state, uint8_t *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, uint8_t *buffer, size_t data_size);
enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t append_size);
int rhizome_write_file(struct rhizome_write *write, const char *filename, off_t offset, uint64_t length);
void rhizome_fail_write(struct rhizome_write *write);
int is_rhizome_write_open(const struct rhizome_write *write);
enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write);
enum rhizome_payload_status rhizome_finish_store(struct rhizome_write *write, rhizome_manifest *m, enum rhizome_payload_status status);
enum rhizome_payload_status rhizome_import_payload_from_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_import_buffer(rhizome_manifest *m, uint8_t *buffer, size_t length);
enum rhizome_payload_status rhizome_stat_payload_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, const char *filepath);
int rhizome_derive_payload_key(rhizome_manifest *m);

enum rhizome_payload_status rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, uint8_t *buffer, size_t len);
enum rhizome_payload_status rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename);
enum rhizome_payload_status rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);

int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset, 
			    const unsigned char *key, const unsigned char *nonce);
enum rhizome_payload_status rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
ssize_t rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
void rhizome_read_close(struct rhizome_read *read);
enum rhizome_payload_status rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state);
enum rhizome_payload_status rhizome_extract_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, uint64_t *lengthp);
ssize_t rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, 
                            uint64_t fileOffset, unsigned char *buffer, size_t length);
int rhizome_cache_close();

int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);

void rhizome_sync_status();

DECLARE_ALARM(rhizome_fetch_status);

/* Rhizome triggers */

DECLARE_TRIGGER(bundle_add, rhizome_manifest*);

#endif //__SERVAL_DNA__RHIZOME_H