From 7ff89afcf4aabdb7e889d11fcc62ae9ca8c5c238 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Wed, 18 Jun 2014 17:25:15 +0930
Subject: [PATCH] Reinstate rhizome database storage limit

- old / large payloads should be evicted to fit more payloads
- if there isn't enough space, new payloads will not be added
---
 commandline.c         |   6 +
 conf_schema.h         |   2 +-
 meshms.c              |  32 ++--
 rhizome.c             |   4 +
 rhizome.h             |  23 +--
 rhizome_database.c    | 433 +++++++-----------------------------------
 rhizome_direct_http.c |   3 +
 rhizome_fetch.c       |   7 +-
 rhizome_restful.c     |   4 +
 rhizome_store.c       | 205 +++++++++++++++++++-
 tests/rhizomeops      |  51 ++++-
 11 files changed, 362 insertions(+), 408 deletions(-)

diff --git a/commandline.c b/commandline.c
index 05619389..24841c83 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1760,6 +1760,11 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     case RHIZOME_PAYLOAD_STATUS_STORED:
     case RHIZOME_PAYLOAD_STATUS_NEW:
       break;
+    case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+    case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
+      status = RHIZOME_BUNDLE_STATUS_DONOTWANT;
+      WHY("Insufficient space to store payload");
+      break;
     case RHIZOME_PAYLOAD_STATUS_ERROR:
       status = RHIZOME_BUNDLE_STATUS_ERROR;
       break;
@@ -1804,6 +1809,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     case RHIZOME_BUNDLE_STATUS_ERROR:
     case RHIZOME_BUNDLE_STATUS_INVALID:
     case RHIZOME_BUNDLE_STATUS_FAKE:
+    case RHIZOME_BUNDLE_STATUS_DONOTWANT:
       break;
     default:
       FATALF("status=%d", status);
diff --git a/conf_schema.h b/conf_schema.h
index 0c8ea2a0..150b8313 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -429,7 +429,7 @@ ATOM(bool_t,                fetch,          1, boolean,, "If false, no new bundl
 ATOM(bool_t,                clean_on_open,  0, boolean,, "If true, Rhizome database is cleaned at start of every command")
 ATOM(bool_t,                clean_on_start, 1, boolean,, "If true, Rhizome database is cleaned at start of daemon")
 STRING(256,                 datastore_path, "", str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory")
-ATOM(uint64_t,              database_size,  1000000, uint64_scaled,, "Size of database in bytes")
+ATOM(uint64_t,              database_size,  0, uint64_scaled,, "Maximum size of database in bytes")
 ATOM(uint32_t,              max_blob_size,  128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
 
 ATOM(uint64_t,              rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")
diff --git a/meshms.c b/meshms.c
index 5891bbdb..140176c4 100644
--- a/meshms.c
+++ b/meshms.c
@@ -334,7 +334,6 @@ static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshm
       case 0:
 	break;
       case -1:
-	status = MESHMS_STATUS_ERROR;
 	goto end;
       default:
 	status = MESHMS_STATUS_PROTOCOL_FAULT;
@@ -346,16 +345,14 @@ static enum meshms_status append_meshms_buffer(const sid_t *my_sid, struct meshm
       goto end;
     }
   } else if (create_ply(my_sid, conv, m) == -1) {
-    status = MESHMS_STATUS_ERROR;
     goto end;
   }
   assert(m->haveSecret);
   assert(m->authorship == AUTHOR_AUTHENTIC);
   enum rhizome_payload_status pstatus = rhizome_append_journal_buffer(m, 0, buffer, len);
-  if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW) {
-    status = MESHMS_STATUS_ERROR;
+  if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
     goto end;
-  }
+  
   enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1);
   if (config.debug.meshms)
     DEBUGF("bstatus=%d", bstatus);
@@ -664,17 +661,20 @@ static enum meshms_status write_known_conversations(rhizome_manifest *m, struct
   rhizome_manifest_set_filehash(m, NULL);
 
   enum rhizome_payload_status pstatus = rhizome_write_open_manifest(&write, m);
-  if (pstatus == RHIZOME_PAYLOAD_STATUS_NEW) {
-    unsigned char version=1;
-    if (rhizome_write_buffer(&write, &version, 1) == -1)
-      goto end;
-    if (write_conversation(&write, conv) == -1)
-      goto end;
-    pstatus = rhizome_finish_write(&write);
-    if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
-      goto end;
-    rhizome_manifest_set_filehash(m, &write.id);
-  }
+  if (pstatus!=RHIZOME_PAYLOAD_STATUS_NEW)
+    // TODO log something?
+    goto end;
+  
+  unsigned char version=1;
+  if (rhizome_write_buffer(&write, &version, 1) == -1)
+    goto end;
+  if (write_conversation(&write, conv) == -1)
+    goto end;
+  pstatus = rhizome_finish_write(&write);
+  if (pstatus != RHIZOME_PAYLOAD_STATUS_NEW)
+    goto end;
+  rhizome_manifest_set_filehash(m, &write.id);
+  
   enum rhizome_bundle_status bstatus = rhizome_manifest_finalise(m, &mout, 1);
   switch (bstatus) {
     case RHIZOME_BUNDLE_STATUS_ERROR:
diff --git a/rhizome.c b/rhizome.c
index f40de05d..85105b4b 100644
--- a/rhizome.c
+++ b/rhizome.c
@@ -167,6 +167,10 @@ enum rhizome_bundle_status rhizome_bundle_import_files(rhizome_manifest *m, rhiz
 	if (rhizome_store_manifest(m) == -1)
 	  return -1;
 	break;
+      case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+      case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
+	status = RHIZOME_BUNDLE_STATUS_DONOTWANT;
+	break;
       case RHIZOME_PAYLOAD_STATUS_ERROR:
       case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
 	return -1;
diff --git a/rhizome.h b/rhizome.h
index 6ba7ed3c..3911f2d5 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -45,14 +45,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 extern time_ms_t rhizome_voice_timeout;
 
-#define RHIZOME_PRIORITY_HIGHEST RHIZOME_PRIORITY_SERVAL_CORE
-#define RHIZOME_PRIORITY_SERVAL_CORE 5
-#define RHIZOME_PRIORITY_SUBSCRIBED 4
-#define RHIZOME_PRIORITY_SERVAL_OPTIONAL 3
-#define RHIZOME_PRIORITY_DEFAULT 2
-#define RHIZOME_PRIORITY_SERVAL_BULK 1
-#define RHIZOME_PRIORITY_NOTINTERESTED 0
-
 #define RHIZOME_IDLE_TIMEOUT 20000
 
 typedef struct rhizome_signature {
@@ -182,8 +174,6 @@ typedef struct rhizome_manifest
     AUTHOR_AUTHENTIC // a local identity is the verified author
   } authorship;
 
-  int fileHighestPriority;
-
   /* Absolute path of the file associated with the manifest */
   const char *dataFileName;
 
@@ -370,6 +360,7 @@ enum rhizome_bundle_status {
     RHIZOME_BUNDLE_STATUS_INVALID = 4, // manifest is invalid
     RHIZOME_BUNDLE_STATUS_FAKE = 5, // manifest signature not valid
     RHIZOME_BUNDLE_STATUS_INCONSISTENT = 6, // manifest filesize/filehash does not match supplied payload
+    RHIZOME_BUNDLE_STATUS_DONOTWANT = 7, // Wont fit or we already have more important bundles
 };
 
 enum rhizome_payload_status {
@@ -380,12 +371,12 @@ enum rhizome_payload_status {
     RHIZOME_PAYLOAD_STATUS_WRONG_SIZE = 3, // payload's size does not match manifest
     RHIZOME_PAYLOAD_STATUS_WRONG_HASH = 4, // payload's hash does not match manifest
     RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL = 5, // cannot encrypt/decrypt (payload key unknown)
+    RHIZOME_PAYLOAD_STATUS_TOO_BIG = 6, // payload will never fit in our store
+    RHIZOME_PAYLOAD_STATUS_UNINITERESTING = 7, // other payloads in our store are more interesting
 };
 
 int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
 int rhizome_manifest_selfsign(rhizome_manifest *m);
-int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
-int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
 int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename);
 int rhizome_manifest_validate(rhizome_manifest *m);
 int rhizome_manifest_parse(rhizome_manifest *m);
@@ -399,7 +390,6 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
 #define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
 
 int rhizome_store_manifest(rhizome_manifest *m);
-int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
 int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
 int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath);
 
@@ -469,6 +459,7 @@ sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc, int log_level, sqlite_ret
 int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
 void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
 int _sqlite_step(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
+int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
 int _sqlite_exec_void(struct __sourceloc, int log_level, const char *sqltext, ...);
 int _sqlite_exec_void_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
 int _sqlite_exec_uint64(struct __sourceloc, uint64_t *result, const char *sqltext, ...);
@@ -530,7 +521,6 @@ int _sqlite_blob_close(struct __sourceloc, int log_level, sqlite3_blob *blob);
 
 double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
 int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
-int rhizome_update_file_priority(const char *fileid);
 enum rhizome_bundle_status rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
 int rhizome_manifest_to_bar(rhizome_manifest *m, rhizome_bar_t *bar);
 int rhizome_is_bar_interesting(const rhizome_bar_t *bar);
@@ -541,6 +531,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
 int rhizome_delete_bundle(const rhizome_bid_t *bidp);
 int rhizome_delete_manifest(const rhizome_bid_t *bidp);
 int rhizome_delete_payload(const rhizome_bid_t *bidp);
+int rhizome_delete_file_id(const char *id);
 int rhizome_delete_file(const rhizome_filehash_t *hashp);
 
 #define RHIZOME_DONTVERIFY 0
@@ -633,7 +624,6 @@ struct rhizome_write
   rhizome_filehash_t id;
   uint64_t temp_id;
   char id_known;
-  int priority;
   uint64_t tail;
   uint64_t file_offset;
   uint64_t written_offset;
@@ -793,6 +783,7 @@ enum rhizome_start_fetch_result {
   SUPERSEDED,
   OLDERBUNDLE,
   NEWERBUNDLE,
+  DONOTWANT,
   IMPORTED,
   SLOTBUSY
 };
@@ -809,7 +800,7 @@ int rhizome_fetch_has_queue_space(unsigned char log2_size);
 /* rhizome storage methods */
 
 int rhizome_exists(const rhizome_filehash_t *hashp);
-enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
+enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length);
 int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
 int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size);
 enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
diff --git a/rhizome_database.c b/rhizome_database.c
index 8e34551c..9792a159 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -32,8 +32,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "server.h"
 
 static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
-static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
-static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
 
 static int create_rhizome_store_dir()
 {
@@ -49,23 +47,6 @@ static int create_rhizome_store_dir()
 sqlite3 *rhizome_db = NULL;
 serval_uuid_t rhizome_db_uuid;
 
-/* XXX Requires a messy join that might be slow. */
-int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
-{
-  uint64_t result = 0;
-  if (sqlite_exec_uint64_retry(retry, &result,
-	"SELECT max(grouplist.priorty) FROM GROUPLIST,MANIFESTS,GROUPMEMBERSHIPS"
-	" WHERE MANIFESTS.id = ?"
-	"   AND GROUPLIST.id = GROUPMEMBERSHIPS.groupid"
-	"   AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id;",
-	RHIZOME_BID_T, bidp,
-	END
-      ) == -1
-  )
-    return -1;
-  return (int) result;
-}
-
 int is_debug_rhizome()
 {
   return config.debug.rhizome;
@@ -226,6 +207,11 @@ int rhizome_opendb()
   
   if (!FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db"))
     RETURN(-1);
+  
+  struct file_meta meta;
+  if (get_file_meta(dbpath, &meta) == -1)
+    RETURN(-1);
+  
   if (sqlite3_open(dbpath,&rhizome_db)){
     RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
   }
@@ -246,13 +232,38 @@ int rhizome_opendb()
   if (version<1){
     /* Create tables as required */
     sqlite_exec_void_loglevel(loglevel, "PRAGMA auto_vacuum=2;", END);
-    if (	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPLIST(id text not null primary key, closed integer,ciphered integer,priority integer);", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, version integer,inserttime integer, filesize integer, filehash text, author text, bar blob, manifest blob);", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILES(id text not null primary key, length integer, highestpriority integer, datavalid integer, inserttime integer);", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS FILEBLOBS(id text not null primary key, data blob);", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "DROP TABLE IF EXISTS FILEMANIFESTS;", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS GROUPMEMBERSHIPS(manifestid text not null, groupid text not null);", END) == -1
-      ||	sqlite_exec_void_retry(&retry, "CREATE TABLE IF NOT EXISTS VERIFICATIONS(sid text not null, did text, name text, starttime integer, endtime integer, signature blob);", END) == -1
+    if (	sqlite_exec_void_retry(&retry, 
+		  "CREATE TABLE IF NOT EXISTS MANIFESTS("
+		      "id text not null primary key, "
+		      "version integer, "
+		      "inserttime integer, "
+		      "filesize integer, "
+		      "filehash text, "
+		      "author text, "
+		      "bar blob, "
+		      "manifest blob, "
+		      "service text, "
+		      "name text, "
+		      "sender text collate nocase, "
+		      "recipient text collate nocase, "
+		      "tail integer"
+		  ");", END) == -1
+      ||	sqlite_exec_void_retry(&retry, 
+		  "CREATE TABLE IF NOT EXISTS FILES("
+		      "id text not null primary key, "
+		      "length integer, "
+		      "datavalid integer, "
+		      "inserttime integer"
+		  ");", END) == -1
+      ||	sqlite_exec_void_retry(&retry, 
+		  "CREATE TABLE IF NOT EXISTS FILEBLOBS("
+		      "id text not null primary key, "
+		      "data blob"
+		  ");", END) == -1
+      ||	sqlite_exec_void_retry(&retry, 
+		  "CREATE TABLE IF NOT EXISTS IDENTITY("
+		      "uuid text not null"
+		  "); ", END) == -1
     ) {
       RETURN(WHY("Failed to create schema"));
     }
@@ -261,12 +272,14 @@ int rhizome_opendb()
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
   }
-  if (version<2){
+  if (version<2 && meta.mtime.tv_sec != -1){
+    // we need to populate these fields on upgrade from very old versions, we can simply re-insert all old manifests
+    // at some point we may deprecate upgrading the database and simply drop it and create a new one
+    // if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN recipient text collate nocase;", END);
-    // if more bundle verification is required in later upgrades, move this to the end, don't run it more than once.
     verify_bundles();
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
   }
@@ -274,14 +287,30 @@ int rhizome_opendb()
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
   }
-  if (version<4){
+  if (version<4 && meta.mtime.tv_sec != -1){
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
   }
-  if (version<5){
+  if (version<5 && meta.mtime.tv_sec != -1){
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
     sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
   }
+  if (version<6){
+    if (meta.mtime.tv_sec != -1){
+      // we've always been at war with eurasia
+      sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPLIST; ", END);
+      sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPMEMBERSHIPS; ", END);
+      sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS VERIFICATIONS; ", END);
+      sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS FILEMANIFESTS;", END);
+    }
+    sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=6;", END);
+  }
+  
+  // TODO recreate tables with collate nocase on all hex columns
+
+  /* Future schema updates should be performed here. 
+   The above schema can be assumed to exist, no matter which version we upgraded from.
+   All changes should attempt to preserve all existing interesting data */
 
   char buf[UUID_STRLEN + 1];
   int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
@@ -306,12 +335,6 @@ int rhizome_opendb()
       DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
   }
 
-  // TODO recreate tables with collate nocase on hex columns
-
-  /* Future schema updates should be performed here. 
-   The above schema can be assumed to exist.
-   All changes should attempt to preserve any existing data */
-
   // We can't delete a file that is being transferred in another process at this very moment...
   if (config.rhizome.clean_on_open)
     rhizome_cleanup(NULL);
@@ -877,7 +900,7 @@ int _sqlite_step(struct __sourceloc __whence, int log_level, sqlite_retry_state
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-static int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
+int _sqlite_exec(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement)
 {
   if (!statement)
     return -1;
@@ -1138,21 +1161,6 @@ int _sqlite_blob_close(struct __sourceloc __whence, int log_level, sqlite3_blob
   return 0;
 }
 
-static uint64_t rhizome_database_used_bytes()
-{
-  uint64_t db_page_size;
-  uint64_t db_page_count;
-  uint64_t db_free_page_count;
-  if (	sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL
-    ||  sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL
-    ||	sqlite_exec_uint64(&db_free_page_count, "PRAGMA free_count;", END) == -1LL
-  ) {
-    WHY("Cannot measure database used bytes");
-    return UINT64_MAX;
-  }
-  return db_page_size * (db_page_count - db_free_page_count);
-}
-
 int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp)
 {
   IN();
@@ -1166,47 +1174,6 @@ int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t versio
   OUT();
 }
 
-static int rhizome_delete_external(const char *id)
-{
-  // attempt to remove any external blob
-  char blob_path[1024];
-  if (!FORMF_RHIZOME_STORE_PATH(blob_path, "%s/%s", RHIZOME_BLOB_SUBDIR, id))
-    return -1;
-  if (unlink(blob_path) == -1) {
-    if (errno != ENOENT)
-      return WHYF_perror("unlink(%s)", alloca_str_toprint(blob_path));
-    return 1;
-  }
-  if (config.debug.rhizome_store)
-    DEBUGF("Deleted blob file %s", blob_path);
-  return 0;
-}
-
-static int rhizome_delete_orphan_fileblobs_retry(sqlite_retry_state *retry)
-{
-  return sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
-      "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
-      END);
-}
-
-int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
-{
-  int ret = 0;
-  if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
-	  "DELETE FROM FILES WHERE id = ? and datavalid = 0;",
-	  RHIZOME_FILEHASH_T, hashp, END
-	) == -1
-  )
-    ret = -1;
-  if (sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, retry,
-	  "DELETE FROM FILEBLOBS WHERE id = ? AND NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
-	  RHIZOME_FILEHASH_T, hashp, END
-	) == -1
-  )
-    ret = -1;
-  return ret;
-}
-
 int rhizome_cleanup(struct rhizome_cleanup_report *report)
 {
   IN();
@@ -1218,20 +1185,15 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
 
   /* For testing, it helps to speed up the cleanup process. */
   const char *orphan_payload_persist_ms = getenv("SERVALD_ORPHAN_PAYLOAD_PERSIST_MS");
-  const char *invalid_payload_persist_ms = getenv("SERVALD_INVALID_PAYLOAD_PERSIST_MS");
   time_ms_t now = gettime_ms();
   time_ms_t insert_horizon_no_manifest = now - (orphan_payload_persist_ms ? atoi(orphan_payload_persist_ms) : 1000); // 1 second ago
-  time_ms_t insert_horizon_not_valid = now - (invalid_payload_persist_ms ? atoi(invalid_payload_persist_ms) : 300000); // 5 minutes ago
 
   // Remove external payload files for stale, incomplete payloads.
-  unsigned candidates = 0;
   sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
-      "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 0;",
-      INT64, insert_horizon_not_valid, END);
+      "SELECT id FROM FILES WHERE datavalid = 0;", END);
   while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
-    candidates++;
     const char *id = (const char *) sqlite3_column_text(statement, 0);
-    if (rhizome_delete_external(id) == 0 && report)
+    if (rhizome_delete_file_id(id)==0 && report)
       ++report->deleted_stale_incoming_files;
   }
   sqlite3_finalize(statement);
@@ -1241,9 +1203,8 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
       "SELECT id FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
       INT64, insert_horizon_no_manifest, END);
   while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
-    candidates++;
     const char *id = (const char *) sqlite3_column_text(statement, 0);
-    if (rhizome_delete_external(id) == 0 && report)
+    if (rhizome_delete_file_id(id)==0 && report)
       ++report->deleted_orphan_files;
   }
   sqlite3_finalize(statement);
@@ -1252,29 +1213,16 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
   // referenced or are stale.  This could take a long time, so for scalability should be done
   // in an incremental background task.  See GitHub issue #50.
  
-  // Remove payload records that are stale and incomplete or old and unreferenced.
-  int ret;
-  if (candidates) {
-    ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
-	"DELETE FROM FILES WHERE inserttime < ? AND datavalid = 0;",
-	INT64, insert_horizon_not_valid, END);
-    if (report && ret > 0)
-      report->deleted_stale_incoming_files += ret;
-    ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
-	"DELETE FROM FILES WHERE inserttime < ? AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
-	INT64, insert_horizon_no_manifest, END);
-    if (report && ret > 0)
-      report->deleted_orphan_files += ret;
-  }
-
   // Remove payload blobs that are no longer referenced.
-  if ((ret = rhizome_delete_orphan_fileblobs_retry(&retry)) > 0 && report)
+  int ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
+      "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );",
+      END);
+  if (ret > 0 && report)
     report->deleted_orphan_fileblobs += ret;
 
   // delete manifests that no longer have payload files
   ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
-      "DELETE FROM MANIFESTS WHERE inserttime < ? AND filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);",
-      INT64, insert_horizon_no_manifest, END);
+      "DELETE FROM MANIFESTS WHERE filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);", END);
   if (report && ret > 0)
     report->deleted_orphan_manifests += ret;
   
@@ -1289,118 +1237,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
   OUT();
 }
 
-int rhizome_make_space(int group_priority, uint64_t bytes)
-{
-  /* Asked for impossibly large amount */
-  const size_t margin = 65536;
-  const uint64_t limit = config.rhizome.database_size > margin ? config.rhizome.database_size - margin : 1;
-  if (bytes >= limit)
-    return WHYF("bytes=%"PRIu64" is too large", bytes);
-
-  uint64_t db_used = rhizome_database_used_bytes();
-  if (db_used == UINT64_MAX)
-    return -1;
-  
-  rhizome_cleanup(NULL);
-  
-  /* If there is already enough space now, then do nothing more */
-  if (db_used + bytes <= limit)
-    return 0;
-
-  /* Okay, not enough space, so free up some. */
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
-      "SELECT id,length FROM FILES WHERE highestpriority < ? ORDER BY DESCENDING LENGTH",
-      INT, group_priority, END);
-  if (!statement)
-    return -1;
-  while (db_used + bytes > limit && sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
-    /* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
-    const char *id;
-    /* Get values */
-    if (sqlite3_column_type(statement, 0)==SQLITE_TEXT)
-      id = (const char *) sqlite3_column_text(statement, 0);
-    else {
-      WHY("Incorrect type in id column of files table");
-      break;
-    }
-    if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER)
-      ; //length=sqlite3_column_int(statement, 1);
-    else {
-      WHY("Incorrect type in length column of files table");
-      break;
-    }
-    rhizome_filehash_t hash;
-    if (str_to_rhizome_filehash_t(&hash, id) == -1)
-      WHYF("invalid field FILES.id=%s -- ignored", alloca_str_toprint(id));
-    else {
-      /* Try to drop this file from storage, discarding any references that do not trump the
-       * priority of this request.  The query done earlier should ensure this, but it doesn't hurt
-       * to be paranoid, and it also protects against inconsistency in the database.
-       */
-      rhizome_drop_stored_file(&hash, group_priority + 1);
-      if ((db_used = rhizome_database_used_bytes()) == UINT64_MAX)
-	break;
-    }
-  }
-  sqlite3_finalize(statement);
-
-  //int64_t equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM
-  //FILES WHERE highestpriority = ? and length > ?", INT, group_priority, INT64, bytes, END);
-  /* XXX Get rid of any equal priority files that are larger than this one */
-
-  /* XXX Get rid of any higher priority files that are not relevant in this time or location */
-
-  /* Couldn't make space */
-  return WHY("Incomplete");
-}
-
-/* Drop the specified file from storage, and any manifests that reference it, provided that none of
- * those manifests are being retained at a higher priority than the maximum specified here.
- */
-int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority)
-{
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  sqlite3_stmt *statement = sqlite_prepare_bind(&retry, "SELECT id FROM MANIFESTS WHERE filehash = ?", RHIZOME_FILEHASH_T, hashp, END);
-  if (!statement)
-    return WHYF("Could not drop stored file id=%s", alloca_tohex_rhizome_filehash_t(*hashp));
-  int can_drop = 1;
-  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
-    /* Find manifests for this file */
-    if (sqlite3_column_type(statement, 0) != SQLITE_TEXT) {
-      WHYF("Incorrect type in id column of manifests table");
-      break;
-    }
-    const char *q_id = (char *) sqlite3_column_text(statement, 0);
-    rhizome_bid_t bid;
-    if (str_to_rhizome_bid_t(&bid, q_id) == -1) {
-      WARNF("malformed column value MANIFESTS.id = %s -- skipped", alloca_str_toprint(q_id));
-      continue;
-    }
-    /* Check that manifest is not part of a higher priority group.
-	If so, we cannot drop the manifest or the file.
-	However, we will keep iterating, as we can still drop any other manifests pointing to this file
-	that are lower priority, and thus free up a little space. */
-    int priority = rhizome_manifest_priority(&retry, &bid);
-    if (priority == -1)
-      WHYF("Cannot drop fileid=%s due to error, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
-    else if (priority > maximum_priority) {
-      WHYF("Cannot drop fileid=%s due to manifest priority, bid=%s", alloca_tohex_rhizome_filehash_t(*hashp), alloca_tohex_rhizome_bid_t(bid));
-      can_drop = 0;
-    } else {
-      if (config.debug.rhizome)
-	DEBUGF("removing stale manifests, groupmemberships");
-      sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &bid, END);
-      sqlite_exec_void_retry(&retry, "DELETE FROM KEYPAIRS WHERE public = ?;", RHIZOME_BID_T, &bid, END);
-      sqlite_exec_void_retry(&retry, "DELETE FROM GROUPMEMBERSHIPS WHERE manifestid = ?;", RHIZOME_BID_T, &bid, END);
-    }
-  }
-  sqlite3_finalize(statement);
-  if (can_drop)
-    rhizome_delete_file_retry(&retry, hashp);
-  return 0;
-}
-
 /*
   Store the specified manifest into the sqlite database.
   We assume that sufficient space has been made for us.
@@ -1419,8 +1255,7 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
   The trick is to insert the blob as all zeroes using a special function, and then
   substitute bytes in the blog progressively.
 
-  We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, 
-   and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
+  We need to also need to create the appropriate row(s) in the MANIFESTS, FILES tables.
  */
 int rhizome_store_manifest(rhizome_manifest *m)
 {
@@ -1493,52 +1328,6 @@ int rhizome_store_manifest(rhizome_manifest *m)
   rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db));
   rhizome_manifest_set_inserttime(m, now);
 
-//  if (serverMode)
-//    rhizome_sync_bundle_inserted(bar);
-
-  // TODO remove old payload?
-  
-#if 0
-  if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
-    int closed=rhizome_manifest_get_ll(m,"closedgroup");
-    if (closed<1) closed=0;
-    int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
-    if (ciphered<1) ciphered=0;
-    if ((stmt = sqlite_prepare_bind(&retry,
-	    "INSERT OR REPLACE INTO GROUPLIST(id,closed,ciphered,priority) VALUES (?,?,?,?);",
-	    RHIZOME_BID_T, &m->cryptoSignPublic,
-	    INT, closed,
-	    INT, ciphered,
-	    INT, RHIZOME_PRIORITY_DEFAULT,
-	    END
-	  )
-      ) == NULL
-    )
-      goto rollback;
-    if (sqlite_step_retry(&retry, stmt) == -1)
-      goto rollback;
-    sqlite3_finalize(stmt);
-    stmt = NULL;
-  }
-#endif
-
-#if 0
-  if (m->group_count > 0) {
-    if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS (manifestid, groupid) VALUES (?, ?);")) == NULL)
-      goto rollback;
-    unsigned i;
-    for (i=0;i<m->group_count;i++){
-      if (sqlite_bind(&retry, stmt, RHIZOME_BID_T, &m->cryptoSignPublic, TEXT, m->groups[i]) == -1)
-	goto rollback;
-      if (sqlite_step_retry(&retry, stmt) == -1)
-	goto rollback;
-      sqlite3_reset(stmt);
-    }
-    sqlite3_finalize(stmt);
-    stmt = NULL;
-  }
-#endif
-
   if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
     // This message used in tests; do not modify or remove.
     INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRIu64,
@@ -1753,29 +1542,6 @@ void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCoun
   (void) tohex(out, byteCount * 2, in);
 }
 
-int rhizome_update_file_priority(const char *fileid)
-{
-  /* work out the highest priority of any referrer */
-  uint64_t highestPriority = 0;
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  if (sqlite_exec_uint64_retry(&retry, &highestPriority,
-	"SELECT max(grouplist.priority) FROM MANIFESTS, GROUPMEMBERSHIPS, GROUPLIST"
-	" WHERE MANIFESTS.filehash = ?"
-	"   AND GROUPMEMBERSHIPS.manifestid = MANIFESTS.id"
-	"   AND GROUPMEMBERSHIPS.groupid = GROUPLIST.id;",
-	TEXT_TOUPPER, fileid, END
-      ) == -1
-  )
-    return -1;
-  if (sqlite_exec_void_retry(&retry,
-	"UPDATE files SET highestPriority = ? WHERE id = ?;",
-	INT64, highestPriority, TEXT_TOUPPER, fileid, END
-      ) == -1
-  )
-    return WHYF("cannot update priority for fileid=%s", fileid);
-  return 0;
-}
-
 /* Search the database for a manifest having the same name and payload content, and if the version
  * is known, having the same version.  Returns RHIZOME_BUNDLE_STATUS_DUPLICATE if a duplicate is found
  * (setting *found to point to the duplicate's manifest), returns RHIZOME_BUNDLE_STATUS_NEW if no
@@ -1961,33 +1727,6 @@ static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizom
   return sqlite3_changes(rhizome_db) ? 0 : 1;
 }
 
-static int rhizome_delete_file_retry(sqlite_retry_state *retry, const rhizome_filehash_t *hashp)
-{
-  int ret = 0;
-  rhizome_delete_external(alloca_tohex_rhizome_filehash_t(*hashp));
-  sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
-  if (!statement || sqlite_exec_retry(retry, statement) == -1)
-    ret = -1;
-  statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", RHIZOME_FILEHASH_T, hashp, END);
-  if (!statement || sqlite_exec_retry(retry, statement) == -1)
-    ret = -1;
-  return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
-}
-
-static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
-{
-  strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
-  int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
-  if (rows == -1)
-    return -1;
-  rhizome_filehash_t hash;
-  if (str_to_rhizome_filehash_t(&hash, strbuf_str(fh)) == -1)
-    return WHYF("invalid field FILES.id=%s", strbuf_str(fh));
-  if (rows && rhizome_delete_file_retry(retry, &hash) == -1)
-    return -1;
-  return 0;
-}
-
 /* Remove a manifest and its bundle from the database, given its manifest ID.
  *
  * Returns 0 if manifest is found and removed and bundle was either absent or removed
@@ -1998,10 +1737,9 @@ static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome
  */
 int rhizome_delete_bundle(const rhizome_bid_t *bidp)
 {
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  if (rhizome_delete_payload_retry(&retry, bidp) == -1)
+  if (rhizome_delete_payload(bidp) == -1)
     return -1;
-  if (rhizome_delete_manifest_retry(&retry, bidp) == -1)
+  if (rhizome_delete_manifest(bidp) == -1)
     return -1;
   return 0;
 }
@@ -2021,35 +1759,6 @@ int rhizome_delete_manifest(const rhizome_bid_t *bidp)
   return rhizome_delete_manifest_retry(&retry, bidp);
 }
 
-/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
- * untouched if present.
- *
- * Returns 0 if manifest is found, its payload is found and removed
- * Returns 1 if manifest or payload is not found
- * Returns -1 on error
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-int rhizome_delete_payload(const rhizome_bid_t *bidp)
-{
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  return rhizome_delete_payload_retry(&retry, bidp);
-}
-
-/* Remove a file from the database, given its file hash.
- *
- * Returns 0 if file is found and removed
- * Returns 1 if file is not found
- * Returns -1 on error
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-int rhizome_delete_file(const rhizome_filehash_t *hashp)
-{
-  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  return rhizome_delete_file_retry(&retry, hashp);
-}
-
 static int is_interesting(const char *id_hex, uint64_t version)
 {
   IN();
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index f413cd06..56316d1d 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -119,6 +119,9 @@ static int rhizome_direct_import_end(struct http_request *hr)
   case RHIZOME_BUNDLE_STATUS_FAKE:
     http_request_simple_response(&r->http, 403, "Manifest not signed");
     return 0;
+  case RHIZOME_BUNDLE_STATUS_DONOTWANT:
+    http_request_simple_response(&r->http, 403, "Not enough space");
+    return 0;
   case RHIZOME_BUNDLE_STATUS_DUPLICATE:
   case RHIZOME_BUNDLE_STATUS_ERROR:
     break;
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index ec0bbc1f..c6ae7df0 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -552,12 +552,14 @@ schedule_fetch(struct rhizome_fetch_slot *slot)
     slot->request_len = strbuf_len(r);
     enum rhizome_payload_status status = rhizome_open_write(&slot->write_state,
 							    &slot->manifest->filehash,
-							    slot->manifest->filesize,
-							    RHIZOME_PRIORITY_DEFAULT);
+							    slot->manifest->filesize);
     switch (status) {
       case RHIZOME_PAYLOAD_STATUS_EMPTY:
       case RHIZOME_PAYLOAD_STATUS_STORED:
 	RETURN(IMPORTED);
+      case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+      case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
+	RETURN(DONOTWANT);
       case RHIZOME_PAYLOAD_STATUS_NEW:
 	break;
       case RHIZOME_PAYLOAD_STATUS_ERROR:
@@ -824,6 +826,7 @@ static void rhizome_start_next_queued_fetch(struct rhizome_fetch_slot *slot)
       case SAMEBUNDLE:
       case SAMEPAYLOAD:
       case SUPERSEDED:
+      case DONOTWANT:
       case NEWERBUNDLE:
       default:
 	// Discard the candidate fetch and loop to try the next in queue.
diff --git a/rhizome_restful.c b/rhizome_restful.c
index ad6f6cad..6f0d32d7 100644
--- a/rhizome_restful.c
+++ b/rhizome_restful.c
@@ -495,6 +495,10 @@ static int restful_rhizome_insert_end(struct http_request *hr)
 	http_request_simple_response(&r->http, 403, strbuf_str(msg));
 	return 403;
       }
+    case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+    case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
+      http_request_simple_response(&r->http, 403, "Not enough space");
+      return 403;
     case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
       http_request_simple_response(&r->http, 403, "Payload hash contradicts manifest");
       return 403;
diff --git a/rhizome_store.c b/rhizome_store.c
index 2b985014..0637beae 100644
--- a/rhizome_store.c
+++ b/rhizome_store.c
@@ -76,14 +76,187 @@ static uint64_t rhizome_create_fileblob(sqlite_retry_state *retry, uint64_t id,
   return rowid;
 }
 
-enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority)
+static int rhizome_delete_external(const char *id)
+{
+  // attempt to remove any external blob
+  char blob_path[1024];
+  if (!FORMF_RHIZOME_STORE_PATH(blob_path, "%s/%s", RHIZOME_BLOB_SUBDIR, id))
+    return -1;
+  if (unlink(blob_path) == -1) {
+    if (errno != ENOENT)
+      return WHYF_perror("unlink(%s)", alloca_str_toprint(blob_path));
+    return 1;
+  }
+  if (config.debug.rhizome_store)
+    DEBUGF("Deleted blob file %s", blob_path);
+  return 0;
+}
+
+static int rhizome_delete_file_id_retry(sqlite_retry_state *retry, const char *id)
+{
+  int ret = 0;
+  rhizome_delete_external(id);
+  sqlite3_stmt *statement = sqlite_prepare_bind(retry, "DELETE FROM fileblobs WHERE id = ?", STATIC_TEXT, id, END);
+  if (!statement || sqlite_exec_retry(retry, statement) == -1)
+    ret = -1;
+  statement = sqlite_prepare_bind(retry, "DELETE FROM files WHERE id = ?", STATIC_TEXT, id, END);
+  if (!statement || sqlite_exec_retry(retry, statement) == -1)
+    ret = -1;
+  return ret == -1 ? -1 : sqlite3_changes(rhizome_db) ? 0 : 1;
+}
+
+static int rhizome_delete_payload_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
+{
+  strbuf fh = strbuf_alloca(RHIZOME_FILEHASH_STRLEN + 1);
+  int rows = sqlite_exec_strbuf_retry(retry, fh, "SELECT filehash FROM manifests WHERE id = ?", RHIZOME_BID_T, bidp, END);
+  if (rows == -1)
+    return -1;
+  if (rows && rhizome_delete_file_id_retry(retry, strbuf_str(fh)) == -1)
+    return -1;
+  return 0;
+}
+
+/* Remove a bundle's payload (file) from the database, given its manifest ID, leaving its manifest
+ * untouched if present.
+ *
+ * Returns 0 if manifest is found, its payload is found and removed
+ * Returns 1 if manifest or payload is not found
+ * Returns -1 on error
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+int rhizome_delete_payload(const rhizome_bid_t *bidp)
+{
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  return rhizome_delete_payload_retry(&retry, bidp);
+}
+
+int rhizome_delete_file_id(const char *id)
+{
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  return rhizome_delete_file_id_retry(&retry, id);
+}
+
+/* Remove a file from the database, given its file hash.
+ *
+ * Returns 0 if file is found and removed
+ * Returns 1 if file is not found
+ * Returns -1 on error
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+int rhizome_delete_file(const rhizome_filehash_t *hashp)
+{
+  return rhizome_delete_file_id(alloca_tohex_rhizome_filehash_t(*hashp));
+}
+
+// TODO readonly version?
+static enum rhizome_payload_status store_make_space(uint64_t bytes)
+{
+  uint64_t external_bytes;
+  uint64_t db_page_size;
+  uint64_t db_page_count;
+  uint64_t db_free_page_count;
+  
+  // TODO limit based on free space?
+  
+  // No limit?
+  if (config.rhizome.database_size==0)
+    return RHIZOME_PAYLOAD_STATUS_NEW;
+    
+  // TODO index external_bytes calculation and/or cache result
+  
+  if (	sqlite_exec_uint64(&db_page_size, "PRAGMA page_size;", END) == -1LL
+    ||  sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END) == -1LL
+    ||	sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END) == -1LL
+    ||  sqlite_exec_uint64(&external_bytes, 
+	  "SELECT SUM(length) "
+	  "FROM FILES  "
+	  "WHERE NOT EXISTS( "
+	    "SELECT 1  "
+	    "FROM FILEBLOBS "
+	    "WHERE FILES.ID = FILEBLOBS.ID "
+	  ");", END) == -1LL
+  )
+    return WHY("Cannot measure database used bytes");
+  
+  if (config.rhizome.database_size < db_page_size*10)
+    return WHYF("rhizome.database_size is too small to store anything");
+  
+  const uint64_t limit = config.rhizome.database_size - db_page_size*4;
+  uint64_t db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
+  
+  if (bytes >= limit){
+    if (config.debug.rhizome)
+      DEBUGF("Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64, 
+	bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
+    return RHIZOME_PAYLOAD_STATUS_TOO_BIG;
+  }
+  
+  // If there is enough space, do nothing
+  if (db_used + bytes <= limit)
+    return RHIZOME_PAYLOAD_STATUS_NEW;
+  
+  // penalise new things by 10 minutes to reduce churn
+  time_ms_t cost = gettime_ms() - 60000 - bytes;
+  
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  // query files by age, penalise larger files so they are removed earlier
+  sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
+      "SELECT id, length, inserttime FROM FILES ORDER BY (inserttime - length)",
+      END);
+  if (!statement)
+    return RHIZOME_PAYLOAD_STATUS_ERROR;
+    
+  while (db_used + bytes > limit && sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
+    const char *id=(const char *) sqlite3_column_text(statement, 0);
+    uint64_t length = sqlite3_column_int(statement, 1);
+    time_ms_t inserttime = sqlite3_column_int64(statement, 2);
+    
+    time_ms_t cost_existing = inserttime - length;
+    
+    if (config.debug.rhizome)
+      DEBUGF("Considering dropping file %s, size %"PRId64" cost %"PRId64" vs %"PRId64" to add %"PRId64" new bytes", 
+	id, length, cost, cost_existing, bytes);
+    // don't allow the new file, we've got more important things to store
+    if (cost < cost_existing)
+      break;
+    
+    // drop the existing content and recalculate used space
+    if (rhizome_delete_external(id)==0)
+      external_bytes -= length;
+    
+    sqlite3_stmt *s = sqlite_prepare_bind(&retry, "DELETE FROM fileblobs WHERE id = ?", STATIC_TEXT, id, END);
+    if (s)
+      sqlite_exec_retry(&retry, s);
+    s = sqlite_prepare_bind(&retry, "DELETE FROM files WHERE id = ?", STATIC_TEXT, id, END);
+    if (s)
+      sqlite_exec_retry(&retry, s);
+      
+    sqlite_exec_uint64(&db_page_count, "PRAGMA page_count;", END);
+    sqlite_exec_uint64(&db_free_page_count, "PRAGMA freelist_count;", END);
+    
+    db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
+  }
+  sqlite3_finalize(statement);
+  
+  if (db_used + bytes <= limit)
+    return RHIZOME_PAYLOAD_STATUS_NEW;
+  
+  if (config.debug.rhizome)
+    DEBUGF("Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64, 
+      bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
+      
+  return RHIZOME_PAYLOAD_STATUS_UNINITERESTING;
+}
+
+enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length)
 {
   if (file_length == 0)
     return RHIZOME_PAYLOAD_STATUS_EMPTY;
 
   write->blob_fd=-1;
   write->sql_blob=NULL;
-  write->priority = priority;
   
   if (expectedHashp){
     if (rhizome_exists(expectedHashp))
@@ -93,6 +266,13 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons
   }else{
     write->id_known=0;
   }
+  
+  if (file_length!=RHIZOME_SIZE_UNSET){
+    enum rhizome_payload_status status = store_make_space(file_length);
+    if (status != RHIZOME_PAYLOAD_STATUS_NEW)
+      return status;
+  }
+  
   time_ms_t now = gettime_ms();
   static uint64_t last_id=0;
   write->temp_id = now;
@@ -452,6 +632,9 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
     if (config.debug.rhizome_store)
       DEBUGF("Wrote %"PRIu64" bytes, set file_length", write->file_offset);
     write->file_length = write->file_offset;
+    status = store_make_space(write->file_length);
+    if (status!=RHIZOME_PAYLOAD_STATUS_NEW)
+      goto failure;
   }
   
   // flush out any remaining buffered pieces to disk
@@ -533,7 +716,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
   }
 
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-  rhizome_remove_file_datainvalid(&retry, &write->id);
+  
   if (rhizome_exists(&write->id)) {
     // we've already got that payload, delete the new copy
     if (write->blob_rowid){
@@ -552,10 +735,9 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
 
     if (sqlite_exec_void_retry(
 	    &retry,
-	    "INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES(?,?,?,1,?);",
+	    "INSERT OR REPLACE INTO FILES(id,length,datavalid,inserttime) VALUES(?,?,1,?);",
 	    RHIZOME_FILEHASH_T, &write->id,
 	    INT64, write->file_length,
-	    INT, write->priority,
 	    INT64, gettime_ms(),
 	    END
 	  ) == -1
@@ -615,7 +797,7 @@ enum rhizome_payload_status rhizome_import_payload_from_file(rhizome_manifest *m
   struct rhizome_write write;
   bzero(&write, sizeof(write));
   
-  enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
+  enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize);
   if (status != RHIZOME_PAYLOAD_STATUS_NEW)
     return status;
   
@@ -644,7 +826,7 @@ enum rhizome_payload_status rhizome_import_buffer(rhizome_manifest *m, unsigned
   struct rhizome_write write;
   bzero(&write, sizeof(write));
   
-  enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
+  enum rhizome_payload_status status = rhizome_open_write(&write, &m->filehash, m->filesize);
   if (status != RHIZOME_PAYLOAD_STATUS_NEW)
     return status;
   
@@ -714,8 +896,7 @@ enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *wr
   enum rhizome_payload_status status = rhizome_open_write(
 	  write,
 	  m->has_filehash ? &m->filehash : NULL,
-	  m->filesize,
-	  RHIZOME_PRIORITY_DEFAULT
+	  m->filesize
 	);
   if (status == RHIZOME_PAYLOAD_STATUS_NEW)
     status = rhizome_write_derive_key(m, write);
@@ -735,6 +916,8 @@ enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, cons
     case RHIZOME_PAYLOAD_STATUS_NEW:
       break;
     case RHIZOME_PAYLOAD_STATUS_STORED:
+    case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+    case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
     case RHIZOME_PAYLOAD_STATUS_ERROR:
     case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
     case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
@@ -765,6 +948,8 @@ enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, cons
     case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
     case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
     case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
+    case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
+    case RHIZOME_PAYLOAD_STATUS_UNINITERESTING:
       break;
     default:
       FATALF("status = %d", status);
@@ -1329,7 +1514,7 @@ enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *wri
   if (advance_by > 0)
     rhizome_manifest_set_tail(m, m->tail + advance_by);
   rhizome_manifest_set_version(m, m->filesize);
-  enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT);
+  enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize);
   if (status == RHIZOME_PAYLOAD_STATUS_NEW && copy_length > 0) {
     // note that we don't need to bother decrypting the existing journal payload
     enum rhizome_payload_status rstatus = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);
diff --git a/tests/rhizomeops b/tests/rhizomeops
index 8255a89e..04297c51 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -1198,7 +1198,7 @@ test_DeleteManifest() {
    assert diff file2 file2x
    rhizome_clean
    assert [ $deleted_files = 1 ]
-   assert [ $deleted_fileblobs = 1 ]
+   assert [ $deleted_fileblobs = 0 ]
    assert [ $deleted_manifests = 0 ]
 }
 
@@ -1257,4 +1257,53 @@ test_DeleteFile() {
    assert_rhizome_list file{2..4}
 }
 
+doc_payloadTooBig="Fail to insert a payload that is larger than the database"
+setup_payloadTooBig() {
+   setup_servald
+   setup_rhizome
+   set_instance +A
+   executeOk_servald config set rhizome.database_size 32K
+}
+test_payloadTooBig(){
+   create_file file1 32K
+   execute $servald rhizome add file $SIDA file1 file1.manifest
+   assertExitStatus '==' 7
+}
+
+doc_payloadUninteresting="Fail to insert a payload that is uninteresting"
+setup_payloadUninteresting() {
+   setup_servald
+   setup_rhizome
+   set_instance +A
+   executeOk_servald config set rhizome.database_size 64K
+}
+test_payloadUninteresting(){
+   create_file file1 32K
+   create_file file2 32K
+   executeOk_servald rhizome add file $SIDA file1 file1.manifest
+   execute $servald rhizome add file $SIDA file2 file2.manifest
+   assertExitStatus '==' 7
+}
+
+doc_evictUninteresting="Evict a large payload to make room for smaller payloads"
+setup_evictUninteresting() {
+   setup_servald
+   setup_rhizome
+   set_instance +A
+   executeOk_servald config set rhizome.database_size 1M
+}
+test_evictUninteresting(){
+   create_file file1 512K
+   create_file file2 256K
+   create_file file3 128K
+   create_file file4 128K
+   executeOk_servald rhizome add file $SIDA file1 file1.manifest
+   executeOk_servald rhizome add file $SIDA file2 file2.manifest
+   executeOk_servald rhizome add file $SIDA file3 file3.manifest
+   executeOk_servald rhizome add file $SIDA file4 file4.manifest
+   rhizome_clean
+   executeOk_servald rhizome list
+   assert_rhizome_list file{2,3,4}
+}
+
 runTests "$@"