Merge latest MeshMS Java API additions into 'development'

conf_schema.h

@@ -427,9 +427,9 @@ STRUCT(rhizome)
 ATOM(bool_t,                enable,         1, boolean,, "If true, server opens Rhizome database when starting")
 ATOM(bool_t,                fetch,          1, boolean,, "If false, no new bundles will be fetched from peers")
 ATOM(bool_t,                clean_on_open,  0, boolean,, "If true, Rhizome database is cleaned at start of every command")
-ATOM(bool_t,                clean_on_start, 1, boolean,, "If true, Rhizome database is cleaned at start of daemon")
 STRING(256,                 datastore_path, "", str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory")
-ATOM(uint64_t,              database_size,  0, uint64_scaled,, "Maximum size of database in bytes")
+ATOM(uint64_t,              database_size,  UINT64_MAX, uint64_scaled,, "Maximum size of database in bytes")
+ATOM(uint64_t,              min_free_space, 100*1024*1024, uint64_scaled,, "Minimum free space to preserve on the disk")
 ATOM(uint32_t,              max_blob_size,  128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
 
 ATOM(uint64_t,              rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")

configure.in

@@ -92,6 +92,8 @@ AC_CHECK_HEADERS(
     sys/mman.h \
     sys/time.h \
     sys/ucred.h \
+    sys/statvfs.h \
+    sys/vfs.h \
     poll.h \
     netdb.h \
     linux/ioctl.h \

java/org/servalproject/servaldna/ServalDCommand.java

@@ -485,6 +485,21 @@ public class ServalDCommand
 		return result.toString();
 	}
 
+	public enum ConfigAction{
+		set,
+		del,
+		sync
+	};
+
+	public static void configActions(Object... arguments) throws ServalDFailureException {
+		// TODO we could verify the types and number of arguments here, though servald is about to do that anyway.
+		String args[] = new String[arguments.length+1];
+		args[0]="config";
+		for (int i=0;i<arguments.length;i++)
+			args[i+1]=arguments[i].toString();
+		ServalDCommand.command(args);
+	}
+
 	public static void deleteConfig(String name) throws ServalDFailureException {
 		ServalDCommand.command("config", "del", name);
 	}

java/org/servalproject/servaldna/ServerControl.java

@@ -101,12 +101,11 @@ public class ServerControl {
 			 */
 			String restfulPassword = ServalDCommand.getConfigItem("rhizome.api.restful.users." + restfulUsername + ".password");
 			if (restfulPassword == null) {
-				String pwd = new BigInteger(130, new SecureRandom()).toString(32);
-				ServalDCommand.setConfigItem("rhizome.api.restful.users." + restfulUsername + ".password", pwd);
-				ServalDCommand.configSync();
-				restfulPassword = ServalDCommand.getConfigItem("rhizome.api.restful.users." + restfulUsername + ".password");
-				if (restfulPassword == null)
-					throw new ServalDInterfaceException("Failed to set restful password");
+				restfulPassword = new BigInteger(130, new SecureRandom()).toString(32);
+				ServalDCommand.configActions(
+						ServalDCommand.ConfigAction.set, "rhizome.api.restful.users." + restfulUsername + ".password", restfulPassword,
+						ServalDCommand.ConfigAction.sync
+				);
 			}
 			client = new ServalDClient(this.httpPort, restfulUsername, restfulPassword);
 		}

msp_proxy.c

@@ -299,6 +299,12 @@ static size_t msp_listener(MSP_SOCKET sock, msp_state_t state, const uint8_t *pa
     // stop listening after the first incoming connection
     msp_stop(listener);
     listener=MSP_SOCKET_NULL;
+    if (service_sock.poll.fd!=-1){
+      if (is_watching(&service_sock))
+	unwatch(&service_sock);
+      mdp_close(service_sock.poll.fd);
+      service_sock.poll.fd=-1;
+    }
   }
   
   struct mdp_sockaddr remote;

rhizome_store.c

@@ -19,6 +19,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 
 #include <assert.h>
+#ifdef HAVE_SYS_STATVFS_H
+#  include <sys/statvfs.h>
+#endif
 #include "serval.h"
 #include "rhizome.h"
 #include "conf.h"
@@ -150,6 +153,43 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp)
   return rhizome_delete_file_id(alloca_tohex_rhizome_filehash_t(*hashp));
 }
 
+static uint64_t store_get_free_space()
+{
+  const char *fake_space = getenv("SERVALD_FREE_SPACE");
+  if (fake_space)
+    return atol(fake_space);
+#ifdef HAVE_SYS_STATVFS_H
+  char store_path[1024];
+  if (!FORMF_RHIZOME_STORE_PATH(store_path, "rhizome.db"))
+    return UINT64_MAX;
+  struct statvfs stats;
+  if (statvfs(store_path, &stats)==-1){
+    WARNF_perror("statvfs(%s)", store_path);
+    return UINT64_MAX;
+  }
+  return stats.f_bsize * stats.f_bfree;
+#else
+  return UINT64_MAX;
+#endif
+}
+
+static uint64_t store_space_limit(uint64_t current_size)
+{
+  uint64_t limit = config.rhizome.database_size;
+  
+  if (config.rhizome.min_free_space!=0){
+    uint64_t free_space = store_get_free_space();
+    if (free_space < config.rhizome.min_free_space){
+      if (current_size + free_space < config.rhizome.min_free_space)
+	limit = 0;
+      else
+	limit = current_size + free_space - config.rhizome.min_free_space;
+    }
+  }
+  
+  return limit;
+}
+
 // TODO readonly version?
 static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizome_cleanup_report *report)
 {
@@ -157,12 +197,11 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
   uint64_t db_page_size;
   uint64_t db_page_count;
   uint64_t db_free_page_count;
+  
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   
-  // TODO limit based on free space?
-  
   // No limit?
-  if (config.rhizome.database_size==0)
+  if (config.rhizome.database_size==UINT64_MAX && config.rhizome.min_free_space==0)
     return RHIZOME_PAYLOAD_STATUS_NEW;
     
   // TODO index external_bytes calculation and/or cache result
@@ -181,13 +220,10 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
   )
     return WHY("Cannot measure database used bytes");
   
-  if (config.rhizome.database_size < db_page_size*10)
-    return WHYF("rhizome.database_size is too small to store anything");
-  
-  const uint64_t limit = config.rhizome.database_size - db_page_size*4;
   uint64_t db_used = external_bytes + db_page_size * (db_page_count - db_free_page_count);
+  const uint64_t limit = store_space_limit(db_used);
   
-  if (bytes >= limit){
+  if (bytes && bytes >= limit){
     if (config.debug.rhizome)
       DEBUGF("Not enough space for %"PRIu64". Used; %"PRIu64" = %"PRIu64" + %"PRIu64" * (%"PRIu64" - %"PRIu64"), Limit; %"PRIu64, 
 	bytes, db_used, external_bytes, db_page_size, db_page_count, db_free_page_count, limit);
@@ -223,7 +259,7 @@ static enum rhizome_payload_status store_make_space(uint64_t bytes, struct rhizo
       DEBUGF("Considering dropping file %s, size %"PRId64" cost %"PRId64" vs %"PRId64" to add %"PRId64" new bytes", 
 	id, length, cost, cost_existing, bytes);
     // don't allow the new file, we've got more important things to store
-    if (cost < cost_existing)
+    if (bytes && cost < cost_existing)
       break;
     
     // drop the existing content and recalculate used space

server.c

@@ -348,14 +348,16 @@ void server_watchdog(struct sched_ent *alarm)
   }
 }
 
-DEFINE_ALARM(rhizome_open_db);
-void rhizome_open_db(struct sched_ent *UNUSED(alarm))
+DEFINE_ALARM(rhizome_clean_db);
+void rhizome_clean_db(struct sched_ent *alarm)
 {
-  if (config.rhizome.enable && !rhizome_db){
-    rhizome_opendb();
-    if (config.rhizome.clean_on_start && !config.rhizome.clean_on_open)
+  if (!config.rhizome.enable || !rhizome_db)
+    return;
+    
+  time_ms_t now = gettime_ms();
   rhizome_cleanup(NULL);
-  }
+  // clean up every 30 minutes or so
+  RESCHEDULE(alarm, now + 30*60*1000, TIME_MS_NEVER_WILL, TIME_MS_NEVER_WILL);
 }
 
 void cf_on_config_change()
@@ -389,7 +391,8 @@ void cf_on_config_change()
     now+config.server.config_reload_interval_ms+100);
 
   if (config.rhizome.enable){
-    RESCHEDULE(&ALARM_STRUCT(rhizome_open_db), now+100, now+100, TIME_MS_NEVER_WILL);
+    rhizome_opendb();
+    RESCHEDULE(&ALARM_STRUCT(rhizome_clean_db), now + 30*60*1000, TIME_MS_NEVER_WILL, TIME_MS_NEVER_WILL);
   }else if(rhizome_db){
     rhizome_close_db();
   }

tests/jni

@@ -125,7 +125,7 @@ teardown_DnaLookup() {
 
 doc_serviceDiscovery="Serval JNI discover network services by name"
 listen_service() {
-   executeOk_servald --timeout=20 msp listen --service=test_name 512 <<EOF
+   executeOk_servald --timeout=20 msp listen --once --service=test_name 512 <<EOF
 Hi!
 EOF
    tfw_cat --stderr
@@ -151,6 +151,10 @@ test_serviceDiscovery() {
    assertStdoutGrep "$SIDB"
    assertStdoutGrep "\<test_name\.msp\.port=512\>"
    tfw_cat --stdout --stderr
+   executeOk_servald --timeout=20 msp connect $SIDB 512 <<EOF
+Hi!
+EOF
+   fork_wait %service
 }
 teardown_serviceDiscovery() {
    stop_all_servald_servers

tests/rhizomeops

@@ -910,7 +910,6 @@ setup_MeshMSAddGrow() {
    setup_rhizome
    executeOk_servald config set rhizome.clean_on_open on
    export SERVALD_ORPHAN_PAYLOAD_PERSIST_MS=0
-   export SERVALD_INVALID_PAYLOAD_PERSIST_MS=0
    echo "Message1" >file1
    echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
 }
@@ -1291,8 +1290,30 @@ setup_evictUninteresting() {
    setup_rhizome
    set_instance +A
    executeOk_servald config set rhizome.database_size 1M
+   create_file file1 512K
+   create_file file2 256K
+   create_file file3 128K
+   create_file file4 128K
 }
 test_evictUninteresting(){
+   executeOk_servald rhizome add file $SIDA file1 file1.manifest
+   executeOk_servald rhizome add file $SIDA file2 file2.manifest
+   executeOk_servald rhizome add file $SIDA file3 file3.manifest
+   executeOk_servald rhizome add file $SIDA file4 file4.manifest
+   tfw_cat --stderr
+   rhizome_clean
+   assert [ $deleted_files = 0 ]
+   assert [ $deleted_fileblobs = 0 ]
+   assert [ $deleted_manifests = 1 ]
+   executeOk_servald rhizome list
+   assert_rhizome_list file{2,3,4}
+}
+
+doc_evictFreeSpace="Reduce database size due to insufficient free space"
+setup_evictFreeSpace() {
+   setup_servald
+   setup_rhizome
+   set_instance +A
    create_file file1 512K
    create_file file2 256K
    create_file file3 128K
@@ -1301,9 +1322,17 @@ test_evictUninteresting(){
    executeOk_servald rhizome add file $SIDA file2 file2.manifest
    executeOk_servald rhizome add file $SIDA file3 file3.manifest
    executeOk_servald rhizome add file $SIDA file4 file4.manifest
+}
+test_evictFreeSpace() {
+   executeOk_servald config \
+      set rhizome.min_free_space 1M
+   # only 640K free...
+   export SERVALD_FREE_SPACE=655360
    rhizome_clean
+   assert [ $deleted_files = 0 ]
+   assert [ $deleted_fileblobs = 0 ]
+   assert [ $deleted_manifests = 1 ]
    executeOk_servald rhizome list
    assert_rhizome_list file{2,3,4}
 }
-
 runTests "$@"