Implement "rhizome add file" and "rhizome list"

commandline.c

@@ -18,7 +18,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
 #include <sys/stat.h>
+#include <sys/time.h>
+#include <math.h>
+#include <string.h>
 #include "serval.h"
+#include "rhizome.h"
 
 typedef struct command_line_option {
   int (*function)(int argc,char **argv,struct command_line_option *o);
@@ -618,11 +622,50 @@ int app_server_get(int argc,char **argv,struct command_line_option *o)
 
 int app_rhizome_add(int argc, char **argv, struct command_line_option *o)
 {
-  char *manifestpath = cli_arg(argc, argv, o, "manifestfilepath", "");
+  const char *filepath = cli_arg(argc, argv, o, "filepath", "");
-  char *filepath = cli_arg(argc, argv, o, "filepath", "");
+  const char *manifestpath = cli_arg(argc, argv, o, "manifestpath", "");
-  printf("manifestpath = \"%s\"\n", manifestpath);
+  /* Ensure the Rhizome database exists and is open */
-  printf("filepath = \"%s\"\n", filepath);
+  if (create_serval_instance_dir() == -1)
-  return 0;
+    return -1;
+  rhizome_datastore_path = serval_instancepath();
+  rhizome_opendb();
+  /* Create a new manifest that will represent the file, having a "name" value of the file's
+   * basename and a "date" value of right now */
+  rhizome_manifest *m = rhizome_new_manifest();
+  const char *name = strrchr(filepath, '/');
+  name = name ? name + 1 : filepath;
+  rhizome_manifest_set(m, "name", name);
+  rhizome_manifest_set_ll(m, "date", overlay_gettime_ms());
+  /* Add the manifest and its associated file to the Rhizome database, generating an "id" in the
+   * process */
+  int ret = rhizome_add_manifest(m, filepath,
+				 NULL, // no groups - XXX should allow them
+				 255, // ttl - XXX should read from somewhere
+				 0, // int verifyP
+				 1, // int checkFileP
+				 1 // int signP
+    );
+  if (ret == -1) {
+    return WHY("Manifest not added to Rhizome database");
+  } else {
+    /* If successfully added, overwrite the manifest file so that the Java component that is
+     * invoking this command can read it to obtain feedback on the result. */
+    if (manifestpath[0] && rhizome_write_manifest_file(m, manifestpath) == -1) {
+      ret = WHY("Could not overwrite manifest file.");
+    }
+  }
+  rhizome_manifest_free(m);
+  return ret;
+}
+
+int app_rhizome_list(int argc, char **argv, struct command_line_option *o)
+{
+  /* Create the instance directory if it does not yet exist */
+  if (create_serval_instance_dir() == -1)
+    return -1;
+  rhizome_datastore_path = serval_instancepath();
+  rhizome_opendb();
+  rhizome_list_manifests(0, 0);
 }
 
 /* NULL marks ends of command structure.
@@ -663,7 +706,9 @@ command_line_option command_line_options[]={
    "Set specified configuration variable."},
   {app_server_get,{"config","get","[<variable>]",NULL},0,
    "Get specified configuration variable."},
-  {app_rhizome_add,{"rhizome","add","<manifestfilepath>","[<filepath>]",NULL},0,
+  {app_rhizome_add,{"rhizome","add","file","<filepath>","[<manifestpath>]",NULL},0,
-   "Add a manifest and file to Rhizome."},
+   "Add a file to Rhizome and optionally write its manifest to the given path"},
+  {app_rhizome_list,{"rhizome","list",NULL},0,
+   "List all manifests and files in Rhizome"},
   {NULL,{NULL}}
 };

dna.c

@@ -609,13 +609,10 @@ int main(int argc,char **argv)
 	  rhizome_opendb();
 	  /* Also set hlr file to be in the Rhizome directory, to save the need to specify it
 	     separately. */
-	  char temp[1024];temp[1023]=0;
+	  char temp[1024];
-	  snprintf(temp,1024,"%s/hlr.dat",optarg); 
+	  if (snprintf(temp, sizeof(temp), "%s/hlr.dat", optarg) >= sizeof(temp))
-	  if (temp[1023]) {
 	    exit(WHY("Rhizome directory name too long."));
-	  }
+	  hlr_file = strdup(temp);
-	  hlr_file=strdup(temp);
-	  
 	  break;
 	case 'M': /* Distribute specified manifest and file pair using Rhizome. */
 	  /* This option assumes that the manifest is locally produced, and will

rhizome.c

@@ -36,51 +36,76 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
    The file should be included in the specified rhizome groups, if possible.
    (some groups may be closed groups that we do not have the private key for.)
 */
+
 int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
 			  int verifyP, int checkFileP, int signP)
 {
   char filename[1024];
   char manifestname[1024];
-  char *buffer;
  
+  if (snprintf(filename, sizeof(filename), "%s/import/file.%s", rhizome_datastore_path, bundle) >= sizeof(filename)
+   || snprintf(manifestname, sizeof(manifestname), "%s/import/manifest.%s", rhizome_datastore_path, bundle) >= sizeof(manifestname)) {
+    return WHY("Manifest bundle name too long");
+  }
 
-  snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
+  /* Read manifest file if no manifest was given */
-  snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
+  rhizome_manifest *m = m_in;
+  if (!m_in)  {
+    m = rhizome_read_manifest_file(manifestname, 0 /* file not buffer */, RHIZOME_VERIFY);
+    if (!m)
+      return WHY("Could not read manifest file.");
+  } else {
+    if (debug&DEBUG_RHIZOMESYNC)
+      fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP);
+  }
 
-  /* Open files */
+  /* Add the manifest and its associated file to the Rhizome database. */
-  rhizome_manifest *m=m_in;
+  int ret = rhizome_add_manifest(m, filename, groups, ttl, verifyP, checkFileP, signP);
+  unlink(filename);
+  if (ret == -1) {
+    unlink(manifestname);
+  } else {
+    /* >>> For testing, write manifest file back to disk and leave it there */
+    // unlink(manifestname);
+    if (rhizome_write_manifest_file(m, manifestname))
+      ret = WHY("Could not write manifest file.");
+  }
+
+  /* If the manifest was allocated in this function, then this function is responsible for freeing
+   * it */
   if (!m_in)
-    m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,RHIZOME_VERIFY);
+    rhizome_manifest_free(m);
-  else
-    if (debug&DEBUG_RHIZOMESYNC) fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP);
 
-  if (!m) return WHY("Could not read manifest file.");
+  return ret;
+}
+
+int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl, int verifyP, int checkFileP, int signP)
+{
+  char *buffer;
   char hexhash[SHA512_DIGEST_STRING_LENGTH];
 
-  /* work out time to live */
-  if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl;
-
   /* Keep associated file name handy for later */
-  m->dataFileName=strdup(filename);
+  m->dataFileName = strdup(filename);
-  if (checkFileP) {
+
+  /* Store time to live */
+  m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
+
+  /* Check file is accessible and discover its length */
+  if (checkFileP || verifyP) {
     struct stat stat;
-    if (lstat(filename,&stat)) {
+    if (lstat(filename,&stat))
       return WHY("Could not stat() associated file");
-      m->fileLength=stat.st_size;
+    m->fileLength = stat.st_size;
-    }
   }
 
-  if (checkFileP||signP) {
+  if (checkFileP || signP) {
-    if (rhizome_hash_file(filename,hexhash)) {
+    if (rhizome_hash_file(filename, hexhash))
-      rhizome_manifest_free(m);
       return WHY("Could not hash file.");
-    }
+    memcpy(&m->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH);
-    memcpy(&m->fileHexHash[0],&hexhash[0],SHA512_DIGEST_STRING_LENGTH);
+    m->fileHashedP = 1;
-    m->fileHashedP=1;
   }
 
-  if (verifyP)
+  if (verifyP) {
-    {
     /* Make sure hashes match.
        Make sure that no signature verification errors were spotted on loading. */
     int verifyErrors=0;
@@ -92,16 +117,11 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
     }
     if (m->errors)
       verifyErrors+=m->errors;
-      if (verifyErrors) {
+    if (verifyErrors)
-	rhizome_manifest_free(m);
-	unlink(manifestname);
-	unlink(filename);
       return WHY("Errors encountered verifying bundle manifest");
   }
-    }
+  else {
-
+    if (!(buffer = rhizome_manifest_get(m, "id", NULL, 0))) {
-  if (!verifyP) {
-    if ((buffer=rhizome_manifest_get(m,"id",NULL,0))!=NULL) {
       /* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system),
 	 so create one, and keep the private key handy. */
       printf("manifest does not have an id\n");
@@ -124,43 +144,34 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
     rhizome_manifest_set(m,"filehash",hexhash);
     if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
       /* Version not set, so set one */
-      rhizome_manifest_set_ll(m,"version",overlay_gettime_ms());
+      rhizome_manifest_set_ll(m,"version", overlay_gettime_ms());
-    rhizome_manifest_set_ll(m,"first_byte",0);
+    rhizome_manifest_set_ll(m,"first_byte", 0);
-    rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename));
+    rhizome_manifest_set_ll(m,"last_byte", m->fileLength);
   }
    
   /* Discard if it is older than the most recent known version */
-  long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
+  long long storedversion = sqlite_exec_int64(
-  if (storedversion>rhizome_manifest_get_ll(m,"version"))
+      "SELECT version from manifests where id='%s';",
-    {
+      rhizome_bytes_to_hex(m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)
-      rhizome_manifest_free(m);
+    );
+  if (storedversion > rhizome_manifest_get_ll(m, "version"))
     return WHY("Newer version exists");
-    }
 					      
   /* Add group memberships */
+  if (groups) {
+    int i;
+    for(i = 0; groups[i]; i++)
+      rhizome_manifest_add_group(m, groups[i]);
+  }
 
-int i;
+  if (rhizome_manifest_finalise(m,signP))
-  if (groups) for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]);
-
-  if (rhizome_manifest_finalise(m,signP)) {
     return WHY("Failed to finalise manifest.\n");
-  }
-
-  /* Write manifest back to disk */
-  if (rhizome_write_manifest_file(m,manifestname)) {
-    rhizome_manifest_free(m);
-    return WHY("Could not write manifest file.");
-  }
 
   /* Okay, it is written, and can be put directly into the rhizome database now */
-  int r=rhizome_store_bundle(m,filename);
+  if (rhizome_store_bundle(m, filename) == -1)
-  if (!r) {
-    // XXX For testing   unlink(manifestname);
-    unlink(filename);
-    return 0;
-  }
-
     return WHY("rhizome_store_bundle() failed.");
+
+  return 0;
 }
 
 /* Update an existing Rhizome bundle */

rhizome.h

@@ -160,16 +160,16 @@ extern sqlite3 *rhizome_db;
 
 int rhizome_opendb();
 int rhizome_manifest_createid(rhizome_manifest *m);
-int rhizome_write_manifest_file(rhizome_manifest *m,char *filename);
+int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
 int rhizome_manifest_sign(rhizome_manifest *m);
 int rhizome_drop_stored_file(char *id,int maximum_priority);
 int rhizome_manifest_priority(char *id);
-rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferPAndSize,int flags);
+rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferPAndSize, int flags);
-int rhizome_hash_file(char *filename,char *hash_out);
+int rhizome_hash_file(const char *filename,char *hash_out);
 char *rhizome_manifest_get(rhizome_manifest *m,char *var,char *value_out,int maxlen);
 long long  rhizome_manifest_get_ll(rhizome_manifest *m,char *var);
 int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value);
-int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value);
+int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value);
 long long rhizome_file_size(char *filename);
 void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line,
 			    rhizome_manifest *m);
@@ -177,13 +177,15 @@ void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line
 rhizome_manifest *_rhizome_new_manifest(const char *file,const char *func,int line);
 #define rhizome_new_manifest() _rhizome_new_manifest(__FILE__,__FUNCTION__,__LINE__)
 int rhizome_manifest_pack_variables(rhizome_manifest *m);
-int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename);
+int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename);
 int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
-int rhizome_store_file(char *file,char *hash,int priortity);
+int rhizome_store_file(const char *file,char *hash,int priortity);
 char *rhizome_safe_encode(unsigned char *in,int len);
 int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
 int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
 			  int verifyP, int checkFileP, int signP);
+int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl,
+			 int verifyP, int checkFileP, int signP);
 int rhizome_manifest_finalise(rhizome_manifest *m,int signP);
 char *rhizome_bytes_to_hex(unsigned char *in,int byteCount);
 int rhizome_hex_to_bytes(char *in,unsigned char *out,int hexChars);

rhizome_bundle.c

@@ -21,7 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "rhizome.h"
 #include <stdlib.h>
 
-rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flags)
+rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferP, int flags)
 {
   if (bufferP>MAX_MANIFEST_BYTES) return NULL;
 
@@ -30,12 +30,15 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
 
   if (bufferP) {
     m->manifest_bytes=bufferP;
-    bcopy(filename,m->manifestdata,m->manifest_bytes);
+    memcpy(m->manifestdata, filename, m->manifest_bytes);
   }
   else {
     FILE *f=fopen(filename,"r");
-    if (!f) { WHY("Could not open manifest file for reading."); 
+    if (!f) {
-      rhizome_manifest_free(m); return NULL; }
+      WHYF("Could not open manifest file %s for reading.", filename); 
+      rhizome_manifest_free(m);
+      return NULL;
+    }
     m->manifest_bytes = fread(m->manifestdata,1,MAX_MANIFEST_BYTES,f);
     fclose(f);
   }
@@ -144,7 +147,7 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
   return m;
 }
 
-int rhizome_hash_file(char *filename,char *hash_out)
+int rhizome_hash_file(const char *filename,char *hash_out)
 {
   /* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
      go.  Trouble is, we need to run Serval DNA on filesystems that lack mmap(),
@@ -212,7 +215,7 @@ double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_
 }
 
 
-int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value)
+int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value)
 {
   int i;
 
@@ -245,18 +248,6 @@ int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value)
   return rhizome_manifest_set(m,var,svalue);
 }
 
-long long rhizome_file_size(char *filename)
-{
-  FILE *f;
-
-  /* XXX really should just use stat instead of opening the file */
-  f=fopen(filename,"r");
-  fseek(f,0,SEEK_END);
-  long long size=ftello(f);
-  fclose(f);
-  return size;
-}
-
 #define MAX_RHIZOME_MANIFESTS 16
 rhizome_manifest manifests[MAX_RHIZOME_MANIFESTS];
 char manifest_free[MAX_RHIZOME_MANIFESTS];
@@ -420,7 +411,7 @@ int rhizome_manifest_sign(rhizome_manifest *m)
   return 0;
 }
 
-int rhizome_write_manifest_file(rhizome_manifest *m,char *filename)
+int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename)
 {
   if (!m) return WHY("Manifest is null.");
   if (!m->finalised) return WHY("Manifest must be finalised before it can be written.");

rhizome_database.c

@@ -290,7 +290,7 @@ int rhizome_drop_stored_file(char *id,int maximum_priority)
   We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, 
   FILEMANIFESTS and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
  */
-int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
+int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename)
 {
   char sqlcmd[1024];
   const char *cmdtail;
@@ -500,13 +500,68 @@ char *rhizome_safe_encode(unsigned char *in,int len)
   return r;
 }
 
+int rhizome_list_manifests(int limit, int offset)
+{
+  char sqlcmd[1024];
+  int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id");
+  if (n >= sizeof(sqlcmd))
+    return WHY("SQL command too long");
+  if (limit) {
+    n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " LIMIT %u", limit);
+    if (n >= sizeof(sqlcmd))
+      return WHY("SQL command too long");
+  }
+  if (offset) {
+    n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " OFFSET %u", offset);
+    if (n >= sizeof(sqlcmd))
+      return WHY("SQL command too long");
+  }
+  sqlite3_stmt *statement;
+  const char *cmdtail;
+  int ret = 0;
+  if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) {
+    sqlite3_finalize(statement);
+    ret = WHY(sqlite3_errmsg(rhizome_db));
+  } else {
+    size_t rows = 0;
+    while (sqlite3_step(statement) == SQLITE_ROW) {
+      ++rows;
+      if (!(   sqlite3_column_count(statement) == 7
+	    && sqlite3_column_type(statement, 0) == SQLITE_TEXT
+	    && sqlite3_column_type(statement, 1) == SQLITE_INTEGER
+	    && sqlite3_column_type(statement, 2) == SQLITE_INTEGER
+	    && sqlite3_column_type(statement, 3) == SQLITE_TEXT
+	    && sqlite3_column_type(statement, 4) == SQLITE_BLOB
+	    && sqlite3_column_type(statement, 5) == SQLITE_INTEGER
+	    && sqlite3_column_type(statement, 6) == SQLITE_INTEGER
+      )) { 
+	ret = WHY("Incorrect statement column");
+	break;
+      }
+      size_t filesize = sqlite3_column_int(statement, 1);
+      size_t manifestblobsize = sqlite3_column_bytes(statement, 4);
+      const char *manifestblob = (char *) sqlite3_column_blob(statement, 4);
+      printf("manifest blob = %s\n", manifestblob);
+      rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, RHIZOME_VERIFY);
+      const char *name = rhizome_manifest_get(m, "name", NULL, 0);
+      printf("file id = %s\nfile length = %u\nfile datavalid = %u\nfile name = \"%s\"\n",
+	  sqlite3_column_text(statement, 0),
+	  filesize,
+	  sqlite3_column_int(statement, 2),
+	  name
+	);
+      rhizome_manifest_free(m);
+    }
+    printf("Found %lu rows\n", (unsigned long) rows);
+  }
+  sqlite3_finalize(statement);
+  return ret;
+}
 
-/* The following function just stores the file (or silently returns if it already
+/* The following function just stores the file (or silently returns if it already exists).
-   exists).
+   The relationships of manifests to this file are the responsibility of the caller. */
-   The relationships of manifests to this file are the responsibility of the
+int rhizome_store_file(const char *file,char *hash,int priority)
-   caller. */
+{
-int rhizome_store_file(char *file,char *hash,int priority) {
-
   int fd=open(file,O_RDONLY);
   if (fd<0) return WHY("Could not open associated file");
   
@@ -695,5 +750,3 @@ int rhizome_update_file_priority(char *fileid)
   }
   return 0;
 }
-
-