Extract files based on the manifest id

Renamed rhizome extract file to rhizome dump file
Added rhizome extract file [manifest] [filepath] [pins]
Modified tests to use the appropriate command, assuming that MeshMS payloads will be encrypted

commandline.c

@@ -1290,26 +1290,51 @@ int app_rhizome_extract_manifest(int argc, const char *const *argv, const struct
 int app_rhizome_extract_file(int argc, const char *const *argv, const struct command_line_option *o, void *context)
 {
   if (config.debug.verbose) DEBUG_argv("command", argc, argv);
-  const char *fileid, *filepath, *keyhex;
-  if (cli_arg(argc, argv, o, "fileid", &fileid, cli_fileid, NULL)
-   || cli_arg(argc, argv, o, "filepath", &filepath, NULL, "") == -1)
-    return -1;
-  cli_arg(argc, argv, o, "key", &keyhex, cli_optional_bundle_crypt_key, "");
-  unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
-  if (keyhex[0] && fromhexstr(key, keyhex, RHIZOME_CRYPT_KEY_BYTES) == -1)
+  const char *fileid, *filepath, *manifestid, *pins;
+  if (cli_arg(argc, argv, o, "manifestid", &manifestid, cli_manifestid, NULL) == -1
+      || cli_arg(argc, argv, o, "filepath", &filepath, NULL, "") == -1
+      || cli_arg(argc, argv, o, "fileid", &fileid, cli_fileid, NULL) == -1
+      || cli_arg(argc, argv, o, "pin,pin...", &pins, NULL, "") == -1)
     return -1;
   /* Ensure the Rhizome database exists and is open */
   if (create_serval_instance_dir() == -1)
     return -1;
   if (rhizome_opendb() == -1)
     return -1;
-  /* Extract the file from the database. */
-  if (!rhizome_exists(fileid)){
-    return 1;
+  
+  int ret=0;
+  
+  if (manifestid){
+    if (!(keyring = keyring_open_with_pins(pins)))
+      return -1;
+    
+    unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
+    if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
+      return WHY("Invalid manifest ID");
+    
+    char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
+    tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
+    
+    rhizome_manifest *m = rhizome_new_manifest();
+    if (m==NULL)
+      return WHY("Out of manifests");
+    
+    ret = rhizome_retrieve_manifest(manifestIdUpper, m);
+    if (ret==0){
+      ret = rhizome_extract_file(m, filepath);
+    }
+    
+    if (m)
+      rhizome_manifest_free(m);
+    
+  }else if(fileid){
+    if (!rhizome_exists(fileid))
+      return 1;
+    
+    ret = rhizome_dump_file(fileid, filepath);
   }
-  if (rhizome_retrieve_file(fileid, filepath, keyhex[0] ? key : NULL))
-    return -1;
-  return 0;
+  
+  return ret;
 }
 
 int app_rhizome_list(int argc, const char *const *argv, const struct command_line_option *o, void *context)
@@ -1946,8 +1971,10 @@ struct command_line_option command_line_options[]={
    "List all manifests and files in Rhizome"},
   {app_rhizome_extract_manifest,{"rhizome","extract","manifest","<manifestid>","[<manifestpath>]","[<pin,pin...>]",NULL},CLIFLAG_STANDALONE,
    "Extract a manifest from Rhizome and write it to the given path"},
-  {app_rhizome_extract_file,{"rhizome","extract","file","<fileid>","[<filepath>]","[<key>]",NULL},CLIFLAG_STANDALONE,
+  {app_rhizome_extract_file,{"rhizome","extract","file","<manifestid>","[<filepath>]","[<pin,pin...>]",NULL},CLIFLAG_STANDALONE,
    "Extract a file from Rhizome and write it to the given path"},
+  {app_rhizome_extract_file,{"rhizome","dump","file","<fileid>","[<filepath>]",NULL},CLIFLAG_STANDALONE,
+   "Extract a file from Rhizome and write it to the given path without attempting decryption"},
   {app_rhizome_direct_sync,{"rhizome","direct","sync","[peer url]",NULL},
    CLIFLAG_STANDALONE,
    "Synchronise with the specified Rhizome Direct server. Return when done."},

rhizome.h

@@ -305,8 +305,6 @@ long long rhizome_bar_version(unsigned char *bar);
 unsigned long long rhizome_bar_bidprefix_ll(unsigned char *bar);
 int rhizome_list_manifests(const char *service, const char *sender_sid, const char *recipient_sid, int limit, int offset);
 int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
-int rhizome_retrieve_file(const char *fileid, const char *filepath,
-			  const unsigned char *key);
 int rhizome_find_manifest_secret(rhizome_manifest *m);
 
 #define RHIZOME_DONTVERIFY 0
@@ -647,5 +645,7 @@ int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stre
 			    const unsigned char *key, unsigned char *nonce);
 int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);
 int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
+int rhizome_extract_file(rhizome_manifest *m, const char *filepath);
+int rhizome_dump_file(const char *id, const char *filepath);
 
 #endif //__SERVALDNA__RHIZOME_H

rhizome_database.c

@@ -1242,6 +1242,7 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m){
     
     m->inserttime = q_inserttime;
   }else{
+    INFOF("Manifest %s was not found", manifestid);
     ret=1;
   }
   
@@ -1249,61 +1250,3 @@ done:
   sqlite3_finalize(statement);
   return ret;  
 }
-
-
-/* Retrieve a file from the database, given its file hash.
- *
- * Returns 0 if file is valid, contents are written to filepath if given.
- * Returns -1 on error.
- */
-int rhizome_retrieve_file(const char *fileid, const char *filepath, const unsigned char *key)
-{
-  int ret=0;
-  
-  if (rhizome_update_file_priority(fileid) == -1)
-    return WHY("Failed to update file priority");
-  
-  struct rhizome_read read_state;
-  bzero(&read_state, sizeof read_state);
-  
-  // for now, always hash the file
-  if (rhizome_open_read(&read_state, fileid, 1))
-    return -1;
-  
-  int fd=-1;
-  
-  if (filepath&&filepath[0]) {
-    fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
-    if (fd == -1)
-      return WHY_perror("open");
-  }
-  
-  if (key){
-    bcopy(key, read_state.key, sizeof(read_state.key));
-    read_state.crypt=1;
-  }
-  
-  unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
-  while((ret=rhizome_read(&read_state, buffer, sizeof(buffer)))>0){
-    if (fd!=-1){
-      if (write(fd,buffer,ret)!=ret) {
-	ret = WHY("Failed to write data to file");
-	break;
-      }
-    }
-  }
-  
-  if (fd!=-1){
-    if (close(fd)==-1)
-      ret=WHY_perror("close");
-  }
-  
-  if (ret>=0){
-    cli_puts("filehash"); cli_delim(":");
-    cli_puts(read_state.id); cli_delim("\n");
-    cli_puts("filesize"); cli_delim(":");
-    cli_printf("%lld", read_state.length); cli_delim("\n");
-  }
-  
-  return ret;
-}

rhizome_store.c

@@ -453,3 +453,59 @@ int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_le
       return -1;
   }while (1);
 }
+
+static int write_file(struct rhizome_read *read, const char *filepath){
+  int fd=-1, ret=0;
+  
+  if (filepath&&filepath[0]) {
+    fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
+    if (fd == -1)
+      return WHY_perror("open");
+  }
+  
+  unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
+  while((ret=rhizome_read(read, buffer, sizeof(buffer)))>0){
+    if (fd!=-1){
+      if (write(fd,buffer,ret)!=ret) {
+	ret = WHY("Failed to write data to file");
+	break;
+      }
+    }
+  }
+  
+  if (fd!=-1){
+    if (close(fd)==-1)
+      ret=WHY_perror("close");
+    if (ret<0){
+      // TODO delete partial file
+    }
+  }
+  
+  return ret;
+}
+
+/* Extract the file related to a manifest to the file system.
+ * The file will be de-crypted and verified while reading.
+ * If filepath is not supplied, the file will still be checked.
+ */
+int rhizome_extract_file(rhizome_manifest *m, const char *filepath){
+  struct rhizome_read read_state;
+  bzero(&read_state, sizeof read_state);
+  
+  // for now, always hash the file
+  if (rhizome_open_read(&read_state, m->fileHexHash, 1))
+    return -1;
+  
+  return write_file(&read_state, filepath);
+}
+
+/* dump the raw contents of a file */
+int rhizome_dump_file(const char *id, const char *filepath){
+  struct rhizome_read read_state;
+  bzero(&read_state, sizeof read_state);
+
+  if (rhizome_open_read(&read_state, id, 1))
+    return -1;
+  
+  return write_file(&read_state, filepath);
+}

testdefs_rhizome.sh

@@ -361,11 +361,11 @@ rhizome_update_file() {
 assert_rhizome_received() {
    [ $# -ne 0 ] || error "missing arguments"
    local name
-   local _hash
+   local _id
    for name; do
       if [ -s "$name" ]; then
-         extract_manifest_filehash _hash "$name.manifest"
-         executeOk_servald rhizome extract file "$_hash" extracted
+         extract_manifest_id _id "$name.manifest"
+         executeOk_servald rhizome extract file "$_id" extracted
          assert cmp "$name" extracted
       fi
    done

tests/rhizomeops

@@ -283,25 +283,32 @@ setup_ExtractFileAfterAdd() {
    tfw_cat --stderr
    executeOk_servald rhizome list ''
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
-   extract_manifest_filehash filehash file1.manifest
+   extract_manifest_id BID file1.manifest
 }
 test_ExtractFileAfterAdd() {
-   executeOk_servald rhizome extract file $filehash file1x
+   executeOk_servald rhizome extract file $BID file1x
+   tfw_cat --stderr
    assert cmp file1 file1x
    local size=$(( $(cat file1 | wc -c) + 0 ))
-   assertStdoutLineCount '==' 2
-   assertStdoutGrep --matches=1 "^filehash:$filehash$"
-   assertStdoutGrep --matches=1 "^filesize:$size$"
+   assertStdoutLineCount '==' 0
+#   assertStdoutGrep --matches=1 "^filehash:$filehash$"
+#   assertStdoutGrep --matches=1 "^filesize:$size$"
 }
 
 doc_ExtractFileMissing="Extract non-existent file"
 setup_ExtractFileMissing() {
    setup_servald
    setup_rhizome
+   BID=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
    filehash=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
 }
 test_ExtractFileMissing() {
-   execute --exit-status=1 $servald rhizome extract file $filehash foo
+   execute --exit-status=1 $servald rhizome extract file $BID foo
+   tfw_cat --stderr
+   assertStdoutLineCount '==' 0
+   assert [ ! -e foo ]
+   execute --exit-status=1 $servald rhizome dump file $filehash foo
+   tfw_cat --stderr
    assertStdoutLineCount '==' 0
    assert [ ! -e foo ]
 }
@@ -312,15 +319,24 @@ setup_ExtractFileInvalidID() {
    setup_rhizome
 }
 test_ExtractFileInvalidID() {
-   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
+   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
    assertStdoutLineCount '==' 0
    assert [ ! -e foo ]
-   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
+   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
    assertStdoutLineCount '==' 0
    assert [ ! -e foo ]
    execute --exit-status=255 $servald rhizome extract file '' foo
    assertStdoutLineCount '==' 0
    assert [ ! -e foo ]
+   execute --exit-status=255 $servald rhizome dump file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
+   assertStdoutLineCount '==' 0
+   assert [ ! -e foo ]
+   execute --exit-status=255 $servald rhizome dump file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
+   assertStdoutLineCount '==' 0
+   assert [ ! -e foo ]
+   execute --exit-status=255 $servald rhizome dump file '' foo
+   assertStdoutLineCount '==' 0
+   assert [ ! -e foo ]
 }
 
 doc_AddDuplicate="Add same manifest detects duplicate"
@@ -507,8 +523,8 @@ test_MeshMSAddCreate() {
    assert_manifest_complete file1.manifest
    executeOk_servald rhizome list ''
    assert_rhizome_list --fromhere=1 file1
-   extract_manifest_filehash filehash file1.manifest
-   executeOk_servald rhizome extract file $filehash file1x
+   extract_manifest_id BID file1.manifest
+   executeOk_servald rhizome extract file $BID file1x
    assert diff file1 file1x
 }
 
@@ -539,14 +555,13 @@ test_MeshMSAddGrow() {
       extract_manifest_id idx file1.manifest
       extract_manifest_filehash filehashx file1.manifest
       extract_manifest_BK bkx file1.manifest
-      compute_filehash filehash file1
       assert --message="manifest ID remains the same" [ "$idx" = "$id" ]
       assert --message="manifest BK remains the same" [ "$bkx" = "$bk" ]
-      assert --message="filehash is for new file" [ "$filehash" = "$filehashx" ]
-      executeOk_servald rhizome extract file "$filehash" file1x
+      assert --message="filehash is for new file" [ "$filehash" != "$filehashx" ]
+      executeOk_servald rhizome extract file "$id" file1x
       assert --message="extracted payload is correct" diff file1 file1x
       for ofilehash in "${ofilehashes[@]}"; do
-         execute --exit-status=1 --stderr $servald rhizome extract file "$ofilehash"
+         execute --exit-status=1 --stderr $servald rhizome dump file "$ofilehash"
       done
    done
 }

tests/rhizomestress

@@ -93,8 +93,9 @@ ammendfile() {
    echo "XXX  ${instance_name}  XXX" >> $instance_dir/file
    executeOk_servald rhizome add file "" "" "$instance_dir/file" "$instance_dir/file.manifest" 
    tfw_cat --stdout --stderr
-   extract_manifest_filehash filehash "$instance_dir/file.manifest"
-   executeOk_servald rhizome extract file $filehash
+   extract_manifest_id id "$instance_dir/file.manifest"
+   # read file contents to verify successful add
+   executeOk_servald rhizome extract file $id
    rm "$instance_dir/file.manifest"
 }