Add support for importing and exporting a file with an appended manifest

commandline.c

@@ -1115,7 +1115,7 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
     return -1;
   
   if (manifestpath[0] 
-      && rhizome_write_manifest_file(mout, manifestpath) == -1)
+      && rhizome_write_manifest_file(mout, manifestpath, 0) == -1)
     ret = WHY("Could not overwrite manifest file.");
   const char *service = rhizome_manifest_get(mout, "service", NULL, 0);
   if (service) {
@@ -1216,6 +1216,32 @@ cleanup:
   return status;
 }
 
+int app_rhizome_append_manifest(int argc, const char *const *argv, const struct command_line_option *o, void *context)
+{
+  if (config.debug.verbose) DEBUG_argv("command", argc, argv);
+  const char *manifestpath, *filepath;
+  if (cli_arg(argc, argv, o, "manifestpath", &manifestpath, NULL, "") == -1
+    || cli_arg(argc, argv, o, "filepath", &filepath, NULL, "") == -1)
+    return -1;
+  
+  rhizome_manifest *m = rhizome_new_manifest();
+  if (!m)
+    return WHY("Out of manifests.");
+  
+  int ret=0;
+  if (rhizome_read_manifest_file(m, manifestpath, 0))
+    ret=-1;
+  // TODO why doesn't read manifest file set finalised???
+  m->finalised=1;
+  
+  if (ret==0 && rhizome_write_manifest_file(m, filepath, 1) == -1)
+    ret = -1;
+  
+  if (m)
+    rhizome_manifest_free(m);
+  return ret;
+}
+
 int app_rhizome_extract_bundle(int argc, const char *const *argv, const struct command_line_option *o, void *context)
 {
   if (config.debug.verbose) DEBUG_argv("command", argc, argv);
@@ -1292,13 +1318,16 @@ int app_rhizome_extract_bundle(int argc, const char *const *argv, const struct c
       cli_write(m->manifestdata, m->manifest_all_bytes);
       cli_delim("\n");
     } else {
-      /* If the manifest has been read in from database, the blob is there,
-       and we can lie and say we are finalised and just want to write it
-       out.  TODO: really should have a dirty/clean flag, so that write
-       works if clean but not finalised. */
-      m->finalised=1;
-      if (rhizome_write_manifest_file(m, manifestpath) == -1)
-	ret = -1;
+      int append = (strcmp(manifestpath, filepath)==0)?1:0;
+      // don't write out the manifest if we were asked to append it and writing the file failed.
+      if ((!append) || retfile==0){
+	/* If the manifest has been read in from database, the blob is there,
+	 and we can lie and say we are finalised and just want to write it
+	 out.  TODO: really should have a dirty/clean flag, so that write
+	 works if clean but not finalised. */
+	m->finalised=1;
+	if (rhizome_write_manifest_file(m, manifestpath, append) == -1)
+	  ret = -1;
       }
     }
   }
@@ -1964,6 +1993,8 @@ struct command_line_option command_line_options[]={
    "Get specified configuration variable."},
   {app_vomp_console,{"console",NULL},0,
     "Test phone call life-cycle from the console"},
+  {app_rhizome_append_manifest, {"rhizome", "append", "manifest", "<filepath>", "<manifestpath>", NULL}, CLIFLAG_STANDALONE,
+    "Append a manifest to the end of the file it belongs to."},
   {app_rhizome_hash_file,{"rhizome","hash","file","<filepath>",NULL},CLIFLAG_STANDALONE,
    "Compute the Rhizome hash of a file"},
   {app_rhizome_add_file,{"rhizome","add","file","<author_sid>","<pin>","<filepath>","[<manifestpath>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,

rhizome.c

@@ -72,7 +72,62 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
 	manifest_path ? alloca_str_toprint(manifest_path) : "NULL",
 	filepath ? alloca_str_toprint(filepath) : "NULL");
   
-  if (rhizome_read_manifest_file(m, manifest_path, 0) == -1)
+  unsigned char buffer[MAX_MANIFEST_BYTES];
+  int buffer_len=0;
+  
+  // manifest has been appended to the end of the file.
+  if (strcmp(manifest_path, filepath)==0){
+    unsigned char marker[4];
+    int ret=0;
+    FILE *f = fopen(filepath, "r");
+    
+    if (f == NULL)
+      return WHYF_perror("Could not open manifest file %s for reading.", filepath);
+    
+    if (fseek(f, -sizeof(marker), SEEK_END))
+      ret=WHY_perror("Unable to seek to end of file");
+    
+    if (ret==0){
+      ret = fread(marker, 1, sizeof(marker), f);
+      if (ret==sizeof(marker))
+	ret=0;
+      else
+	ret=WHY_perror("Unable to read end of manifest marker");
+    }
+    
+    if (ret==0){
+      if (marker[2]!=0x41 || marker[3]!=0x10)
+	ret=WHYF("Expected 0x4110 marker at end of file");
+    }
+    
+    if (ret==0){
+      buffer_len = read_uint16(marker);
+      if (buffer_len < 1 || buffer_len > MAX_MANIFEST_BYTES)
+	ret=WHYF("Invalid manifest length %d", buffer_len);
+    }
+    
+    if (ret==0){
+      if (fseek(f, -(buffer_len+sizeof(marker)), SEEK_END))
+	ret=WHY_perror("Unable to seek to end of file");
+    }
+    
+    if (ret==0){
+      ret = fread(buffer, 1, buffer_len, f);
+      if (ret==buffer_len)
+	ret=0;
+      else
+	ret=WHY_perror("Unable to read manifest contents");
+    }
+    
+    fclose(f);
+    
+    if (ret)
+      return ret;
+    
+    manifest_path=(char*)buffer;
+  }
+  
+  if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1)
     return WHY("could not read manifest file");
   if (rhizome_manifest_verify(m))
     return WHY("could not verify manifest");

rhizome.h

@@ -225,7 +225,7 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
 
 #define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)
 
-int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
+int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
 int rhizome_manifest_selfsign(rhizome_manifest *m);
 int rhizome_drop_stored_file(const char *id,int maximum_priority);
 int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id);

rhizome_bundle.c

@@ -586,23 +586,31 @@ int rhizome_manifest_selfsign(rhizome_manifest *m)
   return 0;
 }
 
-int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename)
+int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append)
 {
   if (config.debug.rhizome) DEBUGF("write manifest (%d bytes) to %s", m->manifest_all_bytes, filename);
   if (!m) return WHY("Manifest is null.");
   if (!m->finalised) return WHY("Manifest must be finalised before it can be written.");
-  FILE *f = fopen(filename, "w");
-  if (f == NULL) {
-    WHY_perror("fopen");
-    return WHYF("Cannot write manifest to %s", filename);
+  FILE *f = fopen(filename, (append?"a":"w"));
+  if (f == NULL)
+    return WHYF_perror("Cannot write manifest to %s", filename);
+  
+  int ret = 0;
+  if (fwrite(m->manifestdata, m->manifest_all_bytes, 1, f)!=1)
+    ret=WHYF_perror("fwrite(%s)", filename);
+  
+  if (ret==0 && append){
+    unsigned char marker[4];
+    write_uint16(marker, m->manifest_all_bytes);
+    marker[2]=0x41;
+    marker[3]=0x10;
+    if (fwrite(marker, 4,1,f)!=1)
+      ret=WHYF_perror("fwrite(%s)", filename);
   }
-  int r1 = fwrite(m->manifestdata, m->manifest_all_bytes, 1, f);
-  int r2 = fclose(f);
-  if (r1 != 1)
-    return WHYF("fwrite(%s) returned %d", filename, r1);
-  if (r2 == EOF)
-    return WHYF("fclose(%s) returned %d", filename, r2);
-  return 0;
+  
+  if (fclose(f))
+    ret=WHYF_perror("fclose(%s)", filename);
+  return ret;
 }
 
 /*

tests/rhizomeops

@@ -805,4 +805,30 @@ test_ImportOwnBundle() {
    assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
 }
 
+doc_ImportCombinedBundle="Can generate a combined bundle, import into another instance and export again"
+setup_ImportCombinedBundle() {
+   setup_servald
+   setup_rhizome
+   set_instance +A
+   echo "Hello from A" >fileA
+   executeOk_servald rhizome add file $SIDA1 '' fileA fileA.manifest
+   assert_stdout_add_file fileA
+   extract_manifest_id manifestid fileA.manifest
+   extract_manifest_filehash filehash fileA.manifest
+   extract_manifest_filesize filesize fileA.manifest
+}
+test_ImportCombinedBundle() {
+   executeOk_servald rhizome append manifest fileA fileA.manifest
+   set_instance +B
+   executeOk_servald rhizome import bundle fileA fileA
+   assertStdoutGrep --matches=1 "^service:file$"
+   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
+   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
+   assertStdoutGrep --matches=1 "^filesize:$filesize\$"
+   executeOk_servald rhizome list ''
+   assert_rhizome_list --fromhere=0 fileA
+   executeOk_servald rhizome extract bundle $manifestid fileAx fileAx
+   assert diff fileA fileAx
+}
+
 runTests "$@"