Add "rhizome add file" --force-new option

Overrides the default behaviour of de-duplicating added manifests

New 'rhizomeops' test case: AddForceDuplicate

commandline.c

@@ -1290,6 +1290,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
     DEBUG_cli_parsed(parsed);
   const char *filepath, *manifestpath, *manifestid, *authorSidHex, *bskhex;
 
+  int force_new = 0 == cli_arg(parsed, "--force-new", NULL, NULL, NULL);
   cli_arg(parsed, "filepath", &filepath, NULL, "");
   if (cli_arg(parsed, "author_sid", &authorSidHex, cli_optional_sid, "") == -1)
     return -1;
@@ -1382,7 +1383,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
   }
   
   rhizome_manifest *mout = NULL;
-  int ret=rhizome_manifest_finalise(m,&mout);
+  int ret=rhizome_manifest_finalise(m, &mout, !force_new);
   if (ret<0){
     rhizome_manifest_free(m);
     return -1;
@@ -2473,7 +2474,7 @@ struct cli_schema command_line_options[]={
     "Append a manifest to the end of the file it belongs to."},
   {app_rhizome_hash_file,{"rhizome","hash","file","<filepath>",NULL}, 0,
    "Compute the Rhizome hash of a file"},
-  {app_rhizome_add_file,{"rhizome","add","file" KEYRING_PIN_OPTIONS,"<author_sid>","<filepath>","[<manifestpath>]","[<bsk>]",NULL}, 0,
+  {app_rhizome_add_file,{"rhizome","add","file" KEYRING_PIN_OPTIONS,"[--force-new]","<author_sid>","<filepath>","[<manifestpath>]","[<bsk>]",NULL}, 0,
 	"Add a file to Rhizome and optionally write its manifest to the given path"},
   {app_rhizome_add_file, {"rhizome", "journal", "append" KEYRING_PIN_OPTIONS, "<author_sid>", "<manifestid>", "<filepath>", "[<bsk>]", NULL}, 0,
 	"Append content to a journal bundle"},

meshms.c

@@ -318,7 +318,7 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv,
   if (rhizome_append_journal_buffer(m, NULL, 0, buffer, len))
     goto end;
   
-  if (rhizome_manifest_finalise(m, &mout))
+  if (rhizome_manifest_finalise(m, &mout, 1))
     goto end;
 
   ret=0;
@@ -594,7 +594,7 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
     goto end;
   strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
   rhizome_manifest_set(m, "filehash", m->fileHexHash);
-  if (rhizome_manifest_finalise(m,&mout))
+  if (rhizome_manifest_finalise(m, &mout, 1))
     goto end;
   
   ret=0;
@@ -962,4 +962,4 @@ end:
     rhizome_manifest_free(m);
   free_conversations(conv);
   return ret;
-}
+}

rhizome.h

@@ -273,7 +273,7 @@ int rhizome_manifest_verify(rhizome_manifest *m);
 int rhizome_manifest_check_sanity(rhizome_manifest *m_in);
 
 int rhizome_manifest_bind_id(rhizome_manifest *m_in);
-int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout);
+int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate);
 int rhizome_add_manifest(rhizome_manifest *m_in,int ttl);
 
 void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount);

rhizome_bundle.c

@@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
 #include <stdlib.h>
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "rhizome.h"
@@ -699,17 +700,15 @@ int rhizome_manifest_dump(rhizome_manifest *m, const char *msg)
   return 0;
 }
 
-int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout)
+int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate)
 {
   IN();
   int ret=0;
   
   // if a manifest was supplied with an ID, don't bother to check for a duplicate.
   // we only want to filter out added files with no existing manifest.
-  if (m->haveSecret==NEW_BUNDLE_ID){
-    if (rhizome_find_duplicate(m, mout)==1)
-      RETURN(2);
-  }
+  if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID && rhizome_find_duplicate(m, mout) == 1)
+    RETURN(2);
   
   *mout=m;
   
@@ -800,6 +799,7 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
       // TODO assert that new version > old version?
     }
   }
+  assert(m->haveSecret);
   
   int crypt = rhizome_manifest_get_ll(m,"crypt"); 
   if (crypt==-1){

rhizome_crypto.c

@@ -38,10 +38,10 @@ unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m)
 
 int rhizome_manifest_createid(rhizome_manifest *m)
 {
-  m->haveSecret=NEW_BUNDLE_ID;
   if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic,m->cryptoSignSecret))
     return WHY("Failed to create keypair for manifest ID.");
   rhizome_manifest_set(m, "id", alloca_tohex_bid(m->cryptoSignPublic));
+  m->haveSecret = NEW_BUNDLE_ID;
   return 0;
 }
 

tests/rhizomeops

@@ -486,7 +486,7 @@ test_ExtractFileInvalidID() {
    assert [ ! -e foo ]
 }
 
-doc_AddDeDuplicate="Add same manifest detects duplicate"
+doc_AddDeDuplicate="Add duplicate file does not create new bundle by default"
 setup_AddDeDuplicate() {
    setup_servald
    setup_rhizome
@@ -517,6 +517,7 @@ test_AddDeDuplicate() {
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
    strip_signatures file1.manifest file1.manifestA
    assert diff file1.manifest file1.manifestA
+   assert [ $file1_secret = $file1_dup_secret ]
    # Repeat for second file.
    execute --exit-status=2 $servald rhizome add file $SIDB1 file2 file2.manifestA
    assert [ -s file2.manifestA ]
@@ -526,10 +527,37 @@ test_AddDeDuplicate() {
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
    strip_signatures file2.manifest file2.manifestA
    assert diff file2.manifest file2.manifestA
-   assert [ $file1_secret = $file1_dup_secret ]
    assert [ $file2_secret = $file2_dup_secret ]
 }
 
+doc_AddForceDuplicate="Add duplicate file with --force-new option"
+setup_AddForceDuplicate() {
+   setup_AddDeDuplicate
+}
+test_AddForceDuplicate() {
+   # Add first file again with the --force-new option.  A new manifest
+   # should be created with a new ID.
+   executeOk_servald rhizome add file --force-new $SIDB1 file1 file1.manifestA
+   assert [ -s file1.manifestA ]
+   assert_stdout_add_file --manifest=file1.manifestA file1
+   extract_stdout_secret file1_dup_secret
+   executeOk_servald rhizome list
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2 --manifest=file1.manifestA file1
+   strip_signatures file1.manifest file1.manifestA
+   assert ! diff file1.manifest file1.manifestA
+   assert [ $file1_secret != $file1_dup_secret ]
+   # Repeat for second file.
+   executeOk_servald rhizome add file --force-new $SIDB1 file2 file2.manifestA
+   assert [ -s file2.manifestA ]
+   assert_stdout_add_file --manifest=file2.manifestA file2
+   extract_stdout_secret file2_dup_secret
+   executeOk_servald rhizome list
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2 --manifest=file1.manifestA file1 --manifest=file2.manifestA file2
+   strip_signatures file2.manifest file2.manifestA
+   assert ! diff file2.manifest file2.manifestA
+   assert [ $file2_secret != $file2_dup_secret ]
+}
+
 doc_AddMismatched="Add mismatched manifest/payload fails"
 setup_AddMismatched() {
    setup_AddDeDuplicate