Merge branch 'master' of github.com:servalproject/serval-dna

commandline.c

@@ -294,7 +294,7 @@ int cli_arg(int argc, const char *const *argv, command_line_option *o, char *arg
     ) {
       const char *value = argv[i];
       if (validator && !(*validator)(value))
-	return setReason("Invalid argument %d '%s': \"%s\"", i + 1, argname, value);
+	return WHYF("Invalid argument %d '%s': \"%s\"", i + 1, argname, value);
       *dst = value;
       return 0;
     }
@@ -1073,6 +1073,18 @@ int cli_optional_sid(const char *arg)
   return !arg[0] || validateSid(arg);
 }
 
+int app_rhizome_hash_file(int argc, const char *const *argv, struct command_line_option *o)
+{
+  const char *filepath;
+  cli_arg(argc, argv, o, "filepath", &filepath, NULL, "");
+  char hexhash[SHA512_DIGEST_STRING_LENGTH];
+  if (rhizome_hash_file(filepath, hexhash))
+    return -1;
+  cli_puts(hexhash);
+  cli_delim("\n");
+  return 0;
+}
+
 int app_rhizome_add_file(int argc, const char *const *argv, struct command_line_option *o)
 {
   const char *filepath, *manifestpath, *authorSid, *pin;
@@ -1571,6 +1583,8 @@ command_line_option command_line_options[]={
    "Set specified configuration variable."},
   {app_config_get,{"config","get","[<variable>]",NULL},CLIFLAG_STANDALONE,
    "Get specified configuration variable."},
+  {app_rhizome_hash_file,{"rhizome","hash","file","<filepath>",NULL},CLIFLAG_STANDALONE,
+   "Compute the Rhizome hash of a file"},
   {app_rhizome_add_file,{"rhizome","add","file","<author_sid>","<pin>","<filepath>","[<manifestpath>]",NULL},CLIFLAG_STANDALONE,
    "Add a file to Rhizome and optionally write its manifest to the given path"},
   {app_rhizome_list,{"rhizome","list","[<service>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,

monitor.c

@@ -627,7 +627,7 @@ int monitor_call_status(vomp_call_state *call)
 int monitor_announce_peer(unsigned char *sid)
 {
   unsigned char msg[1024];
-  snprintf((char *)msg,1024,"\nnewpeer:%s\n",overlay_render_sid(sid));
+  snprintf((char *)msg,1024,"\nNEWPEER:%s\n",overlay_render_sid(sid));
   monitor_tell_clients(msg,strlen((char *)msg),MONITOR_PEERS);
   return 0;
 }

rhizome_bundle.c

@@ -174,17 +174,25 @@ int rhizome_hash_file(const char *filename,char *hash_out)
      implementation.
   */
   FILE *f = fopen(filename, "r");
-  if (!f)
+  if (!f) {
-    return WHYF("Could not read %s to calculate SHA512 hash.", filename);
+    WHY_perror("fopen");
+    return WHYF("Could not open %s to calculate SHA512 hash.", filename);
+  }
   SHA512_CTX context;
   SHA512_Init(&context);
   while (!feof(f)) {
     unsigned char buffer[8192];
     int r = fread(buffer, 1, 8192, f);
+    if (r == -1) {
+      WHY_perror("fread");
+      fclose(f);
+      return WHYF("Error reading %s to calculate SHA512 hash", filename);
+    }
     if (r > 0)
       SHA512_Update(&context, buffer, r);
   }
   SHA512_End(&context, (char *)hash_out);
+  fclose(f);
   return 0;
 }
 

rhizome_database.c

@@ -584,7 +584,20 @@ int rhizome_list_manifests(const char *service, const char *sender_sid, const ch
       size_t manifestblobsize = sqlite3_column_bytes(statement, 3); // must call after sqlite3_column_blob()
       rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0);
       const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
-      if (!service[0] || (blob_service && strcasecmp(service, blob_service) == 0)) {
+      int match = 1;
+      if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0))
+	match = 0;
+      if (match && sender_sid[0]) {
+	const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0);
+	if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0))
+	  match = 0;
+      }
+      if (match && recipient_sid[0]) {
+	const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
+	if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0))
+	  match = 0;
+      }
+      if (match) {
 	const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
 	long long blob_date = rhizome_manifest_get_ll(m, "date");
 	cli_puts(blob_service ? blob_service : ""); cli_delim(":");

rhizome_packetformats.c

@@ -322,7 +322,7 @@ int overlay_rhizome_saw_advertisements(int i,overlay_frame *f, long long now)
 	m=rhizome_read_manifest_file((char *)&f->payload->bytes[ofs],
 				     manifest_length,RHIZOME_DONTVERIFY);
 	int importManifest=0;
-	if (!m->errors)
+	if (m&&(!m->errors))
 	  {
 	    /* Manifest is okay, so see if it is worth storing */
 	    if (rhizome_manifest_version_cache_lookup(m)) {
@@ -349,7 +349,7 @@ int overlay_rhizome_saw_advertisements(int i,overlay_frame *f, long long now)
 	  {
 	    if (debug&DEBUG_RHIZOME) fprintf(stderr,"Unverified manifest has errors - so not processing any further.\n");
 	  }
-	rhizome_manifest_free(m);
+	if (m) rhizome_manifest_free(m);
 	m=NULL;
 
 	if (importManifest) {

serval.h

@@ -1206,7 +1206,7 @@ typedef struct overlay_mdp_addrlist {
    More lets us include preemptive retransmissions.
    Less reduces the chance of packets getting lost, and reduces
    the bandwidth used. */
-#define VOMP_STUFF_BYTES 1024
+#define VOMP_STUFF_BYTES 800
 /* elements sorted by size for alignment */
 typedef struct overlay_mdp_vompevent {
   /* Once a call has been established, this is how the MDP/VoMP server

tests/dna_rhizome

@@ -25,20 +25,28 @@ setup_servald_rhizome() {
    setup_servald "$@"
    executeOk $servald config set debug.rhizome on
    executeOk $servald keyring add
+   executeOk $servald keyring add
+   executeOk $servald keyring add
+   executeOk $servald keyring add
    executeOk $servald keyring list
    sid=$(replayStdout | sed -ne '1s/^\([0-9a-fA-F]\{64\}\):.*$/\1/p')
-   assert --message='identity known' [ -n "$sid" ]
+   sid1=$(replayStdout | sed -ne '2s/^\([0-9a-fA-F]\{64\}\):.*$/\1/p')
+   sid2=$(replayStdout | sed -ne '3s/^\([0-9a-fA-F]\{64\}\):.*$/\1/p')
+   sid3=$(replayStdout | sed -ne '4s/^\([0-9a-fA-F]\{64\}\):.*$/\1/p')
+   assert --message='main identity known' [ -n "$sid" ]
+   assert --message='identity 1 known' [ -n "$sid1" ]
+   assert --message='identity 2 known' [ -n "$sid2" ]
+   assert --message='identity 3 known' [ -n "$sid3" ]
 }
 
 assert_rhizome_list() {
-   executeOk $servald rhizome list
    assertStdoutLineCount '==' $(($# + 2))
    assertStdoutIs --line=1 -e '8\n'
    assertStdoutIs --line=2 -e 'service:fileid:manifestid:version:inserttime:length:date:name\n'
    local filename
    for filename; do
       unpack_manifest_for_grep "$filename"
-      assertStdoutGrep --matches=1 "^file:$re_filehash:$re_manifestid:.*:$re_name\$"
+      assertStdoutGrep --matches=1 "^$re_service:$re_filehash:$re_manifestid:.*:$re_name\$"
    done
 }
 
@@ -46,7 +54,7 @@ assert_stdout_add_file() {
    local filename="$1"
    unpack_manifest_for_grep "$filename"
    assertStdoutLineCount '==' 5
-   assertStdoutGrep --matches=1 '^service:file$'
+   assertStdoutGrep --matches=1 "^service:$re_service\$"
    assertStdoutGrep --matches=1 "^name:${2:-$re_name}\$"
    assertStdoutGrep --matches=1 "^manifestid:$re_manifestid\$"
    assertStdoutGrep --matches=1 "^filehash:$re_filehash\$"
@@ -55,18 +63,30 @@ assert_stdout_add_file() {
 
 unpack_manifest_for_grep() {
    local filename="$1"
+   re_service='[A-Za-z0-9_]\+'
    re_size=$(( $(cat "$filename" | wc -c) + 0 ))
-   re_filehash='[0-9a-fA-F]\{128\}'
+   re_filehash=$($servald rhizome hash file "$filename")
+   if [ -z "$re_filehash" ]; then
+      error "Could not compute rhizome hash of $filename"
+   fi
    re_manifestid='[0-9a-fA-F]\{64\}'
-   re_name="${filename##*/}" # should escape grep metacharacters
+   re_name="${filename##*/}" # TODO should escape grep metacharacters
    # If there is a manifest file that looks like it matches this payload
    # file, then use its file hash to check the rhizome list output.
-   if [ -r "$filename.manifest" ]; then
+   local filehash=$(sed -n -e '/^filehash=/s///p' "$filename.manifest" 2>/dev/null)
-      local name=$(sed -n -e '/^name=/s///p' "$filename.manifest")
+   if [ "$filehash" = "$re_filehash" ]; then
-      if [ "$name" == "${filename##*/}" ]; then
-         re_filehash=$(sed -n -e '/^filehash=/s///p' "$filename.manifest")
       re_manifestid=$(sed -n -e '/^id=/s///p' "$filename.manifest")
-      fi
+      # TODO should escape grep metacharacters, although service names should be tame
+      re_service=$(sed -n -e '/^service=/s///p' "$filename.manifest")
+      case "$re_service" in
+      file)
+         # TODO should escape grep metacharacters
+         re_name=$(sed -n -e '/^name=/s///p' "$filename.manifest")
+         ;;
+      *)
+         re_name=
+         ;;
+      esac
    fi
 }
 
@@ -109,6 +129,10 @@ extract_manifest_id() {
    extract_manifest "$1" "$2" id '[0-9a-fA-F]\{64\}'
 }
 
+extract_manifest_BK() {
+   extract_manifest "$1" "$2" BK '[0-9a-fA-F]\{128\}'
+}
+
 extract_manifest_filehash() {
    extract_manifest "$1" "$2" filehash '[0-9a-fA-F]\{128\}'
 }
@@ -122,12 +146,14 @@ setup_InitialEmptyList() {
    setup_servald_rhizome
 }
 test_InitialEmptyList() {
+   executeOk $servald rhizome list
    assert_rhizome_list
 }
 
 doc_AddNoAuthorNoManifest="Add with no author and no manifest file"
 setup_AddNoAuthorNoManifest() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo "Another test file" >file2
@@ -140,6 +166,7 @@ test_AddNoAuthorNoManifest() {
 doc_AddNoManifest="Add with no manifest file"
 setup_AddNoManifest() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo "Another test file" >file2
@@ -152,6 +179,7 @@ test_AddNoManifest() {
 doc_AddNonExistManifest="Add with non-existent manifest file"
 setup_AddNonExistManifest() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo "Another test file" >file2
@@ -175,6 +203,7 @@ test_AddNonExistManifest() {
 doc_AddManifest="Add with minimal manifest file"
 setup_AddManifest() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo -e 'name=wah\ndate=12345' >file1.manifest
@@ -197,6 +226,7 @@ test_AddManifest() {
 doc_AddThenList="List contains one file after one add"
 setup_AddThenList() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo "Another test file" >file2
@@ -204,9 +234,11 @@ setup_AddThenList() {
 test_AddThenList() {
    # Add first file
    executeOk $servald rhizome add file $sid '' file1 file1.manifest
+   executeOk $servald rhizome list
    assert_rhizome_list file1
    # Add second file
    executeOk $servald rhizome add file $sid '' file2 file2.manifest
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
 }
 
@@ -215,6 +247,7 @@ setup_AddThenExtractManifest() {
    setup_servald_rhizome
    echo "A test file" >file1
    executeOk $servald rhizome add file $sid '' file1 file1.manifest
+   executeOk $servald rhizome list
    assert_rhizome_list file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
@@ -265,6 +298,7 @@ setup_AddThenExtractFile() {
    setup_servald_rhizome
    echo "A test file" >file1
    executeOk $servald rhizome add file $sid '' file1 file1.manifest
+   executeOk $servald rhizome list
    assert_rhizome_list file1
    extract_manifest_filehash filehash file1.manifest
 }
@@ -307,6 +341,7 @@ test_ExtractFileInvalidID() {
 doc_AddDuplicate="Add same manifest detects duplicate"
 setup_AddDuplicate() {
    setup_servald_rhizome
+   executeOk $servald rhizome list
    assert_rhizome_list
    echo "A test file" >file1
    echo "Another test file" >file2
@@ -316,6 +351,7 @@ setup_AddDuplicate() {
    # Add second file
    executeOk $servald rhizome add file $sid '' file2 file2.manifest
    # Make sure they are both in the list.
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
 }
 test_AddDuplicate() {
@@ -325,6 +361,7 @@ test_AddDuplicate() {
    execute --exit-status=2 $servald rhizome add file $sid '' file1 file1.manifestA
    assert [ -s file1.manifestA ]
    assert_stdout_add_file file1
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
    strip_signatures file1.manifest file1.manifestA
    assert diff file1.manifest file1.manifestA
@@ -332,6 +369,7 @@ test_AddDuplicate() {
    execute --exit-status=2 $servald rhizome add file $sid '' file2 file2.manifestA
    assert [ -s file2.manifestA ]
    assert_stdout_add_file file2
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
    strip_signatures file2.manifest file2.manifestA
    assert diff file2.manifest file2.manifestA
@@ -349,10 +387,11 @@ test_AddMismatched() {
    assertExitStatus '!=' 0
    assert cmp file1.manifest file1_2.manifest
    # And rhizome store should be unchanged.
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
 }
 
-doc_AddUpdateSameVersion="Add new payload to existing manifest with same version"
+doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails"
 setup_AddUpdateSameVersion() {
    setup_AddDuplicate
    cp file1.manifest file1_2.manifest
@@ -371,6 +410,7 @@ test_AddUpdateSameVersion() {
    tfw_cat -v file1_2.manifest
    assert cmp file1_2.manifest file1_2.manifest.orig
    # And rhizome store should be unchanged.
+   executeOk $servald rhizome list
    assert_rhizome_list file1 file2
 }
 
@@ -388,8 +428,8 @@ test_AddUpdateNewVersion() {
    assert_stdout_add_file file1_2 file1
    assert_manifest_newer file1.manifest file1_2.manifest
    # Rhizome store contents reflect new payload.
-   mv -f file1_2.manifest file1.manifest
+   executeOk $servald rhizome list
-   assert_rhizome_list file1 file2
+   assert_rhizome_list file1_2 file2
 }
 
 doc_AddUpdateAutoVersion="Add new payload to existing manifest with automatic version"
@@ -404,8 +444,139 @@ test_AddUpdateAutoVersion() {
    executeOk $servald rhizome add file $sid '' file1_2 file1_2.manifest
    assert_manifest_newer file1.manifest file1_2.manifest
    # Rhizome store contents reflect new payload.
-   mv -f file1_2.manifest file1.manifest
+   executeOk $servald rhizome list
-   assert_rhizome_list file1 file2
+   assert_rhizome_list file1_2 file2
+}
+
+doc_AddUnsupportedService="Add with unsupported service fails"
+setup_AddUnsupportedService() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo -e 'service=Fubar' >file1.manifest
+}
+test_AddUnsupportedService() {
+   execute $servald rhizome add file $sid '' file1 file1.manifest
+   assertExitStatus '!=' 0
+}
+
+doc_AddMeshMSCreate="First add MeshMS creates manifest"
+setup_AddMeshMSCreate() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
+}
+test_AddMeshMSCreate() {
+   executeOk $servald rhizome add file $sid '' file1 file1.manifest
+   executeOk $servald rhizome list
+   assert_rhizome_list file1
+   extract_manifest_filehash filehash file1.manifest
+   executeOk $servald rhizome extract file $filehash file1x
+   assert diff file1 file1x
+}
+
+doc_AddMeshMSGrow="Second add MeshMS updates manifest"
+setup_AddMeshMSGrow() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo "Message2" >file2
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
+   executeOk $servald rhizome add file $sid '' file1 file1.manifest
+   extract_manifest_id id file1.manifest
+   extract_manifest_BK bk file1.manifest
+   echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$sid\nrecipient=$sid1" >file2.manifest
+}
+test_AddMeshMSGrow() {
+   executeOk $servald rhizome add file $sid '' file2 file2.manifest
+   extract_manifest_filehash filehash file2.manifest
+   executeOk $servald rhizome list
+   assert_rhizome_list file2
+   executeOk $servald rhizome extract file $filehash file2x
+   assert diff file2 file2x
+}
+
+doc_AddMeshMSMissingSender="Add MeshMS without sender fails"
+setup_AddMeshMSMissingSender() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo -e "service=MeshMS1\nrecipient=$sid1" >file1.manifest
+}
+test_AddMeshMSMissingSender() {
+   execute $servald rhizome add file $sid '' file1 file1.manifest
+   assertExitStatus '!=' 0
+}
+
+doc_AddMeshMSMissingRecipient="Add MeshMS without recipient fails"
+setup_AddMeshMSMissingRecipient() {
+   setup_servald_rhizome
+   executeOk $servald rhizome list
+   assert_rhizome_list
+   echo "Message1" >file1
+   echo -e "service=MeshMS1\nsender=$sid" >file1.manifest
+}
+test_AddMeshMSMissingRecipient() {
+   execute $servald rhizome add file $sid '' file1 file1.manifest
+   assertExitStatus '!=' 0
+}
+
+doc_AddMeshMSMissingSender="Add MeshMS without author uses sender"
+setup_AddMeshMSMissingSender() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
+}
+test_AddMeshMSMissingSender() {
+   executeOk $servald rhizome add file '' '' file1 file1.manifest
+   executeOk $servald rhizome list
+   assert_rhizome_list file1
+}
+
+doc_ListMeshMSFilter="List MeshMS manifests by filter"
+setup_ListMeshMSFilter() {
+   setup_servald_rhizome
+   echo "Message1" >file1
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid1" >file1.manifest
+   echo "Message2" >file2
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid2" >file2.manifest
+   echo "Message3" >file3
+   echo -e "service=MeshMS1\nsender=$sid\nrecipient=$sid3" >file3.manifest
+   echo "Message3" >file4
+   echo -e "service=MeshMS1\nsender=$sid1\nrecipient=$sid2" >file4.manifest
+   executeOk $servald rhizome add file '' '' file1 file1.manifest
+   executeOk $servald rhizome add file '' '' file2 file2.manifest
+   executeOk $servald rhizome add file '' '' file3 file3.manifest
+   executeOk $servald rhizome add file '' '' file4 file4.manifest
+   executeOk $servald rhizome list
+   assert_rhizome_list file1 file2 file3 file4
+}
+test_ListMeshMSFilter() {
+   executeOk $servald rhizome list file
+   assert_rhizome_list
+   executeOk $servald rhizome list MeshMS1
+   assert_rhizome_list file1 file2 file3 file4
+   executeOk $servald rhizome list '' $sid
+   assert_rhizome_list file1 file2 file3
+   executeOk $servald rhizome list '' $sid1
+   assert_rhizome_list file4
+   executeOk $servald rhizome list '' $sid2
+   assert_rhizome_list
+   executeOk $servald rhizome list '' $sid3
+   assert_rhizome_list
+   executeOk $servald rhizome list '' '' $sid
+   assert_rhizome_list
+   executeOk $servald rhizome list '' '' $sid1
+   assert_rhizome_list file1
+   executeOk $servald rhizome list '' '' $sid2
+   assert_rhizome_list file2 file4
+   executeOk $servald rhizome list file '' $sid2
+   assert_rhizome_list
+   executeOk $servald rhizome list '' '' $sid3
+   assert_rhizome_list file3
+   executeOk $servald rhizome list '' $sid $sid3
+   assert_rhizome_list file3
+   executeOk $servald rhizome list '' $sid1 $sid3
+   assert_rhizome_list
+   executeOk $servald rhizome list '' $sid1 $sid2
+   assert_rhizome_list file4
 }
 
 runTests "$@"

vomp.c

@@ -273,8 +273,6 @@ int vomp_send_status(vomp_call_state *call,int flags,overlay_mdp_frame *arg)
 	}
       }
 
-      WHYF("sending VoMP frame to far end: state=%d.%d",
-	   call->local.state,call->remote.state);
       overlay_mdp_send(&mdp,0,0);
       
       call->local.sequence++;