Improve testdefs: foreach_instance() function

The 'rhizomeprotocol' tests now use create_single_identity() in fixtures (which
tests the 'keyring list' command).

Use foreach_instance() function in various places, and replace functions that
used to loop over instances with functions that work on the current instance.

testdefs.sh

@@ -131,7 +131,8 @@ set_instance() {
       error "missing instance name argument"
       ;;
    +[A-Z])
-      instance_name="${1#+}"
+      instance_arg="${1}"
+      instance_name="${instance_arg#+}"
       instance_number=$((36#$instance_name - 9))
       tfw_log "# set instance = $instance_name, number = $instance_number"
       export instance_dir="${servald_instances_dir?:}/$instance_name"
@@ -146,6 +147,48 @@ set_instance() {
    esac
 }
 
+# Composition function:
+#  - invoke a command once in many instances
+#  - in "--all" mode (default), returns the count of commands that returned
+#    nonzero (ie, failure count); this returns zero only if all commands in all
+#    instances return zero, ie, is an AND relation on success; this is
+#    guaranteed to invoke the command in all instances (unless terminated
+#    by a failed assertion)
+#  - in "--any" mode, returns zero as soon as any command returns zero; ie, is
+#    an OR relation on success; N.B. this may not invoke the command in all
+#    instances
+foreach_instance() {
+   mode=all
+   case "$1" in
+   --any) mode=any; shift;;
+   --all) mode=all; shift;;
+   esac
+   local -a instances=()
+   while [ $# -ne 0 ]; do
+      case "$1" in
+      +[A-Z]) instances+=("$1"); shift;;
+      *) break;;
+      esac
+   done
+   push_instance
+   local ret=0
+   local I
+   for I in ${instances[*]}; do
+      set_instance $I
+      if "$@"; then
+         case $mode in
+         any) break;;
+         esac
+      else
+         case $mode in
+         all) let ++ret;;
+         esac
+      fi
+   done
+   pop_instance
+   return $ret
+}
+
 # Utility function for setting up servald JNI fixtures:
 #  - check that libservald.so is present
 #  - set LD_LIBRARY_PATH so that libservald.so can be found
@@ -424,12 +467,12 @@ assert_all_servald_servers_no_errors() {
 #  - set the DID{I} variable, eg DIDA, to the phone number of the new identity
 #  - set the NAME{I} variable, eg NAMEA, to the name of the new identity
 create_single_identity() {
-   DID${instance_name}1="${1-$((5550000 + $instance_number))}"
-   NAME${instance_name}1="${2-Agent $instance_name Smith}"
-   create_identities 1
    local sidvar=SID${instance_name}1
    local didvar=DID${instance_name}1
    local namevar=NAME${instance_name}1
+   eval "$didvar=\${1-\$((5550000 + \$instance_number))}"
+   eval "$namevar=\${2-Agent \$instance_name Smith}"
+   create_identities 1
    eval SID$instance_name="${!sidvar}"
    eval DID$instance_name="${!didvar}"
    eval NAME$instance_name="${!namevar}"
@@ -463,12 +506,12 @@ create_identities() {
       # them, otherwise extract the DID and NAME automatically generated by
       # servald.
       if [ -n "${!didvar}" -o -n "${!namevar}" ]; then
-         executeOk_servald set did $SID "${!didvar}" "${!namevar}"
+         executeOk_servald set did "${!sidvar}" "${!didvar}" "${!namevar}"
          eval "$didvar=\${!didvar}"
          eval "$namevar=\${!namevar}"
       else
-         extract_stdout_keyvalue_optional DID$instance_name$i did "$rexp_did" && tfw_log "$didvar=${!didvar}"
+         extract_stdout_keyvalue_optional $didvar did "$rexp_did" && tfw_log "$didvar=${!didvar}"
-         extract_stdout_keyvalue_optional NAME$instance_name$i name ".*" && tfw_log "$namevar=${!namevar}"
+         extract_stdout_keyvalue_optional $namevar name ".*" && tfw_log "$namevar=${!namevar}"
       fi
    done
    for ((i = 1; i <= N; ++i)); do
@@ -525,25 +568,39 @@ start_servald_instances() {
 }
 
 # Assertion function:
-# - asserts that all running instances report peer lists that contain all the SIDs of all the other
+# - asserts that the current instance reports a peer list that contains all the
-#   running instances
+#   SIDs of all the other instances
 # - uses the SID{I}{1..N} variables set by create_instances()
-assert_all_instance_peers_complete() {
+assert_peers_are_instances() {
-   push_instance
+   local I N
-   local I J N
+   executeOk_servald id allpeers
    for I; do
-      set_instance $I
+      for ((N=1; 1; ++N)); do
-      executeOk_servald id allpeers
+         local sidvar=SID${I#+}$N
-      for J; do
+         [ -n "${!sidvar}" ] || break
-         [ $I = $J ] && continue
+         assertStdoutGrep "${!sidvar}"
-         for ((N=1; 1; ++N)); do
-            local sidvar=SID${J#+}$N
-            [ -n "${!sidvar}" ] || break
-            assertStdoutGrep "${!sidvar}"
-         done
       done
    done
-   pop_instance
+}
+
+# Predicate function:
+# - useful in combination with assert() and wait_until()
+# - return true if the current instance has logged that it has seen all other instances via the
+#   selfannounce mechanism
+has_seen_instances() {
+   local I N
+   local logvar=LOG$instance_name
+   for I; do
+      [ $I = $instance_arg ] && continue
+      for ((N=1; 1; ++N)); do
+         local sidvar=SID${I#+}$N
+         [ -n "${!sidvar}" ] || break
+         if ! grep "ADD OVERLAY NODE sid=${!sidvar}" "${!logvar}"; then
+            return 1
+         fi
+      done
+   done
+   return 0
 }
 
 # Predicate function:
@@ -551,19 +608,5 @@ assert_all_instance_peers_complete() {
 # - return true if all instances have logged that they have seen all other instances via the
 #   selfannounce mechanism
 instances_see_each_other() {
-   local I J N
+   foreach_instance "$@" has_seen_instances "$@"
-   for I; do
-      local logvar=LOG${I#+}
-      for J; do
-         [ $I = $J ] && continue
-         for ((N=1; 1; ++N)); do
-            local sidvar=SID${J#+}$N
-            [ -n "${!sidvar}" ] || break
-            if ! grep "ADD OVERLAY NODE sid=${!sidvar}" "${!logvar}"; then
-               return 1
-            fi
-         done
-      done
-   done
-   return 0
 }

tests/rhizomeprotocol

@@ -31,10 +31,8 @@ teardown() {
 }
 
 setup_rhizome() {
-   set_instance +A
+   foreach_instance +A +B create_single_identity
-   create_identities 1
    set_instance +B
-   create_identities 1
 }
 
 # Called by start_servald_instances for each instance.
@@ -110,6 +108,8 @@ setup_FileTransfer() {
    set_instance +A
    add_file file1
    start_servald_instances +A +B
+   foreach_instance +A assert_peers_are_instances +B
+   foreach_instance +B assert_peers_are_instances +A
 }
 test_FileTransfer() {
    wait_until bundle_received_by +B
@@ -131,6 +131,8 @@ setup_FileTransferBig() {
    ls -l file1
    add_file file1
    start_servald_instances +A +B
+   foreach_instance +A assert_peers_are_instances +B
+   foreach_instance +B assert_peers_are_instances +A
 }
 test_FileTransferBig() {
    wait_until bundle_received_by +B
@@ -156,6 +158,10 @@ setup_FileTransferMulti() {
    set_instance +A
    add_file file1
    start_servald_instances +A +B +C +D +E
+   foreach_instance +A assert_peers_are_instances +B +C +D +E
+   foreach_instance +B assert_peers_are_instances +A +C +D +E
+   foreach_instance +C assert_peers_are_instances +A +B +D +E
+   foreach_instance +D assert_peers_are_instances +A +B +C +E
 }
 test_FileTransferMulti() {
    wait_until bundle_received_by +B +C +D +E
@@ -172,6 +178,8 @@ setup_FileTransferDelete() {
    set_instance +A
    add_file file1
    start_servald_instances +A +B
+   foreach_instance +A assert_peers_are_instances +B
+   foreach_instance +B assert_peers_are_instances +A
    wait_until bundle_received_by +B
    set_instance +A
    >file1_2