Refactor multi-instance server start code into test defs

testdefs.sh

@@ -101,7 +101,8 @@ set_instance() {
       ;;
    +[A-Z])
       instance_name="${1#+}"
-      tfw_log "# set instance = $instance_name"
+      instance_number=$((36#$instance_name - 9))
+      tfw_log "# set instance = $instance_name, number = $instance_number"
       export instance_dir="${servald_instances_dir?:}/$instance_name"
       mkdir -p "$instance_dir"
       export SERVALINSTANCE_PATH="$instance_dir/servald"
@@ -378,3 +379,93 @@ assert_all_servald_servers_no_errors() {
    pop_instance
 }
 
+# Utility function
+#  - create an identity
+#  - assign a phone number (DID) and name to the new identity, use defaults
+#    if not specified by arg1 and arg2
+#  - set the SID variable to the SID of the new identity
+#  - set the NUMBER variable to the phone number of the new identity
+#  - set the NAME variable to the name of the new identity
+create_identity() {
+   executeOk_servald keyring add
+   assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ]
+   executeOk_servald keyring list
+   SID=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p")
+   assert --message='main identity known' [ -n "$SID" ]
+   NUMBER="${1-$((5550000 + $instance_number))}"
+   NAME="${2-Agent $instance_name Smith}"
+   executeOk_servald set did $SID "$NUMBER" "$NAME"
+   tfw_log "Identity $instance_name: $SID $NUMBER $NAME"
+}
+
+# Utility function, to be overridden as needed:
+#  - set up the configuration immediately prior to starting a servald server
+#    process
+#  - called by start_servald_instances
+configure_servald_server() {
+   :
+}
+
+# Utility function:
+#  - start a set of servald server processes running on a shared dummy interface
+#    and with its own private monitor and MDP abstract socket names
+#  - set variable DUMMYNET to the full path name of shared dummy interface
+#  - set variables SIDx to the SID of instance x: SIDA, SIDB, etc.
+#  - set variables LOGx to the full path of server log file for instance x: LOGA,
+#    LOGB, etc,
+#  - wait for all instances to detect each other
+#  - assert that all instances are in each others' peer lists
+start_servald_instances() {
+   push_instance
+   tfw_log "# start servald instances $*"
+   DUMMYNET=$SERVALD_VAR/dummy
+   >$DUMMYNET
+   local I J
+   for I; do
+      set_instance $I
+      create_identity
+      executeOk_servald config set interfaces "+>$DUMMYNET"
+      executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.$TFWUNIQUE.$instance_name"
+      executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.$TFWUNIQUE.$instance_name"
+      configure_servald_server
+      start_servald_server
+      eval SID${I#+}="$SID"
+      eval LOG${I#+}="$instance_servald_log"
+   done
+   # Now wait until they see each other.
+   local timeout_seconds=30
+   local timeout_retry_seconds=0.25
+   sleep $timeout_seconds &
+   local timeout_pid=$!
+   tfw_log "# wait for instances to see each other"
+   while true; do
+      local allseen=true
+      for I; do
+         for J; do
+            [ $I = $J ] && continue
+            local logvar=LOG${I#+}
+            local sidvar=SID${J#+}
+            if ! grep "ADD OVERLAY NODE sid=${!sidvar}" "${!logvar}"; then
+               allseen=false
+               break 2
+            fi
+         done
+      done
+      $allseen && break
+      kill -0 $timeout_pid 2>/dev/null || fail "timeout"
+      tfw_log "sleep $timeout_retry_seconds"
+      sleep $timeout_retry_seconds
+   done
+   tfw_log "# dummynet file:" $(ls -l $DUMMYNET)
+   for I; do
+      set_instance $I
+      executeOk_servald id allpeers
+      assertStdoutLineCount '==' $(($# - 1))
+      for J; do
+         [ $I = $J ] && continue
+         local sidvar=SID${J#+}
+         assertStdoutGrep "${!sidvar}"
+      done
+   done
+   pop_instance
+}

tests/dnahelper

@@ -24,17 +24,23 @@ source "${0%/*}/../testdefs.sh"
 setup() {
    setup_servald
    assert_no_servald_processes
+   setup_dnahelper
 }
 
 teardown() {
-# TODO Disabled until stopping multiple-instances are supported
+   stop_all_servald_servers
-# TODO  stop_all_servald_servers
    kill_all_servald_processes
    assert_no_servald_processes
 }
 
-setup_servald_instance() {
+# Called by start_servald_instances immediately before starting the server
-    cat > $SERVALINSTANCE_PATH/dnahelper <<EOF
+# process in each instance.
+configure_servald_server() {
+   executeOk_servald config set dna.helper "$TFWTMP/dnahelper"
+}
+
+setup_dnahelper() {
+    cat >"$TFWTMP/dnahelper" <<EOF
 #!/usr/bin/env python
 # Sample DNA Helper application for testing
 
@@ -88,53 +94,11 @@ if __name__ == "__main__":
     main()
 
 EOF
-
-
-   push_instance
-   set_instance "$1"
-   executeOk_servald config set interfaces "+>$2"
-   executeOk_servald config set dna.helper "$SERVALINSTANCE_PATH/dnahelper"
-   executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.$1"
-   executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.$1"
-   executeOk_servald keyring add
-   assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ]
-   executeOk_servald keyring list
-   tfw_cat --stdout
-   sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p")
-   assert --message='main identity known' [ -n "$sid" ]
-   executeOk_servald set did $sid 5550001 "Agent $instance_name Smith"
-   start_servald_server
-   executeOk_servald status
-   tfw_cat --stdout
-   pop_instance
 }
 
-setup_servald_instances() {
+doc_MultiServer="Start three servald servers with dna helper"
-   setup_servald
+test_MultiServer() {
-   DUMMYNET=`pwd`/dummy.dat
+   start_servald_instances +A +B +C
-   rm $DUMMYNET
-   touch $DUMMYNET
-   assert [ -e $DUMMYNET ]
-   setup_servald_instance +A $DUMMYNET
-   SIDA=$sid
-   setup_servald_instance +B $DUMMYNET
-   SIDB=$sid
-   # Now make sure that they can see each other
-   sleep 10 # Should be plenty of time
-   set_instance +A
-   echo "Dummynet file $DUMMYNET after 5 seconds: "`ls -l $DUMMYNET`
-   executeOk_servald id peers
-   assertStdoutLineCount '==' 1
-}
-
-doc_MultipleServalDTest="Start two servald instances with dna helpers"
-
-setup_MultipleServalDTest() {
-   setup
-}
-
-test_MultipleServalDTest() {
-   setup_servald_instances
 }
 
 runTests "$@"

tests/dnaprotocol

@@ -24,6 +24,8 @@ source "${0%/*}/../testdefs.sh"
 setup() {
    setup_servald
    assert_no_servald_processes
+   start_servald_instances +A +B
+   set_instance +A
 }
 
 teardown() {
@@ -32,78 +34,29 @@ teardown() {
    assert_no_servald_processes
 }
 
-start_servald_instance() {
+# Called by start_servald_instances immediately before starting the server
-   push_instance
+# process in each instance.
-   set_instance "$1"
+configure_servald_server() {
+   executeOk_servald config set mdp.wifi.tick_ms 100
+   executeOk_servald config set mdp.selfannounce.ticks_per_full_address 1
    executeOk_servald config set debug.interfaces Yes
    executeOk_servald config set debug.packetformats No
    executeOk_servald config set debug.routing Yes
    executeOk_servald config set debug.tx No
    executeOk_servald config set debug.rx No
    executeOk_servald config set debug.mdprequests Yes
-   executeOk_servald config set interfaces "+>$2"
-   executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.$TFWUNIQUE.$instance_name"
-   executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.$TFWUNIQUE.$instance_name"
-   executeOk_servald config set mdp.wifi.tick_ms 100
-   executeOk_servald config set mdp.selfannounce.ticks_per_full_address 1
-   executeOk_servald keyring add
-   assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ]
-   executeOk_servald keyring list
-   tfw_cat --stdout
-   sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p")
-   assert --message='main identity known' [ -n "$sid" ]
-   executeOk_servald set did $sid 5550001 "Agent $instance_name Smith"
-   start_servald_server
-   log="$instance_servald_log"
-   pop_instance
 }
 
-start_servald_instances() {
+doc_MultiServer="Start three servald servers with dummy interface"
-   DUMMYNET=$PWD/dummy
-   >$DUMMYNET
-   start_servald_instance +A $DUMMYNET
-   SIDA=$sid
-   LOGA="$log"
-   start_servald_instance +B $DUMMYNET
-   SIDB=$sid
-   LOGB="$log"
-   # Now wait until they see each other.
-   local timeout_seconds=30
-   local timeout_retry_seconds=0.25
-   sleep $timeout_seconds &
-   local timeout_pid=$!
-   tfw_log "Wait for instances to see each other"
-   while ! grep "^INFO:.*ADD OVERLAY NODE sid=$SIDB" "$LOGA" || ! grep "^INFO:.*ADD OVERLAY NODE sid=$SIDA" "$LOGB"; do
-      kill -0 $timeout_pid 2>/dev/null || fail "timeout"
-      tfw_log "sleep $timeout_retry_seconds"
-      sleep $timeout_retry_seconds
-   done
-   tfw_log "# Dummynet file:" $(ls -l $DUMMYNET)
-   set_instance +A
-   executeOk_servald id allpeers
-   assertStdoutLineCount '==' 1
-   assertStdoutGrep "$SIDB"
-   set_instance +B
-   executeOk_servald id allpeers
-   assertStdoutLineCount '==' 1
-   assertStdoutGrep "$SIDA"
-   set_instance +A
-}
-
-doc_MultiServer="Start two servald instances with dummy interface"
 setup_MultiServer() {
-   setup
+   setup_servald
+   assert_no_servald_processes
 }
 test_MultiServer() {
-   start_servald_instances
+   start_servald_instances +A +B +C
 }
 
 doc_LookupWildcard="Lookup by wildcard"
-setup_LookupWildcard() {
-   setup
-   start_servald_instances
-   set_instance +A
-}
 test_LookupWildcard() {
    executeOk_servald dna lookup "*"
    assertStdoutLineCount '==' 2
@@ -112,11 +65,6 @@ test_LookupWildcard() {
 }
 
 doc_LookupEmpty="Lookup by empty string"
-setup_LookupEmpty() {
-   setup
-   start_servald_instances
-   set_instance +A
-}
 test_LookupEmpty() {
    executeOk_servald dna lookup ""
    assertStdoutLineCount '==' 2 
@@ -125,11 +73,6 @@ test_LookupEmpty() {
 }
 
 doc_LookupNumber="Lookup by phone number"
-setup_LookupNumber() {
-   setup
-   start_servald_instances
-   set_instance +A
-}
 test_LookupNumber() {
    # Make sure we get no false positives
    executeOk_servald dna lookup "5551234"
@@ -146,11 +89,6 @@ test_LookupNumber() {
 }
 
 doc_NodeinfoLocal="Node info auto-resolves for local identities"
-setup_NodeinfoLocal() {
-   setup
-   start_servald_instances
-   set_instance +A
-}
 test_NodeinfoLocal() {
    # node info for a local identity returns DID/Name since it is free, even
    # if it isn't asked for.
@@ -161,11 +99,6 @@ test_NodeinfoLocal() {
 }
 
 doc_NodeinfoRemote="Node info resolves remote identities"
-setup_NodeinfoRemote() {
-   setup
-   start_servald_instances
-   set_instance +A
-}
 test_NodeinfoRemote() {
    # if resolvedid is not specified for a remote identity, then don't resolve
    # it.