serval-dna/tests/routing

#!/bin/bash

# Tests for Route discovery
#
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.


source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"

interface_up() {
   $GREP "Interface .* is up" $instance_servald_log || return 1
   return 0
}

has_link() {
   interface_ex=".*"
   case "$1" in
   dummy*) interface_ex="$1"; shift;;
   esac
   set_instance $1
   executeOk_servald route print
   tfw_log "Looking for link from $1 to $2 ($4)"
   if ! grep "^${4}:\(BROADCAST\|UNICAST\):${interface_ex}:0*\$" $_tfw_tmp/stdout; then
      tfw_log "Link not found"
#      tfw_log "^${4}:\(BROADCAST\|UNICAST\):${interface_ex}:0*\$"
#      tfw_cat --stdout --stderr
      return 1
   fi
   [ $4 = $5 ] && return 0;
   tfw_log "Path from $1 to $3 should be via $2 ($5, $4)"
   if ! grep "^${5}:INDIRECT::${4}\$" $_tfw_tmp/stdout; then
      tfw_log "No path found"
#      tfw_log "^${5}:INDIRECT::${4}\$"
#      tfw_cat --stdout --stderr
      return 1
   fi
}

path_exists() {
   local dest
   eval dest=\$$#
   local dest_sidvar=SID${dest#+}
   local next_inst=$1
   shift
   local I
   for I; do
      local sidvar=SID${I#+}
      [ -n "${!sidvar}" ] || break
      has_link $next_inst $I $dest ${!sidvar} ${!dest_sidvar} || return 1
      next_inst=$I
   done
   return 0
}

start_routing_instance() {
   executeOk_servald config \
      set debug.mdprequests yes \
      set debug.linkstate yes \
      set debug.verbose yes \
      set debug.overlayrouting yes \
      set log.console.level debug \
      set log.console.show_pid on \
      set log.console.show_time on \
      set rhizome.enable no
   start_servald_server
   wait_until interface_up
}

log_routing_table() {
   executeOk_servald route print
   tfw_cat --stdout --stderr
}

teardown() {
   foreach_instance_with_pidfile log_routing_table
   stop_all_servald_servers
   kill_all_servald_processes
   assert_no_servald_processes
   report_all_servald_servers
}

doc_single_link="Start 2 instances on one link"
setup_single_link() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B start_routing_instance
}
test_single_link() {
   wait_until --timeout=10 path_exists +A +B
   wait_until --timeout=5 path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}

doc_multiple_ids="Route between multiple identities"
setup_multiple_ids() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_identities 2
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B start_routing_instance
}
test_multiple_ids() {
   wait_until path_exists +A +B
   wait_until has_link +A +B +B $SIDB1 $SIDB2
   wait_until path_exists +B +A
   wait_until has_link +B +A +A $SIDA1 $SIDA2
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB2 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB1:BROADCAST:dummy.*:0*"
   assertStdoutGrep --matches=1 "^$SIDB2:INDIRECT::$SIDB1"
   set_instance +B
   executeOk_servald mdp ping --timeout=3 $SIDA2 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDA1:BROADCAST:dummy.*:0*"
   assertStdoutGrep --matches=1 "^$SIDA2:INDIRECT::$SIDA1"
}

doc_unlock_ids="Routes appear and disappear as identities are [un]locked"
setup_unlock_ids() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   set_instance +A
   executeOk_servald keyring add 'entry-pin'
   extract_stdout_keyvalue SIDX sid "$rexp_sid"
   foreach_instance +A +B start_routing_instance
}
test_unlock_ids() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald id enter pin 'entry-pin'
   wait_until has_link +B +A +A $SIDA $SIDX
   set_instance +A
   executeOk_servald id relinquish pin 'entry-pin'
   set_instance +B
   wait_until --timeout=30 sid_offline $SIDX
}

doc_migrate_id="Unlocking the same identity triggers migration"
setup_migrate_id() {
   setup_servald
   assert_no_servald_processes
   set_instance +A
   executeOk_servald keyring add 'entry-pin'
   extract_stdout_keyvalue SIDX sid "$rexp_sid"
   executeOk_servald keyring dump --entry-pin=entry-pin --secret sidx
   tfw_cat sidx
   set_instance +B
   executeOk_servald keyring load sidx '' 'entry-pin'
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B start_routing_instance
}
test_migrate_id() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald id enter pin 'entry-pin'
   wait_until has_link +B +A +A $SIDA $SIDX
   set_instance +B
   executeOk_servald id enter pin 'entry-pin'
   wait_until --timeout=30 has_link +A +B +B $SIDB $SIDX
}

doc_single_mdp="Use single MDP per packet encapsulation"
setup_single_mdp() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B executeOk_servald config set interfaces.1.encapsulation single
   foreach_instance +A +B start_routing_instance
}
test_single_mdp() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}

doc_mismatched_encap="Mismatched MDP packet encapsulation"
setup_mismatched_encap() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A executeOk_servald config set interfaces.1.encapsulation single
   foreach_instance +A +B start_routing_instance
}
test_mismatched_encap() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}

doc_single_p2p="Start 2 instances on a point to point link"
setup_single_p2p() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B \
     executeOk_servald config \
       set interfaces.1.encapsulation single \
       set interfaces.1.point_to_point on
   foreach_instance +A +B start_routing_instance
}
test_single_p2p() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   assertGrep "$instance_servald_log" 'Established point to point link'
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}

doc_slip_encoding="Test slip encoding and decoding"
setup_slip_encoding() {
   setup_servald
   assert_no_servald_processes
}
test_slip_encoding() {
   executeOk_servald test slip --seed=1 --iterations=2000
}
start_fakeradio() {
   $servald_build_root/fakeradio 4 1 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
   FAKERADIO_PID=$!
   wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
   local _line=`head "$SERVALD_VAR/radioout" -n 1`
   END1="${_line#*:}"
   _line=`tail "$SERVALD_VAR/radioout" -n 1`
   END2="${_line#*:}"
   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
}

doc_simulate_extender="Simulate a mesh extender radio link"
setup_simulate_extender() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   start_fakeradio
   set_instance +A
   executeOk_servald config \
      set interfaces.1.file "$END1"
   set_instance +B
   executeOk_servald config \
      set interfaces.1.file "$END2"
   foreach_instance +A +B \
      executeOk_servald config \
         set debug.throttling on \
         set debug.packetradio on \
         set debug.radio_link on \
         set interfaces.1.type CATEAR \
         set interfaces.1.mdp.tick_ms 5000 \
         set interfaces.1.socket_type STREAM \
         set interfaces.1.encapsulation SINGLE \
         set interfaces.1.point_to_point on
   foreach_instance +A +B start_routing_instance
}
test_simulate_extender() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}
teardown_simulate_extender() {
   teardown
   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
   kill $FAKERADIO_PID
   tfw_cat "$SERVALD_VAR/radioerr"
}

doc_multiple_nodes="Multiple nodes on one link"
setup_multiple_nodes() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B +C +D create_single_identity
   foreach_instance +A +B +C +D add_servald_interface 1
   foreach_instance +A +B +C +D start_routing_instance
}
test_multiple_nodes() {
   wait_until path_exists +A +B
   wait_until path_exists +A +C
   wait_until path_exists +A +D
   wait_until path_exists +B +A
   wait_until path_exists +C +A
   wait_until path_exists +D +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
   executeOk_servald mdp ping --timeout=3 $SIDC 1
   tfw_cat --stdout --stderr
   executeOk_servald mdp ping --timeout=3 $SIDD 1
   tfw_cat --stdout --stderr
}

doc_scan="Network scan with isolated clients"
setup_scan() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B +C create_single_identity
  foreach_instance +A +B +C add_servald_interface --file 1
  foreach_instance +A +B +C \
    executeOk_servald config \
      set interfaces.1.drop_broadcasts on
  foreach_instance +A +B +C start_routing_instance
}
test_scan() {
   set_instance +A
   executeOk_servald scan
   wait_until scan_completed
   wait_until --timeout=10 has_seen_instances +B +C
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB:UNICAST:"
   assertStdoutGrep --matches=1 "^$SIDC:UNICAST:"
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
   set_instance +B
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDA:UNICAST:"
   assertStdoutGrep --matches=1 "^$SIDC:INDIRECT:"
   executeOk_servald mdp ping --timeout=3 $SIDC 1
   tfw_cat --stdout --stderr
}

doc_scan_one="Network scan a single address"
setup_scan_one() {
  setup_scan
}
test_scan_one() {
   set_instance +A
   executeOk_servald scan 127.0.1.2
   wait_until scan_completed
   wait_until --timeout=10 has_seen_instances +B
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB:UNICAST:"
   assertStdoutGrep --matches=0 "^$SIDC:"
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
}

scan_completed() {
  grep "Scan completed" $instance_servald_log || return 1
  return 0
}

doc_single_filter="Single device with a broadcast filter"
setup_single_filter() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  set_instance +B
  executeOk_servald config \
      set interfaces.1.drop_broadcasts on
  foreach_instance +A +B start_routing_instance
}
test_single_filter() {
   wait_until --timeout=10 path_exists +A +B
   wait_until --timeout=5 path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB:UNICAST:"
   set_instance +B
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDA:BROADCAST:"
}

doc_broadcast_only="Broadcast packets only"
setup_broadcast_only() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface --file 1
   foreach_instance +A +B \
      executeOk_servald config set interfaces.1.drop_unicasts 1
   foreach_instance +A +B start_routing_instance
}
test_broadcast_only() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB:BROADCAST:"
}

doc_prefer_unicast="Prefer unicast packets"
setup_prefer_unicast() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +A +B \
      executeOk_servald config \
         set interfaces.1.prefer_unicast 1 \
         set debug.overlayframes 1
   foreach_instance +A +B start_routing_instance
}
test_prefer_unicast() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDB 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDB:UNICAST:"
}

doc_multihop_linear="Start 4 instances in a linear arrangement"
setup_multihop_linear() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B +C +D create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +B +C add_servald_interface 2
   foreach_instance +C +D add_servald_interface 3
   foreach_instance +A +B +C +D start_routing_instance
}
test_multihop_linear() {
   wait_until path_exists +A +B +C +D
   wait_until path_exists +D +C +B +A
   set_instance +A
   executeOk_servald --stdout --stderr mdp ping --timeout=3 $SIDD 1
   tfw_cat --stdout --stderr
   executeOk_servald mdp trace $SIDD
   assertStdoutGrep --matches=1 "^0:$SIDA\$"
   assertStdoutGrep --matches=1 "^1:$SIDB\$"
   assertStdoutGrep --matches=1 "^2:$SIDC\$"
   assertStdoutGrep --matches=1 "^3:$SIDD\$"
   assertStdoutGrep --matches=1 "^4:$SIDC\$"
   assertStdoutGrep --matches=1 "^5:$SIDB\$"
   assertStdoutGrep --matches=1 "^6:$SIDA\$"
}

doc_unicast_route="Route across unicast links"
setup_unicast_route() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B +C +D create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +B +C add_servald_interface --file 2
  foreach_instance +C +D add_servald_interface --file 3
  set_instance +A
  executeOk_servald config \
      set interfaces.1.drop_broadcasts on
  set_instance +C
  executeOk_servald config \
      set interfaces.2.drop_broadcasts on \
      set interfaces.3.drop_broadcasts on
  foreach_instance +A +B +C +D start_routing_instance
}
test_unicast_route() {
   wait_until --timeout=20 path_exists +A +B +C +D
   wait_until --timeout=5 path_exists +D +C +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDD 1
   tfw_cat --stdout --stderr
}

setup_offline() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B +C +D create_single_identity
   foreach_instance +A +B +C add_servald_interface 1
   foreach_instance +A +B +D add_servald_interface 2
   foreach_instance +A +B +C +D start_routing_instance
}

sid_offline() {
   if ! grep "NOT REACHABLE sid=${1}" $instance_servald_log; then
      return 1
   fi
}

instance_offline() {
   local I N
   for I; do
      [ $I = $instance_arg ] && continue
      for ((N=1; 1; ++N)); do
         local sidvar=SID${I#+}$N
         [ -n "${!sidvar}" ] || break
         if sid_offline ${!sidvar}; then
            return 1
         fi
      done
   done
}

doc_offline="One node going offline"
test_offline() {
   foreach_instance +A +B +C +D \
      wait_until has_seen_instances +A +B +C +D
   stop_servald_server +C
   foreach_instance +A +B +D \
      wait_until --timeout=30 instance_offline +C
}

doc_lose_neighbours="Lose and regain neighbours"
setup_lose_neighbours() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B +C create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +B +C add_servald_interface 2
   foreach_instance +A +B +C start_routing_instance
}
test_lose_neighbours() {
   wait_until path_exists +A +B +C
   wait_until path_exists +C +B +A
   stop_servald_server +B
   foreach_instance +A +C \
      wait_until --timeout=30 instance_offline +B
   set_instance +A
   wait_until --timeout=30 instance_offline +C
   start_servald_server +B
   wait_until --timeout=20 path_exists +A +B +C
   wait_until --timeout=20 path_exists +C +B +A
}

setup_multi_interface() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   foreach_instance +A +B add_servald_interface --wifi 1
   foreach_instance +A +B add_servald_interface --ethernet 2
   foreach_instance +A +B start_routing_instance
}

doc_multi_interface="Multiple links of different types to the same neighbour"
test_multi_interface() {
   set_instance +A
   wait_until has_link "dummy2.*" +A +B +B $SIDB $SIDB
   set_instance +B
   wait_until has_link "dummy2.*" +B +A +A $SIDA $SIDA
   set_instance +A
   executeOk_servald config set interfaces.2.exclude 1
   wait_until has_link "dummy1.*" +A +B +B $SIDB $SIDB
   set_instance +B
   wait_until has_link "dummy1.*" +B +A +A $SIDA $SIDA
   set_instance +A
   executeOk_servald config del interfaces.2.exclude
   wait_until has_link "dummy2.*" +A +B +B $SIDB $SIDB
   set_instance +B
   wait_until has_link "dummy2.*" +B +A +A $SIDA $SIDA
}

# TODO assert that all packets arrive? assert that no duplicates arrive?
doc_ping_unreliable="Ping over a 1-hop unreliable link"
setup_ping_unreliable() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +A +B \
    executeOk_servald config \
      set interfaces.1.drop_packets 40
  foreach_instance +A +B start_routing_instance
}
test_ping_unreliable() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --interval=0.100 --timeout=3 $SIDB 100
   tfw_cat --stdout --stderr
}

doc_ping_unreliable2="Ping over a 2-hop unreliable link"
setup_ping_unreliable2() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B +C create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +A +B \
    executeOk_servald config \
      set interfaces.1.drop_packets 40
  foreach_instance +B +C add_servald_interface --file 2
  foreach_instance +B +C \
    executeOk_servald config \
      set interfaces.2.drop_packets 40
  foreach_instance +A +B +C start_routing_instance
}
test_ping_unreliable2() {
   wait_until path_exists +A +B +C
   wait_until path_exists +C +B +A
   set_instance +A
   executeOk_servald mdp ping --interval=0.100 --timeout=3 $SIDC 100
   tfw_cat --stdout --stderr
}

doc_brping_unreliable="Broadcast ping over a 1-hop unreliable link"
setup_brping_unreliable() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +A +B \
    executeOk_servald config \
      set interfaces.1.drop_packets 20
  foreach_instance +A +B start_routing_instance
}
test_brping_unreliable() {
   wait_until path_exists +A +B
   wait_until path_exists +B +A
   set_instance +A
   executeOk_servald mdp ping --interval=0.100 --timeout=2 broadcast 100
   tfw_cat --stdout --stderr
}

doc_unreliable_links="Prefer a longer, better path vs an unreliable link"
setup_unreliable_links() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B +C create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +B +C add_servald_interface --file 2
  foreach_instance +A +C add_servald_interface --file 3
  set_instance +A
  executeOk_servald config \
    set interfaces.1.drop_packets 5 \
    set interfaces.3.drop_packets 70
  set_instance +B
  executeOk_servald config \
    set interfaces.1.drop_packets 5 \
    set interfaces.2.drop_packets 5
  set_instance +C
  executeOk_servald config \
    set interfaces.2.drop_packets 5 \
    set interfaces.3.drop_packets 70
  foreach_instance +A +B +C start_routing_instance
}
test_unreliable_links() {
   wait_until path_exists +A +B +C
   wait_until path_exists +C +B +A
   set_instance +A
   executeOk_servald mdp ping --interval=0.100 --timeout=3 $SIDC 30
   tfw_cat --stdout --stderr
   wait_until path_exists +A +B +C
   wait_until path_exists +C +B +A
}

doc_unreliable_links2="Choose the best multihop path with some unreliable links"
setup_unreliable_links2() {
  setup_servald
  assert_no_servald_processes
  foreach_instance +A +B +C +D create_single_identity
  foreach_instance +A +B add_servald_interface --file 1
  foreach_instance +A +C add_servald_interface --file 2
  foreach_instance +A +D add_servald_interface --file 3
  foreach_instance +B +C add_servald_interface --file 4
  foreach_instance +B +D add_servald_interface --file 5
  foreach_instance +C +D add_servald_interface --file 6
  set_instance +A
  executeOk_servald config \
    set interfaces.1.drop_packets 5 \
    set interfaces.2.drop_packets 40 \
    set interfaces.3.drop_packets 90
  set_instance +B
  executeOk_servald config \
    set interfaces.1.drop_packets 5 \
    set interfaces.4.drop_packets 5 \
    set interfaces.5.drop_packets 40
  set_instance +C
  executeOk_servald config \
    set interfaces.2.drop_packets 40 \
    set interfaces.4.drop_packets 5 \
    set interfaces.6.drop_packets 5
  set_instance +D
  executeOk_servald config \
    set interfaces.3.drop_packets 90 \
    set interfaces.5.drop_packets 40 \
    set interfaces.6.drop_packets 5
  foreach_instance +A +B +C +D start_routing_instance
}
test_unreliable_links2() {
   wait_until path_exists +A +B +C +D
   wait_until path_exists +D +C +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDD 5
   tfw_cat --stdout --stderr
}

setup_circle() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B +C +D +E +F +G +H create_single_identity
   foreach_instance +A +B add_servald_interface 1
   foreach_instance +B +C add_servald_interface 2
   foreach_instance +C +D add_servald_interface 3
   foreach_instance +D +E add_servald_interface 4
   foreach_instance +E +F add_servald_interface 5
   foreach_instance +F +G add_servald_interface 6
   foreach_instance +G +H add_servald_interface 7
   foreach_instance +H +A add_servald_interface 8
   foreach_instance +A +B +C +D +E +F +G +H start_routing_instance
}

doc_circle="Circle of nodes with one going offline"
test_circle() {
   foreach_instance +A +B +C +D +E +F +G +H \
      wait_until has_seen_instances +A +B +C +D +E +F +G +H
   wait_until path_exists +A +B +C
   wait_until path_exists +C +B +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDC 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDC:INDIRECT:"
   stop_servald_server +B
   foreach_instance +A +C \
      wait_until --timeout=10 instance_offline +B
   wait_until path_exists +A +H +G +F +E +D +C
   wait_until path_exists +C +D +E +F +G +H +A
   set_instance +A
   executeOk_servald mdp ping --timeout=3 $SIDC 1
   tfw_cat --stdout --stderr
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDC:INDIRECT:"
}

setup_crowded_mess() {
   setup_servald
   assert_no_servald_processes
# BCDE & DEFG form squares, ABC & FGH form triangles
   foreach_instance +A +B +C +D +E +F +G +H create_single_identity
   foreach_instance +A +B +C add_servald_interface 1
   foreach_instance +B +D add_servald_interface 2
   foreach_instance +C +E add_servald_interface 3
   foreach_instance +D +E add_servald_interface 4
   foreach_instance +D +F add_servald_interface 5
   foreach_instance +E +G add_servald_interface 6
   foreach_instance +F +G +H add_servald_interface 7
   foreach_instance +A +B +C +D +E +F +G +H \
      executeOk_servald config \
         set mdp.iftype.wifi.reachable_timeout_ms 60000
   foreach_instance +A +B +C +D +E +F +G +H start_routing_instance
}

doc_crowded_mess="Multiple possible paths"
test_crowded_mess() {
   foreach_instance +A +H \
      wait_until has_seen_instances +A +H
   set_instance +A
   executeOk_servald route print
   assertStdoutGrep --matches=1 "^$SIDH:INDIRECT:"
   executeOk_servald mdp ping --timeout=3 $SIDH 1
   tfw_cat --stdout --stderr
}

runTests "$@"