serval-dna/tests/rhizomeprotocol

#!/bin/bash

# Tests for Serval rhizome protocol.
#
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
source "${0%/*}/../testdefs_rhizome.sh"

shopt -s extglob

finally() {
   stop_all_servald_servers
}

teardown() {
   kill_all_servald_processes
   assert_no_servald_processes
   report_all_servald_servers
}

default_config() {
   executeOk_servald config \
      set log.console.level debug \
      set log.console.show_pid on \
      set log.console.show_time on \
      set debug.rhizome on \
      set debug.http_server on \
      set debug.httpd on \
      set debug.rhizome_manifest on \
      set debug.rhizome_ads on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on
}

# Called by start_servald_instances for each instance.
configure_servald_server() {
   add_servald_interface
   default_config
}

setup_common() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   set_instance +B
}

receive_and_update_bundle() {
   wait_until "$@" bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   assert_rhizome_received file1
   set_instance +A
   rhizome_update_file file1 file2
   set_instance +B
   wait_until "$@" bundle_received_by $BID:$VERSION +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file2
   assert_rhizome_received file2
}

doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
   setup_common
   set_instance +A
   rhizome_add_file file1 250000
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_FileTransfer() {
   receive_and_update_bundle
}

doc_EncryptedTransfer="Encrypted payload can be opened by destination"
setup_EncryptedTransfer() {
   setup_common
   set_instance +A
   echo "Clear Text" >file1
   echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB" >file1.manifest
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_manifest_id BID file1.manifest
   extract_manifest_version VERSION file1.manifest
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_EncryptedTransfer() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome extract file $BID file1x
   assert diff file1 file1x
}

doc_NoFetch="Disabling fetching still allows pushing bundles"
setup_NoFetch() {
   setup_common
   set_instance +A
   rhizome_add_file file1
   BID1=$BID
   VERSION1=$VERSION
   set_instance +B
   rhizome_add_file file2
   BID2=$BID
   VERSION2=$VERSION
   executeOk_servald config set rhizome.fetch 0
   start_servald_instances +A +B
}
test_NoFetch() {
   wait_until bundle_received_by $BID2:$VERSION2 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1 --fromhere=0 file2
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file2
}

doc_DisablingHTTPServer="Disabling HTTP rhizome transports works"
setup_DisablingHTTPServer() {
   setup_common
   set_instance +A
   rhizome_add_file file1 2048
   executeOk_servald config set rhizome.http.enable 0
   start_servald_instances +A
}
test_DisablingHTTPServer() {
   !rhizome_http_server_started
}

doc_HTTPTransport="Rhizome over HTTP transport"
setup_HTTPTransport() {
   setup_common
   set_instance +B
   executeOk_servald config set rhizome.mdp.enable 0
   set_instance +A
   executeOk_servald config set rhizome.mdp.enable 0
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_HTTPTransport() {
   receive_and_update_bundle
}

doc_MDPTransport="Rhizome over MDP transport"
setup_MDPTransport() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransport() {
   receive_and_update_bundle
}

doc_MDPTransportFailOver="Rhizome fails over to MDP transport"
setup_MDPTransportFailOver() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set debug.nohttptx 1 \
       set rhizome.mdp.enable 1
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransportFailOver() {
   receive_and_update_bundle
}

doc_UnicastTransfer="Rhizome over unicast MDP link"
setup_UnicastTransfer() {
   configure_servald_server() {
      default_config
      executeOk_servald config \
         set rhizome.http.enable 0
   }
   setup_common
   set_instance +A
   add_servald_interface --file
   rhizome_add_file file1 2048
   set_instance +B
   add_servald_interface --file
   executeOk_servald config \
      set interfaces.1.broadcast.drop on
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_UnicastTransfer() {
   receive_and_update_bundle
}


doc_journalMDP="Transfer and update a journal bundle via MDP"
setup_journalMDP() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0
   set_instance +A
   create_file file1 2048
   create_file file2 2048
   executeOk_servald rhizome journal append $SIDA "" file1
   extract_stdout_manifestid BID
   extract_stdout_version VERSION
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_journalMDP() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   executeOk_servald rhizome journal append $SIDA $BID file2
   extract_stdout_version VERSION2
   set_instance +B
   wait_until bundle_received_by $BID:$VERSION2 +B
   assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}

doc_journalHTTP="Transfer and update a journal bundle via HTTP"
setup_journalHTTP() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.mdp.enable 0
   set_instance +A
   create_file file1 2048
   create_file file2 2048
   executeOk_servald rhizome journal append $SIDA "" file1
   extract_stdout_manifestid BID
   extract_stdout_version VERSION
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_journalHTTP() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   executeOk_servald rhizome journal append $SIDA $BID file2
   extract_stdout_version VERSION2
   set_instance +B
   wait_until bundle_received_by $BID:$VERSION2 +B
   assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}

#common setup and test routines for transferring a 1MB file
setup_bigfile_common() {
   set_instance +A
   dd if=/dev/urandom of=file1 bs=1k count=1k 2>&1
   echo x >>file1
   ls -l file1
   rhizome_add_file file1
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
bigfile_common_test() {
   set_instance +B
   wait_until --timeout=120 bundle_received_by $BID:$VERSION +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   assert_rhizome_received file1
}

doc_FileTransferBigMDP="Big new bundle transfers to one node via MDP"
setup_FileTransferBigMDP() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config set rhizome.http.enable 0
   setup_bigfile_common
}
test_FileTransferBigMDP() {
   bigfile_common_test
}

doc_FileTransferUnreliableBigMDP="Big new bundle over unreliable MDP transport"
setup_FileTransferUnreliableBigMDP() {
   configure_servald_server() {
      add_servald_interface --file
      default_config
      executeOk_servald config \
         set rhizome.http.enable 0 \
         set interfaces.1.drop_packets 10
   }
   setup_common
   setup_bigfile_common
}
test_FileTransferUnreliableBigMDP() {
   bigfile_common_test
}


doc_FileTransferBig="Big new bundle transfers to one node via HTTP"
setup_FileTransferBig() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config set rhizome.mdp.enable 0
   setup_bigfile_common
}
test_FileTransferBig() {
   bigfile_common_test
}

doc_FileTransferBigMDPExtBlob="Big new bundle transfers to one node via MDP, external blob file"
setup_FileTransferBigMDPExtBlob() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.http.enable 0 \
         set rhizome.max_blob_size 0 \
	 set debug.rhizome_store 1
   setup_bigfile_common
}
test_FileTransferBigMDPExtBlob() {
   bigfile_common_test
}

doc_FileTransferBigHTTPExtBlob="Big new bundle transfers to one node via HTTP, external blob file"
setup_FileTransferBigHTTPExtBlob() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.mdp.enable 0 \
         set rhizome.max_blob_size 0 \
	 set debug.rhizome_store 1
   setup_bigfile_common
}
test_FileTransferBigHTTPExtBlob() {
   bigfile_common_test
}

# common setup and test routines for transfers to 4 nodes
setup_multitransfer_common() {
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B +C +D +E
   set_instance +A
   assert_peers_are_instances +B +C +D +E
   set_instance +B
   assert_peers_are_instances +A +C +D +E
   set_instance +C
   assert_peers_are_instances +A +B +D +E
   set_instance +D
   assert_peers_are_instances +A +B +C +E
   set_instance +E
   assert_peers_are_instances +A +B +C +D
}
multitransfer_common_test() {
   wait_until bundle_received_by $BID:$VERSION +B +C +D +E
   for i in B C D E; do
      set_instance +$i
      executeOk_servald rhizome list
      assert_rhizome_list --fromhere=0 file1
      assert_rhizome_received file1
   done
}

doc_FileTransferMulti="New bundle transfers to four nodes via HTTP"
setup_FileTransferMulti() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config set rhizome.mdp.enable 0
   setup_multitransfer_common
}
test_FileTransferMulti() {
   multitransfer_common_test
}

doc_FileTransferMultiMDP="New bundle transfers to four nodes via MDP"
setup_FileTransferMultiMDP() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config set rhizome.http.enable 0
   setup_multitransfer_common
}
test_FileTransferMultiMDP() {
   multitransfer_common_test
}

doc_FileTransferMultiMDPExtBlob="New bundle transfers to four nodes via MDP, external blob files"
setup_FileTransferMultiMDPExtBlob() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config \
         set rhizome.http.enable 0 \
         set rhizome.max_blob_size 0 \
	 set debug.rhizome_store 1
   setup_multitransfer_common
}
test_FileTransferMultiMDPExtBlob() {
   multitransfer_common_test
}

doc_FileTransferMultiHTTPExtBlob="New bundle transfers to four nodes via HTTP, external blob files"
setup_FileTransferMultiHTTPExtBlob() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config \
         set rhizome.mdp.enable 0 \
         set rhizome.max_blob_size 0 \
	 set debug.rhizome_store 1
   setup_multitransfer_common
}
test_FileTransferMultiHTTPExtBlob() {
   multitransfer_common_test
}

doc_FileTransferDelete="Payload deletion transfers to one node"
setup_FileTransferDelete() {
   setup_common
   set_instance +A
   rhizome_add_file file1
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   >file1_2
   rhizome_update_file file1 file1_2
}
test_FileTransferDelete() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1_2
   assert_rhizome_received file1_2
}

doc_CorruptPayload="A corrupted payload should be re-fetched"
setup_CorruptPayload() {
   setup_common
   set_instance +A
   executeOk_servald config \
      set rhizome.max_blob_size 0 \
      set debug.rhizome_store 1
   rhizome_add_file file1 1024
   start_servald_instances +A +B
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   stop_servald_server
   assert cmp file1 "$SERVALINSTANCE_PATH/blob/$FILEHASH"
   create_file file2 1024
   assert --error-on-fail ! cmp file1 file2
   cp file2 "$SERVALINSTANCE_PATH/blob/$FILEHASH"
   execute --exit-status=255 $servald rhizome extract file $BID file1a
   # TODO at the moment, the re-fetch is only triggered by restarting the
   # daemon.  Eventually (when the Rhizome Rank is implemented), the re-fetch
   # shoud be automatic and immediate without restarting the daemon.
   start_servald_server
}
test_CorruptPayload() {
   wait_until grep -i "Stored file $FILEHASH" $LOGA
   executeOk $servald rhizome extract file $BID file1a
}

doc_MissingPayload="A missing payload should be re-fetched"
setup_MissingPayload() {
   setup_common
   set_instance +A
   executeOk_servald config \
      set rhizome.max_blob_size 0 \
      set debug.rhizome_store 1
   rhizome_add_file file1 1024
   assert cmp file1 "$SERVALINSTANCE_PATH/blob/$FILEHASH"
   start_servald_instances +A +B
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   stop_servald_server
   rm -f "$SERVALINSTANCE_PATH/blob/$FILEHASH"
   execute --exit-status=1 --stderr $servald rhizome extract file $BID file1a
   # TODO at the moment, the re-fetch is only triggered by restarting the
   # daemon.  Eventually (when the Rhizome Rank is implemented), the re-fetch
   # shoud be automatic and immediate without restarting the daemon.
   start_servald_server
}
test_MissingPayload() {
   wait_until grep -i "Stored file $FILEHASH" $LOGA
   executeOk $servald rhizome extract file $BID file1a
}

doc_ConnectOnEnable="Enable and disable rhizome while fetching"
setup_ConnectOnEnable(){
   setup_common
   set_instance +A
   dd if=/dev/urandom of=file1 bs=1k count=1k 2>&1
   echo x >>file1
   rhizome_add_file file1
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.enable off
   start_servald_instances +A +B
}
test_ConnectOnEnable(){
   # TODO try to force disabling rhizome mid transfer
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.enable on \
         sync
   sleep 2
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.enable off \
         sync
   sleep 3
}

doc_HttpFetchRange="Fetch a file range using HTTP GET"
setup_HttpFetchRange() {
   setup_curl 7
   setup_common
   set_instance +A
   rhizome_add_file file1 100
   tail --bytes +33 file1 >file1.tail
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpFetchRange() {
   executeOk curl \
         --silent --fail --show-error \
         --output http.output \
         --dump-header http.headers \
         --write-out '%{http_code}\n' \
	 --continue-at 32 \
         "http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH"
   tfw_cat -v http.headers http.output
   assertGrep http.headers "^Content-Range: bytes 32-99/100
$"
   assertGrep http.headers "^Content-Length: 68
$"
   tfw_cat -v file1.tail http.output
   assert cmp file1.tail http.output
}

doc_HttpImport="Import bundle using HTTP POST multi-part form."
setup_HttpImport() {
   setup_curl 7
   setup_common
   cat >README.WHYNOTSIPS <<'EOF'
When we were looking at implementing secure calls for OpenBTS it was suggested
that we configure Asterisk to use SIPS/ZRTP. This would have been relatively
easy to setup, however there are a few problems.
.
Number one is that when Asterisk checks the certificates it will either
validate the certificate (checking the chain of trust and so on) and then
check that the common name attribute on the certificate matches the hostname
of the peer, or it will do none of these checks. This code is in main/tcptls.c
line 206 (in version 1.8.14.1).
.
This is undesirable in a setup where there is limited or no infrastructure as
there is not likely to be a DNS server setup, or even rigid IP assignments
that would allow a static hosts file based setup. This situation would force
the administrator to disable the checks completely which would allow a trivial
man in the middle attack.
.
It would be possible to modify Asterisk to have a third way where it validates
the certificate and checks the chain of trust but does not look at the common
name. We decided against this approach as the VOMP channel driver was written
in time to avoid it.
EOF
   set_instance +B
   executeOk_servald rhizome add file $SIDB README.WHYNOTSIPS README.WHYNOTSIPS.manifest
   assert_manifest_complete README.WHYNOTSIPS.manifest
   assert_stdout_add_file README.WHYNOTSIPS
   set_instance +A
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpImport() {
   executeOk curl \
         --silent --fail --show-error \
         --output http.output \
         --dump-header http.headers \
         --write-out '%{http_code}\n' \
         --form 'data=@README.WHYNOTSIPS' \
         --form 'manifest=@README.WHYNOTSIPS.manifest' \
         "$addr_localhost:$PORTA/rhizome/import"
   tfw_cat http.headers http.output
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 README.WHYNOTSIPS
   assert_rhizome_received README.WHYNOTSIPS
}

doc_HttpAddLocal="Add file locally using HTTP, returns manifest"
setup_HttpAddLocal() {
   setup_curl 7
   setup_common
   set_instance +A
   executeOk_servald config \
      set rhizome.api.addfile.uri_path "/rhizome/secretaddfile" \
      set rhizome.api.addfile.default_author $SIDA
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpAddLocal() {
   echo 'File file1' >file1
   executeOk curl --silent --form 'data=@file1' "http://${addr_localhost}:$PORTA/rhizome/secretaddfile" --output file1.manifest
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_name name file1.manifest
   assert [ "$name" = file1 ]
   assert_rhizome_received file1
}

setup_direct() {
   set_instance +A
   rhizome_add_file fileA1 1000
   BID_A1=$BID
   VERSION_A1=$VERSION
   rhizome_add_file fileA2 10000
   BID_A2=$BID
   VERSION_A2=$VERSION
   rhizome_add_file fileA3 100000
   BID_A3=$BID
   VERSION_A3=$VERSION
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
   set_instance +B
   executeOk_servald config \
      set log.console.level debug \
      set log.console.show_time on \
      set debug.rhizome on \
      set debug.httpd on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on
   rhizome_add_file fileB1 2000
   BID_B1=$BID
   VERSION_B1=$VERSION
   rhizome_add_file fileB2 20000
   BID_B2=$BID
   VERSION_B2=$VERSION
   rhizome_add_file fileB3 200000
   BID_B3=$BID
   VERSION_B3=$VERSION
}

setup_direct_peer() {
   executeOk_servald config \
      set rhizome.direct.peer.0 "http://${addr_localhost}:${PORTA}"
}

doc_DirectPush="One way direct push bundles to configured peer"
setup_DirectPush() {
   setup_common
   setup_direct
   setup_direct_peer
   executeOk ls -l
   tfw_cat --stdout
}
test_DirectPush() {
   set_instance +B
   executeOk_servald rhizome direct push
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}

doc_DirectPushArg="One way direct push bundles to given peer URL"
setup_DirectPushArg() {
   setup_common
   setup_direct
}
test_DirectPushArg() {
   set_instance +B
   executeOk_servald rhizome direct push "http://${addr_localhost}:${PORTA}"
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}

doc_DirectPull="One way direct pull bundle from configured peer"
setup_DirectPull() {
   setup_common
   setup_direct
   setup_direct_peer
}
test_DirectPull() {
   set_instance +B
   executeOk_servald rhizome direct pull
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
   assert_rhizome_received fileA1
   assert_rhizome_received fileA2
   assert_rhizome_received fileA3
}

doc_DirectSync="Two-way direct sync bundles with configured peer"
setup_DirectSync() {
   setup_common
   setup_direct
   setup_direct_peer
}
test_DirectSync() {
   set_instance +B
   executeOk_servald rhizome direct sync
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
   assert_rhizome_received fileA1
   assert_rhizome_received fileA2
   assert_rhizome_received fileA3
}

interface_up() {
   $GREP "Interface .* is up" $instance_servald_log || return 1
   return 0
}
start_radio_instance() {
   executeOk_servald config \
      set debug.rhizome on \
      set debug.rhizome_ads on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on \
      set debug.radio_link on \
      set rhizome.advertise.interval 5000 \
      set rhizome.rhizome_mdp_block_size 375 \
      set log.console.level debug \
      set log.console.show_pid on \
      set log.console.show_time on \
      set interfaces.1.type CATEAR \
      set interfaces.1.broadcast.tick_ms 5000 \
      set interfaces.1.socket_type STREAM \
      set interfaces.1.broadcast.encapsulation SINGLE \
      set interfaces.1.point_to_point on
   start_servald_server
   wait_until interface_up
}

start_fakeradio() {
   $servald_build_root/fakeradio $1 $2 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
   FAKERADIO_PID=$!
   wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
   local _line=`head "$SERVALD_VAR/radioout" -n 1`
   END1="${_line#*:}"
   _line=`tail "$SERVALD_VAR/radioout" -n 1`
   END2="${_line#*:}"
   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
}

doc_SimulatedRadio="MDP Transfer over simulated radio link (~90% packet arrival)"
setup_SimulatedRadio() {
   setup_common
   start_fakeradio 4 0.9
   set_instance +A
   rhizome_add_file file1 5000
   executeOk_servald config \
      set interfaces.1.file "$END1"
   set_instance +B
   executeOk_servald config \
      set interfaces.1.file "$END2"
   foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio() {
   receive_and_update_bundle
}
teardown_SimulatedRadio() {
   teardown
   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
   kill $FAKERADIO_PID
   tfw_cat "$SERVALD_VAR/radioerr"
}

doc_SimulatedRadio2="MDP Transfer over simulated radio link (~50% packet arrival)"
setup_SimulatedRadio2() {
   setup_common
   start_fakeradio 4 0.5
   set_instance +A
   rhizome_add_file file1 5000
   executeOk_servald config \
      set interfaces.1.file "$END1"
   set_instance +B
   executeOk_servald config \
      set interfaces.1.file "$END2"
   foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio2() {
   receive_and_update_bundle --timeout=120
}
teardown_SimulatedRadio2() {
   teardown
   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
   kill $FAKERADIO_PID
   tfw_cat "$SERVALD_VAR/radioerr"
}

runTests "$@"