serval-dna/tests/rhizomeprotocol

#!/bin/bash

# Tests for Serval rhizome protocol.
#
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
source "${0%/*}/../testdefs_rhizome.sh"

shopt -s extglob

finally() {
   stop_all_servald_servers
}

teardown() {
   kill_all_servald_processes
   assert_no_servald_processes
   report_all_servald_servers
}

# Called by start_servald_instances for each instance.
configure_servald_server() {
   executeOk_servald config \
      set log.console.level debug \
      set log.console.show_pid on \
      set log.console.show_time on \
      set debug.rhizome on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on \
      set server.respawn_on_crash off \
      set mdp.iftype.wifi.tick_ms 500
}

setup_curl_7() {
   case "$(curl --version | tr '\n' ' ')" in
   curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
   '') fail "curl(1) command is not present";;
   *) fail "curl(1) version is not adequate (expecting 7 or higher)";;
   esac
   unset http_proxy
   unset HTTP_PROXY
   unset HTTPS_PROXY
   unset ALL_PROXY
}

setup_common() {
   setup_servald
   assert_no_servald_processes
   foreach_instance +A +B create_single_identity
   set_instance +B
}

receive_and_update_bundle() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   assert_rhizome_received file1
   set_instance +A
   rhizome_update_file file1 file2
   set_instance +B
   wait_until bundle_received_by $BID:$VERSION +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file2
   assert_rhizome_received file2
}

doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
   setup_common
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_FileTransfer() {
   receive_and_update_bundle
}

doc_EncryptedTransfer="Encrypted payload can be opened by destination"
setup_EncryptedTransfer() {
   setup_common
   set_instance +A
   echo "Clear Text" >file1
   echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB" >file1.manifest
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_manifest_id BID file1.manifest
   extract_manifest_version VERSION file1.manifest
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_EncryptedTransfer() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome extract file $BID file1x
   assert diff file1 file1x
}

doc_NoFetch="Disabling fetching still allows pushing bundles"
setup_NoFetch() {
   setup_common
   set_instance +A
   rhizome_add_file file1
   BID1=$BID
   VERSION1=$VERSION
   set_instance +B
   rhizome_add_file file2
   BID2=$BID
   VERSION2=$VERSION
   executeOk_servald config set rhizome.fetch 0
   start_servald_instances +A +B
}
test_NoFetch() {
   wait_until bundle_received_by $BID2:$VERSION2 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1 --fromhere=0 file2
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file2
}

doc_DisablingHTTPServer="Disabling HTTP rhizome transports works"
setup_DisablingHTTPServer() {
   setup_common
   set_instance +A
   rhizome_add_file file1 2048
   executeOk_servald config set rhizome.http.enable 0
   start_servald_instances +A
}
test_DisablingHTTPServer() {
   !rhizome_http_server_started
}

doc_HTTPTransport="Rhizome over HTTP transport"
setup_HTTPTransport() {
   setup_common
   set_instance +B
   executeOk_servald config set rhizome.mdp.enable 0
   set_instance +A
   executeOk_servald config set rhizome.mdp.enable 0
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_HTTPTransport() {
   receive_and_update_bundle
}

doc_MDPTransport="Rhizome over MDP transport"
setup_MDPTransport() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransport() {
   receive_and_update_bundle
}

doc_MDPTransportFailOver="Rhizome fails over to MDP transport"
setup_MDPTransportFailOver() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set debug.rhizome_nohttptx 1 \
       set rhizome.mdp.enable 1
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransportFailOver() {
   receive_and_update_bundle
}

doc_UnicastTransfer="Rhizome over unicast MDP link"
setup_UnicastTransfer() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0
   set_instance +A
   rhizome_add_file file1 2048
   set_instance +B
   executeOk_servald config \
      set interfaces.1.file foo \
      set interfaces.1.drop_broadcasts on
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_UnicastTransfer() {
   receive_and_update_bundle
}

doc_SimulatedRadio="MDP Transfer over simulated radio link"
interface_up() {
   $GREP "Interface .* is up" $instance_servald_log || return 1
   return 0
}
start_radio_instance() {
   executeOk_servald config \
      set monitor.socket "org.servalproject.servald.monitor.socket.$TFWUNIQUE.$instance_name" \
      set mdp.socket "org.servalproject.servald.mdp.socket.$TFWUNIQUE.$instance_name" \
      set debug.rhizome on \
      set debug.rhizome_ads on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on \
      set rhizome.advertise.interval 1000 \
      set rhizome.rhizome_mdp_block_size 200 \
      set log.console.level debug \
      set log.console.show_pid on \
      set log.console.show_time on \
      set interfaces.1.type CATEAR \
      set interfaces.1.mdp_tick_ms 5000 \
      set interfaces.1.socket_type STREAM \
      set interfaces.1.encapsulation SINGLE \
      set interfaces.1.point_to_point on \
      set interfaces.1.packet_interval 5000 \
      set interfaces.1.burst_size 100 \
      set interfaces.1.throttle 32000
   start_servald_server
   wait_until interface_up
}
setup_SimulatedRadio() {
   setup_common
   $servald_build_root/fakeradio 64 10000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
   FAKERADIO_PID=$!
   sleep 1
   local END1=`head "$SERVALD_VAR/radioout" -n 1`
   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
   set_instance +A
   rhizome_add_file file1 2048
   executeOk_servald config \
      set interfaces.1.file "$END1"
   set_instance +B
   executeOk_servald config \
      set interfaces.1.file "$END2"
   foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio() {
   receive_and_update_bundle
}
teardown_SimulatedRadio() {
   teardown
   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
   kill $FAKERADIO_PID
   tfw_cat "$SERVALD_VAR/radioerr"
}

doc_journalMDP="Transfer and update a journal bundle via MDP"
setup_journalMDP() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0
   set_instance +A
   create_file file1 2048
   create_file file2 2048
   executeOk_servald rhizome journal append $SIDA "" file1
   extract_stdout_manifestid BID
   extract_stdout_version VERSION
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_journalMDP() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   executeOk_servald rhizome journal append $SIDA $BID file2
   extract_stdout_version VERSION2
   set_instance +B
   wait_until bundle_received_by $BID:$VERSION2 +B
   assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}

doc_journalHTTP="Transfer and update a journal bundle via HTTP"
setup_journalHTTP() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.mdp.enable 0
   set_instance +A
   create_file file1 2048
   create_file file2 2048
   executeOk_servald rhizome journal append $SIDA "" file1
   extract_stdout_manifestid BID
   extract_stdout_version VERSION
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
test_journalHTTP() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   executeOk_servald rhizome journal append $SIDA $BID file2
   extract_stdout_version VERSION2
   set_instance +B
   wait_until bundle_received_by $BID:$VERSION2 +B
   assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}

#common setup and test routines for transferring a 1MB file
setup_bigfile_common() {
   set_instance +A
   dd if=/dev/urandom of=file1 bs=1k count=1k 2>&1
   echo x >>file1
   ls -l file1
   rhizome_add_file file1
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
}
bigfile_common_test() {
   set_instance +B
   wait_until --timeout=120 bundle_received_by $BID:$VERSION +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   assert_rhizome_received file1
}

doc_FileTransferBigMDP="Big new bundle transfers to one node via MDP"
setup_FileTransferBigMDP() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config set rhizome.http.enable 0
   setup_bigfile_common
}
test_FileTransferBigMDP() {
   bigfile_common_test
}

doc_FileTransferUnreliableBigMDP="Big new bundle over unreliable MDP transport"
setup_FileTransferUnreliableBigMDP() {
   setup_common
   foreach_instance +A +B \
     executeOk_servald config \
       set rhizome.http.enable 0 \
       set interfaces.1.file dummy \
       set interfaces.1.drop_packets 10
   setup_bigfile_common
}
test_FileTransferUnreliableBigMDP() {
   bigfile_common_test
}


doc_FileTransferBig="Big new bundle transfers to one node via HTTP"
setup_FileTransferBig() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config set rhizome.mdp.enable 0
   setup_bigfile_common
}
test_FileTransferBig() {
   bigfile_common_test
}

doc_FileTransferBigMDPExtBlob="Big new bundle transfers to one node via MDP, external blob file"
setup_FileTransferBigMDPExtBlob() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.http.enable 0 \
         set rhizome.external_blobs 1 \
	 set debug.externalblobs 1
   setup_bigfile_common
}
test_FileTransferBigMDPExtBlob() {
   bigfile_common_test
}

doc_FileTransferBigHTTPExtBlob="Big new bundle transfers to one node via HTTP, external blob file"
setup_FileTransferBigHTTPExtBlob() {
   setup_common
   foreach_instance +A +B \
      executeOk_servald config \
         set rhizome.mdp.enable 0 \
         set rhizome.external_blobs 1 \
	 set debug.externalblobs 1
   setup_bigfile_common
}
test_FileTransferBigHTTPExtBlob() {
   bigfile_common_test
}

# common setup and test routines for transfers to 4 nodes
setup_multitransfer_common() {
   set_instance +A
   rhizome_add_file file1 2048
   start_servald_instances +A +B +C +D +E
   set_instance +A
   assert_peers_are_instances +B +C +D +E
   set_instance +B
   assert_peers_are_instances +A +C +D +E
   set_instance +C
   assert_peers_are_instances +A +B +D +E
   set_instance +D
   assert_peers_are_instances +A +B +C +E
   set_instance +E
   assert_peers_are_instances +A +B +C +D
}
multitransfer_common_test() {
   wait_until bundle_received_by $BID:$VERSION +B +C +D +E
   for i in B C D E; do
      set_instance +$i
      executeOk_servald rhizome list
      assert_rhizome_list --fromhere=0 file1
      assert_rhizome_received file1
   done
}

doc_FileTransferMulti="New bundle transfers to four nodes via HTTP"
setup_FileTransferMulti() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config set rhizome.mdp.enable 0
   setup_multitransfer_common
}
test_FileTransferMulti() {
   multitransfer_common_test
}

doc_FileTransferMultiMDP="New bundle transfers to four nodes via MDP"
setup_FileTransferMultiMDP() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config set rhizome.http.enable 0
   setup_multitransfer_common
}
test_FileTransferMultiMDP() {
   multitransfer_common_test
}

doc_FileTransferMultiMDPExtBlob="New bundle transfers to four nodes via MDP, external blob files"
setup_FileTransferMultiMDPExtBlob() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config \
         set rhizome.http.enable 0 \
         set rhizome.external_blobs 1 \
	 set debug.externalblobs 1
   setup_multitransfer_common
}
test_FileTransferMultiMDPExtBlob() {
   multitransfer_common_test
}

doc_FileTransferMultiHTTPExtBlob="New bundle transfers to four nodes via HTTP, external blob files"
setup_FileTransferMultiHTTPExtBlob() {
   setup_common
   foreach_instance +A +B +C +D +E \
      executeOk_servald config \
         set rhizome.mdp.enable 0 \
         set rhizome.external_blobs 1 \
	 set debug.externalblobs 1
   setup_multitransfer_common
}
test_FileTransferMultiHTTPExtBlob() {
   multitransfer_common_test
}

doc_FileTransferDelete="Payload deletion transfers to one node"
setup_FileTransferDelete() {
   setup_common
   set_instance +A
   rhizome_add_file file1
   start_servald_instances +A +B
   foreach_instance +A assert_peers_are_instances +B
   foreach_instance +B assert_peers_are_instances +A
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +A
   >file1_2
   rhizome_update_file file1 file1_2
}
test_FileTransferDelete() {
   wait_until bundle_received_by $BID:$VERSION +B
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1_2
   assert_rhizome_received file1_2
}

doc_CorruptPayload="A corrupted payload should be re-fetched"
setup_CorruptPayload() {
   setup_common
   set_instance +A
   executeOk_servald config set rhizome.external_blobs 1
   rhizome_add_file file1 1024
   create_file file2 1024
   start_servald_instances +A +B
   wait_until bundle_received_by $BID:$VERSION +B
}
test_CorruptPayload() {
   set_instance +A
   cp file2 $SERVALINSTANCE_PATH/$FILEHASH
   execute --exit-status=255 $servald rhizome extract file $BID file1a
   stop_servald_server
   start_servald_server
   wait_until grep -i "Stored file $FILEHASH" $LOGA
}

doc_HttpFetchRange="Fetch a file range using HTTP GET."
setup_HttpFetchRange() {
   setup_curl_7
   setup_common
   set_instance +A
   rhizome_add_file file1
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpFetchRange() {
   executeOk curl \
         --silent --fail --show-error \
         --output http.output \
         --dump-header http.headers \
         --write-out '%{http_code}\n' \
	 --continue-at 32 \
         "http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH"
   tfw_cat http.headers http.output
   assertGrep http.headers "Content-range:"
   assertGrep http.headers "Content-length:"
}

doc_HttpImport="Import bundle using HTTP POST multi-part form."
setup_HttpImport() {
   setup_curl_7
   setup_common
   cat >README.WHYNOTSIPS <<'EOF'
When we were looking at implementing secure calls for OpenBTS it was suggested
that we configure Asterisk to use SIPS/ZRTP. This would have been relatively
easy to setup, however there are a few problems.
.
Number one is that when Asterisk checks the certificates it will either
validate the certificate (checking the chain of trust and so on) and then
check that the common name attribute on the certificate matches the hostname
of the peer, or it will do none of these checks. This code is in main/tcptls.c
line 206 (in version 1.8.14.1).
.
This is undesirable in a setup where there is limited or no infrastructure as
there is not likely to be a DNS server setup, or even rigid IP assignments
that would allow a static hosts file based setup. This situation would force
the administrator to disable the checks completely which would allow a trivial
man in the middle attack.
.
It would be possible to modify Asterisk to have a third way where it validates
the certificate and checks the chain of trust but does not look at the common
name. We decided against this approach as the VOMP channel driver was written
in time to avoid it.
EOF
   set_instance +B
   executeOk_servald rhizome add file $SIDB README.WHYNOTSIPS README.WHYNOTSIPS.manifest
   assert_manifest_complete README.WHYNOTSIPS.manifest
   assert_stdout_add_file README.WHYNOTSIPS
   set_instance +A
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpImport() {
   executeOk curl \
         --silent --fail --show-error \
         --output http.output \
         --dump-header http.headers \
         --write-out '%{http_code}\n' \
         --form 'data=@README.WHYNOTSIPS' \
         --form 'manifest=@README.WHYNOTSIPS.manifest' \
         "$addr_localhost:$PORTA/rhizome/import"
   tfw_cat http.headers http.output
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 README.WHYNOTSIPS
   assert_rhizome_received README.WHYNOTSIPS
}

doc_HttpAddLocal="Add file locally using HTTP, returns manifest"
setup_HttpAddLocal() {
   setup_curl_7
   setup_common
   set_instance +A
   executeOk_servald config \
      set rhizome.api.addfile.uri_path "/rhizome/secretaddfile" \
      set rhizome.api.addfile.default_author $SIDA
   start_servald_instances +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
}
test_HttpAddLocal() {
   echo 'File file1' >file1
   executeOk curl --silent --form 'data=@file1' "http://${addr_localhost}:$PORTA/rhizome/secretaddfile" --output file1.manifest
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_name name file1.manifest
   assert [ "$name" = file1 ]
   assert_rhizome_received file1
}

setup_direct() {
   set_instance +A
   rhizome_add_file fileA1 1000
   BID_A1=$BID
   VERSION_A1=$VERSION
   rhizome_add_file fileA2 10000
   BID_A2=$BID
   VERSION_A2=$VERSION
   rhizome_add_file fileA3 100000
   BID_A3=$BID
   VERSION_A3=$VERSION
   start_servald_instances dummy1 +A
   wait_until rhizome_http_server_started +A
   get_rhizome_server_port PORTA +A
   set_instance +B
   executeOk_servald config \
      set log.console.level debug \
      set log.console.show_time on \
      set debug.rhizome on \
      set debug.rhizome_tx on \
      set debug.rhizome_rx on
   rhizome_add_file fileB1 2000
   BID_B1=$BID
   VERSION_B1=$VERSION
   rhizome_add_file fileB2 20000
   BID_B2=$BID
   VERSION_B2=$VERSION
   rhizome_add_file fileB3 200000
   BID_B3=$BID
   VERSION_B3=$VERSION
}

setup_direct_peer() {
   executeOk_servald config \
      set rhizome.direct.peer.0 "http://${addr_localhost}:${PORTA}"
}

doc_DirectPush="One way direct push bundles to configured peer"
setup_DirectPush() {
   setup_common
   setup_direct
   setup_direct_peer
}
test_DirectPush() {
   set_instance +B
   executeOk_servald rhizome direct push
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}

doc_DirectPushArg="One way direct push bundles to given peer URL"
setup_DirectPushArg() {
   setup_common
   setup_direct
}
test_DirectPushArg() {
   set_instance +B
   executeOk_servald rhizome direct push "http://${addr_localhost}:${PORTA}"
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}

doc_DirectPull="One way direct pull bundle from configured peer"
setup_DirectPull() {
   setup_common
   setup_direct
   setup_direct_peer
}
test_DirectPull() {
   set_instance +B
   executeOk_servald rhizome direct pull
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
   assert_rhizome_received fileA1
   assert_rhizome_received fileA2
   assert_rhizome_received fileA3
}

doc_DirectSync="Two-way direct sync bundles with configured peer"
setup_DirectSync() {
   setup_common
   setup_direct
   setup_direct_peer
}
test_DirectSync() {
   set_instance +B
   executeOk_servald rhizome direct sync
   tfw_cat --stdout --stderr
   assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr
   set_instance +A
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
   assert_rhizome_received fileB1
   assert_rhizome_received fileB2
   assert_rhizome_received fileB3
   set_instance +B
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
   assert_rhizome_received fileA1
   assert_rhizome_received fileA2
   assert_rhizome_received fileA3
}

runTests "$@"