serval-dna/tests/rhizomeprotocol
Andrew Bettison c9f9025581 Fix intermittent 'routing' test failures
Caused by tests running so slowly that nodes become unreachable due to
timeout before the assertion of reachability is made.  The timeout was
hard coded at five times the tick interval (2.5 seconds).

Introduced new config options: mdp.iftype.*.reachable_timeout_ms and
interface.*.mdp.reachable_timeout_ms to allow the timeout to be
configured.  Configure the timeout to 60 seconds in 'routing' tests.

Changed the config schema to move options interfaces.*.mdp_tick_ms and
interfaces.*.packet_interval into interfaces.*.mdp.tick_ms and
interfaces.*.mdp.packet_interval (introduced new sub-struct 'mdp').
2013-10-08 16:36:07 +10:30

751 lines
22 KiB
Bash
Executable File

#!/bin/bash
# Tests for Serval rhizome protocol.
#
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
source "${0%/*}/../testdefs_rhizome.sh"
shopt -s extglob
finally() {
stop_all_servald_servers
}
teardown() {
kill_all_servald_processes
assert_no_servald_processes
report_all_servald_servers
}
# Called by start_servald_instances for each instance.
configure_servald_server() {
executeOk_servald config \
set log.console.level debug \
set log.console.show_pid on \
set log.console.show_time on \
set debug.rhizome on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set server.respawn_on_crash off \
set mdp.iftype.wifi.tick_ms 500
}
setup_curl_7() {
case "$(curl --version | tr '\n' ' ')" in
curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
'') fail "curl(1) command is not present";;
*) fail "curl(1) version is not adequate (expecting 7 or higher)";;
esac
unset http_proxy
unset HTTP_PROXY
unset HTTPS_PROXY
unset ALL_PROXY
}
setup_common() {
setup_servald
assert_no_servald_processes
foreach_instance +A +B create_single_identity
set_instance +B
}
receive_and_update_bundle() {
wait_until bundle_received_by $BID:$VERSION +B
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
assert_rhizome_received file1
set_instance +A
rhizome_update_file file1 file2
set_instance +B
wait_until bundle_received_by $BID:$VERSION +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file2
assert_rhizome_received file2
}
doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
setup_common
set_instance +A
rhizome_add_file file1 2048
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_FileTransfer() {
receive_and_update_bundle
}
doc_EncryptedTransfer="Encrypted payload can be opened by destination"
setup_EncryptedTransfer() {
setup_common
set_instance +A
echo "Clear Text" >file1
echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB" >file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_manifest_id BID file1.manifest
extract_manifest_version VERSION file1.manifest
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_EncryptedTransfer() {
wait_until bundle_received_by $BID:$VERSION +B
set_instance +B
executeOk_servald rhizome extract file $BID file1x
assert diff file1 file1x
}
doc_NoFetch="Disabling fetching still allows pushing bundles"
setup_NoFetch() {
setup_common
set_instance +A
rhizome_add_file file1
BID1=$BID
VERSION1=$VERSION
set_instance +B
rhizome_add_file file2
BID2=$BID
VERSION2=$VERSION
executeOk_servald config set rhizome.fetch 0
start_servald_instances +A +B
}
test_NoFetch() {
wait_until bundle_received_by $BID2:$VERSION2 +A
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 file1 --fromhere=0 file2
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 file2
}
doc_DisablingHTTPServer="Disabling HTTP rhizome transports works"
setup_DisablingHTTPServer() {
setup_common
set_instance +A
rhizome_add_file file1 2048
executeOk_servald config set rhizome.http.enable 0
start_servald_instances +A
}
test_DisablingHTTPServer() {
!rhizome_http_server_started
}
doc_HTTPTransport="Rhizome over HTTP transport"
setup_HTTPTransport() {
setup_common
set_instance +B
executeOk_servald config set rhizome.mdp.enable 0
set_instance +A
executeOk_servald config set rhizome.mdp.enable 0
rhizome_add_file file1 2048
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_HTTPTransport() {
receive_and_update_bundle
}
doc_MDPTransport="Rhizome over MDP transport"
setup_MDPTransport() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0
set_instance +A
rhizome_add_file file1 2048
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransport() {
receive_and_update_bundle
}
doc_MDPTransportFailOver="Rhizome fails over to MDP transport"
setup_MDPTransportFailOver() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set debug.rhizome_nohttptx 1 \
set rhizome.mdp.enable 1
set_instance +A
rhizome_add_file file1 2048
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_MDPTransportFailOver() {
receive_and_update_bundle
}
doc_UnicastTransfer="Rhizome over unicast MDP link"
setup_UnicastTransfer() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0
set_instance +A
rhizome_add_file file1 2048
set_instance +B
executeOk_servald config \
set interfaces.1.file foo \
set interfaces.1.drop_broadcasts on
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_UnicastTransfer() {
receive_and_update_bundle
}
doc_SimulatedRadio="MDP Transfer over simulated radio link"
interface_up() {
$GREP "Interface .* is up" $instance_servald_log || return 1
return 0
}
start_radio_instance() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_ads on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set debug.throttling on \
set debug.mavlink on \
set rhizome.advertise.interval 5000 \
set rhizome.rhizome_mdp_block_size 350 \
set log.console.level debug \
set log.console.show_pid on \
set log.console.show_time on \
set interfaces.1.type CATEAR \
set interfaces.1.mdp.tick_ms 5000 \
set interfaces.1.socket_type STREAM \
set interfaces.1.encapsulation SINGLE \
set interfaces.1.point_to_point on
start_servald_server
wait_until interface_up
}
setup_SimulatedRadio() {
setup_common
$servald_build_root/fakeradio 6 10000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
FAKERADIO_PID=$!
sleep 1
local END1=`head "$SERVALD_VAR/radioout" -n 1`
local END2=`tail "$SERVALD_VAR/radioout" -n 1`
tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
set_instance +A
rhizome_add_file file1 10000
executeOk_servald config \
set interfaces.1.file "$END1"
set_instance +B
executeOk_servald config \
set interfaces.1.file "$END2"
foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio() {
receive_and_update_bundle
}
teardown_SimulatedRadio() {
teardown
tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
kill $FAKERADIO_PID
tfw_cat "$SERVALD_VAR/radioerr"
}
doc_journalMDP="Transfer and update a journal bundle via MDP"
setup_journalMDP() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0
set_instance +A
create_file file1 2048
create_file file2 2048
executeOk_servald rhizome journal append $SIDA "" file1
extract_stdout_manifestid BID
extract_stdout_version VERSION
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_journalMDP() {
wait_until bundle_received_by $BID:$VERSION +B
set_instance +A
executeOk_servald rhizome journal append $SIDA $BID file2
extract_stdout_version VERSION2
set_instance +B
wait_until bundle_received_by $BID:$VERSION2 +B
assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}
doc_journalHTTP="Transfer and update a journal bundle via HTTP"
setup_journalHTTP() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.mdp.enable 0
set_instance +A
create_file file1 2048
create_file file2 2048
executeOk_servald rhizome journal append $SIDA "" file1
extract_stdout_manifestid BID
extract_stdout_version VERSION
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
test_journalHTTP() {
wait_until bundle_received_by $BID:$VERSION +B
set_instance +A
executeOk_servald rhizome journal append $SIDA $BID file2
extract_stdout_version VERSION2
set_instance +B
wait_until bundle_received_by $BID:$VERSION2 +B
assertGrep $instance_servald_log "Copying [0-9]\+ bytes from previous journal"
}
#common setup and test routines for transferring a 1MB file
setup_bigfile_common() {
set_instance +A
dd if=/dev/urandom of=file1 bs=1k count=1k 2>&1
echo x >>file1
ls -l file1
rhizome_add_file file1
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
}
bigfile_common_test() {
set_instance +B
wait_until --timeout=120 bundle_received_by $BID:$VERSION +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
assert_rhizome_received file1
}
doc_FileTransferBigMDP="Big new bundle transfers to one node via MDP"
setup_FileTransferBigMDP() {
setup_common
foreach_instance +A +B \
executeOk_servald config set rhizome.http.enable 0
setup_bigfile_common
}
test_FileTransferBigMDP() {
bigfile_common_test
}
doc_FileTransferUnreliableBigMDP="Big new bundle over unreliable MDP transport"
setup_FileTransferUnreliableBigMDP() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0 \
set interfaces.1.file dummy \
set interfaces.1.drop_packets 10
setup_bigfile_common
}
test_FileTransferUnreliableBigMDP() {
bigfile_common_test
}
doc_FileTransferBig="Big new bundle transfers to one node via HTTP"
setup_FileTransferBig() {
setup_common
foreach_instance +A +B \
executeOk_servald config set rhizome.mdp.enable 0
setup_bigfile_common
}
test_FileTransferBig() {
bigfile_common_test
}
doc_FileTransferBigMDPExtBlob="Big new bundle transfers to one node via MDP, external blob file"
setup_FileTransferBigMDPExtBlob() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.http.enable 0 \
set rhizome.external_blobs 1 \
set debug.externalblobs 1
setup_bigfile_common
}
test_FileTransferBigMDPExtBlob() {
bigfile_common_test
}
doc_FileTransferBigHTTPExtBlob="Big new bundle transfers to one node via HTTP, external blob file"
setup_FileTransferBigHTTPExtBlob() {
setup_common
foreach_instance +A +B \
executeOk_servald config \
set rhizome.mdp.enable 0 \
set rhizome.external_blobs 1 \
set debug.externalblobs 1
setup_bigfile_common
}
test_FileTransferBigHTTPExtBlob() {
bigfile_common_test
}
# common setup and test routines for transfers to 4 nodes
setup_multitransfer_common() {
set_instance +A
rhizome_add_file file1 2048
start_servald_instances +A +B +C +D +E
set_instance +A
assert_peers_are_instances +B +C +D +E
set_instance +B
assert_peers_are_instances +A +C +D +E
set_instance +C
assert_peers_are_instances +A +B +D +E
set_instance +D
assert_peers_are_instances +A +B +C +E
set_instance +E
assert_peers_are_instances +A +B +C +D
}
multitransfer_common_test() {
wait_until bundle_received_by $BID:$VERSION +B +C +D +E
for i in B C D E; do
set_instance +$i
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
assert_rhizome_received file1
done
}
doc_FileTransferMulti="New bundle transfers to four nodes via HTTP"
setup_FileTransferMulti() {
setup_common
foreach_instance +A +B +C +D +E \
executeOk_servald config set rhizome.mdp.enable 0
setup_multitransfer_common
}
test_FileTransferMulti() {
multitransfer_common_test
}
doc_FileTransferMultiMDP="New bundle transfers to four nodes via MDP"
setup_FileTransferMultiMDP() {
setup_common
foreach_instance +A +B +C +D +E \
executeOk_servald config set rhizome.http.enable 0
setup_multitransfer_common
}
test_FileTransferMultiMDP() {
multitransfer_common_test
}
doc_FileTransferMultiMDPExtBlob="New bundle transfers to four nodes via MDP, external blob files"
setup_FileTransferMultiMDPExtBlob() {
setup_common
foreach_instance +A +B +C +D +E \
executeOk_servald config \
set rhizome.http.enable 0 \
set rhizome.external_blobs 1 \
set debug.externalblobs 1
setup_multitransfer_common
}
test_FileTransferMultiMDPExtBlob() {
multitransfer_common_test
}
doc_FileTransferMultiHTTPExtBlob="New bundle transfers to four nodes via HTTP, external blob files"
setup_FileTransferMultiHTTPExtBlob() {
setup_common
foreach_instance +A +B +C +D +E \
executeOk_servald config \
set rhizome.mdp.enable 0 \
set rhizome.external_blobs 1 \
set debug.externalblobs 1
setup_multitransfer_common
}
test_FileTransferMultiHTTPExtBlob() {
multitransfer_common_test
}
doc_FileTransferDelete="Payload deletion transfers to one node"
setup_FileTransferDelete() {
setup_common
set_instance +A
rhizome_add_file file1
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
wait_until bundle_received_by $BID:$VERSION +B
set_instance +A
>file1_2
rhizome_update_file file1 file1_2
}
test_FileTransferDelete() {
wait_until bundle_received_by $BID:$VERSION +B
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1_2
assert_rhizome_received file1_2
}
doc_CorruptPayload="A corrupted payload should be re-fetched"
setup_CorruptPayload() {
setup_common
set_instance +A
executeOk_servald config set rhizome.external_blobs 1
rhizome_add_file file1 1024
create_file file2 1024
start_servald_instances +A +B
wait_until bundle_received_by $BID:$VERSION +B
}
test_CorruptPayload() {
set_instance +A
cp file2 $SERVALINSTANCE_PATH/$FILEHASH
execute --exit-status=255 $servald rhizome extract file $BID file1a
stop_servald_server
start_servald_server
wait_until grep -i "Stored file $FILEHASH" $LOGA
}
doc_HttpFetchRange="Fetch a file range using HTTP GET."
setup_HttpFetchRange() {
setup_curl_7
setup_common
set_instance +A
rhizome_add_file file1
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
}
test_HttpFetchRange() {
executeOk curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
--write-out '%{http_code}\n' \
--continue-at 32 \
"http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH"
tfw_cat http.headers http.output
assertGrep http.headers "Content-range:"
assertGrep http.headers "Content-length:"
}
doc_HttpImport="Import bundle using HTTP POST multi-part form."
setup_HttpImport() {
setup_curl_7
setup_common
cat >README.WHYNOTSIPS <<'EOF'
When we were looking at implementing secure calls for OpenBTS it was suggested
that we configure Asterisk to use SIPS/ZRTP. This would have been relatively
easy to setup, however there are a few problems.
.
Number one is that when Asterisk checks the certificates it will either
validate the certificate (checking the chain of trust and so on) and then
check that the common name attribute on the certificate matches the hostname
of the peer, or it will do none of these checks. This code is in main/tcptls.c
line 206 (in version 1.8.14.1).
.
This is undesirable in a setup where there is limited or no infrastructure as
there is not likely to be a DNS server setup, or even rigid IP assignments
that would allow a static hosts file based setup. This situation would force
the administrator to disable the checks completely which would allow a trivial
man in the middle attack.
.
It would be possible to modify Asterisk to have a third way where it validates
the certificate and checks the chain of trust but does not look at the common
name. We decided against this approach as the VOMP channel driver was written
in time to avoid it.
EOF
set_instance +B
executeOk_servald rhizome add file $SIDB README.WHYNOTSIPS README.WHYNOTSIPS.manifest
assert_manifest_complete README.WHYNOTSIPS.manifest
assert_stdout_add_file README.WHYNOTSIPS
set_instance +A
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
}
test_HttpImport() {
executeOk curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
--write-out '%{http_code}\n' \
--form 'data=@README.WHYNOTSIPS' \
--form 'manifest=@README.WHYNOTSIPS.manifest' \
"$addr_localhost:$PORTA/rhizome/import"
tfw_cat http.headers http.output
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 README.WHYNOTSIPS
assert_rhizome_received README.WHYNOTSIPS
}
doc_HttpAddLocal="Add file locally using HTTP, returns manifest"
setup_HttpAddLocal() {
setup_curl_7
setup_common
set_instance +A
executeOk_servald config \
set rhizome.api.addfile.uri_path "/rhizome/secretaddfile" \
set rhizome.api.addfile.default_author $SIDA
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
}
test_HttpAddLocal() {
echo 'File file1' >file1
executeOk curl --silent --form 'data=@file1' "http://${addr_localhost}:$PORTA/rhizome/secretaddfile" --output file1.manifest
assert_manifest_complete file1.manifest
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 file1
extract_manifest_name name file1.manifest
assert [ "$name" = file1 ]
assert_rhizome_received file1
}
setup_direct() {
set_instance +A
rhizome_add_file fileA1 1000
BID_A1=$BID
VERSION_A1=$VERSION
rhizome_add_file fileA2 10000
BID_A2=$BID
VERSION_A2=$VERSION
rhizome_add_file fileA3 100000
BID_A3=$BID
VERSION_A3=$VERSION
start_servald_instances dummy1 +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
set_instance +B
executeOk_servald config \
set log.console.level debug \
set log.console.show_time on \
set debug.rhizome on \
set debug.rhizome_tx on \
set debug.rhizome_rx on
rhizome_add_file fileB1 2000
BID_B1=$BID
VERSION_B1=$VERSION
rhizome_add_file fileB2 20000
BID_B2=$BID
VERSION_B2=$VERSION
rhizome_add_file fileB3 200000
BID_B3=$BID
VERSION_B3=$VERSION
}
setup_direct_peer() {
executeOk_servald config \
set rhizome.direct.peer.0 "http://${addr_localhost}:${PORTA}"
}
doc_DirectPush="One way direct push bundles to configured peer"
setup_DirectPush() {
setup_common
setup_direct
setup_direct_peer
}
test_DirectPush() {
set_instance +B
executeOk_servald rhizome direct push
tfw_cat --stdout --stderr
assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
assert_rhizome_received fileB1
assert_rhizome_received fileB2
assert_rhizome_received fileB3
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}
doc_DirectPushArg="One way direct push bundles to given peer URL"
setup_DirectPushArg() {
setup_common
setup_direct
}
test_DirectPushArg() {
set_instance +B
executeOk_servald rhizome direct push "http://${addr_localhost}:${PORTA}"
tfw_cat --stdout --stderr
assert bundle_received_by $BID_B1:$VERSION_B1 $BID_B2:$VERSION_B2 $BID_B3:$VERSION_B3 +A
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
assert_rhizome_received fileB1
assert_rhizome_received fileB2
assert_rhizome_received fileB3
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileB1 fileB2 fileB3
}
doc_DirectPull="One way direct pull bundle from configured peer"
setup_DirectPull() {
setup_common
setup_direct
setup_direct_peer
}
test_DirectPull() {
set_instance +B
executeOk_servald rhizome direct pull
tfw_cat --stdout --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
assert_rhizome_received fileA1
assert_rhizome_received fileA2
assert_rhizome_received fileA3
}
doc_DirectSync="Two-way direct sync bundles with configured peer"
setup_DirectSync() {
setup_common
setup_direct
setup_direct_peer
}
test_DirectSync() {
set_instance +B
executeOk_servald rhizome direct sync
tfw_cat --stdout --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3
assert_rhizome_received fileB1
assert_rhizome_received fileB2
assert_rhizome_received fileB3
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 fileA1 fileA2 fileA3 --fromhere=1 fileB1 fileB2 fileB3
assert_rhizome_received fileA1
assert_rhizome_received fileA2
assert_rhizome_received fileA3
}
runTests "$@"