From 7317f9b56c16abaf071deaea81f770064c8daca1 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrewb@zip.com.au>
Date: Wed, 17 Oct 2012 18:10:39 +1030
Subject: [PATCH] Add a new 'rhizomeprotocol' test case: stress test

Refactored some testdefs.sh and testdefs_rhizome.sh, not yet complete.
---
 testdefs.sh             |  72 ++++++++++++++-----
 testdefs_rhizome.sh     |  26 ++++---
 testframework.sh        |   8 +++
 tests/directory_service |   6 +-
 tests/dnahelper         |   6 +-
 tests/rhizomeops        |  64 ++++++++---------
 tests/rhizomeprotocol   | 155 +++++++++++++++++++++++++++-------------
 7 files changed, 227 insertions(+), 110 deletions(-)

diff --git a/testdefs.sh b/testdefs.sh
index 487435b2..854d2c23 100644
--- a/testdefs.sh
+++ b/testdefs.sh
@@ -209,6 +209,23 @@ foreach_instance() {
    return $ret
 }
 
+# Composition function:
+#  - invoke a command once in every instance that contains a server pidfile
+#  - takes the same options as foreach_instance()
+foreach_instance_with_pidfile() {
+   push_instance
+   local -a instances=()
+   if pushd "${servald_instances_dir?:}" >/dev/null; then
+      for name in *; do
+         set_instance "+$name"
+         get_servald_server_pidfile && instances+=("+$name")
+      done
+      popd >/dev/null
+   fi
+   pop_instance
+   foreach_instance "${instances[@]}" "$@"
+}
+
 # Utility function for setting up servald JNI fixtures:
 #  - check that libservald.so is present
 #  - set LD_LIBRARY_PATH so that libservald.so can be found
@@ -307,13 +324,6 @@ stop_servald_server() {
          assert --message="servald process still running" [ "$apid" -ne "$servald_pid" ]
       done
    fi
-   # Append the server log file to the test log.
-   [ -s "$instance_servald_log" ] && tfw_cat "$instance_servald_log"
-   # Append a core dump backtrace to the test log.
-   if [ -s "$instance_dir/core" ]; then
-      tfw_core_backtrace "$servald" "$instance_dir/core"
-      rm -f "$instance_dir/core"
-   fi
    # Check there is at least one fewer servald processes running.
    for bpid in ${before_pids[*]}; do
       local isgone=true
@@ -327,6 +337,18 @@ stop_servald_server() {
          tfw_log "# ended servald process: pid=$bpid"
       fi
    done
+}
+
+# Utility function:
+#  - cat a servald server log file and core dump information into the test log
+report_servald_server() {
+   # Append the server log file to the test log.
+   [ -s "$instance_servald_log" ] && tfw_cat "$instance_servald_log"
+   # Append a core dump backtrace to the test log.
+   if [ -s "$instance_dir/core" ]; then
+      tfw_core_backtrace "$servald" "$instance_dir/core"
+      rm -f "$instance_dir/core"
+   fi
    pop_instance
 }
 
@@ -372,18 +394,36 @@ assert_servald_server_pidfile() {
    assert get_servald_server_pidfile "$@"
 }
 
+# Assertion function:
+#  - assert that the given instance's server has the given status ('running' or 'stopped')
+assert_servald_server_status() {
+   push_instance
+   set_instance_fromarg "$1" && shift
+   [ $# -eq 1 ] || error "invalid arguments"
+   executeOk_servald status
+   local status
+   extract_stdout_keyvalue status status '.*'
+   assert --message="instance +$instance_name servald server status is '$1'" [ "$status" = "$1" ]
+   pop_instance
+}
+
+# Assertion function:
+#  - asserts that all servald instances with a pidfile have a server in a given
+#    state
+assert_status_all_servald_servers() {
+   foreach_instance_with_pidfile assert_servald_server_status "$@"
+}
+
 # Utility function for tearing down servald fixtures:
 #  - stop all servald server process instances in an orderly fashion
 stop_all_servald_servers() {
-   push_instance
-   if pushd "${servald_instances_dir?:}" >/dev/null; then
-      for name in *; do
-         set_instance "+$name"
-         get_servald_server_pidfile && stop_servald_server
-      done
-      popd >/dev/null
-   fi
-   pop_instance
+   foreach_instance_with_pidfile stop_servald_server
+}
+
+# Utility function for tearing down servald fixtures:
+#  - log a report of the execution of all servald server process instances
+report_all_servald_servers() {
+   foreach_instance_with_pidfile report_servald_server
 }
 
 # Utility function for tearing down servald fixtures:
diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh
index 7f586647..98c97b9c 100644
--- a/testdefs_rhizome.sh
+++ b/testdefs_rhizome.sh
@@ -45,25 +45,31 @@ assert_manifest_complete() {
    fi
 }
 
+# Assertion function:
+# - assert that the output of a "servald rhizome list" command exactly describes the given files
 assert_rhizome_list() {
-   assertStdoutLineCount --stderr '==' $(($# + 2))
    assertStdoutIs --stderr --line=1 -e '12\n'
    assertStdoutIs --stderr --line=2 -e 'service:id:version:date:.inserttime:.author:.fromhere:filesize:filehash:sender:recipient:name\n'
    local filename
-   local re__author="\($rexp_sid\)\{0,1\}"
-   local re__fromhere
+   local exactly=true
    local re__inserttime="$rexp_date"
+   local re__fromhere='[01]'
+   local re__author="\($rexp_sid\)\{0,1\}"
+   local files=0
    for filename; do
-      re__fromhere=1
       case "$filename" in
-      *@*) re__author="${filename##*@}"; filename="${filename%@*}";;
+      --fromhere=*) re__fromhere="${filename#*=}";;
+      --author=*) re__author="${filename#*=}";;
+      --and-others) exactly=false;;
+      --*) error "unsupported option: $filename";;
+      *)
+         unpack_manifest_for_grep "$filename"
+         assertStdoutGrep --stderr --matches=1 "^$re_service:$re_manifestid:$re_version:$re_date:$re__inserttime:$re__author:$re__fromhere:$re_filesize:$re_filehash:$re_sender:$re_recipient:$re_name\$"
+         let files+=1
+         ;;
       esac
-      case "$filename" in
-      *!) re__fromhere=0; filename="${filename%!}";;
-      esac
-      unpack_manifest_for_grep "$filename"
-      assertStdoutGrep --stderr --matches=1 "^$re_service:$re_manifestid:$re_version:$re_date:$re__inserttime:$re__author:$re__fromhere:$re_filesize:$re_filehash:$re_sender:$re_recipient:$re_name\$"
    done
+   $exactly && assertStdoutLineCount --stderr '==' $(($files + 2))
 }
 
 assert_stdout_add_file() {
diff --git a/testframework.sh b/testframework.sh
index 0af1c9b0..158a25cd 100644
--- a/testframework.sh
+++ b/testframework.sh
@@ -478,6 +478,12 @@ escape_grep_extended() {
    echo "$re"
 }
 
+# Return true if all the arguments arg2... match the given grep(1) regular
+# expression arg1.
+matches_rexp() {
+   _tfw_matches_rexp "$@"
+}
+
 # Executes its arguments as a command:
 #  - captures the standard output and error in temporary files for later
 #    examination
@@ -886,6 +892,7 @@ _tfw_assert() {
 declare -a _tfw_opt_dump_on_fail
 
 _tfw_dump_on_fail() {
+   local arg
    for arg; do
       local _found=false
       local _f
@@ -959,6 +966,7 @@ _tfw_getopts() {
 _tfw_matches_rexp() {
    local rexp="$1"
    shift
+   local arg
    for arg; do
       if ! echo "$arg" | $GREP -q -e "$rexp"; then
          return 1
diff --git a/tests/directory_service b/tests/directory_service
index 0170ce8a..bec77499 100755
--- a/tests/directory_service
+++ b/tests/directory_service
@@ -45,9 +45,9 @@ setup() {
 }
 
 teardown() {
-   stop_all_servald_servers
    kill_all_servald_processes
    assert_no_servald_processes
+   report_all_servald_servers
 }
 
 is_published() {
@@ -75,7 +75,7 @@ test_publish() {
    assertStdoutLineCount '==' 2
    assertStdoutGrep --matches=1 "^sid://$SIDC/local/$DIDC:$DIDC:$NAMEC\$"
    assertStdoutGrep --matches=1 "^sid://$SIDD/local/$DIDD:$DIDD:$NAMED\$"
-   return
+   assert_status_all_servald_servers running
 }
 
 start_routing_instance() {
@@ -130,5 +130,7 @@ test_routing() {
    assertStdoutGrep --matches=1 "^sid://$SIDB/local/$DIDB:$DIDB:$NAMEB\$"
    executeOk_servald mdp ping $SIDB 3
    tfw_cat --stdout --stderr
+   assert_status_all_servald_servers running
 }
+
 runTests "$@"
diff --git a/tests/dnahelper b/tests/dnahelper
index c4b6599c..5f98d9f8 100755
--- a/tests/dnahelper
+++ b/tests/dnahelper
@@ -29,10 +29,14 @@ setup() {
    assert_all_instance_peers_complete +A
 }
 
-teardown() {
+finally() {
    stop_all_servald_servers
+}
+
+teardown() {
    kill_all_servald_processes
    assert_no_servald_processes
+   report_all_servald_servers
 }
 
 # Called by start_servald_instances immediately before starting the server
diff --git a/tests/rhizomeops b/tests/rhizomeops
index 396e7fb9..5ed5af1e 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -141,7 +141,7 @@ test_AddEmpty() {
    assertGrep .manifest '^name=$'
    assertGrep .manifest '^filesize=0$'
    executeOk_servald rhizome list ''
-   assert_rhizome_list @$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 ''
 }
 
 doc_AddThenList="List contains one file after one add"
@@ -157,11 +157,11 @@ test_AddThenList() {
    # Add first file
    executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    # Add second file
    executeOk_servald rhizome add file $SIDB1 '' file2 file2.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 
 doc_ExtractManifestAfterAdd="Extract manifest after one add"
@@ -171,7 +171,7 @@ setup_ExtractManifestAfterAdd() {
    echo "A test file" >file1
    executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
@@ -198,7 +198,7 @@ setup_ExtractManifestAfterAddNoAuthor() {
    echo "A test file" >file1
    executeOk_servald rhizome add file '' '' file1 file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1!
+   assert_rhizome_list --fromhere=0 file1
    extract_manifest_id manifestid file1.manifest
    extract_manifest_version version file1.manifest
    extract_manifest_filehash filehash file1.manifest
@@ -254,7 +254,7 @@ setup_ExtractFileAfterAdd() {
    executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
    tfw_cat --stderr
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
    extract_manifest_filehash filehash file1.manifest
 }
 test_ExtractFileAfterAdd() {
@@ -312,7 +312,7 @@ setup_AddDuplicate() {
    extract_stdout_secret file2_secret
    # Make sure they are both in the list.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 test_AddDuplicate() {
    # Add first file again - nothing should change in its manifests, and it
@@ -322,7 +322,7 @@ test_AddDuplicate() {
    assert [ -s file1.manifestA ]
    assert_stdout_add_file file1
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
    strip_signatures file1.manifest file1.manifestA
    assert diff file1.manifest file1.manifestA
    # Repeat for second file.
@@ -330,7 +330,7 @@ test_AddDuplicate() {
    assert [ -s file2.manifestA ]
    assert_stdout_add_file file2
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
    strip_signatures file2.manifest file2.manifestA
    assert diff file2.manifest file2.manifestA
 }
@@ -348,7 +348,7 @@ test_AddMismatched() {
    assert cmp file1.manifest file1_2.manifest
    # And rhizome store should be unchanged.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 
 doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails"
@@ -371,7 +371,7 @@ test_AddUpdateSameVersion() {
    assert cmp file1_2.manifest file1_2.manifest.orig
    # And rhizome store should be unchanged.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 
 doc_AddUpdateNewVersion="Add new payload to existing manifest with new version"
@@ -390,7 +390,7 @@ test_AddUpdateNewVersion() {
    assert_manifest_newer file1.manifest file1_2.manifest
    # Rhizome store contents reflect new payload.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1_2@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
 }
 
 doc_AddUpdateDiscoverAuthor="Add new payload to manifest with author discovery"
@@ -403,7 +403,7 @@ test_AddUpdateDiscoverAuthor() {
    tfw_cat --stderr
    # Rhizome store contents have new payload.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1_2@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
 }
 
 doc_AddUpdateNoAuthor="Cannot add new payload to authorless manifest"
@@ -418,7 +418,7 @@ test_AddUpdateNoAuthor() {
    assertExitStatus '!=' 0
    # Rhizome store contents have old payload.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 
 doc_AddUpdateNoAuthorWithSecret="Add new payload to authorless manifest with bundle secret"
@@ -431,7 +431,7 @@ test_AddUpdateNoAuthorWithSecret() {
    tfw_cat --stderr
    # Rhizome store contents have new payload.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1_2@$SIDB1 file2@$SIDB1
+   assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
 }
 
 doc_AddUpdateAutoVersion="Add new payload to existing manifest with automatic version"
@@ -447,7 +447,7 @@ test_AddUpdateAutoVersion() {
    assert_manifest_newer file1.manifest file1_2.manifest
    # Rhizome store contents reflect new payload.
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1_2 file2
+   assert_rhizome_list --fromhere=1 file1_2 file2
 }
 
 doc_AddUnsupportedService="Add with unsupported service fails"
@@ -474,7 +474,7 @@ test_MeshMSAddCreate() {
    assert_stdout_add_file file1
    assert_manifest_complete file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
    extract_manifest_filehash filehash file1.manifest
    executeOk_servald rhizome extract file $filehash file1x
    assert diff file1 file1x
@@ -492,7 +492,7 @@ test_MeshMSAddGrow() {
    assert_stdout_add_file file1
    assert_manifest_complete file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
    extract_manifest_id id file1.manifest
    extract_manifest_filehash filehash file1.manifest
    extract_manifest_BK bk file1.manifest
@@ -503,7 +503,7 @@ test_MeshMSAddGrow() {
       echo "Message$m" >>file1
       executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
       executeOk_servald rhizome list ''
-      assert_rhizome_list file1
+      assert_rhizome_list --fromhere=1 file1
       extract_manifest_id idx file1.manifest
       extract_manifest_filehash filehashx file1.manifest
       extract_manifest_BK bkx file1.manifest
@@ -557,7 +557,7 @@ test_MeshMSAddMissingAuthor() {
    assert_stdout_add_file file1
    assert_manifest_complete file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
 }
 
 doc_MeshMSListFilter="List MeshMS manifests by filter"
@@ -585,17 +585,17 @@ setup_MeshMSListFilter() {
    assert_stdout_add_file file4
    assert_manifest_complete file4.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1 file2 file3 file4
+   assert_rhizome_list --fromhere=1 file1 file2 file3 file4
 }
 test_MeshMSListFilter() {
    executeOk_servald rhizome list '' file
    assert_rhizome_list
    executeOk_servald rhizome list '' MeshMS1
-   assert_rhizome_list file1 file2 file3 file4
+   assert_rhizome_list --fromhere=1 file1 file2 file3 file4
    executeOk_servald rhizome list '' '' $SIDB1
-   assert_rhizome_list file1 file2 file3
+   assert_rhizome_list --fromhere=1 file1 file2 file3
    executeOk_servald rhizome list '' '' $SIDB2
-   assert_rhizome_list file4
+   assert_rhizome_list --fromhere=1 file4
    executeOk_servald rhizome list '' '' $SIDB3
    assert_rhizome_list
    executeOk_servald rhizome list '' '' $SIDB4
@@ -603,19 +603,19 @@ test_MeshMSListFilter() {
    executeOk_servald rhizome list '' '' '' $SIDB1
    assert_rhizome_list
    executeOk_servald rhizome list '' '' '' $SIDB2
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
    executeOk_servald rhizome list '' '' '' $SIDB3
-   assert_rhizome_list file2 file4
+   assert_rhizome_list --fromhere=1 file2 file4
    executeOk_servald rhizome list '' file '' $SIDB3
    assert_rhizome_list
    executeOk_servald rhizome list '' '' '' $SIDB4
-   assert_rhizome_list file3
+   assert_rhizome_list --fromhere=1 file3
    executeOk_servald rhizome list '' '' $SIDB1 $SIDB4
-   assert_rhizome_list file3
+   assert_rhizome_list --fromhere=1 file3
    executeOk_servald rhizome list '' '' $SIDB2 $SIDB4
    assert_rhizome_list
    executeOk_servald rhizome list '' '' $SIDB2 $SIDB3
-   assert_rhizome_list file4
+   assert_rhizome_list --fromhere=1 file4
 }
 
 doc_ImportForeignBundle="Can import a bundle created by another instance"
@@ -632,7 +632,7 @@ test_ImportForeignBundle() {
    executeOk_servald rhizome import bundle fileA fileA.manifest
    assert_stdout_import_bundle fileA
    executeOk_servald rhizome list ''
-   assert_rhizome_list fileA!
+   assert_rhizome_list --fromhere=0 fileA
 }
 
 doc_ImportOwnBundle="Can import a bundle created by same instance"
@@ -654,7 +654,7 @@ test_ImportOwnBundle() {
    assert_stdout_import_bundle fileB
    # Bundle author and sender are unknown, so appears not to be from here
    executeOk_servald rhizome list ''
-   assert_rhizome_list fileB!
+   assert_rhizome_list --fromhere=0 fileB
    # Extracting the manifest discovers that it is ours.
    executeOk_servald rhizome extract manifest $manifestid fileBx.manifest
    tfw_cat --stderr
@@ -671,7 +671,7 @@ test_ImportOwnBundle() {
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
    # Now bundle author is known, so appears to be from here
    executeOk_servald rhizome list ''
-   assert_rhizome_list fileB@$SIDB2
+   assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
 }
 
 runTests "$@"
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 54c5afb0..d0947e6d 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -25,9 +25,9 @@ source "${0%/*}/../testdefs_rhizome.sh"
 shopt -s extglob
 
 teardown() {
-   stop_all_servald_servers
    kill_all_servald_processes
    assert_no_servald_processes
+   report_all_servald_processes
 }
 
 setup_rhizome() {
@@ -49,27 +49,45 @@ configure_servald_server() {
 }
 
 # Predicate function:
-#  - return true if the file bundle identified by arg1=BID and arg2=VERSION has been
-#    received by all the given instances
+#  - return true if the file bundles identified by args BID[:VERSION] has been
+#    received by all the given instances args +I
 #  - does this by examining the server log files of the respective instances
 #    for tell-tale INFO messages
 bundle_received_by() {
-   local BID="$1"
-   local VERSION="$2"
-   shift 2
-   local rexp="RHIZOME ADD MANIFEST service=file bid=$BID version=$VERSION"
-   local I
-   for I; do
-      case "$I" in
-      +*)
-         local logvar="LOG${I#+}"
-         grep "$rexp" "${!logvar}" || return 1
+   local -a rexps
+   local restart=true
+   local arg bid version rexp
+   for arg; do
+      case "$arg" in
+      +([0-9A-F])?(:+([0-9])))
+         $restart && rexps=()
+         restart=false
+         bid="${arg%%:*}"
+         matches_rexp "$rexp_manifestid" "$bid" || error "invalid bundle ID: $bid"
+         if [ "$bid" = "$arg" ]; then
+            rexps+=("RHIZOME ADD MANIFEST service=file bid=$bid")
+         else
+            version="${arg#*:}"
+            rexps+=("RHIZOME ADD MANIFEST service=file bid=$bid version=$version")
+         fi
+         ;;
+      +[A-Z])
+         local logvar="LOG${arg#+}"
+         for rexp in "${rexps[@]}"; do
+            echo grep "$rexp" "${!logvar}"
+            grep "$rexp" "${!logvar}" || return 1
+         done
+         restart=true
          ;;
       --stderr)
-         replayStderr | grep "$rexp" || return 1
+         for rexp in "${rexps[@]}"; do
+            echo replayStderr \| grep "$rexp"
+            replayStderr | grep "$rexp" || return 1
+         done
+         restart=true
          ;;
       *)
-         error "invalid instance argument: $I"
+         error "invalid argument: $arg"
          return 1
          ;;
       esac
@@ -112,7 +130,7 @@ add_file() {
    local sidvar="SID$instance_name"
    executeOk_servald rhizome add file "${!sidvar}" '' "$name" "$name.manifest"
    executeOk_servald rhizome list ''
-   assert_rhizome_list "$name"
+   assert_rhizome_list --fromhere=1 --author="${!sidvar}" "$name" --and-others
    extract_manifest_vars "$name.manifest"
 }
 
@@ -125,18 +143,21 @@ update_file() {
    $SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d;/^version=/d;/^name=/d' "$new_name.manifest"
    executeOk_servald rhizome add file "${!sidvar}" '' "$new_name" "$new_name.manifest"
    executeOk_servald rhizome list ''
-   assert_rhizome_list "$new_name"
+   assert_rhizome_list --fromhere=1 "$new_name"
    extract_manifest_vars "$new_name.manifest"
 }
 
 assert_received() {
-   local name="${1?}"
+   [ $# -ne 0 ] || error "missing arguments"
+   local name
    local _hash
-   if [ -s "$name" ]; then
-      extract_manifest_filehash _hash "$name.manifest"
-      executeOk_servald rhizome extract file "$_hash" extracted
-      assert cmp "$name" extracted
-   fi
+   for name; do
+      if [ -s "$name" ]; then
+         extract_manifest_filehash _hash "$name.manifest"
+         executeOk_servald rhizome extract file "$_hash" extracted
+         assert cmp "$name" extracted
+      fi
+   done
 }
 
 doc_FileTransfer="New bundle and update transfer to one node"
@@ -149,17 +170,17 @@ setup_FileTransfer() {
    foreach_instance +B assert_peers_are_instances +A
 }
 test_FileTransfer() {
-   wait_until bundle_received_by $BID $VERSION +B
+   wait_until bundle_received_by $BID:$VERSION +B
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1!
+   assert_rhizome_list --fromhere=0 file1
    assert_received file1
    set_instance +A
    update_file file1 file2
    set_instance +B
-   wait_until bundle_received_by $BID $VERSION +B
+   wait_until bundle_received_by $BID:$VERSION +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file2!
+   assert_rhizome_list --fromhere=0 file2
    assert_received file2
 }
 
@@ -176,10 +197,10 @@ setup_FileTransferBig() {
    foreach_instance +B assert_peers_are_instances +A
 }
 test_FileTransferBig() {
-   wait_until bundle_received_by $BID $VERSION +B
+   wait_until bundle_received_by $BID:$VERSION +B
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1!
+   assert_rhizome_list --fromhere=0 file1
    assert_received file1
 }
 
@@ -195,12 +216,11 @@ setup_FileTransferMulti() {
    foreach_instance +D assert_peers_are_instances +A +B +C +E
 }
 test_FileTransferMulti() {
-   wait_until bundle_received_by $BID $VERSION +B +C +D +E
-   local I
-   for I in +B +C +D +E; do
-      set_instance $I
+   wait_until bundle_received_by $BID:$VERSION +B +C +D +E
+   for i in B C D E; do
+      set_instance +$i
       executeOk_servald rhizome list ''
-      assert_rhizome_list file1!
+      assert_rhizome_list --fromhere=0 file1
       assert_received file1
    done
 }
@@ -213,16 +233,16 @@ setup_FileTransferDelete() {
    start_servald_instances +A +B
    foreach_instance +A assert_peers_are_instances +B
    foreach_instance +B assert_peers_are_instances +A
-   wait_until bundle_received_by $BID $VERSION +B
+   wait_until bundle_received_by $BID:$VERSION +B
    set_instance +A
    >file1_2
    update_file file1 file1_2
 }
 test_FileTransferDelete() {
-   wait_until bundle_received_by $BID $VERSION +B
+   wait_until bundle_received_by $BID:$VERSION +B
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1_2!
+   assert_rhizome_list --fromhere=0 file1_2
    assert_received file1_2
 }
 
@@ -272,7 +292,7 @@ test_HttpImport() {
          "$addr_localhost:$PORTA/rhizome/import"
    tfw_cat http.headers http.output
    executeOk_servald rhizome list ''
-   assert_rhizome_list README.WHYNOTSIPS!
+   assert_rhizome_list --fromhere=0 README.WHYNOTSIPS
    assert_received README.WHYNOTSIPS
 }
 
@@ -292,7 +312,7 @@ test_HttpAddLocal() {
    executeOk curl --silent --form 'data=@file1' "http://$addr_localhost:$PORTA/rhizome/secretaddfile" --output file1.manifest
    assert_manifest_complete file1.manifest
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
    extract_manifest_name name file1.manifest
    assert [ "$name" = file1 ]
    assert_received file1
@@ -327,14 +347,14 @@ test_DirectPush() {
    set_instance +B
    executeOk_servald rhizome direct push
    tfw_cat --stdout --stderr
-   assert bundle_received_by $BID2 $VERSION2 +A
+   assert bundle_received_by $BID2:$VERSION2 +A
    set_instance +A
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1 file2!
+   assert_rhizome_list --fromhere=1 file1 --fromhere=0 file2
    assert_received file2
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file2
+   assert_rhizome_list --fromhere=1 file2
 }
 
 doc_DirectPull="One way pull bundle from unconnected node"
@@ -346,13 +366,13 @@ test_DirectPull() {
    set_instance +B
    executeOk_servald rhizome direct pull
    tfw_cat --stdout --stderr
-   assert bundle_received_by $BID1 $VERSION1 --stderr
+   assert bundle_received_by $BID1:$VERSION1 --stderr
    set_instance +A
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1
+   assert_rhizome_list --fromhere=1 file1
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1! file2
+   assert_rhizome_list --fromhere=0 file1 --fromhere=1 file2
    assert_received file1
 }
 
@@ -365,16 +385,53 @@ test_DirectSync() {
    set_instance +B
    executeOk_servald rhizome direct sync
    tfw_cat --stdout --stderr
-   assert bundle_received_by $BID1 $VERSION1 --stderr
-   assert bundle_received_by $BID2 $VERSION2 +A
+   assert bundle_received_by $BID1:$VERSION1 --stderr $BID2:$VERSION2 +A
    set_instance +A
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1 file2!
+   assert_rhizome_list --fromhere=1 file1 --fromhere=0 file2
    assert_received file2
    set_instance +B
    executeOk_servald rhizome list ''
-   assert_rhizome_list file1! file2
+   assert_rhizome_list --fromhere=0 file1 --fromhere=1 file2
    assert_received file1
 }
 
+doc_FileTransferStress="Many bundle transfers between five nodes"
+setup_FileTransferStress() {
+   setup_servald
+   assert_no_servald_processes
+   foreach_instance +A +B +C +D +E create_single_identity
+   for i in A B C D E
+   do
+      eval "bundles$i=()"
+      set_instance +$i
+      for n in 1 2 3 4 5 6 7 8 9 a b c d e f g
+      do
+         add_file file-$i-$n
+         eval "bundles$i+=(\$BID:\$VERSION)"
+      done
+   done
+   start_servald_instances +A +B +C +D +E
+   foreach_instance +A assert_peers_are_instances +B +C +D +E
+   foreach_instance +B assert_peers_are_instances +A +C +D +E
+   foreach_instance +C assert_peers_are_instances +A +B +D +E
+   foreach_instance +D assert_peers_are_instances +A +B +C +E
+}
+test_FileTransferStress() {
+   wait_until bundle_received_by \
+      $bundlesA +B +C +D +E \
+      $bundlesB +A +C +D +E \
+      $bundlesC +A +B +D +E \
+      $bundlesD +A +B +C +E \
+      $bundlesE +A +B +C +D
+   local i
+   for i in A B C D E; do
+      set_instance +$i
+      executeOk_servald rhizome list ''
+      assert_rhizome_list --fromhere=1 file-$i-* --fromhere=0 file-!($i)-*
+      assert_received file-!($i)-*
+   done
+   assert_status_all_servald_servers running
+}
+
 runTests "$@"