Improve test framework

- new execute() option: --exit-status=N
   equivalent to assertExitStatus --stderr '==' N
 - new executeOk() function, shortcut for execute --exit-status=0

testframework.sh

@@ -150,35 +150,22 @@ realpath() {
 }
 
 execute() {
-   _tfw_last_argv0="$1"
    echo "# execute $*"
-   /usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr
+   _tfw_getopts execute "$@"
-   _tfw_exitStatus=$?
+   shift $_tfw_getopts_shift
-   echo "# Exit status = $_tfw_exitStatus"
+   _tfw_execute "$@"
-   if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then
+}
-      echo "# Times file contains spurious data:"
+
-      tfw_cat --header=times $_tfw_tmp/times
+executeOk() {
-      if [ $_tfw_exitStatus -eq 0 ]; then
+   echo "# executeOk $*"
-         _tfw_exitStatus=255
+   _tfw_getopts executeok "$@"
-         echo "# Deeming exit status of command to be $_tfw_exitStatus"
+   _tfw_opt_exit_status=0
-      fi
+   shift $_tfw_getopts_shift
-      realtime=0
+   _tfw_execute "$@"
-      usertime=0
-      systime=0
-      realtime_ms=0
-      #usertime_ms=0
-      #systime_ms=0
-   else
-      source $_tfw_tmp/times
-      realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }")
-      #usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }")
-      #systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }")
-   fi
-   return 0
 }
 
 assert() {
-   _tfw_getopts_assert assert "$@"
+   _tfw_getopts assert "$@"
    shift $_tfw_getopts_shift
    _tfw_assert "$@" || _tfw_failexit
    echo "# assert $*"
@@ -186,7 +173,7 @@ assert() {
 }
 
 assertExpr() {
-   _tfw_getopts_assert assertexpr "$@"
+   _tfw_getopts assertexpr "$@"
    shift $_tfw_getopts_shift
    local awkexpr=$(_tfw_expr_to_awkexpr "$@")
    _tfw_message="${_tfw_message+$_tfw_message }($awkexpr)"
@@ -196,7 +183,7 @@ assertExpr() {
 }
 
 fail() {
-   _tfw_getopts_assert fail "$@"
+   _tfw_getopts fail "$@"
    shift $_tfw_getopts_shift
    [ $# -ne 0 ] && _tfw_failmsg "$1"
    _tfw_backtrace >&2
@@ -204,7 +191,7 @@ fail() {
 }
 
 error() {
-   _tfw_getopts_assert error "$@"
+   _tfw_getopts error "$@"
    shift $_tfw_getopts_shift
    [ $# -ne 0 ] && _tfw_errormsg "$1"
    _tfw_backtrace >&2
@@ -224,14 +211,14 @@ tfw_cat() {
    for file; do
       case $file in
       --stdout) 
-         echo "#--- ${header:-stdout of ${_tfw_last_argv0##*/}} ---"
+         echo "#--- ${header:-stdout of ${_tfw_execute_argv0##*/}} ---"
          /bin/cat $show_nonprinting $_tfw_tmp/stdout
          echo "#---"
          header=
          show_nonprinting=
          ;;
       --stderr) 
-         echo "#--- ${header:-stderr of ${_tfw_last_argv0##*/}} ---"
+         echo "#--- ${header:-stderr of ${_tfw_execute_argv0##*/}} ---"
          /bin/cat $show_nonprinting $_tfw_tmp/stderr
          echo "#---"
          header=
@@ -251,18 +238,18 @@ tfw_cat() {
 }
 
 assertExitStatus() {
-   _tfw_getopts_assert exitstatus "$@"
+   _tfw_getopts assertexitstatus "$@"
    shift $_tfw_getopts_shift
-   [ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_last_argv0##*/} ($_tfw_exitStatus) $*"
+   [ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) $*"
    _tfw_assertExpr "$_tfw_exitStatus" "$@" || _tfw_failexit
    echo "# assert $_tfw_message"
    return 0
 }
 
 assertRealTime() {
-   _tfw_getopts_assert realtime "$@"
+   _tfw_getopts assertrealtime "$@"
    shift $_tfw_getopts_shift
-   [ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_last_argv0##*/} ($realtime) $*"
+   [ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_execute_argv0##*/} ($realtime) $*"
    _tfw_assertExpr "$realtime" "$@" || _tfw_failexit
    echo "# assert $_tfw_message"
    return 0
@@ -301,7 +288,7 @@ assertStderrGrep() {
 }
 
 assertGrep() {
-   _tfw_getopts_assert filegrep "$@"
+   _tfw_getopts assertgrep "$@"
    shift $_tfw_getopts_shift
    if [ $# -ne 2 ]; then
       _tfw_error "incorrect arguments"
@@ -456,6 +443,40 @@ _tfw_teardown() {
    /bin/rm -rf $_tfw_tmp
 }
 
+_tfw_execute() {
+   _tfw_execute_argv0="$1"
+   /usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr
+   _tfw_exitStatus=$?
+   if [ -n "$_tfw_opt_exit_status" ]; then
+      _tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) is $_tfw_opt_exit_status"
+      _tfw_dump_stderr_on_fail=true
+      _tfw_assert [ "$_tfw_exitStatus" -eq "$_tfw_opt_exit_status" ] || _tfw_failexit
+      echo "# assert $_tfw_message"
+   else
+      echo "# exit status of ${_tfw_execute_argv0##*/} = $_tfw_exitStatus"
+   fi
+   if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then
+      echo "# times file contains spurious data:"
+      tfw_cat --header=times $_tfw_tmp/times
+      if [ $_tfw_exitStatus -eq 0 ]; then
+         _tfw_exitStatus=255
+         echo "# deeming exit status of command to be $_tfw_exitStatus"
+      fi
+      realtime=0
+      usertime=0
+      systime=0
+      realtime_ms=0
+      #usertime_ms=0
+      #systime_ms=0
+   else
+      source $_tfw_tmp/times
+      realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }")
+      #usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }")
+      #systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }")
+   fi
+   return 0
+}
+
 _tfw_assert() {
    if ! "$@"; then
       _tfw_failmsg "assertion failed: ${_tfw_message:-$*}"
@@ -465,20 +486,22 @@ _tfw_assert() {
    return 0
 }
 
-_tfw_getopts_assert() {
+_tfw_getopts() {
    local context="$1"
    shift
    _tfw_message=
    _tfw_dump_stdout_on_fail=false
    _tfw_dump_stderr_on_fail=false
+   _tfw_opt_exit_status=
    _tfw_opt_matches=
    _tfw_getopts_shift=0
    while [ $# -ne 0 ]; do
       case "$context:$1" in
-      *:--message=*) _tfw_message="${1#*=}";;
       *:--stdout) _tfw_dump_stdout_on_fail=true;;
       *:--stderr) _tfw_dump_stderr_on_fail=true;;
-      filegrep:--matches=*) _tfw_opt_matches="${1#*=}";;
+      execute:--exit-status=*) _tfw_opt_exit_status="${1#*=}";;
+      assert*:--message=*) _tfw_message="${1#*=}";;
+      assertgrep:--matches=*) _tfw_opt_matches="${1#*=}";;
       *:--) let _tfw_getopts_shift=_tfw_getopts_shift+1; shift; break;;
       *:--*) _tfw_error "unsupported option: $1";;
       *) break;;
@@ -529,13 +552,13 @@ _tfw_assertExpr() {
 _tfw_assert_stdxxx_is() {
    local qual="$1"
    shift
-   _tfw_getopts_assert filecontent --$qual "$@"
+   _tfw_getopts assertfilecontent --$qual "$@"
    shift $((_tfw_getopts_shift - 1))
    if [ $# -lt 1 ]; then
       _tfw_error "incorrect arguments"
       return 254
    fi
-   local message="${_tfw_message:-$qual of ${_tfw_last_argv0##*/} is $*}"
+   local message="${_tfw_message:-$qual of ${_tfw_execute_argv0##*/} is $*}"
    echo -n "$@" >$_tfw_tmp/stdxxx_is.tmp
    if ! /usr/bin/cmp --quiet $_tfw_tmp/stdxxx_is.tmp "$_tfw_tmp/$qual"; then
       _tfw_failmsg "assertion failed: $message"
@@ -549,7 +572,7 @@ _tfw_assert_stdxxx_is() {
 _tfw_assert_stdxxx_linecount() {
    local qual="$1"
    shift
-   _tfw_getopts_assert filecontent --$qual "$@"
+   _tfw_getopts assertfilecontent --$qual "$@"
    shift $((_tfw_getopts_shift - 1))
    if [ $# -lt 1 ]; then
       _tfw_error "incorrect arguments"
@@ -565,13 +588,13 @@ _tfw_assert_stdxxx_linecount() {
 _tfw_assert_stdxxx_grep() {
    local qual="$1"
    shift
-   _tfw_getopts_assert filegrep --$qual "$@"
+   _tfw_getopts assertgrep --$qual "$@"
    shift $((_tfw_getopts_shift - 1))
    if [ $# -ne 1 ]; then
       _tfw_error "incorrect arguments"
       return 254
    fi
-   _tfw_assert_grep "$qual of ${_tfw_last_argv0##*/}" $_tfw_tmp/$qual "$@"
+   _tfw_assert_grep "$qual of ${_tfw_execute_argv0##*/}" $_tfw_tmp/$qual "$@"
 }
 
 _tfw_assert_grep() {

tests/dna_biterrors

@@ -40,8 +40,7 @@ test_BitErrorCreateSID() {
    local totaltime_ms=0
    local maxtime_ms=0
    while [ $i -lt $iterations ]; do
-      execute $dna -B $ber -d 0427679796 -C
+      executeOk $dna -B $ber -d 0427679796 -C
-      assertExitStatus '==' 0
       sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
       let totaltime_ms=totaltime_ms+realtime_ms
       [ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms
@@ -70,11 +69,9 @@ setup_SetVarBigValueBitErrors() {
    echo_long_message >$DNATMP/dnatest.in
 }
 test_SetVarBigValueBitErrors() {
-   execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
+   executeOk $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
-   assertExitStatus '==' 0
    assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
-   execute $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note
+   executeOk $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note
-   assertExitStatus '==' 0
    assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
    assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out
 }

tests/dna_hlr

@@ -28,16 +28,14 @@ teardown() {
 
 # Utility function
 create_sid() {
-   execute $dna "$@" -d 0427679796 -C
+   executeOk $dna "$@" -d 0427679796 -C
-   assertExitStatus '==' 0
    assertStdoutGrep --matches=1 '^OK:'
    sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
 }
 
 # Utility function
 set_short_var() {
-   execute $dna -s $sid -i 0 -W note="a short literal value"
+   executeOk $dna -s $sid -i 0 -W note="a short literal value"
-   assertExitStatus '==' 0
 }
 
 # Utility function
@@ -69,8 +67,7 @@ setup_GetShortVar() {
    create_sid
 }
 test_GetShortVar() {
-   execute $dna -d 0427679796 -R dids
+   executeOk $dna -d 0427679796 -R dids
-   assertExitStatus '==' 0
    assertRealTime '>' 2.9 # Waited for all replies
    assertStdoutGrep --message='received an existing DID' "^DIDS:${sid}:[0-9]\+:0:0427679796"
    assertStdoutGrep --matches=1 --message='filtered out duplicate responses' "^DIDS:${sid}:[0-9]\+:0:0427679796"
@@ -98,8 +95,7 @@ setup_SetGetShortVar() {
    set_short_var
 }
 test_SetGetShortVar() {
-   execute $dna -s $sid -i 0 -R note
+   executeOk $dna -s $sid -i 0 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='read variable correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value"
 }
@@ -115,8 +111,7 @@ setup_GetShortVarToFile() {
 test_GetShortVarToFile() {
    echo "WARNING: Known issue: output to file does not work with DID lists"
    dnatest_dat=$DNATMP/dnatest.dat
-   execute $dna -s $sid -O $dnatest_dat -i 0 -R note
+   executeOk $dna -s $sid -O $dnatest_dat -i 0 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='read variable into file' "^NOTE:$sid:[0-9]\+:0"
    assertFileContents $dnatest_dat 'a short literal value'
@@ -131,12 +126,10 @@ setup_SetMultiShortVar() {
    set_short_var
 }
 test_SetMultiShortVar() {
-   execute $dna -s $sid -i 1 -W note='$414243'
+   executeOk $dna -s $sid -i 1 -W note='$414243'
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
-   execute $dna -s $sid -i 1 -R note
+   executeOk $dna -s $sid -i 1 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='new variable correctly set in hex' "^NOTE:$sid:[0-9]\+:1:ABC$"
 }
@@ -148,14 +141,11 @@ setup_GetMultiShortVar() {
    start_dna_server
    create_sid
    set_short_var
-   execute $dna -s $sid -i 1 -W note='$414243'
+   executeOk $dna -s $sid -i 1 -W note='$414243'
-   assertExitStatus '==' 0
+   executeOk $dna -s $sid -i 2 -W note
-   execute $dna -s $sid -i 2 -W note
-   assertExitStatus '==' 0
 }
 test_GetMultiShortVar() {
-   execute $dna -s $sid -m -R note
+   executeOk $dna -s $sid -m -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='first variable read correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value$"
    assertStdoutGrep --matches=1 --message='second variable read correctly' "^NOTE:$sid:[0-9]\+:1:ABC$"
@@ -171,12 +161,10 @@ setup_OverwriteVarNoUpdate() {
    set_short_var
 }
 test_OverwriteVarNoUpdate() {
-   execute $dna -s $sid -i 0 -W note="replacement short literal value"
+   executeOk $dna -s $sid -i 0 -W note="replacement short literal value"
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=0 --message='variable write not confirmed' "^WROTE:$sid$"
-   execute $dna -s $sid -i 0 -R note
+   executeOk $dna -s $sid -i 0 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable retains original value' "^NOTE:$sid:[0-9]\+:0:a short literal value"
 }
@@ -190,12 +178,10 @@ setup_UpdateVar() {
    set_short_var
 }
 test_UpdateVar() {
-   execute $dna -s $sid -i 0 -U note="replacement short literal value"
+   executeOk $dna -s $sid -i 0 -U note="replacement short literal value"
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
-   execute $dna -s $sid -i 0 -R note
+   executeOk $dna -s $sid -i 0 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable has new value' "^NOTE:$sid:[0-9]\+:0:replacement short literal value"
 }
@@ -210,12 +196,10 @@ setup_UpdateVarBigValue() {
    echo_long_message >$DNATMP/dnatest.in
 }
 test_UpdateVarBigValue() {
-   execute $dna -s $sid -i 0 -U note="@dnatest.in"
+   executeOk $dna -s $sid -i 0 -U note="@dnatest.in"
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
-   execute $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note
+   executeOk $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note
-   assertExitStatus '==' 0
    assertRealTime '<' 0.5
    assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
    assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out

tests/dna_rhizome

@@ -22,8 +22,7 @@ source "${0%/*}/../testframework.sh"
 source "${0%/*}/../testdefs.sh"
 
 assert_rhizome_list_empty() {
-   execute $dna rhizome list
+   executeOk $dna rhizome list
-   assertExitStatus --stdout --stderr '==' 0
    assertStdoutIs -e "Found 0 rows\n"
 }
 
@@ -44,8 +43,7 @@ setup_AddManifest() {
    echo "Another test file" >file2
 }
 test_AddManifest() {
-   execute $dna rhizome add file file1 file1.manifest
+   executeOk $dna rhizome add file file1 file1.manifest
-   assertExitStatus --stderr '==' 0
    tfw_cat --stdout
    tfw_cat --stderr
    tfw_cat -v file1.manifest
@@ -67,20 +65,16 @@ setup_AddThenList() {
 }
 test_AddThenList() {
    # Add first file
-   execute $dna rhizome add file file1
+   executeOk $dna rhizome add file file1
-   assertExitStatus --stderr '==' 0
+   executeOk $dna rhizome list
-   execute $dna rhizome list
-   assertExitStatus --stdout --stderr '==' 0
    assertStdoutGrep "^Found 1 rows$"
    assertStdoutLineCount '==' 6
    assertStdoutGrep --matches=1 '^file name = "file1"$'
    assertStdoutGrep --matches=0 '^file name = "file2"$'
    replayStdout >add1.stdout
    # Add second file
-   execute $dna rhizome add file file2
+   executeOk $dna rhizome add file file2
-   assertExitStatus --stderr '==' 0
+   executeOk $dna rhizome list
-   execute $dna rhizome list
-   assertExitStatus --stdout --stderr '==' 0
    assertStdoutGrep "^Found 2 rows$"
    assertStdoutLineCount '==' 11
    assertStdoutGrep --matches=1 '^file name = "file1"$'
@@ -94,24 +88,19 @@ setup_AddDuplicate() {
    echo "A test file" >file1
    echo "Another test file" >file2
    # Add first file
-   execute $dna rhizome add file file1
+   executeOk $dna rhizome add file file1
-   assertExitStatus --stderr '==' 0
    # Add second file
-   execute $dna rhizome add file file2
+   executeOk $dna rhizome add file file2
-   assertExitStatus --stderr '==' 0
    # Make sure they are both in the list.
-   execute $dna rhizome list
+   executeOk $dna rhizome list
-   assertExitStatus --stdout --stderr '==' 0
    assertStdoutGrep "^Found 2 rows$"
    assertStdoutGrep --matches=1 '^file name = "file1"$'
    assertStdoutGrep --matches=1 '^file name = "file2"$'
 }
 test_AddDuplicate() {
    # Add first file again (nothing should change except its date).
-   execute $dna rhizome add file file1
+   executeOk $dna rhizome add file file1
-   assertExitStatus --stderr '==' 0
+   executeOk $dna rhizome list
-   execute $dna rhizome list
-   assertExitStatus --stdout --stderr '==' 0
    assertStdoutGrep "^Found 2 rows$"
    assertStdoutLineCount '==' 11
    assertStdoutGrep --matches=1 '^file name = "file1"$'