From c4e32498390b29baf6c3a6f78f356cf0b460cd8f Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Thu, 12 Apr 2012 17:04:37 +0930 Subject: [PATCH] Improve test framework - new execute() option: --exit-status=N equivalent to assertExitStatus --stderr '==' N - new executeOk() function, shortcut for execute --exit-status=0 --- testframework.sh | 109 +++++++++++++++++++++++++++----------------- tests/dna_biterrors | 9 ++-- tests/dna_hlr | 48 +++++++------------ tests/dna_rhizome | 33 +++++--------- 4 files changed, 96 insertions(+), 103 deletions(-) diff --git a/testframework.sh b/testframework.sh index 22369240..4abd776c 100644 --- a/testframework.sh +++ b/testframework.sh @@ -150,35 +150,22 @@ realpath() { } execute() { - _tfw_last_argv0="$1" echo "# execute $*" - /usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr - _tfw_exitStatus=$? - echo "# Exit status = $_tfw_exitStatus" - if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then - echo "# Times file contains spurious data:" - tfw_cat --header=times $_tfw_tmp/times - if [ $_tfw_exitStatus -eq 0 ]; then - _tfw_exitStatus=255 - echo "# Deeming exit status of command to be $_tfw_exitStatus" - fi - realtime=0 - usertime=0 - systime=0 - realtime_ms=0 - #usertime_ms=0 - #systime_ms=0 - else - source $_tfw_tmp/times - realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }") - #usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }") - #systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }") - fi - return 0 + _tfw_getopts execute "$@" + shift $_tfw_getopts_shift + _tfw_execute "$@" +} + +executeOk() { + echo "# executeOk $*" + _tfw_getopts executeok "$@" + _tfw_opt_exit_status=0 + shift $_tfw_getopts_shift + _tfw_execute "$@" } assert() { - _tfw_getopts_assert assert "$@" + _tfw_getopts assert "$@" shift $_tfw_getopts_shift _tfw_assert "$@" || _tfw_failexit echo "# assert $*" @@ -186,7 +173,7 @@ assert() { } assertExpr() { - _tfw_getopts_assert assertexpr "$@" + _tfw_getopts assertexpr "$@" shift $_tfw_getopts_shift local awkexpr=$(_tfw_expr_to_awkexpr "$@") _tfw_message="${_tfw_message+$_tfw_message }($awkexpr)" @@ -196,7 +183,7 @@ assertExpr() { } fail() { - _tfw_getopts_assert fail "$@" + _tfw_getopts fail "$@" shift $_tfw_getopts_shift [ $# -ne 0 ] && _tfw_failmsg "$1" _tfw_backtrace >&2 @@ -204,7 +191,7 @@ fail() { } error() { - _tfw_getopts_assert error "$@" + _tfw_getopts error "$@" shift $_tfw_getopts_shift [ $# -ne 0 ] && _tfw_errormsg "$1" _tfw_backtrace >&2 @@ -224,14 +211,14 @@ tfw_cat() { for file; do case $file in --stdout) - echo "#--- ${header:-stdout of ${_tfw_last_argv0##*/}} ---" + echo "#--- ${header:-stdout of ${_tfw_execute_argv0##*/}} ---" /bin/cat $show_nonprinting $_tfw_tmp/stdout echo "#---" header= show_nonprinting= ;; --stderr) - echo "#--- ${header:-stderr of ${_tfw_last_argv0##*/}} ---" + echo "#--- ${header:-stderr of ${_tfw_execute_argv0##*/}} ---" /bin/cat $show_nonprinting $_tfw_tmp/stderr echo "#---" header= @@ -251,18 +238,18 @@ tfw_cat() { } assertExitStatus() { - _tfw_getopts_assert exitstatus "$@" + _tfw_getopts assertexitstatus "$@" shift $_tfw_getopts_shift - [ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_last_argv0##*/} ($_tfw_exitStatus) $*" + [ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) $*" _tfw_assertExpr "$_tfw_exitStatus" "$@" || _tfw_failexit echo "# assert $_tfw_message" return 0 } assertRealTime() { - _tfw_getopts_assert realtime "$@" + _tfw_getopts assertrealtime "$@" shift $_tfw_getopts_shift - [ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_last_argv0##*/} ($realtime) $*" + [ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_execute_argv0##*/} ($realtime) $*" _tfw_assertExpr "$realtime" "$@" || _tfw_failexit echo "# assert $_tfw_message" return 0 @@ -301,7 +288,7 @@ assertStderrGrep() { } assertGrep() { - _tfw_getopts_assert filegrep "$@" + _tfw_getopts assertgrep "$@" shift $_tfw_getopts_shift if [ $# -ne 2 ]; then _tfw_error "incorrect arguments" @@ -456,6 +443,40 @@ _tfw_teardown() { /bin/rm -rf $_tfw_tmp } +_tfw_execute() { + _tfw_execute_argv0="$1" + /usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr + _tfw_exitStatus=$? + if [ -n "$_tfw_opt_exit_status" ]; then + _tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) is $_tfw_opt_exit_status" + _tfw_dump_stderr_on_fail=true + _tfw_assert [ "$_tfw_exitStatus" -eq "$_tfw_opt_exit_status" ] || _tfw_failexit + echo "# assert $_tfw_message" + else + echo "# exit status of ${_tfw_execute_argv0##*/} = $_tfw_exitStatus" + fi + if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then + echo "# times file contains spurious data:" + tfw_cat --header=times $_tfw_tmp/times + if [ $_tfw_exitStatus -eq 0 ]; then + _tfw_exitStatus=255 + echo "# deeming exit status of command to be $_tfw_exitStatus" + fi + realtime=0 + usertime=0 + systime=0 + realtime_ms=0 + #usertime_ms=0 + #systime_ms=0 + else + source $_tfw_tmp/times + realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }") + #usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }") + #systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }") + fi + return 0 +} + _tfw_assert() { if ! "$@"; then _tfw_failmsg "assertion failed: ${_tfw_message:-$*}" @@ -465,20 +486,22 @@ _tfw_assert() { return 0 } -_tfw_getopts_assert() { +_tfw_getopts() { local context="$1" shift _tfw_message= _tfw_dump_stdout_on_fail=false _tfw_dump_stderr_on_fail=false + _tfw_opt_exit_status= _tfw_opt_matches= _tfw_getopts_shift=0 while [ $# -ne 0 ]; do case "$context:$1" in - *:--message=*) _tfw_message="${1#*=}";; *:--stdout) _tfw_dump_stdout_on_fail=true;; *:--stderr) _tfw_dump_stderr_on_fail=true;; - filegrep:--matches=*) _tfw_opt_matches="${1#*=}";; + execute:--exit-status=*) _tfw_opt_exit_status="${1#*=}";; + assert*:--message=*) _tfw_message="${1#*=}";; + assertgrep:--matches=*) _tfw_opt_matches="${1#*=}";; *:--) let _tfw_getopts_shift=_tfw_getopts_shift+1; shift; break;; *:--*) _tfw_error "unsupported option: $1";; *) break;; @@ -529,13 +552,13 @@ _tfw_assertExpr() { _tfw_assert_stdxxx_is() { local qual="$1" shift - _tfw_getopts_assert filecontent --$qual "$@" + _tfw_getopts assertfilecontent --$qual "$@" shift $((_tfw_getopts_shift - 1)) if [ $# -lt 1 ]; then _tfw_error "incorrect arguments" return 254 fi - local message="${_tfw_message:-$qual of ${_tfw_last_argv0##*/} is $*}" + local message="${_tfw_message:-$qual of ${_tfw_execute_argv0##*/} is $*}" echo -n "$@" >$_tfw_tmp/stdxxx_is.tmp if ! /usr/bin/cmp --quiet $_tfw_tmp/stdxxx_is.tmp "$_tfw_tmp/$qual"; then _tfw_failmsg "assertion failed: $message" @@ -549,7 +572,7 @@ _tfw_assert_stdxxx_is() { _tfw_assert_stdxxx_linecount() { local qual="$1" shift - _tfw_getopts_assert filecontent --$qual "$@" + _tfw_getopts assertfilecontent --$qual "$@" shift $((_tfw_getopts_shift - 1)) if [ $# -lt 1 ]; then _tfw_error "incorrect arguments" @@ -565,13 +588,13 @@ _tfw_assert_stdxxx_linecount() { _tfw_assert_stdxxx_grep() { local qual="$1" shift - _tfw_getopts_assert filegrep --$qual "$@" + _tfw_getopts assertgrep --$qual "$@" shift $((_tfw_getopts_shift - 1)) if [ $# -ne 1 ]; then _tfw_error "incorrect arguments" return 254 fi - _tfw_assert_grep "$qual of ${_tfw_last_argv0##*/}" $_tfw_tmp/$qual "$@" + _tfw_assert_grep "$qual of ${_tfw_execute_argv0##*/}" $_tfw_tmp/$qual "$@" } _tfw_assert_grep() { diff --git a/tests/dna_biterrors b/tests/dna_biterrors index 3d43e667..4c313b22 100755 --- a/tests/dna_biterrors +++ b/tests/dna_biterrors @@ -40,8 +40,7 @@ test_BitErrorCreateSID() { local totaltime_ms=0 local maxtime_ms=0 while [ $i -lt $iterations ]; do - execute $dna -B $ber -d 0427679796 -C - assertExitStatus '==' 0 + executeOk $dna -B $ber -d 0427679796 -C sid=`replayStdout | grep '^OK:' | cut -f2 -d:` let totaltime_ms=totaltime_ms+realtime_ms [ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms @@ -70,11 +69,9 @@ setup_SetVarBigValueBitErrors() { echo_long_message >$DNATMP/dnatest.in } test_SetVarBigValueBitErrors() { - execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in" - assertExitStatus '==' 0 + executeOk $dna -B $ber -s $sid -i 0 -W note="@dnatest.in" assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0" assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out } diff --git a/tests/dna_hlr b/tests/dna_hlr index 4ea3adc1..0b3505f2 100755 --- a/tests/dna_hlr +++ b/tests/dna_hlr @@ -28,16 +28,14 @@ teardown() { # Utility function create_sid() { - execute $dna "$@" -d 0427679796 -C - assertExitStatus '==' 0 + executeOk $dna "$@" -d 0427679796 -C assertStdoutGrep --matches=1 '^OK:' sid=`replayStdout | grep '^OK:' | cut -f2 -d:` } # Utility function set_short_var() { - execute $dna -s $sid -i 0 -W note="a short literal value" - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -W note="a short literal value" } # Utility function @@ -69,8 +67,7 @@ setup_GetShortVar() { create_sid } test_GetShortVar() { - execute $dna -d 0427679796 -R dids - assertExitStatus '==' 0 + executeOk $dna -d 0427679796 -R dids assertRealTime '>' 2.9 # Waited for all replies assertStdoutGrep --message='received an existing DID' "^DIDS:${sid}:[0-9]\+:0:0427679796" assertStdoutGrep --matches=1 --message='filtered out duplicate responses' "^DIDS:${sid}:[0-9]\+:0:0427679796" @@ -98,8 +95,7 @@ setup_SetGetShortVar() { set_short_var } test_SetGetShortVar() { - execute $dna -s $sid -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='read variable correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value" } @@ -115,8 +111,7 @@ setup_GetShortVarToFile() { test_GetShortVarToFile() { echo "WARNING: Known issue: output to file does not work with DID lists" dnatest_dat=$DNATMP/dnatest.dat - execute $dna -s $sid -O $dnatest_dat -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -O $dnatest_dat -i 0 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='read variable into file' "^NOTE:$sid:[0-9]\+:0" assertFileContents $dnatest_dat 'a short literal value' @@ -131,12 +126,10 @@ setup_SetMultiShortVar() { set_short_var } test_SetMultiShortVar() { - execute $dna -s $sid -i 1 -W note='$414243' - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 1 -W note='$414243' assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -s $sid -i 1 -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 1 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='new variable correctly set in hex' "^NOTE:$sid:[0-9]\+:1:ABC$" } @@ -148,14 +141,11 @@ setup_GetMultiShortVar() { start_dna_server create_sid set_short_var - execute $dna -s $sid -i 1 -W note='$414243' - assertExitStatus '==' 0 - execute $dna -s $sid -i 2 -W note - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 1 -W note='$414243' + executeOk $dna -s $sid -i 2 -W note } test_GetMultiShortVar() { - execute $dna -s $sid -m -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -m -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='first variable read correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value$" assertStdoutGrep --matches=1 --message='second variable read correctly' "^NOTE:$sid:[0-9]\+:1:ABC$" @@ -171,12 +161,10 @@ setup_OverwriteVarNoUpdate() { set_short_var } test_OverwriteVarNoUpdate() { - execute $dna -s $sid -i 0 -W note="replacement short literal value" - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -W note="replacement short literal value" assertRealTime '<' 0.5 assertStdoutGrep --matches=0 --message='variable write not confirmed' "^WROTE:$sid$" - execute $dna -s $sid -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable retains original value' "^NOTE:$sid:[0-9]\+:0:a short literal value" } @@ -190,12 +178,10 @@ setup_UpdateVar() { set_short_var } test_UpdateVar() { - execute $dna -s $sid -i 0 -U note="replacement short literal value" - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -U note="replacement short literal value" assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -s $sid -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable has new value' "^NOTE:$sid:[0-9]\+:0:replacement short literal value" } @@ -210,12 +196,10 @@ setup_UpdateVarBigValue() { echo_long_message >$DNATMP/dnatest.in } test_UpdateVarBigValue() { - execute $dna -s $sid -i 0 -U note="@dnatest.in" - assertExitStatus '==' 0 + executeOk $dna -s $sid -i 0 -U note="@dnatest.in" assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note - assertExitStatus '==' 0 + executeOk $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0" assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out diff --git a/tests/dna_rhizome b/tests/dna_rhizome index 5b6eb8e8..7f753805 100755 --- a/tests/dna_rhizome +++ b/tests/dna_rhizome @@ -22,8 +22,7 @@ source "${0%/*}/../testframework.sh" source "${0%/*}/../testdefs.sh" assert_rhizome_list_empty() { - execute $dna rhizome list - assertExitStatus --stdout --stderr '==' 0 + executeOk $dna rhizome list assertStdoutIs -e "Found 0 rows\n" } @@ -44,8 +43,7 @@ setup_AddManifest() { echo "Another test file" >file2 } test_AddManifest() { - execute $dna rhizome add file file1 file1.manifest - assertExitStatus --stderr '==' 0 + executeOk $dna rhizome add file file1 file1.manifest tfw_cat --stdout tfw_cat --stderr tfw_cat -v file1.manifest @@ -67,20 +65,16 @@ setup_AddThenList() { } test_AddThenList() { # Add first file - execute $dna rhizome add file file1 - assertExitStatus --stderr '==' 0 - execute $dna rhizome list - assertExitStatus --stdout --stderr '==' 0 + executeOk $dna rhizome add file file1 + executeOk $dna rhizome list assertStdoutGrep "^Found 1 rows$" assertStdoutLineCount '==' 6 assertStdoutGrep --matches=1 '^file name = "file1"$' assertStdoutGrep --matches=0 '^file name = "file2"$' replayStdout >add1.stdout # Add second file - execute $dna rhizome add file file2 - assertExitStatus --stderr '==' 0 - execute $dna rhizome list - assertExitStatus --stdout --stderr '==' 0 + executeOk $dna rhizome add file file2 + executeOk $dna rhizome list assertStdoutGrep "^Found 2 rows$" assertStdoutLineCount '==' 11 assertStdoutGrep --matches=1 '^file name = "file1"$' @@ -94,24 +88,19 @@ setup_AddDuplicate() { echo "A test file" >file1 echo "Another test file" >file2 # Add first file - execute $dna rhizome add file file1 - assertExitStatus --stderr '==' 0 + executeOk $dna rhizome add file file1 # Add second file - execute $dna rhizome add file file2 - assertExitStatus --stderr '==' 0 + executeOk $dna rhizome add file file2 # Make sure they are both in the list. - execute $dna rhizome list - assertExitStatus --stdout --stderr '==' 0 + executeOk $dna rhizome list assertStdoutGrep "^Found 2 rows$" assertStdoutGrep --matches=1 '^file name = "file1"$' assertStdoutGrep --matches=1 '^file name = "file2"$' } test_AddDuplicate() { # Add first file again (nothing should change except its date). - execute $dna rhizome add file file1 - assertExitStatus --stderr '==' 0 - execute $dna rhizome list - assertExitStatus --stdout --stderr '==' 0 + executeOk $dna rhizome add file file1 + executeOk $dna rhizome list assertStdoutGrep "^Found 2 rows$" assertStdoutLineCount '==' 11 assertStdoutGrep --matches=1 '^file name = "file1"$'