Improve test framework

- new execute() option: --exit-status=N
   equivalent to assertExitStatus --stderr '==' N
 - new executeOk() function, shortcut for execute --exit-status=0
This commit is contained in:
Andrew Bettison 2012-04-12 17:04:37 +09:30
parent fab1df61e1
commit c4e3249839
4 changed files with 96 additions and 103 deletions

View File

@ -150,35 +150,22 @@ realpath() {
}
execute() {
_tfw_last_argv0="$1"
echo "# execute $*"
/usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr
_tfw_exitStatus=$?
echo "# Exit status = $_tfw_exitStatus"
if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then
echo "# Times file contains spurious data:"
tfw_cat --header=times $_tfw_tmp/times
if [ $_tfw_exitStatus -eq 0 ]; then
_tfw_exitStatus=255
echo "# Deeming exit status of command to be $_tfw_exitStatus"
fi
realtime=0
usertime=0
systime=0
realtime_ms=0
#usertime_ms=0
#systime_ms=0
else
source $_tfw_tmp/times
realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }")
#usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }")
#systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }")
fi
return 0
_tfw_getopts execute "$@"
shift $_tfw_getopts_shift
_tfw_execute "$@"
}
executeOk() {
echo "# executeOk $*"
_tfw_getopts executeok "$@"
_tfw_opt_exit_status=0
shift $_tfw_getopts_shift
_tfw_execute "$@"
}
assert() {
_tfw_getopts_assert assert "$@"
_tfw_getopts assert "$@"
shift $_tfw_getopts_shift
_tfw_assert "$@" || _tfw_failexit
echo "# assert $*"
@ -186,7 +173,7 @@ assert() {
}
assertExpr() {
_tfw_getopts_assert assertexpr "$@"
_tfw_getopts assertexpr "$@"
shift $_tfw_getopts_shift
local awkexpr=$(_tfw_expr_to_awkexpr "$@")
_tfw_message="${_tfw_message+$_tfw_message }($awkexpr)"
@ -196,7 +183,7 @@ assertExpr() {
}
fail() {
_tfw_getopts_assert fail "$@"
_tfw_getopts fail "$@"
shift $_tfw_getopts_shift
[ $# -ne 0 ] && _tfw_failmsg "$1"
_tfw_backtrace >&2
@ -204,7 +191,7 @@ fail() {
}
error() {
_tfw_getopts_assert error "$@"
_tfw_getopts error "$@"
shift $_tfw_getopts_shift
[ $# -ne 0 ] && _tfw_errormsg "$1"
_tfw_backtrace >&2
@ -224,14 +211,14 @@ tfw_cat() {
for file; do
case $file in
--stdout)
echo "#--- ${header:-stdout of ${_tfw_last_argv0##*/}} ---"
echo "#--- ${header:-stdout of ${_tfw_execute_argv0##*/}} ---"
/bin/cat $show_nonprinting $_tfw_tmp/stdout
echo "#---"
header=
show_nonprinting=
;;
--stderr)
echo "#--- ${header:-stderr of ${_tfw_last_argv0##*/}} ---"
echo "#--- ${header:-stderr of ${_tfw_execute_argv0##*/}} ---"
/bin/cat $show_nonprinting $_tfw_tmp/stderr
echo "#---"
header=
@ -251,18 +238,18 @@ tfw_cat() {
}
assertExitStatus() {
_tfw_getopts_assert exitstatus "$@"
_tfw_getopts assertexitstatus "$@"
shift $_tfw_getopts_shift
[ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_last_argv0##*/} ($_tfw_exitStatus) $*"
[ -z "$_tfw_message" ] && _tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) $*"
_tfw_assertExpr "$_tfw_exitStatus" "$@" || _tfw_failexit
echo "# assert $_tfw_message"
return 0
}
assertRealTime() {
_tfw_getopts_assert realtime "$@"
_tfw_getopts assertrealtime "$@"
shift $_tfw_getopts_shift
[ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_last_argv0##*/} ($realtime) $*"
[ -z "$_tfw_message" ] && _tfw_message="real execution time of ${_tfw_execute_argv0##*/} ($realtime) $*"
_tfw_assertExpr "$realtime" "$@" || _tfw_failexit
echo "# assert $_tfw_message"
return 0
@ -301,7 +288,7 @@ assertStderrGrep() {
}
assertGrep() {
_tfw_getopts_assert filegrep "$@"
_tfw_getopts assertgrep "$@"
shift $_tfw_getopts_shift
if [ $# -ne 2 ]; then
_tfw_error "incorrect arguments"
@ -456,6 +443,40 @@ _tfw_teardown() {
/bin/rm -rf $_tfw_tmp
}
_tfw_execute() {
_tfw_execute_argv0="$1"
/usr/bin/time --format='realtime=%e;usertime=%U;systime=%S' --output=$_tfw_tmp/times "$@" >$_tfw_tmp/stdout 2>$_tfw_tmp/stderr
_tfw_exitStatus=$?
if [ -n "$_tfw_opt_exit_status" ]; then
_tfw_message="exit status of ${_tfw_execute_argv0##*/} ($_tfw_exitStatus) is $_tfw_opt_exit_status"
_tfw_dump_stderr_on_fail=true
_tfw_assert [ "$_tfw_exitStatus" -eq "$_tfw_opt_exit_status" ] || _tfw_failexit
echo "# assert $_tfw_message"
else
echo "# exit status of ${_tfw_execute_argv0##*/} = $_tfw_exitStatus"
fi
if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then
echo "# times file contains spurious data:"
tfw_cat --header=times $_tfw_tmp/times
if [ $_tfw_exitStatus -eq 0 ]; then
_tfw_exitStatus=255
echo "# deeming exit status of command to be $_tfw_exitStatus"
fi
realtime=0
usertime=0
systime=0
realtime_ms=0
#usertime_ms=0
#systime_ms=0
else
source $_tfw_tmp/times
realtime_ms=$(/usr/bin/awk "BEGIN { print int($realtime * 1000) }")
#usertime_ms=$(/usr/bin/awk "BEGIN { print int($usertime * 1000) }")
#systime_ms=$(/usr/bin/awk "BEGIN { print int($systime * 1000) }")
fi
return 0
}
_tfw_assert() {
if ! "$@"; then
_tfw_failmsg "assertion failed: ${_tfw_message:-$*}"
@ -465,20 +486,22 @@ _tfw_assert() {
return 0
}
_tfw_getopts_assert() {
_tfw_getopts() {
local context="$1"
shift
_tfw_message=
_tfw_dump_stdout_on_fail=false
_tfw_dump_stderr_on_fail=false
_tfw_opt_exit_status=
_tfw_opt_matches=
_tfw_getopts_shift=0
while [ $# -ne 0 ]; do
case "$context:$1" in
*:--message=*) _tfw_message="${1#*=}";;
*:--stdout) _tfw_dump_stdout_on_fail=true;;
*:--stderr) _tfw_dump_stderr_on_fail=true;;
filegrep:--matches=*) _tfw_opt_matches="${1#*=}";;
execute:--exit-status=*) _tfw_opt_exit_status="${1#*=}";;
assert*:--message=*) _tfw_message="${1#*=}";;
assertgrep:--matches=*) _tfw_opt_matches="${1#*=}";;
*:--) let _tfw_getopts_shift=_tfw_getopts_shift+1; shift; break;;
*:--*) _tfw_error "unsupported option: $1";;
*) break;;
@ -529,13 +552,13 @@ _tfw_assertExpr() {
_tfw_assert_stdxxx_is() {
local qual="$1"
shift
_tfw_getopts_assert filecontent --$qual "$@"
_tfw_getopts assertfilecontent --$qual "$@"
shift $((_tfw_getopts_shift - 1))
if [ $# -lt 1 ]; then
_tfw_error "incorrect arguments"
return 254
fi
local message="${_tfw_message:-$qual of ${_tfw_last_argv0##*/} is $*}"
local message="${_tfw_message:-$qual of ${_tfw_execute_argv0##*/} is $*}"
echo -n "$@" >$_tfw_tmp/stdxxx_is.tmp
if ! /usr/bin/cmp --quiet $_tfw_tmp/stdxxx_is.tmp "$_tfw_tmp/$qual"; then
_tfw_failmsg "assertion failed: $message"
@ -549,7 +572,7 @@ _tfw_assert_stdxxx_is() {
_tfw_assert_stdxxx_linecount() {
local qual="$1"
shift
_tfw_getopts_assert filecontent --$qual "$@"
_tfw_getopts assertfilecontent --$qual "$@"
shift $((_tfw_getopts_shift - 1))
if [ $# -lt 1 ]; then
_tfw_error "incorrect arguments"
@ -565,13 +588,13 @@ _tfw_assert_stdxxx_linecount() {
_tfw_assert_stdxxx_grep() {
local qual="$1"
shift
_tfw_getopts_assert filegrep --$qual "$@"
_tfw_getopts assertgrep --$qual "$@"
shift $((_tfw_getopts_shift - 1))
if [ $# -ne 1 ]; then
_tfw_error "incorrect arguments"
return 254
fi
_tfw_assert_grep "$qual of ${_tfw_last_argv0##*/}" $_tfw_tmp/$qual "$@"
_tfw_assert_grep "$qual of ${_tfw_execute_argv0##*/}" $_tfw_tmp/$qual "$@"
}
_tfw_assert_grep() {

View File

@ -40,8 +40,7 @@ test_BitErrorCreateSID() {
local totaltime_ms=0
local maxtime_ms=0
while [ $i -lt $iterations ]; do
execute $dna -B $ber -d 0427679796 -C
assertExitStatus '==' 0
executeOk $dna -B $ber -d 0427679796 -C
sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
let totaltime_ms=totaltime_ms+realtime_ms
[ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms
@ -70,11 +69,9 @@ setup_SetVarBigValueBitErrors() {
echo_long_message >$DNATMP/dnatest.in
}
test_SetVarBigValueBitErrors() {
execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
assertExitStatus '==' 0
executeOk $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out
}

View File

@ -28,16 +28,14 @@ teardown() {
# Utility function
create_sid() {
execute $dna "$@" -d 0427679796 -C
assertExitStatus '==' 0
executeOk $dna "$@" -d 0427679796 -C
assertStdoutGrep --matches=1 '^OK:'
sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
}
# Utility function
set_short_var() {
execute $dna -s $sid -i 0 -W note="a short literal value"
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -W note="a short literal value"
}
# Utility function
@ -69,8 +67,7 @@ setup_GetShortVar() {
create_sid
}
test_GetShortVar() {
execute $dna -d 0427679796 -R dids
assertExitStatus '==' 0
executeOk $dna -d 0427679796 -R dids
assertRealTime '>' 2.9 # Waited for all replies
assertStdoutGrep --message='received an existing DID' "^DIDS:${sid}:[0-9]\+:0:0427679796"
assertStdoutGrep --matches=1 --message='filtered out duplicate responses' "^DIDS:${sid}:[0-9]\+:0:0427679796"
@ -98,8 +95,7 @@ setup_SetGetShortVar() {
set_short_var
}
test_SetGetShortVar() {
execute $dna -s $sid -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='read variable correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value"
}
@ -115,8 +111,7 @@ setup_GetShortVarToFile() {
test_GetShortVarToFile() {
echo "WARNING: Known issue: output to file does not work with DID lists"
dnatest_dat=$DNATMP/dnatest.dat
execute $dna -s $sid -O $dnatest_dat -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -O $dnatest_dat -i 0 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='read variable into file' "^NOTE:$sid:[0-9]\+:0"
assertFileContents $dnatest_dat 'a short literal value'
@ -131,12 +126,10 @@ setup_SetMultiShortVar() {
set_short_var
}
test_SetMultiShortVar() {
execute $dna -s $sid -i 1 -W note='$414243'
assertExitStatus '==' 0
executeOk $dna -s $sid -i 1 -W note='$414243'
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -s $sid -i 1 -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -i 1 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='new variable correctly set in hex' "^NOTE:$sid:[0-9]\+:1:ABC$"
}
@ -148,14 +141,11 @@ setup_GetMultiShortVar() {
start_dna_server
create_sid
set_short_var
execute $dna -s $sid -i 1 -W note='$414243'
assertExitStatus '==' 0
execute $dna -s $sid -i 2 -W note
assertExitStatus '==' 0
executeOk $dna -s $sid -i 1 -W note='$414243'
executeOk $dna -s $sid -i 2 -W note
}
test_GetMultiShortVar() {
execute $dna -s $sid -m -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -m -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='first variable read correctly' "^NOTE:$sid:[0-9]\+:0:a short literal value$"
assertStdoutGrep --matches=1 --message='second variable read correctly' "^NOTE:$sid:[0-9]\+:1:ABC$"
@ -171,12 +161,10 @@ setup_OverwriteVarNoUpdate() {
set_short_var
}
test_OverwriteVarNoUpdate() {
execute $dna -s $sid -i 0 -W note="replacement short literal value"
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -W note="replacement short literal value"
assertRealTime '<' 0.5
assertStdoutGrep --matches=0 --message='variable write not confirmed' "^WROTE:$sid$"
execute $dna -s $sid -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable retains original value' "^NOTE:$sid:[0-9]\+:0:a short literal value"
}
@ -190,12 +178,10 @@ setup_UpdateVar() {
set_short_var
}
test_UpdateVar() {
execute $dna -s $sid -i 0 -U note="replacement short literal value"
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -U note="replacement short literal value"
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -s $sid -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable has new value' "^NOTE:$sid:[0-9]\+:0:replacement short literal value"
}
@ -210,12 +196,10 @@ setup_UpdateVarBigValue() {
echo_long_message >$DNATMP/dnatest.in
}
test_UpdateVarBigValue() {
execute $dna -s $sid -i 0 -U note="@dnatest.in"
assertExitStatus '==' 0
executeOk $dna -s $sid -i 0 -U note="@dnatest.in"
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertExitStatus '==' 0
executeOk $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out

View File

@ -22,8 +22,7 @@ source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
assert_rhizome_list_empty() {
execute $dna rhizome list
assertExitStatus --stdout --stderr '==' 0
executeOk $dna rhizome list
assertStdoutIs -e "Found 0 rows\n"
}
@ -44,8 +43,7 @@ setup_AddManifest() {
echo "Another test file" >file2
}
test_AddManifest() {
execute $dna rhizome add file file1 file1.manifest
assertExitStatus --stderr '==' 0
executeOk $dna rhizome add file file1 file1.manifest
tfw_cat --stdout
tfw_cat --stderr
tfw_cat -v file1.manifest
@ -67,20 +65,16 @@ setup_AddThenList() {
}
test_AddThenList() {
# Add first file
execute $dna rhizome add file file1
assertExitStatus --stderr '==' 0
execute $dna rhizome list
assertExitStatus --stdout --stderr '==' 0
executeOk $dna rhizome add file file1
executeOk $dna rhizome list
assertStdoutGrep "^Found 1 rows$"
assertStdoutLineCount '==' 6
assertStdoutGrep --matches=1 '^file name = "file1"$'
assertStdoutGrep --matches=0 '^file name = "file2"$'
replayStdout >add1.stdout
# Add second file
execute $dna rhizome add file file2
assertExitStatus --stderr '==' 0
execute $dna rhizome list
assertExitStatus --stdout --stderr '==' 0
executeOk $dna rhizome add file file2
executeOk $dna rhizome list
assertStdoutGrep "^Found 2 rows$"
assertStdoutLineCount '==' 11
assertStdoutGrep --matches=1 '^file name = "file1"$'
@ -94,24 +88,19 @@ setup_AddDuplicate() {
echo "A test file" >file1
echo "Another test file" >file2
# Add first file
execute $dna rhizome add file file1
assertExitStatus --stderr '==' 0
executeOk $dna rhizome add file file1
# Add second file
execute $dna rhizome add file file2
assertExitStatus --stderr '==' 0
executeOk $dna rhizome add file file2
# Make sure they are both in the list.
execute $dna rhizome list
assertExitStatus --stdout --stderr '==' 0
executeOk $dna rhizome list
assertStdoutGrep "^Found 2 rows$"
assertStdoutGrep --matches=1 '^file name = "file1"$'
assertStdoutGrep --matches=1 '^file name = "file2"$'
}
test_AddDuplicate() {
# Add first file again (nothing should change except its date).
execute $dna rhizome add file file1
assertExitStatus --stderr '==' 0
execute $dna rhizome list
assertExitStatus --stdout --stderr '==' 0
executeOk $dna rhizome add file file1
executeOk $dna rhizome list
assertStdoutGrep "^Found 2 rows$"
assertStdoutLineCount '==' 11
assertStdoutGrep --matches=1 '^file name = "file1"$'