From ce5dc011b888efcabc3aff99a23bfb9d6a109ae5 Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Tue, 10 Apr 2012 12:41:15 +0930 Subject: [PATCH] Split dna bit error tests into separate script - Put common utility functions into testdefs.sh - Add abspath() and realpath() functions to test framework - Test framework now executes all tests with working directory in a temporary directory, to avoid pollution of caller's working directory - Improve test log verbosity - Name test log files by test script name --- .gitignore | 2 +- testdefs.sh | 73 ++++++++++++++ testframework.sh | 158 +++++++++++++++++++++--------- tests/dna_biterrors | 82 ++++++++++++++++ tests/{dna_create_hlr => dna_hlr} | 130 ++++-------------------- 5 files changed, 284 insertions(+), 161 deletions(-) create mode 100644 testdefs.sh create mode 100755 tests/dna_biterrors rename tests/{dna_create_hlr => dna_hlr} (64%) diff --git a/.gitignore b/.gitignore index a470e691..64d66c31 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ nacl/naclinc.txt nacl/nacllib.txt serval.c dna -test.log +test.*.log diff --git a/testdefs.sh b/testdefs.sh new file mode 100644 index 00000000..0e8c2717 --- /dev/null +++ b/testdefs.sh @@ -0,0 +1,73 @@ +# Common definitions for all test suites in test/* + +this=$(abspath "${BASH_SOURCE[0]}") + +# Utility function for setting up a fixture with a DNA server process: +# - Ensure that no dna processes are running +# - Start a dna server process +# - Ensure that it is still running after one second +start_dna_server() { + check_no_dna_processes + # Start DNA server + $dna -v verbose -f $hlr_dat -S 1 -n "$@" >$DNATMP/dna.log 2>&1 & + sleep 1 + pid=$(ps -u$UID | awk '$4 == "dna" {print $1}') + if [ -z "$pid" ]; then + echo "dna server did not start" + tfw_cat --header=dna.log $SERVALINSTANCE_PATH/dna.log + fail + fi + if ! [ -s $SERVALINSTANCE_PATH/serval.pid ] && kill -0 $(cat $SERVALINSTANCE_PATH/serval.pid); then + echo "serval.pid was not created" + tfw_cat --header=dna.log $SERVALINSTANCE_PATH/dna.log + fail + fi + echo "# Started dna server process, pid=$pid" +} + +# Utility function for tearing down DNA fixtures: +# - If a dna server process is running, then kill it +# - Cat any dna log file into the test log +# - Ensure that no dna processes are running +stop_dna_server() { + if [ -s $SERVALINSTANCE_PATH/serval.pid ]; then + local pid=$(cat $SERVALINSTANCE_PATH/serval.pid) + if kill $pid; then + echo "# Killed dna process pid=$pid" + else + error "# Dna process pid=$pid was not running" + fi + fi + if [ -s $DNATMP/dna.log ]; then + tfw_cat --header=dna.log $DNATMP/dna.log + fi + check_no_dna_processes +} + +# Utility function for creating DNA fixtures: +# - Create a temporary directory to contain all dna-related files +# - set $dna and $hlr_dat variables +# - set SERVALINSTANCE_PATH environment variable +setup_dna() { + dna=$(abspath "${this%/*}/dna") # The DNA executable under test + if ! [ -x "$dna" ]; then + error "dna executable not present: $dna" + return 1 + fi + export DNATMP=$TFWTMP/dnatmp + mkdir $DNATMP + export SERVALINSTANCE_PATH=$DNATMP + hlr_dat=$SERVALINSTANCE_PATH/hlr.dat +} + +# Utility function for managing DNA fixtures: +# - Ensure there are no existing DNA server processes +check_no_dna_processes() { + local pids=$(ps -u$UID | awk '$4 == "dna" {print $1}') + if [ -n "$pids" ]; then + error "cannot run test: dna process already running with pid: $pids" + return 1 + fi + echo "# No other dna processes running for uid=$UID" + return 0 +} diff --git a/testframework.sh b/testframework.sh index 5f35b678..fd5014b8 100644 --- a/testframework.sh +++ b/testframework.sh @@ -63,7 +63,10 @@ runTests() { _tfw_checkBashVersion _tfw_trace=false _tfw_verbose=false - _tfw_logfile=test.log + _tfw_invoking_script=$(abspath "${BASH_SOURCE[1]}") + _tfw_suite_name="${_tfw_invoking_script##*/}" + _tfw_cwd=$(abspath "$PWD") + _tfw_logfile="$_tfw_cwd/test.$_tfw_suite_name.log" local allargs="$*" local filter= while [ $# -ne 0 ]; do @@ -86,22 +89,23 @@ runTests() { # Iterate through all test cases. local testcount=0 local passcount=0 - for testName in `_tfw_findTests` + local testName + for testName in `_tfw_find_tests` do - _tfw_testname=$testName - if [ -z "$filter" -o "${testName#$filter}" != "$testName" ]; then + _tfw_test_name="$testName" + if [ -z "$filter" -o "${_tfw_test_name#$filter}" != "$_tfw_test_name" ]; then let testcount=testcount+1 ( - local docvar="doc_$testName" - _tfw_echo -n "$testcount. ${!docvar:-$testName}..." + local docvar="doc_$_tfw_test_name" + _tfw_echo -n "$testcount. ${!docvar:-$_tfw_test_name}..." trap '_tfw_status=$?; _tfw_teardown; exit $_tfw_status' 0 1 2 15 _tfw_result=ERROR _tfw_setup _tfw_result=FAIL _tfw_phase=testcase - echo "# call test_$testName()" + echo "# call test_$_tfw_test_name()" $_tfw_trace && set -x - test_$testName + test_$_tfw_test_name _tfw_result=PASS exit 0 ) @@ -133,6 +137,18 @@ teardown() { # The following functions are provided to facilitate writing test cases and # fixtures. +# Echo the absolute path (containing symlinks if given) of the given +# file/directory, which does not have to exist or even be accessible. +abspath() { + _tfw_abspath -L "$1" +} + +# Echo the absolute path (resolving all symlinks) of the given file/directory, +# which does not have to exist or even be accessible. +realpath() { + _tfw_abspath -P "$1" +} + execute() { _tfw_last_argv0="$1" echo "# execute $*" @@ -141,7 +157,7 @@ execute() { echo "# Exit status = $_tfw_exitStatus" if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then echo "# Times file contains spurious data:" - _tfw_cat $_tfw_tmp/times + tfw_cat --header=times $_tfw_tmp/times if [ $_tfw_exitStatus -eq 0 ]; then _tfw_exitStatus=255 echo "# Deeming exit status of command to be $_tfw_exitStatus" @@ -202,6 +218,33 @@ fatal() { _tfw_fatalexit } +tfw_cat() { + local $header + for file; do + case $file in + --stdout) + echo "#--- ${header:-stdout of ${_tfw_last_argv0##*/}} ---" + /bin/cat $_tfw_tmp/stdout + echo "#---" + header= + ;; + --stderr) + echo "#--- ${header:-stderr of ${_tfw_last_argv0##*/}} ---" + /bin/cat $_tfw_tmp/stderr + echo "#---" + header= + ;; + --header=*) header="${1#*=}";; + *) + echo "#--- ${header:-$file} ---" + /bin/cat "$file" + echo "#---" + header= + ;; + esac + done +} + assertExitStatus() { _tfw_getopts_assert exitstatus "$@" shift $_tfw_getopts_shift @@ -247,6 +290,43 @@ assertStderrGrep() { # Internal (private) functions that are not to be invoked directly from test # scripts. +# Echo the absolute path of the given path, using only Bash builtins. +_tfw_abspath() { + cdopt=-L + if [ $# -gt 1 -a "${1:0:1}" = - ]; then + cdopt="$1" + shift + fi + case "$1" in + */) + builtin echo $(_tfw_abspath $cdopt "${1%/}")/ + ;; + /*/*) + if [ -d "$1" ]; then + (CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD") + else + builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}" + fi + ;; + /*) + echo "$1" + ;; + */*) + if [ -d "$1" ]; then + (CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD") + else + builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}" + fi + ;; + . | ..) + (CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD") + ;; + *) + (CDPATH= builtin cd $cdopt . && builtin echo "$PWD/$1") + ;; + esac +} + _tfw_setup() { _tfw_phase=setup _tfw_tmp=/tmp/_tfw-$$ @@ -262,18 +342,19 @@ _tfw_setup() { fi export TFWTMP=$_tfw_tmp/tmp /bin/mkdir $TFWTMP + cd $TFWTMP echo '# SETUP' - case `type -t setup_$testName` in + case `type -t setup_$_tfw_test_name` in function) - echo "# call setup_$testName()" + echo "# call setup_$_tfw_test_name()" $_tfw_trace && set -x - setup_$testName $testName + setup_$_tfw_test_name $_tfw_test_name set +x ;; *) - echo "# call setup($testName)" + echo "# call setup($_tfw_test_name)" $_tfw_trace && set -x - setup $testName + setup $_tfw_test_name set +x ;; esac @@ -283,23 +364,23 @@ _tfw_setup() { _tfw_teardown() { _tfw_phase=teardown echo '# TEARDOWN' - case `type -t teardown_$testName` in + case `type -t teardown_$_tfw_test_name` in function) - echo "# call teardown_$testName()" + echo "# call teardown_$_tfw_test_name()" $_tfw_trace && set -x - teardown_$testName + teardown_$_tfw_test_name set +x ;; *) - echo "# call teardown($testName)" + echo "# call teardown($_tfw_test_name)" $_tfw_trace && set -x - teardown $testName + teardown $_tfw_test_name set +x ;; esac echo '# END TEARDOWN' { - local banner="==================== $_tfw_testname ====================" + local banner="==================== $_tfw_test_name ====================" echo "$banner" echo "TEST RESULT: $_tfw_result" echo '++++++++++ log.stdout ++++++++++' @@ -457,28 +538,6 @@ _tfw_echoerr() { fi } -_tfw_cat() { - for file; do - case $file in - --stdout) - echo "--- stdout of ${_tfw_last_argv0##*/} ---" - /bin/cat $_tfw_tmp/stdout - echo "---" - ;; - --stderr) - echo "--- stderr of ${_tfw_last_argv0##*/} ---" - /bin/cat $_tfw_tmp/stderr - echo "---" - ;; - *) - echo "--- $file ---" - /bin/cat "$file" - echo "---" - ;; - esac - done -} - _tfw_checkBashVersion() { case $BASH_VERSION in [56789].* | 4.[23456789].*) ;; @@ -487,7 +546,7 @@ _tfw_checkBashVersion() { esac } -_tfw_findTests() { +_tfw_find_tests() { builtin declare -F | sed -n -e '/^declare -f test_..*/s/^declare -f test_//p' | sort } @@ -519,8 +578,8 @@ _tfw_backtrace() { _tfw_failexit() { # When exiting a test case due to a failure, log any diagnostic output that # has been requested. - $_tfw_dump_stdout_on_fail && _tfw_cat --stdout - $_tfw_dump_stderr_on_fail && _tfw_cat --stderr + $_tfw_dump_stdout_on_fail && tfw_cat --stdout + $_tfw_dump_stderr_on_fail && tfw_cat --stderr # A failure during setup or teardown is treated as an error. case $_tfw_phase in testcase) exit 1;; @@ -531,7 +590,8 @@ _tfw_failexit() { # An "error" event prevents a test from running, so it neither passes nor fails. # Other tests may still proceed. -_tfw_error() { +_tfw_errormsg() { + [ $# -eq 0 ] && set -- "(no message)" local -i up=1 while true; do case ${FUNCNAME[$up]} in @@ -539,7 +599,11 @@ _tfw_error() { *) break;; esac done - echo "ERROR in ${FUNCNAME[$up]}: $*" >&2 + echo "ERROR in ${FUNCNAME[$up]}: $*" +} + +_tfw_error() { + _tfw_errormsg "$@" >&2 _tfw_backtrace >&2 _tfw_errorexit } diff --git a/tests/dna_biterrors b/tests/dna_biterrors new file mode 100755 index 00000000..3d43e667 --- /dev/null +++ b/tests/dna_biterrors @@ -0,0 +1,82 @@ +#!/bin/bash + +# Tests for Serval DNA bit error performance. +# +# Copyright 2012 Paul Gardner-Stephen +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +source "${0%/*}/../testframework.sh" +source "${0%/*}/../testdefs.sh" + +# Default teardown function +teardown() { + stop_dna_server +} + +# Test case, transcribed from ../testdna +doc_BitErrorCreateSID='Create new SID despite bit errors' +setup_BitErrorCreateSID() { + ber=0.001 + setup_dna + start_dna_server -B $ber +} +test_BitErrorCreateSID() { + local iterations=10 + local i=0 + local fails=0 + local totaltime_ms=0 + local maxtime_ms=0 + while [ $i -lt $iterations ]; do + execute $dna -B $ber -d 0427679796 -C + assertExitStatus '==' 0 + sid=`replayStdout | grep '^OK:' | cut -f2 -d:` + let totaltime_ms=totaltime_ms+realtime_ms + [ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms + if [ -z "$sid" ]; then + let fails=fails+1 + echo "# iteration $i: no sid after ${realtime_ms} ms" + else + echo "# iteration $i: sid=$sid after ${realtime_ms} ms" + fi + let i=i+1 + done + let meantime_ms=totaltime_ms/iterations + echo "# average time = $meantime_ms ms" + echo "# maximum time = $maxtime_ms ms" + assertExpr --message='packet loss was properly injected' $meantime_ms '>' 25 + assertExpr --message="reliable at BER=$ber packet loss" $fails '<=' 4 + assertExpr --message="timeout works at BER=$ber packet loss" $totaltime_ms '<=' 3000 '&&' $maxtime_ms '<=' 330 +} + +# Test case, transcribed from ../testdna +doc_SetVarBigValueBitErrors='Can set variable with multi-packet value despite bit errors' +setup_SetVarBigValueBitErrors() { + ber=0.00001 + setup_dna + start_dna_server -B $ber + echo_long_message >$DNATMP/dnatest.in +} +test_SetVarBigValueBitErrors() { + execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in" + assertExitStatus '==' 0 + assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" + execute $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note + assertExitStatus '==' 0 + assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0" + assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out +} + +runTests "$@" diff --git a/tests/dna_create_hlr b/tests/dna_hlr similarity index 64% rename from tests/dna_create_hlr rename to tests/dna_hlr index 4aff77b3..4ea3adc1 100755 --- a/tests/dna_create_hlr +++ b/tests/dna_hlr @@ -1,6 +1,7 @@ #!/bin/bash -# Test script for Serval DNA executable. +# Tests for Serval DNA HLR operations. +# # Copyright 2012 Paul Gardner-Stephen # # This program is free software; you can redistribute it and/or @@ -18,57 +19,13 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. source "${0%/*}/../testframework.sh" -dna="${0%/*}/../dna" - -# Default setup function -setup() { - export SERVALINSTANCE_PATH=$TFWTMP/dna - mkdir $SERVALINSTANCE_PATH - hlr_dat=$SERVALINSTANCE_PATH/hlr.dat -} +source "${0%/*}/../testdefs.sh" # Default teardown function teardown() { stop_dna_server } -# Utility function -start_dna_server() { - # Ensure there are no existing DNA servers - pids=$(ps -u$UID | awk '$4 == "dna" {print $1}') - if [ -n "$pids" ]; then - error "cannot run test: dna process already running with pid: $pids" - fi - # Start DNA server - $dna -v verbose -f $hlr_dat -S 1 -n "$@" >$TFWTMP/dna.log 2>&1 & - sleep 1 - pids=$(ps -u$UID | awk '$4 == "dna" {print $1}') - if [ -z "$pids" ]; then - echo "dna server did not start" - echo "--- dna.log ---" - cat $SERVALINSTANCE_PATH/dna.log - echo "---" - fail - fi - if ! [ -s $SERVALINSTANCE_PATH/serval.pid ] && kill -0 $(cat $SERVALINSTANCE_PATH/serval.pid); then - echo "serval.pid was not created" - echo "--- dna.log ---" - cat $SERVALINSTANCE_PATH/dna.log - echo "---" - fail - fi -} - -# Utility function -stop_dna_server() { - [ -s $SERVALINSTANCE_PATH/serval.pid ] && kill $(cat $SERVALINSTANCE_PATH/serval.pid) - if [ -s $TFWTMP/dna.log ]; then - echo '--- dna.log ---' - cat $TFWTMP/dna.log - echo '---' - fi -} - # Utility function create_sid() { execute $dna "$@" -d 0427679796 -C @@ -95,7 +52,7 @@ echo_long_message() { # Test case, transcribed from ../testdna doc_CreateSID='Create new SID' setup_CreateSID() { - setup + setup_dna start_dna_server } test_CreateSID() { @@ -107,7 +64,7 @@ test_CreateSID() { # Test case, transcribed from ../testdna doc_GetShortVar='Get short variable' setup_GetShortVar() { - setup + setup_dna start_dna_server create_sid } @@ -122,7 +79,7 @@ test_GetShortVar() { # Test case, transcribed from ../testdna doc_SetShortVar='Set short variable' setup_SetShortVar() { - setup + setup_dna start_dna_server create_sid } @@ -135,7 +92,7 @@ test_SetShortVar() { # Test case, transcribed from ../testdna doc_SetGetShortVar='Set then get short variable' setup_SetGetShortVar() { - setup + setup_dna start_dna_server create_sid set_short_var @@ -150,14 +107,14 @@ test_SetGetShortVar() { # Test case, transcribed from ../testdna doc_GetShortVarToFile='Get variable into a file' setup_GetShortVarToFile() { - setup + setup_dna start_dna_server create_sid set_short_var } test_GetShortVarToFile() { echo "WARNING: Known issue: output to file does not work with DID lists" - dnatest_dat=$TFWTMP/dnatest.dat + dnatest_dat=$DNATMP/dnatest.dat execute $dna -s $sid -O $dnatest_dat -i 0 -R note assertExitStatus '==' 0 assertRealTime '<' 0.5 @@ -168,7 +125,7 @@ test_GetShortVarToFile() { # Test case, transcribed from ../testdna doc_SetMultiShortVar='Set multiple instances of a short variable' setup_SetMultiShortVar() { - setup + setup_dna start_dna_server create_sid set_short_var @@ -187,7 +144,7 @@ test_SetMultiShortVar() { # Test case, transcribed from ../testdna doc_GetMultiShortVar='Get multiple instances of a short variable' setup_GetMultiShortVar() { - setup + setup_dna start_dna_server create_sid set_short_var @@ -208,7 +165,7 @@ test_GetMultiShortVar() { # Test case, transcribed from ../testdna doc_OverwriteVarNoUpdate='Set does not overwrite variable' setup_OverwriteVarNoUpdate() { - setup + setup_dna start_dna_server create_sid set_short_var @@ -227,7 +184,7 @@ test_OverwriteVarNoUpdate() { # Test case, transcribed from ../testdna doc_UpdateVar='Update overwrites variable' setup_UpdateVar() { - setup + setup_dna start_dna_server create_sid set_short_var @@ -246,75 +203,22 @@ test_UpdateVar() { # Test case, transcribed from ../testdna doc_UpdateVarBigValue='Can set variable with multi-packet value' setup_UpdateVarBigValue() { - setup + setup_dna start_dna_server create_sid set_short_var - echo_long_message >$TFWTMP/dnatest.in + echo_long_message >$DNATMP/dnatest.in } test_UpdateVarBigValue() { execute $dna -s $sid -i 0 -U note="@dnatest.in" assertExitStatus '==' 0 assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -v verbose -s $sid -O $TFWTMP/dnatest.out -i 0 -R note + execute $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note assertExitStatus '==' 0 assertRealTime '<' 0.5 assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0" - assert --message='long value read correctly' cmp --quiet $TFWTMP/dnatest.in $TFWTMP/dnatest.out -} - -# Test case, transcribed from ../testdna -doc_BitErrorCreateSID='Create new SID despite bit errors' -setup_BitErrorCreateSID() { - ber=0.001 - setup - start_dna_server -B $ber -} -test_BitErrorCreateSID() { - local iterations=10 - local i=0 - local fails=0 - local totaltime_ms=0 - local maxtime_ms=0 - while [ $i -lt $iterations ]; do - execute $dna -B $ber -d 0427679796 -C - assertExitStatus '==' 0 - sid=`replayStdout | grep '^OK:' | cut -f2 -d:` - let totaltime_ms=totaltime_ms+realtime_ms - [ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms - if [ -z "$sid" ]; then - let fails=fails+1 - echo "# iteration $i: no sid after ${realtime_ms} ms" - else - echo "# iteration $i: sid=$sid after ${realtime_ms} ms" - fi - let i=i+1 - done - let meantime_ms=totaltime_ms/iterations - echo "# average time = $meantime_ms ms" - echo "# maximum time = $maxtime_ms ms" - assertExpr --message='packet loss was properly injected' $meantime_ms '>' 25 - assertExpr --message="reliable at BER=$ber packet loss" $fails '<=' 4 - assertExpr --message="timeout works at BER=$ber packet loss" $totaltime_ms '<=' 3000 '&&' $maxtime_ms '<=' 330 -} - -# Test case, transcribed from ../testdna -doc_SetVarBigValueBitErrors='Can set variable with multi-packet value despite bit errors' -setup_SetVarBigValueBitErrors() { - ber=0.00001 - setup - start_dna_server -B $ber - echo_long_message >$TFWTMP/dnatest.in -} -test_SetVarBigValueBitErrors() { - execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in" - assertExitStatus '==' 0 - assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$" - execute $dna -v verbose -B $ber -s $sid -O $TFWTMP/dnatest.out -i 0 -R note - assertExitStatus '==' 0 - assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0" - assert --message='long value read correctly' cmp --quiet $TFWTMP/dnatest.in $TFWTMP/dnatest.out + assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out } runTests "$@"