Split dna bit error tests into separate script

- Put common utility functions into testdefs.sh
 - Add abspath() and realpath() functions to test framework
 - Test framework now executes all tests with working directory in a temporary
   directory, to avoid pollution of caller's working directory
 - Improve test log verbosity
 - Name test log files by test script name
This commit is contained in:
Andrew Bettison 2012-04-10 12:41:15 +09:30
parent f5acb301d8
commit ce5dc011b8
5 changed files with 284 additions and 161 deletions

2
.gitignore vendored
View File

@ -16,4 +16,4 @@ nacl/naclinc.txt
nacl/nacllib.txt
serval.c
dna
test.log
test.*.log

73
testdefs.sh Normal file
View File

@ -0,0 +1,73 @@
# Common definitions for all test suites in test/*
this=$(abspath "${BASH_SOURCE[0]}")
# Utility function for setting up a fixture with a DNA server process:
# - Ensure that no dna processes are running
# - Start a dna server process
# - Ensure that it is still running after one second
start_dna_server() {
check_no_dna_processes
# Start DNA server
$dna -v verbose -f $hlr_dat -S 1 -n "$@" >$DNATMP/dna.log 2>&1 &
sleep 1
pid=$(ps -u$UID | awk '$4 == "dna" {print $1}')
if [ -z "$pid" ]; then
echo "dna server did not start"
tfw_cat --header=dna.log $SERVALINSTANCE_PATH/dna.log
fail
fi
if ! [ -s $SERVALINSTANCE_PATH/serval.pid ] && kill -0 $(cat $SERVALINSTANCE_PATH/serval.pid); then
echo "serval.pid was not created"
tfw_cat --header=dna.log $SERVALINSTANCE_PATH/dna.log
fail
fi
echo "# Started dna server process, pid=$pid"
}
# Utility function for tearing down DNA fixtures:
# - If a dna server process is running, then kill it
# - Cat any dna log file into the test log
# - Ensure that no dna processes are running
stop_dna_server() {
if [ -s $SERVALINSTANCE_PATH/serval.pid ]; then
local pid=$(cat $SERVALINSTANCE_PATH/serval.pid)
if kill $pid; then
echo "# Killed dna process pid=$pid"
else
error "# Dna process pid=$pid was not running"
fi
fi
if [ -s $DNATMP/dna.log ]; then
tfw_cat --header=dna.log $DNATMP/dna.log
fi
check_no_dna_processes
}
# Utility function for creating DNA fixtures:
# - Create a temporary directory to contain all dna-related files
# - set $dna and $hlr_dat variables
# - set SERVALINSTANCE_PATH environment variable
setup_dna() {
dna=$(abspath "${this%/*}/dna") # The DNA executable under test
if ! [ -x "$dna" ]; then
error "dna executable not present: $dna"
return 1
fi
export DNATMP=$TFWTMP/dnatmp
mkdir $DNATMP
export SERVALINSTANCE_PATH=$DNATMP
hlr_dat=$SERVALINSTANCE_PATH/hlr.dat
}
# Utility function for managing DNA fixtures:
# - Ensure there are no existing DNA server processes
check_no_dna_processes() {
local pids=$(ps -u$UID | awk '$4 == "dna" {print $1}')
if [ -n "$pids" ]; then
error "cannot run test: dna process already running with pid: $pids"
return 1
fi
echo "# No other dna processes running for uid=$UID"
return 0
}

View File

@ -63,7 +63,10 @@ runTests() {
_tfw_checkBashVersion
_tfw_trace=false
_tfw_verbose=false
_tfw_logfile=test.log
_tfw_invoking_script=$(abspath "${BASH_SOURCE[1]}")
_tfw_suite_name="${_tfw_invoking_script##*/}"
_tfw_cwd=$(abspath "$PWD")
_tfw_logfile="$_tfw_cwd/test.$_tfw_suite_name.log"
local allargs="$*"
local filter=
while [ $# -ne 0 ]; do
@ -86,22 +89,23 @@ runTests() {
# Iterate through all test cases.
local testcount=0
local passcount=0
for testName in `_tfw_findTests`
local testName
for testName in `_tfw_find_tests`
do
_tfw_testname=$testName
if [ -z "$filter" -o "${testName#$filter}" != "$testName" ]; then
_tfw_test_name="$testName"
if [ -z "$filter" -o "${_tfw_test_name#$filter}" != "$_tfw_test_name" ]; then
let testcount=testcount+1
(
local docvar="doc_$testName"
_tfw_echo -n "$testcount. ${!docvar:-$testName}..."
local docvar="doc_$_tfw_test_name"
_tfw_echo -n "$testcount. ${!docvar:-$_tfw_test_name}..."
trap '_tfw_status=$?; _tfw_teardown; exit $_tfw_status' 0 1 2 15
_tfw_result=ERROR
_tfw_setup
_tfw_result=FAIL
_tfw_phase=testcase
echo "# call test_$testName()"
echo "# call test_$_tfw_test_name()"
$_tfw_trace && set -x
test_$testName
test_$_tfw_test_name
_tfw_result=PASS
exit 0
)
@ -133,6 +137,18 @@ teardown() {
# The following functions are provided to facilitate writing test cases and
# fixtures.
# Echo the absolute path (containing symlinks if given) of the given
# file/directory, which does not have to exist or even be accessible.
abspath() {
_tfw_abspath -L "$1"
}
# Echo the absolute path (resolving all symlinks) of the given file/directory,
# which does not have to exist or even be accessible.
realpath() {
_tfw_abspath -P "$1"
}
execute() {
_tfw_last_argv0="$1"
echo "# execute $*"
@ -141,7 +157,7 @@ execute() {
echo "# Exit status = $_tfw_exitStatus"
if grep --quiet --invert-match --regexp='^realtime=[0-9.]*;usertime=[0-9.]*;systime=[0-9.]*$' $_tfw_tmp/times; then
echo "# Times file contains spurious data:"
_tfw_cat $_tfw_tmp/times
tfw_cat --header=times $_tfw_tmp/times
if [ $_tfw_exitStatus -eq 0 ]; then
_tfw_exitStatus=255
echo "# Deeming exit status of command to be $_tfw_exitStatus"
@ -202,6 +218,33 @@ fatal() {
_tfw_fatalexit
}
tfw_cat() {
local $header
for file; do
case $file in
--stdout)
echo "#--- ${header:-stdout of ${_tfw_last_argv0##*/}} ---"
/bin/cat $_tfw_tmp/stdout
echo "#---"
header=
;;
--stderr)
echo "#--- ${header:-stderr of ${_tfw_last_argv0##*/}} ---"
/bin/cat $_tfw_tmp/stderr
echo "#---"
header=
;;
--header=*) header="${1#*=}";;
*)
echo "#--- ${header:-$file} ---"
/bin/cat "$file"
echo "#---"
header=
;;
esac
done
}
assertExitStatus() {
_tfw_getopts_assert exitstatus "$@"
shift $_tfw_getopts_shift
@ -247,6 +290,43 @@ assertStderrGrep() {
# Internal (private) functions that are not to be invoked directly from test
# scripts.
# Echo the absolute path of the given path, using only Bash builtins.
_tfw_abspath() {
cdopt=-L
if [ $# -gt 1 -a "${1:0:1}" = - ]; then
cdopt="$1"
shift
fi
case "$1" in
*/)
builtin echo $(_tfw_abspath $cdopt "${1%/}")/
;;
/*/*)
if [ -d "$1" ]; then
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
else
builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}"
fi
;;
/*)
echo "$1"
;;
*/*)
if [ -d "$1" ]; then
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
else
builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}"
fi
;;
. | ..)
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
;;
*)
(CDPATH= builtin cd $cdopt . && builtin echo "$PWD/$1")
;;
esac
}
_tfw_setup() {
_tfw_phase=setup
_tfw_tmp=/tmp/_tfw-$$
@ -262,18 +342,19 @@ _tfw_setup() {
fi
export TFWTMP=$_tfw_tmp/tmp
/bin/mkdir $TFWTMP
cd $TFWTMP
echo '# SETUP'
case `type -t setup_$testName` in
case `type -t setup_$_tfw_test_name` in
function)
echo "# call setup_$testName()"
echo "# call setup_$_tfw_test_name()"
$_tfw_trace && set -x
setup_$testName $testName
setup_$_tfw_test_name $_tfw_test_name
set +x
;;
*)
echo "# call setup($testName)"
echo "# call setup($_tfw_test_name)"
$_tfw_trace && set -x
setup $testName
setup $_tfw_test_name
set +x
;;
esac
@ -283,23 +364,23 @@ _tfw_setup() {
_tfw_teardown() {
_tfw_phase=teardown
echo '# TEARDOWN'
case `type -t teardown_$testName` in
case `type -t teardown_$_tfw_test_name` in
function)
echo "# call teardown_$testName()"
echo "# call teardown_$_tfw_test_name()"
$_tfw_trace && set -x
teardown_$testName
teardown_$_tfw_test_name
set +x
;;
*)
echo "# call teardown($testName)"
echo "# call teardown($_tfw_test_name)"
$_tfw_trace && set -x
teardown $testName
teardown $_tfw_test_name
set +x
;;
esac
echo '# END TEARDOWN'
{
local banner="==================== $_tfw_testname ===================="
local banner="==================== $_tfw_test_name ===================="
echo "$banner"
echo "TEST RESULT: $_tfw_result"
echo '++++++++++ log.stdout ++++++++++'
@ -457,28 +538,6 @@ _tfw_echoerr() {
fi
}
_tfw_cat() {
for file; do
case $file in
--stdout)
echo "--- stdout of ${_tfw_last_argv0##*/} ---"
/bin/cat $_tfw_tmp/stdout
echo "---"
;;
--stderr)
echo "--- stderr of ${_tfw_last_argv0##*/} ---"
/bin/cat $_tfw_tmp/stderr
echo "---"
;;
*)
echo "--- $file ---"
/bin/cat "$file"
echo "---"
;;
esac
done
}
_tfw_checkBashVersion() {
case $BASH_VERSION in
[56789].* | 4.[23456789].*) ;;
@ -487,7 +546,7 @@ _tfw_checkBashVersion() {
esac
}
_tfw_findTests() {
_tfw_find_tests() {
builtin declare -F | sed -n -e '/^declare -f test_..*/s/^declare -f test_//p' | sort
}
@ -519,8 +578,8 @@ _tfw_backtrace() {
_tfw_failexit() {
# When exiting a test case due to a failure, log any diagnostic output that
# has been requested.
$_tfw_dump_stdout_on_fail && _tfw_cat --stdout
$_tfw_dump_stderr_on_fail && _tfw_cat --stderr
$_tfw_dump_stdout_on_fail && tfw_cat --stdout
$_tfw_dump_stderr_on_fail && tfw_cat --stderr
# A failure during setup or teardown is treated as an error.
case $_tfw_phase in
testcase) exit 1;;
@ -531,7 +590,8 @@ _tfw_failexit() {
# An "error" event prevents a test from running, so it neither passes nor fails.
# Other tests may still proceed.
_tfw_error() {
_tfw_errormsg() {
[ $# -eq 0 ] && set -- "(no message)"
local -i up=1
while true; do
case ${FUNCNAME[$up]} in
@ -539,7 +599,11 @@ _tfw_error() {
*) break;;
esac
done
echo "ERROR in ${FUNCNAME[$up]}: $*" >&2
echo "ERROR in ${FUNCNAME[$up]}: $*"
}
_tfw_error() {
_tfw_errormsg "$@" >&2
_tfw_backtrace >&2
_tfw_errorexit
}

82
tests/dna_biterrors Executable file
View File

@ -0,0 +1,82 @@
#!/bin/bash
# Tests for Serval DNA bit error performance.
#
# Copyright 2012 Paul Gardner-Stephen
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
# Default teardown function
teardown() {
stop_dna_server
}
# Test case, transcribed from ../testdna
doc_BitErrorCreateSID='Create new SID despite bit errors'
setup_BitErrorCreateSID() {
ber=0.001
setup_dna
start_dna_server -B $ber
}
test_BitErrorCreateSID() {
local iterations=10
local i=0
local fails=0
local totaltime_ms=0
local maxtime_ms=0
while [ $i -lt $iterations ]; do
execute $dna -B $ber -d 0427679796 -C
assertExitStatus '==' 0
sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
let totaltime_ms=totaltime_ms+realtime_ms
[ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms
if [ -z "$sid" ]; then
let fails=fails+1
echo "# iteration $i: no sid after ${realtime_ms} ms"
else
echo "# iteration $i: sid=$sid after ${realtime_ms} ms"
fi
let i=i+1
done
let meantime_ms=totaltime_ms/iterations
echo "# average time = $meantime_ms ms"
echo "# maximum time = $maxtime_ms ms"
assertExpr --message='packet loss was properly injected' $meantime_ms '>' 25
assertExpr --message="reliable at BER=$ber packet loss" $fails '<=' 4
assertExpr --message="timeout works at BER=$ber packet loss" $totaltime_ms '<=' 3000 '&&' $maxtime_ms '<=' 330
}
# Test case, transcribed from ../testdna
doc_SetVarBigValueBitErrors='Can set variable with multi-packet value despite bit errors'
setup_SetVarBigValueBitErrors() {
ber=0.00001
setup_dna
start_dna_server -B $ber
echo_long_message >$DNATMP/dnatest.in
}
test_SetVarBigValueBitErrors() {
execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
assertExitStatus '==' 0
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -v verbose -B $ber -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertExitStatus '==' 0
assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out
}
runTests "$@"

View File

@ -1,6 +1,7 @@
#!/bin/bash
# Test script for Serval DNA executable.
# Tests for Serval DNA HLR operations.
#
# Copyright 2012 Paul Gardner-Stephen
#
# This program is free software; you can redistribute it and/or
@ -18,57 +19,13 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
source "${0%/*}/../testframework.sh"
dna="${0%/*}/../dna"
# Default setup function
setup() {
export SERVALINSTANCE_PATH=$TFWTMP/dna
mkdir $SERVALINSTANCE_PATH
hlr_dat=$SERVALINSTANCE_PATH/hlr.dat
}
source "${0%/*}/../testdefs.sh"
# Default teardown function
teardown() {
stop_dna_server
}
# Utility function
start_dna_server() {
# Ensure there are no existing DNA servers
pids=$(ps -u$UID | awk '$4 == "dna" {print $1}')
if [ -n "$pids" ]; then
error "cannot run test: dna process already running with pid: $pids"
fi
# Start DNA server
$dna -v verbose -f $hlr_dat -S 1 -n "$@" >$TFWTMP/dna.log 2>&1 &
sleep 1
pids=$(ps -u$UID | awk '$4 == "dna" {print $1}')
if [ -z "$pids" ]; then
echo "dna server did not start"
echo "--- dna.log ---"
cat $SERVALINSTANCE_PATH/dna.log
echo "---"
fail
fi
if ! [ -s $SERVALINSTANCE_PATH/serval.pid ] && kill -0 $(cat $SERVALINSTANCE_PATH/serval.pid); then
echo "serval.pid was not created"
echo "--- dna.log ---"
cat $SERVALINSTANCE_PATH/dna.log
echo "---"
fail
fi
}
# Utility function
stop_dna_server() {
[ -s $SERVALINSTANCE_PATH/serval.pid ] && kill $(cat $SERVALINSTANCE_PATH/serval.pid)
if [ -s $TFWTMP/dna.log ]; then
echo '--- dna.log ---'
cat $TFWTMP/dna.log
echo '---'
fi
}
# Utility function
create_sid() {
execute $dna "$@" -d 0427679796 -C
@ -95,7 +52,7 @@ echo_long_message() {
# Test case, transcribed from ../testdna
doc_CreateSID='Create new SID'
setup_CreateSID() {
setup
setup_dna
start_dna_server
}
test_CreateSID() {
@ -107,7 +64,7 @@ test_CreateSID() {
# Test case, transcribed from ../testdna
doc_GetShortVar='Get short variable'
setup_GetShortVar() {
setup
setup_dna
start_dna_server
create_sid
}
@ -122,7 +79,7 @@ test_GetShortVar() {
# Test case, transcribed from ../testdna
doc_SetShortVar='Set short variable'
setup_SetShortVar() {
setup
setup_dna
start_dna_server
create_sid
}
@ -135,7 +92,7 @@ test_SetShortVar() {
# Test case, transcribed from ../testdna
doc_SetGetShortVar='Set then get short variable'
setup_SetGetShortVar() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
@ -150,14 +107,14 @@ test_SetGetShortVar() {
# Test case, transcribed from ../testdna
doc_GetShortVarToFile='Get variable into a file'
setup_GetShortVarToFile() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
}
test_GetShortVarToFile() {
echo "WARNING: Known issue: output to file does not work with DID lists"
dnatest_dat=$TFWTMP/dnatest.dat
dnatest_dat=$DNATMP/dnatest.dat
execute $dna -s $sid -O $dnatest_dat -i 0 -R note
assertExitStatus '==' 0
assertRealTime '<' 0.5
@ -168,7 +125,7 @@ test_GetShortVarToFile() {
# Test case, transcribed from ../testdna
doc_SetMultiShortVar='Set multiple instances of a short variable'
setup_SetMultiShortVar() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
@ -187,7 +144,7 @@ test_SetMultiShortVar() {
# Test case, transcribed from ../testdna
doc_GetMultiShortVar='Get multiple instances of a short variable'
setup_GetMultiShortVar() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
@ -208,7 +165,7 @@ test_GetMultiShortVar() {
# Test case, transcribed from ../testdna
doc_OverwriteVarNoUpdate='Set does not overwrite variable'
setup_OverwriteVarNoUpdate() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
@ -227,7 +184,7 @@ test_OverwriteVarNoUpdate() {
# Test case, transcribed from ../testdna
doc_UpdateVar='Update overwrites variable'
setup_UpdateVar() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
@ -246,75 +203,22 @@ test_UpdateVar() {
# Test case, transcribed from ../testdna
doc_UpdateVarBigValue='Can set variable with multi-packet value'
setup_UpdateVarBigValue() {
setup
setup_dna
start_dna_server
create_sid
set_short_var
echo_long_message >$TFWTMP/dnatest.in
echo_long_message >$DNATMP/dnatest.in
}
test_UpdateVarBigValue() {
execute $dna -s $sid -i 0 -U note="@dnatest.in"
assertExitStatus '==' 0
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -v verbose -s $sid -O $TFWTMP/dnatest.out -i 0 -R note
execute $dna -v verbose -s $sid -O $DNATMP/dnatest.out -i 0 -R note
assertExitStatus '==' 0
assertRealTime '<' 0.5
assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
assert --message='long value read correctly' cmp --quiet $TFWTMP/dnatest.in $TFWTMP/dnatest.out
}
# Test case, transcribed from ../testdna
doc_BitErrorCreateSID='Create new SID despite bit errors'
setup_BitErrorCreateSID() {
ber=0.001
setup
start_dna_server -B $ber
}
test_BitErrorCreateSID() {
local iterations=10
local i=0
local fails=0
local totaltime_ms=0
local maxtime_ms=0
while [ $i -lt $iterations ]; do
execute $dna -B $ber -d 0427679796 -C
assertExitStatus '==' 0
sid=`replayStdout | grep '^OK:' | cut -f2 -d:`
let totaltime_ms=totaltime_ms+realtime_ms
[ $realtime_ms -gt $maxtime_ms ] && maxtime_ms=$realtime_ms
if [ -z "$sid" ]; then
let fails=fails+1
echo "# iteration $i: no sid after ${realtime_ms} ms"
else
echo "# iteration $i: sid=$sid after ${realtime_ms} ms"
fi
let i=i+1
done
let meantime_ms=totaltime_ms/iterations
echo "# average time = $meantime_ms ms"
echo "# maximum time = $maxtime_ms ms"
assertExpr --message='packet loss was properly injected' $meantime_ms '>' 25
assertExpr --message="reliable at BER=$ber packet loss" $fails '<=' 4
assertExpr --message="timeout works at BER=$ber packet loss" $totaltime_ms '<=' 3000 '&&' $maxtime_ms '<=' 330
}
# Test case, transcribed from ../testdna
doc_SetVarBigValueBitErrors='Can set variable with multi-packet value despite bit errors'
setup_SetVarBigValueBitErrors() {
ber=0.00001
setup
start_dna_server -B $ber
echo_long_message >$TFWTMP/dnatest.in
}
test_SetVarBigValueBitErrors() {
execute $dna -B $ber -s $sid -i 0 -W note="@dnatest.in"
assertExitStatus '==' 0
assertStdoutGrep --matches=1 --message='variable write confirmed' "^WROTE:$sid$"
execute $dna -v verbose -B $ber -s $sid -O $TFWTMP/dnatest.out -i 0 -R note
assertExitStatus '==' 0
assertStdoutGrep --matches=1 --message='variable new value returned' "^NOTE:$sid:[0-9]\+:0"
assert --message='long value read correctly' cmp --quiet $TFWTMP/dnatest.in $TFWTMP/dnatest.out
assert --message='long value read correctly' cmp --quiet $DNATMP/dnatest.in $DNATMP/dnatest.out
}
runTests "$@"