mirror of
https://github.com/servalproject/serval-dna.git
synced 2024-12-19 13:17:56 +00:00
684735b11c
Ensure that timestamp always advances by at least one second every time the config file is modified, add test case Refactor test defs: detecting started HTTPD is not specific to Rhizome Refactor struct file_meta and associated functions into "io.h" and io.c Add various strbuf helper functions for formatting struct timespec and struct file_meta to assist debug logging
514 lines
16 KiB
Bash
514 lines
16 KiB
Bash
# Common definitions for rhizome test suites.
|
|
# Copyright 2012 The Serval Project, Inc.
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
# Some useful regular expressions. These must work for grep(1) (as basic
|
|
# expressions) and also in sed(1).
|
|
rexp_service='[A-Za-z0-9_]\+'
|
|
rexp_manifestid='[0-9a-fA-F]\{64\}'
|
|
rexp_bundlekey='[0-9a-fA-F]\{64\}'
|
|
rexp_bundlesecret="$rexp_bundlekey"
|
|
rexp_filehash='[0-9a-fA-F]\{128\}'
|
|
rexp_filesize='[0-9]\{1,\}'
|
|
rexp_version='[0-9]\{1,\}'
|
|
rexp_crypt='[01]'
|
|
rexp_date='[0-9]\{1,\}'
|
|
rexp_rowid='[0-9]\{1,\}'
|
|
|
|
assert_manifest_complete() {
|
|
local manifest="$1"
|
|
tfw_cat -v "$manifest"
|
|
assertGrep "$manifest" "^service=$rexp_service\$"
|
|
assertGrep "$manifest" "^id=$rexp_manifestid\$"
|
|
assertGrep "$manifest" "^BK=$rexp_bundlekey\$"
|
|
assertGrep "$manifest" "^date=$rexp_date\$"
|
|
assertGrep "$manifest" "^version=$rexp_version\$"
|
|
assertGrep "$manifest" "^filesize=$rexp_filesize\$"
|
|
if $GREP -q '^filesize=0$' "$manifest"; then
|
|
assertGrep --matches=0 "$manifest" "^filehash="
|
|
else
|
|
assertGrep "$manifest" "^filehash=$rexp_filehash\$"
|
|
fi
|
|
if $GREP -q '^service=file$' "$manifest"; then
|
|
assertGrep "$manifest" "^name="
|
|
fi
|
|
}
|
|
|
|
# Assertion function:
|
|
# - assert that the output of a "servald rhizome list" command exactly describes the given files
|
|
assert_rhizome_list() {
|
|
assertStdoutIs --stderr --line=1 -e '13\n'
|
|
assertStdoutIs --stderr --line=2 -e '_id:service:id:version:date:.inserttime:.author:.fromhere:filesize:filehash:sender:recipient:name\n'
|
|
local exactly=true
|
|
local re__inserttime="$rexp_date"
|
|
local re__fromhere='[01]'
|
|
local re__author="\($rexp_sid\)\{0,1\}"
|
|
local files=0
|
|
local manifestname=
|
|
local arg
|
|
for arg; do
|
|
case "$arg" in
|
|
--fromhere=*) re__fromhere="${arg#*=}";;
|
|
--author=*) re__author="${arg#*=}";;
|
|
--manifest=*) manifestname="${arg#*=}";;
|
|
--and-others) exactly=false;;
|
|
--*) error "unsupported option: $arg";;
|
|
*)
|
|
unpack_manifest_for_grep "$arg" "$manifestname"
|
|
assertStdoutGrep --stderr --matches=1 "^$rexp_rowid:$re_service:$re_manifestid:$re_version:$re_date:$re__inserttime:$re__author:$re__fromhere:$re_filesize:$re_filehash:$re_sender:$re_recipient:$re_name\$"
|
|
let files+=1
|
|
manifestname=
|
|
;;
|
|
esac
|
|
done
|
|
$exactly && assertStdoutLineCount --stderr '==' $(($files + 2))
|
|
rhizome_list_file_count=$(( $(replayStdout | wc -l) - 2 ))
|
|
}
|
|
|
|
rhizome_list_dump() {
|
|
local ncols
|
|
local -a headers
|
|
local readvars=
|
|
read ncols
|
|
local oIFS="$IFS"
|
|
IFS=:
|
|
read -r -a headers
|
|
IFS="$oIFS"
|
|
local hdr
|
|
local colvar
|
|
for hdr in "${headers[@]}"; do
|
|
readvars="$readvars __col_${hdr//[^A-Za-z0-9_]/_}"
|
|
done
|
|
eval local $readvars
|
|
for colvar in $readvars; do
|
|
eval "$colvar=ok"
|
|
done
|
|
local echoargs
|
|
for colname; do
|
|
case $colname in
|
|
manifestid|bundleid) colvar=__col_id;;
|
|
*) colvar="__col_${colname//[^A-Za-z0-9_]/_}";;
|
|
esac
|
|
eval [ -n "\"\${$colvar+ok}\"" ] || error "invalid column name: $colname" || return $?
|
|
echoargs="$echoargs $colname=\"\${$colvar}\""
|
|
done
|
|
IFS=:
|
|
while eval read -r $readvars; do
|
|
eval echo $echoargs
|
|
done
|
|
IFS="$oIFS"
|
|
}
|
|
|
|
assert_stdout_add_file() {
|
|
local manifestname=
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--manifest=*) manifestname="${1#*=}"; shift;;
|
|
--*) error "unsupported option: $1"; return 1;;
|
|
--) shift; break;;
|
|
*) break;;
|
|
esac
|
|
done
|
|
[ $# -ge 1 ] || error "missing filename arg"
|
|
local filename="$1"
|
|
shift
|
|
unpack_manifest_for_grep "$filename" "$manifestname"
|
|
compute_filehash actual_filehash "$filename" actual_filesize
|
|
opt_service=
|
|
opt_manifestid=
|
|
opt_author=
|
|
opt_secret=
|
|
opt_BK=
|
|
opt_filesize=
|
|
opt_name=false
|
|
if replayStdout | $GREP -q '^service:file$'; then
|
|
opt_name=true
|
|
fi
|
|
opt_filehash=true
|
|
if [ "$re_crypt" = 1 ]; then
|
|
opt_filehash=false
|
|
fi
|
|
fieldnames='service|manifestid|author|secret|BK|filesize|filehash|name'
|
|
for arg; do
|
|
case "$arg" in
|
|
!+($fieldnames))
|
|
fieldname="${arg#!}"
|
|
eval opt_$fieldname=false
|
|
;;
|
|
+($fieldnames)=*)
|
|
value="${arg#*=}"
|
|
fieldname="${arg%%=*}"
|
|
assertStdoutGrep --matches=1 "^$fieldname:$value\$"
|
|
;;
|
|
*)
|
|
error "unsupported argument: $arg"
|
|
;;
|
|
esac
|
|
done
|
|
${opt_service:-true} && assertStdoutGrep --matches=1 "^service:$re_service\$"
|
|
${opt_manifestid:-true} && assertStdoutGrep --matches=1 "^manifestid:$re_manifestid\$"
|
|
${opt_author:-true} && assertStdoutGrep --matches=1 "^\.author:$rexp_sid\$"
|
|
${opt_secret:-true} && assertStdoutGrep --matches=1 "^\.secret:$re_secret\$"
|
|
${opt_BK:-true} && assertStdoutGrep --matches=1 "^BK:$re_BK\$"
|
|
${opt_filesize:-true} && assertStdoutGrep --matches=1 "^filesize:$actual_filesize\$"
|
|
if replayStdout | $GREP -q '^filesize:0$'; then
|
|
assertStdoutGrep --matches=0 "^filehash:"
|
|
else
|
|
${opt_filehash:-true} && assertStdoutGrep --matches=1 "^filehash:$actual_filehash\$"
|
|
fi
|
|
${opt_name:-true} && assertStdoutGrep --matches=1 "^name:$re_name\$"
|
|
}
|
|
|
|
assert_stdout_import_bundle() {
|
|
# Output of "import bundle" is the same as "add file" but without the secret
|
|
# or author fields.
|
|
assert_stdout_add_file "$@" '!secret' '!author'
|
|
}
|
|
|
|
unpack_manifest_for_grep() {
|
|
local filename="$1"
|
|
local manifestname="${2:-$filename.manifest}"
|
|
re_service="$rexp_service"
|
|
re_manifestid="$rexp_manifestid"
|
|
re_version="$rexp_version"
|
|
re_date="$rexp_date"
|
|
re_secret="$rexp_bundlesecret"
|
|
re_BK="$rexp_bundlekey"
|
|
re_sender="\($rexp_sid\)\{0,1\}"
|
|
re_recipient="\($rexp_sid\)\{0,1\}"
|
|
re_filesize="$rexp_filesize"
|
|
re_filehash="\($rexp_filehash\)\{0,1\}"
|
|
re_name=$(escape_grep_basic "${filename##*/}")
|
|
if [ -e "$manifestname" ]; then
|
|
re_filesize=$($SED -n -e '/^filesize=/s///p' "$manifestname")
|
|
if [ "$filesize" = 0 ]; then
|
|
re_filehash=
|
|
else
|
|
re_filehash=$($SED -n -e '/^filehash=/s///p' "$manifestname")
|
|
fi
|
|
re_secret="$rexp_bundlesecret"
|
|
re_service=$($SED -n -e '/^service=/s///p' "$manifestname")
|
|
re_service=$(escape_grep_basic "$re_service")
|
|
re_manifestid=$($SED -n -e '/^id=/s///p' "$manifestname")
|
|
re_version=$($SED -n -e '/^version=/s///p' "$manifestname")
|
|
re_date=$($SED -n -e '/^date=/s///p' "$manifestname")
|
|
re_crypt=$($SED -n -e '/^crypt=/s///p' "$manifestname")
|
|
re_name=$($SED -n -e '/^name=/s///p' "$manifestname")
|
|
re_name=$(escape_grep_basic "$re_name")
|
|
re_BK=$($SED -n -e '/^BK=/s///p' "$manifestname")
|
|
re_sender=$($SED -n -e '/^sender=/s///p' "$manifestname")
|
|
re_recipient=$($SED -n -e '/^recipient=/s///p' "$manifestname")
|
|
fi
|
|
}
|
|
|
|
assert_manifest_newer() {
|
|
local manifest1="$1"
|
|
local manifest2="$2"
|
|
# The new manifest must have a higher version than the original.
|
|
extract_manifest_version oldversion "$manifest1"
|
|
extract_manifest_version newversion "$manifest2"
|
|
assert [ $newversion -gt $oldversion ]
|
|
# The new manifest must have a different filehash from the original.
|
|
extract_manifest_filehash oldfilehash "$manifest1"
|
|
extract_manifest_filehash newfilehash "$manifest2"
|
|
assert [ $oldfilehash != $newfilehash ]
|
|
}
|
|
|
|
strip_signatures() {
|
|
for file; do
|
|
cat -v "$file" | $SED -e '/^^@/,$d' >"tmp.$file" && mv -f "tmp.$file" "$file"
|
|
done
|
|
}
|
|
|
|
extract_stdout_manifestid() {
|
|
extract_stdout_keyvalue "$1" manifestid "$rexp_manifestid"
|
|
}
|
|
|
|
extract_stdout_version() {
|
|
extract_stdout_keyvalue "$1" version "$rexp_version"
|
|
}
|
|
|
|
extract_stdout_author() {
|
|
extract_stdout_keyvalue "$1" .author "$rexp_author"
|
|
}
|
|
|
|
extract_stdout_secret() {
|
|
extract_stdout_keyvalue "$1" .secret "$rexp_bundlesecret"
|
|
}
|
|
|
|
extract_stdout_rowid() {
|
|
extract_stdout_keyvalue "$1" .rowid "$rexp_rowid"
|
|
}
|
|
|
|
extract_stdout_inserttime() {
|
|
extract_stdout_keyvalue "$1" .inserttime "$rexp_date"
|
|
}
|
|
|
|
extract_stdout_BK() {
|
|
extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
|
|
}
|
|
|
|
extract_stdout_date() {
|
|
extract_stdout_keyvalue "$1" date "$rexp_date"
|
|
}
|
|
|
|
extract_stdout_filesize() {
|
|
extract_stdout_keyvalue "$1" filesize "$rexp_filesize"
|
|
}
|
|
|
|
extract_stdout_filehash() {
|
|
extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
|
|
}
|
|
|
|
extract_stdout_crypt() {
|
|
extract_stdout_keyvalue "$1" crypt "$rexp_crypt"
|
|
}
|
|
|
|
extract_manifest() {
|
|
local __var="$1"
|
|
local __manifestfile="$2"
|
|
local __label="$3"
|
|
local __rexp="${4:-[^=]*}"
|
|
local __value=$($SED -n -e "/^$__label=$__rexp\$/s/^$__label=//p" "$__manifestfile")
|
|
[ -n "$__var" ] && eval $__var="\$__value"
|
|
}
|
|
|
|
assert_manifest_fields() {
|
|
local manifestfile="$1"
|
|
shift
|
|
assert --message="manifest file $manifestfile is readable" [ -r "$manifestfile" ]
|
|
[ $# -gt 0 ] || error "missing arguments"
|
|
local arg label value
|
|
for arg; do
|
|
case "$arg" in
|
|
!*)
|
|
assertGrep \
|
|
--matches=0 \
|
|
--message="$manifestfile contains no '$arg=' line" \
|
|
--dump-on-fail="$manifestfile" \
|
|
"$manifestfile" "^$arg="
|
|
;;
|
|
*=*)
|
|
label="${arg%%=*}"
|
|
value="${arg#*=}"
|
|
local mvalue
|
|
extract_manifest mvalue "$manifestfile" "$label"
|
|
assert \
|
|
--message="$manifestfile contains '$label=$value' line" \
|
|
--dump-on-fail="$manifestfile" \
|
|
[ "$mvalue" = "$value" ]
|
|
;;
|
|
*)
|
|
assertGrep \
|
|
--message="$manifestfile contains '$arg=' line" \
|
|
--dump-on-fail="$manifestfile" \
|
|
"$manifestfile" "^$arg="
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
extract_manifest_service() {
|
|
extract_manifest "$1" "$2" service "$rexp_service"
|
|
}
|
|
|
|
extract_manifest_id() {
|
|
extract_manifest "$1" "$2" id "$rexp_manifestid"
|
|
}
|
|
|
|
extract_manifest_BK() {
|
|
extract_manifest "$1" "$2" BK "$rexp_bundlekey"
|
|
}
|
|
|
|
extract_manifest_filesize() {
|
|
extract_manifest "$1" "$2" filesize "$rexp_filesize"
|
|
}
|
|
|
|
extract_manifest_filehash() {
|
|
extract_manifest "$1" "$2" filehash "$rexp_filehash"
|
|
}
|
|
|
|
extract_manifest_name() {
|
|
extract_manifest "$1" "$2" name ".*"
|
|
}
|
|
|
|
extract_manifest_version() {
|
|
extract_manifest "$1" "$2" version "$rexp_version"
|
|
}
|
|
|
|
extract_manifest_date() {
|
|
extract_manifest "$1" "$2" date "$rexp_date"
|
|
}
|
|
|
|
extract_manifest_crypt() {
|
|
extract_manifest "$1" "$2" crypt "$rexp_crypt"
|
|
}
|
|
|
|
compute_filehash() {
|
|
local _filehashvar="$1"
|
|
local _file="$2"
|
|
local _filesizevar="$3"
|
|
local _hash=
|
|
local _size=0
|
|
if [ -s "$_file" ]; then
|
|
local _hash=$($servald rhizome hash file "$_file") || error "$servald failed to compute file hash"
|
|
[ -z "${_hash//[0-9a-fA-F]/}" ] || error "file hash contains non-hex: $_hash"
|
|
[ "${#_hash}" -eq 128 ] || error "file hash incorrect length: $_hash"
|
|
local _size=$(( $(cat "$filename" | wc -c) + 0 ))
|
|
fi
|
|
[ -n "$_filehashvar" ] && eval $_filehashvar="\$_hash"
|
|
[ -n "$_filesizevar" ] && eval $_filesizevar="\$_size"
|
|
}
|
|
|
|
rhizome_http_server_started() {
|
|
local logvar=LOG${1#+}
|
|
$GREP 'HTTP SERVER START.*port=[0-9].*services=[^ ]*\<Rhizome\>' "${!logvar}"
|
|
}
|
|
|
|
get_rhizome_server_port() {
|
|
get_servald_http_server_port "$@"
|
|
}
|
|
|
|
# Predicate function:
|
|
# - return true if the file bundles identified by args BID[:VERSION] has been
|
|
# received by all the given instances args +I
|
|
# - does this by examining the server log files of the respective instances
|
|
# for tell-tale INFO messages
|
|
bundle_received_by() {
|
|
local -a rexps bundles
|
|
local restart=true
|
|
local arg bid version rexp bundle bundlefile i
|
|
local ret=0
|
|
for arg; do
|
|
case "$arg" in
|
|
+([0-9A-F])?(:+([0-9])))
|
|
$restart && rexps=() bundles=()
|
|
restart=false
|
|
bid="${arg%%:*}"
|
|
matches_rexp "$rexp_manifestid" "$bid" || error "invalid bundle ID: $bid" || return $?
|
|
bundles+=("$arg")
|
|
if [ "$bid" = "$arg" ]; then
|
|
rexps+=("RHIZOME ADD MANIFEST service=.* bid=$bid")
|
|
else
|
|
version="${arg#*:}"
|
|
rexps+=("RHIZOME ADD MANIFEST service=.* bid=$bid version=$version")
|
|
fi
|
|
;;
|
|
+[A-Z])
|
|
tfw_nolog push_and_set_instance $arg || return $?
|
|
tfw_nolog assert_servald_server_status running
|
|
for ((i = 0; i < ${#bundles[*]}; ++i)); do
|
|
bundle="${bundles[$i]}"
|
|
rexp="${rexps[$i]}"
|
|
bundledir="$instance_dir/cache/bundles_received"
|
|
bundlefile="$bundledir/$bundle"
|
|
if [ ! -s "$bundlefile" ]; then
|
|
[ -d "$bundledir" ] || mkdir -p "$bundledir" || error "mkdir failed"
|
|
if grep "$rexp" "$instance_servald_log" >"$bundlefile"; then
|
|
tfw_log "bundle $bundle received by instance +$instance_name"
|
|
else
|
|
ret=1
|
|
fi
|
|
fi
|
|
done
|
|
restart=true
|
|
pop_instance
|
|
;;
|
|
--stderr)
|
|
for ((i = 0; i < ${#bundles[*]}; ++i)); do
|
|
bundle="${bundles[$i]}"
|
|
rexp="${rexps[$i]}"
|
|
if replayStderr | grep "$rexp" >/dev/null; then
|
|
tfw_log "bundle $bundle received by ($executed)"
|
|
else
|
|
ret=1
|
|
fi
|
|
done
|
|
restart=true
|
|
;;
|
|
*)
|
|
error "invalid argument: $arg"
|
|
return 1
|
|
;;
|
|
esac
|
|
done
|
|
return $ret
|
|
}
|
|
|
|
extract_manifest_vars() {
|
|
local manifest="${1?}"
|
|
extract_manifest_id BID "$manifest"
|
|
extract_manifest_version VERSION "$manifest"
|
|
extract_manifest_filesize FILESIZE "$manifest"
|
|
FILEHASH=
|
|
if [ "$FILESIZE" != '0' ]; then
|
|
extract_manifest_filehash FILEHASH "$manifest"
|
|
fi
|
|
}
|
|
|
|
rhizome_add_file() {
|
|
local name="$1"
|
|
local size="${2:-64}"
|
|
rhizome_add_files --size="$size" "$name"
|
|
extract_manifest_vars "$name.manifest"
|
|
}
|
|
|
|
rhizome_add_files() {
|
|
local size=64
|
|
local sidvar="SID$instance_name"
|
|
local -a names=()
|
|
for arg; do
|
|
case "$arg" in
|
|
--size=*)
|
|
size="${arg##*=}"
|
|
;;
|
|
*)
|
|
local name="$arg"
|
|
[ -e "$name" ] || create_file "$name" $size
|
|
executeOk_servald rhizome add file "${!sidvar}" "$name" "$name.manifest"
|
|
names+=("$name")
|
|
esac
|
|
done
|
|
executeOk_servald rhizome list
|
|
assert_rhizome_list --fromhere=1 --author="${!sidvar}" "${names[@]}" --and-others
|
|
}
|
|
|
|
rhizome_update_file() {
|
|
local orig_name="$1"
|
|
local new_name="$2"
|
|
[ -e "$new_name" ] || echo 'File $new_name' >"$new_name"
|
|
local sidvar="SID$instance_name"
|
|
[ "$new_name" != "$orig_name" ] && cp "$orig_name.manifest" "$new_name.manifest"
|
|
$SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d;/^version=/d;/^name=/d' "$new_name.manifest"
|
|
executeOk_servald rhizome add file "${!sidvar}" "$new_name" "$new_name.manifest"
|
|
executeOk_servald rhizome list
|
|
assert_rhizome_list --fromhere=1 "$new_name"
|
|
extract_manifest_vars "$new_name.manifest"
|
|
}
|
|
|
|
assert_rhizome_received() {
|
|
[ $# -ne 0 ] || error "missing arguments"
|
|
local name
|
|
local _id
|
|
for name; do
|
|
if [ -s "$name" ]; then
|
|
extract_manifest_id _id "$name.manifest"
|
|
executeOk_servald rhizome extract file "$_id" extracted
|
|
assert cmp "$name" extracted
|
|
fi
|
|
done
|
|
}
|