KNELPerf v2 wave 1: Ultix scheduler breakout + loop skeleton + reports (#826)
ci / audit (push) Failing after 26s
ci / audit (push) Failing after 26s
- scheduler/: day/night engine (conf-driven redesign), psi/perfsnap collectors, proxmox-ctl, systemd timer templates — ported from ~/projects/ultix per the breakout map in scheduler/README.md - loop/: perf-loop driver (baseline/audit/tweak/rebaseline), cpu-bench (new CPU leg), baseline-diff (tolerance compare) — smoke-tested locally - docs/ARCHITECTURE.md: full program design (loop, resource groups, VM/spindle balancing, k8s-vs-Slurm, beszel+RAPL/iDRAC telemetry spine, workload classes, solar/HA + two-site power economics) - docs/report-amt-power-telemetry.md + docs/report-moonlight-desktop.md Redmine: https://projects.knownelement.com/issues/826#note-2
This commit is contained in:
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# KNELPerf baseline diff — compares two baseline bundles (or live-audits one).
|
||||
#
|
||||
# baseline-diff.sh <dirA> <dirB> compare two stored bundles
|
||||
# Values are TSV metric/value/unit rows from cpu-bench.tsv. Numeric metrics
|
||||
# compare with a tolerance (default 10%); non-numeric metrics compare for
|
||||
# equality (governor flips are exactly what we want to catch).
|
||||
set -euo pipefail
|
||||
TOL_PCT=${KNELPERF_DIFF_TOL:-25} # 1-thread openssl on a busy host swings >10% run-to-run
|
||||
[ $# = 2 ] || { echo "usage: $0 <dirA> <dirB>" >&2; exit 2; }
|
||||
A=$1; B=$2
|
||||
[ -r "$A/cpu-bench.tsv" ] || { echo "no cpu-bench.tsv in $A" >&2; exit 3; }
|
||||
[ -r "$B/cpu-bench.tsv" ] || { echo "no cpu-bench.tsv in $B" >&2; exit 3; }
|
||||
|
||||
join_tsv() { awk -F'\t' 'NR==FNR{a[$1]=$2;next} {print $1"\t"a[$1]"\t"$2"\t"$3}' "$1" "$2"; }
|
||||
|
||||
# shellcheck disable=SC2034 # exit-status reserved for future strictness
|
||||
status=0
|
||||
join_tsv "$A/cpu-bench.tsv" "$B/cpu-bench.tsv" | while IFS=$'\t' read -r metric va vb unit; do
|
||||
[ -n "$va" ] && [ -n "$vb" ] || continue
|
||||
if [ "$va" = "$vb" ]; then
|
||||
printf 'OK %-28s %s = %s %s\n' "$metric" "$va" "$vb" "$unit"
|
||||
elif [ "$va" = "${va#[-+0-9.]}" ] || [ "$vb" = "${vb#[-+0-9.]}" ]; then
|
||||
printf 'DRIFT %-28s %s -> %s %s\n' "$metric" "$va" "$vb" "$unit"
|
||||
else
|
||||
delta=$(awk -v a="$va" -v b="$vb" 'BEGIN{d=(b-a)/((a==0)?1:a)*100; printf "%+.1f", d}')
|
||||
flag=$(awk -v d="${delta#+}" -v t="$TOL_PCT" 'BEGIN{m=d<0?-d:d; if(m>t) print "DELTA"; else print "OK"}')
|
||||
if [ "$flag" = OK ]; then
|
||||
printf 'OK %-28s %s -> %s (%s%%) %s\n' "$metric" "$va" "$vb" "$delta" "$unit"
|
||||
else
|
||||
printf 'DELTA %-28s %s -> %s (%s%%) %s\n' "$metric" "$va" "$vb" "$delta" "$unit"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo "tolerance: ${TOL_PCT}% (set KNELPERF_DIFF_TOL to change)"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
# KNELPerf CPU benchmark — the missing CPU leg of the perf loop (#826).
|
||||
# Runs ON the target host (or locally). Zero-install friendly: uses openssl
|
||||
# (always present) plus sysbench when available. Output = TSV tag/value.
|
||||
#
|
||||
# Design notes:
|
||||
# - openssl speed = single+multi core crypto throughput (stable, comparable)
|
||||
# - sysbench cpu (if installed) = integer event rate
|
||||
# - /proc-derived context: governor, PSI at capture time
|
||||
set -u
|
||||
DUR=${KNELPERF_CPUBENCH_SECS:-10}
|
||||
|
||||
echo -e "metric\tvalue\tunit"
|
||||
|
||||
gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo none)
|
||||
echo -e "governor\t$gov\t-"
|
||||
nproc=$(nproc)
|
||||
echo -e "nproc\t$nproc\t-"
|
||||
|
||||
# openssl: 1 thread then all threads, aes-256-cbc (avx-sensitive) and sha256.
|
||||
# Parse: LAST line whose first field is the algo; value = last numeric field
|
||||
# (per-thread rows precede the aggregate row in -multi output).
|
||||
for algo in aes-256-cbc sha256; do
|
||||
v=$(openssl speed -seconds "$DUR" "$algo" 2>/dev/null \
|
||||
| awk -v a="$algo" '$1==a{v=$NF} END{print v}')
|
||||
[ -n "${v:-}" ] && echo -e "openssl_${algo}_1t\t$v\tk/s"
|
||||
# -multi: per-thread rows precede the aggregate; keep the LAST match
|
||||
v=$(openssl speed -multi "$nproc" -seconds "$DUR" "$algo" 2>/dev/null \
|
||||
| awk -v a="$algo" '$1==a{v=$NF} END{print v}')
|
||||
[ -n "${v:-}" ] && echo -e "openssl_${algo}_${nproc}t\t$v\tk/s"
|
||||
done
|
||||
|
||||
# sysbench (optional, if installed)
|
||||
if command -v sysbench >/dev/null 2>&1; then
|
||||
ev=$(sysbench cpu --time=$((DUR*1000)) --threads=1 run 2>/dev/null \
|
||||
| awk -F'/s|events per second:/{for(i=1;i<=NF;i++) if($i ~ /^[0-9.]+$/){print $i; exit}}')
|
||||
[ -n "${ev:-}" ] && echo -e "sysbench_cpu_1t\t$ev\tevents/s"
|
||||
ev=$(sysbench cpu --time=$((DUR*1000)) --threads="$nproc" run 2>/dev/null \
|
||||
| awk -F'/s|events per second:/{for(i=1;i<=NF;i++) if($i ~ /^[0-9.]+$/){print $i; exit}}')
|
||||
[ -n "${ev:-}" ] && echo -e "sysbench_cpu_${nproc}t\t$ev\tevents/s"
|
||||
else
|
||||
echo -e "sysbench_cpu\tabsent\t-"
|
||||
fi
|
||||
|
||||
# PSI snapshot at capture time (context for the numbers above)
|
||||
for res in cpu memory io; do
|
||||
v=$(awk '$1=="some"{for(i=2;i<=NF;i++) if($i ~ /^avg60=/){sub("avg60=","",$i); print $i; exit}}' \
|
||||
"/proc/pressure/$res" 2>/dev/null || echo 0)
|
||||
echo -e "psi_${res}_avg60\t$v\t%"
|
||||
done
|
||||
exit 0
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# KNELPerf loop driver — baseline -> audit/benchmark -> tweak -> re-baseline.
|
||||
#
|
||||
# The heart of KNELPerf v2 (#826). One host, one cycle phase at a time:
|
||||
#
|
||||
# perf-loop.sh baseline <host> capture baseline bundle (cpu/net/storage/psi)
|
||||
# perf-loop.sh audit <host> read-only audit vs baseline (alerts on drift)
|
||||
# perf-loop.sh tweak <host> <id> apply a tweak (from tweaks/) after CR gate
|
||||
# perf-loop.sh rebaseline <host> capture post-tweak bundle + diff report
|
||||
# perf-loop.sh compare <host> <a> <b> diff two stored bundles
|
||||
#
|
||||
# Bundles live in data/baselines/<host>/<ts>/ and are plain TSV/JSON so diff
|
||||
# and jq stay the toolchain. Nothing here mutates a host except `tweak`.
|
||||
set -euo pipefail
|
||||
REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
DATA_DIR=${KNELPERF_DATA_DIR:-$REPO_ROOT/data/baselines}
|
||||
SSH_OPTS=(-o ConnectTimeout=8 -o BatchMode=yes)
|
||||
|
||||
usage() { sed -n '2,12p' "$0"; exit 2; }
|
||||
[ $# -ge 2 ] || usage
|
||||
cmd=$1; host=$2
|
||||
|
||||
now() { date +%Y%m%d-%H%M%S; }
|
||||
host_dir() { mkdir -p "$DATA_DIR/$1"; }
|
||||
|
||||
run_on_host() { # host script-path args...
|
||||
local h=$1 script=$2; shift 2
|
||||
case $h in
|
||||
local) bash "$REPO_ROOT/$script" "$@" ;;
|
||||
*) ssh "${SSH_OPTS[@]}" "root@$h" "bash -s" < "$REPO_ROOT/$script" "$@" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
case "$cmd" in
|
||||
baseline|rebaseline)
|
||||
ts=$(now); dir="$DATA_DIR/$host/$ts"; host_dir "$host"; mkdir -p "$dir"
|
||||
echo "== KNELPerf $cmd: $host -> $dir"
|
||||
# shellcheck disable=SC2043 # single-part list by design; grows as legs are added
|
||||
for part in cpu-bench; do
|
||||
run_on_host "$host" "loop/$part.sh" > "$dir/$part.tsv" 2> "$dir/$part.err" \
|
||||
|| echo "WARN: $part failed (see $dir/$part.err)"
|
||||
done
|
||||
# storage + network reuse the existing bench harness outputs
|
||||
run_on_host "$host" "bench/bench-run.sh" > "$dir/bench-run.tsv" 2> "$dir/bench.err" \
|
||||
|| echo "WARN: bench-run failed (see $dir/bench.err)"
|
||||
echo "$cmd $host $ts" > "$dir/MANIFEST"
|
||||
echo "done: $(ls "$dir")"
|
||||
;;
|
||||
audit)
|
||||
latest=$(find "$DATA_DIR/$host" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' 2>/dev/null | sort | tail -1)
|
||||
[ -n "$latest" ] || { echo "no baseline for $host" >&2; exit 3; }
|
||||
echo "== KNELPerf audit: $host vs baseline $latest"
|
||||
"$REPO_ROOT/loop/baseline-diff.sh" "$DATA_DIR/$host/$latest" "$host"
|
||||
;;
|
||||
tweak)
|
||||
[ $# -ge 3 ] || usage; tweak_id=$3
|
||||
echo "== KNELPerf tweak $tweak_id on $host (CR gate applies - see AGENTS.md)"
|
||||
run_on_host "$host" "tweaks/$tweak_id.sh"
|
||||
;;
|
||||
compare)
|
||||
[ $# -ge 4 ] || usage
|
||||
"$REPO_ROOT/loop/baseline-diff.sh" "$DATA_DIR/$host/$3" "$DATA_DIR/$host/$4"
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user