vptechops
d18e467484
Adversarial review verdict FAIL -> addressed: runtime-low + shutdown
chain re-poll /5 min (edge-crossing could miss a page mid-outage);
suppression stamp set after notify; plan body names all 7 hosts
explicitly; test fixes (pipefail SIGPIPE, comment-only assertion).
Reviewer P1 'tripp_lite slug' refuted with live entity registry
evidence (entity_id derives from name; sensor exists, 8f48730 aligned).
On-box ha core check passed with the changed files.
Review thread: #3
93 lines
4.3 KiB
Bash
93 lines
4.3 KiB
Bash
#!/bin/bash
|
|
# SDLC regression tests (#778) — run in CI (see .gitea/workflows/ci.yml).
|
|
# Each test guards a rule that exists because of a real incident or ruling.
|
|
set -uo pipefail
|
|
cd "$(dirname "$0")/.." || exit 1
|
|
|
|
FAIL=0
|
|
t() {
|
|
# $1 = label, $2 = shell command string
|
|
if eval "$2"; then
|
|
echo " OK $1"
|
|
else
|
|
echo " FAIL $1"
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
echo "=== alert twins (every alert ships a recovery twin) ==="
|
|
t "utility power restored twin" \
|
|
"grep -q 'id: pfv_plant_ups_power_restored' automations.yaml"
|
|
t "temperature recovered twin" \
|
|
"grep -q 'id: pfv_plant_temperature_recovered' automations.yaml"
|
|
|
|
echo "=== alert deep links (taps land on dashboards, #344) ==="
|
|
ALERTS="POWER OUTAGE|UPS BATTERY LOW|UPS COMMS LOST|UPS RUNTIME LOW|Power restored|HIGH TEMPERATURE|SENSOR WATCHDOG|TEMP RISING FAST|TEMPERATURE RECOVERED"
|
|
DL_OK=1
|
|
IFS='|' read -ra ALERT_ARR <<< "$ALERTS"
|
|
for a in "${ALERT_ARR[@]}"; do
|
|
if ! grep -A2 "title: $a$" automations.yaml | grep -q " url: "; then
|
|
echo " (missing deep link: $a)"
|
|
DL_OK=0
|
|
fi
|
|
done
|
|
t "all alerts deep-linked" "[ \"$DL_OK\" -eq 1 ]"
|
|
|
|
echo "=== watchdog coverage (#789) ==="
|
|
t "watchdog watches the deploy sensor" \
|
|
"grep -q 'sensor.pfv_deploy_gitea_release_head, 30' automations.yaml"
|
|
t "watchdog automation exists" \
|
|
"grep -q 'id: pfv_watchdog_stale_entities' automations.yaml"
|
|
|
|
echo "=== ghost entities (dead references never return) ==="
|
|
t "no tsys9_ups ghost refs" \
|
|
"! grep -rq 'tsys9_ups' automations.yaml packages/ dashboards/"
|
|
|
|
echo "=== secrets ==="
|
|
t "no private keys in tree" \
|
|
"! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ."
|
|
|
|
echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ==="
|
|
# The 30-min sweep used to page every cycle even when the fleet was all-OK
|
|
# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate.
|
|
t "plant watchdog notify gated (dead-only + cooldown input)" \
|
|
"grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml"
|
|
|
|
# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed
|
|
# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT).
|
|
t "trend sensors fed from smoothed sources" \
|
|
"grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'"
|
|
|
|
# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's
|
|
# shutdown window was unmonitored.
|
|
t "runtime-low covers Tripp Lite" \
|
|
"sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'"
|
|
t "runtime-low status check maps per-UPS" \
|
|
"sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q \"sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status'\""
|
|
|
|
echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ==="
|
|
t "shutdown chain package present, dry-run default ON" \
|
|
"grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'"
|
|
t "armed switch default OFF" \
|
|
"grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'"
|
|
# Host coverage must hold in the PLAN BODY, not the header comment —
|
|
# scope the grep to the plan template variable.
|
|
PLAN_OK=1
|
|
for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do
|
|
# full-read grep (not -q): the script runs with pipefail — -q's early
|
|
# exit SIGPIPEs sed and flips the pipeline status to 141
|
|
sed -n '/plan: >-/,/mode_note:/p' packages/ups_shutdown.yaml | grep "$h" >/dev/null || PLAN_OK=0
|
|
done
|
|
t "plan covers all seven hypervisors (plan body)" "[ \$PLAN_OK -eq 1 ]"
|
|
t "pfv-bms VM 100 ordered last of the VMs (#455)" \
|
|
"grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml"
|
|
|
|
echo "=== level-based re-checks (review finding: edge-only triggers miss pages) ==="
|
|
t "runtime-low re-polls every 5 min" \
|
|
"sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'minutes: /5'"
|
|
t "shutdown chain re-polls every 5 min" \
|
|
"grep -A30 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml | grep -q 'minutes: /5'"
|
|
|
|
echo "=== SDLC SUITE: $FAIL failures ==="
|
|
exit "$FAIL"
|