feat(sweep): daily maintenance sweep script [#777]
HA health, sensor freshness (watchdog classes), deploy pipeline, runner, NUT both UPSes, DNS, disk. Exit = failure count.
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
# Daily maintenance sweep (#777) — run from the pfv workstation.
|
||||
# Checks HA health, pipeline, sensor freshness, NUT, DNS, runner.
|
||||
# Exit code = number of failures. Designed for a daily cron + report.
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.." || exit 1
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILED_ITEMS=""
|
||||
|
||||
ok() {
|
||||
PASS=$((PASS + 1))
|
||||
echo " OK $1"
|
||||
}
|
||||
|
||||
bad() {
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_ITEMS="${FAILED_ITEMS} - $1"$'\n'
|
||||
echo " FAIL $1"
|
||||
}
|
||||
|
||||
chk() {
|
||||
# $1 = label, $2 = shell command string to evaluate
|
||||
if eval "$2" >/dev/null 2>&1; then
|
||||
ok "$1"
|
||||
else
|
||||
bad "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# optional: environment overrides for remote.sh routing
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
[ -f ~/.creds/homeassistant.env ] && source ~/.creds/homeassistant.env
|
||||
set +a
|
||||
|
||||
ha_api() {
|
||||
timeout 20 curl -sS -H "Authorization: Bearer $HASS_TOKEN" \
|
||||
"$HASS_URL$1" 2>/dev/null
|
||||
}
|
||||
|
||||
echo "=== Home Assistant ==="
|
||||
MSG=$(ha_api /api/ | jq -r .message 2>/dev/null)
|
||||
if [ "$MSG" = "API running." ]; then ok "HA API up"; else bad "HA API: '$MSG'"; fi
|
||||
|
||||
W=$(ha_api /api/states/input_boolean.pfv_watchdog_auto_restart 2>/dev/null | jq -r .state)
|
||||
if [ "$W" = "on" ] || [ "$W" = "off" ]; then ok "watchdog present ($W)"; else bad "watchdog switch state: $W"; fi
|
||||
|
||||
STALE=$(ha_api /api/states 2>/dev/null | jq -r '
|
||||
def epoch: sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601;
|
||||
[ .[]
|
||||
| select(.entity_id | test("tsys1_ups_status|tsys1_triplite_status|pfv_tsys[0-9]_cpu_temperature"))
|
||||
| select(.last_updated != null)
|
||||
| {e: .entity_id, age: ((now - (.last_updated | epoch)) / 60)} ]
|
||||
| map(select(.age > 20))
|
||||
| if length > 0 then (map(.e) | join(", ")) else "0" end' 2>/dev/null)
|
||||
if [ "$STALE" = "0" ]; then ok "critical sensors fresh"; else bad "stale critical sensors: ${STALE:-query-failed}"; fi
|
||||
|
||||
REL=$(ha_api /api/states/sensor.pfv_deploy_gitea_release_head 2>/dev/null)
|
||||
RELSTATE=$(echo "$REL" | jq -r '.state[:7]' 2>/dev/null)
|
||||
RELAGE=$(echo "$REL" | jq -r '
|
||||
def epoch: sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601;
|
||||
((now - (.last_updated | epoch)) / 60) | floor' 2>/dev/null)
|
||||
if [ -n "$RELSTATE" ] && [ "${RELAGE:-999}" -le 30 ]; then
|
||||
ok "deploy sensor fresh (release ${RELSTATE}, ${RELAGE} min)"
|
||||
else
|
||||
bad "deploy sensor stale/missing (${RELSTATE:-?} @ ${RELAGE:-?} min)"
|
||||
fi
|
||||
|
||||
echo "=== Pipeline / CI ==="
|
||||
chk "gitea reachable" "curl -sS -o /dev/null https://git.knownelement.com/api/v1/version"
|
||||
chk "runner container up" "docker ps --format '{{.Names}}' | grep -q ukrrs-pfv-gitea-runner"
|
||||
|
||||
cd /home/reachableceo/projects/KNEL/pfv-bms 2>/dev/null && {
|
||||
D=$(git rev-parse --short=7 dev 2>/dev/null)
|
||||
R=$(git rev-parse --short=7 origin/release 2>/dev/null)
|
||||
if [ -n "$D" ] && [ "$D" = "$R" ]; then ok "dev == release (${D})"; else
|
||||
echo " NOTE dev ${D:-?} != release ${R:-?} (pending promotion)"
|
||||
fi
|
||||
cd - >/dev/null || true
|
||||
}
|
||||
|
||||
echo "=== Infra ==="
|
||||
chk "DNS pfv-bms.knel.net" "getent hosts pfv-bms.knel.net"
|
||||
chk "DNS git.knownelement.com" "getent hosts git.knownelement.com"
|
||||
chk "gitea version API" "curl -sS https://git.knownelement.com/api/v1/version"
|
||||
|
||||
NUTCHK=$(VM_IP=pfv-tsys1.knel.net VM_USER=root timeout 45 bash tests/remote.sh vm \
|
||||
'upsc tsys1_ups@localhost battery.charge >/dev/null 2>&1 && upsc tsys1_triplite@localhost battery.charge >/dev/null 2>&1 && echo NUT-OK' 2>/dev/null | grep -c NUT-OK)
|
||||
if [ "${NUTCHK:-0}" -ge 1 ]; then
|
||||
ok "NUT: both UPSes responding on tsys1"
|
||||
else
|
||||
bad "NUT check on tsys1"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2016 # remote awk must see literal $5 on the far side
|
||||
DISKCHK=$(VM_IP=pfv-bms-lan.knel.net VM_USER=root VM_PORT=22222 timeout 45 bash tests/remote.sh vm \
|
||||
'df -m /mnt/data | awk "NR==2 {if (\$5+0 < 80) print \"DISK-OK\"}"' 2>/dev/null | grep -c DISK-OK)
|
||||
if [ "${DISKCHK:-0}" -ge 1 ]; then
|
||||
ok "pfv-bms data disk <80% used"
|
||||
else
|
||||
bad "pfv-bms data disk >=80% or unreachable"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=== SWEEP SUMMARY: $PASS pass, $FAIL fail ==="
|
||||
if [ -n "$FAILED_ITEMS" ]; then
|
||||
echo "FAILED ITEMS:"
|
||||
echo "$FAILED_ITEMS"
|
||||
fi
|
||||
exit "$FAIL"
|
||||
Reference in New Issue
Block a user