diff --git a/scripts/daily-sweep.sh b/scripts/daily-sweep.sh new file mode 100644 index 0000000..26bc8a1 --- /dev/null +++ b/scripts/daily-sweep.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Daily maintenance sweep (#777) — run from the pfv workstation. +# Checks HA health, pipeline, sensor freshness, NUT, DNS, runner. +# Exit code = number of failures. Designed for a daily cron + report. +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0 +FAIL=0 +FAILED_ITEMS="" + +ok() { + PASS=$((PASS + 1)) + echo " OK $1" +} + +bad() { + FAIL=$((FAIL + 1)) + FAILED_ITEMS="${FAILED_ITEMS} - $1"$'\n' + echo " FAIL $1" +} + +chk() { + # $1 = label, $2 = shell command string to evaluate + if eval "$2" >/dev/null 2>&1; then + ok "$1" + else + bad "$1" + fi +} + +# optional: environment overrides for remote.sh routing +set -a +# shellcheck disable=SC1091 +[ -f ~/.creds/homeassistant.env ] && source ~/.creds/homeassistant.env +set +a + +ha_api() { + timeout 20 curl -sS -H "Authorization: Bearer $HASS_TOKEN" \ + "$HASS_URL$1" 2>/dev/null +} + +echo "=== Home Assistant ===" +MSG=$(ha_api /api/ | jq -r .message 2>/dev/null) +if [ "$MSG" = "API running." ]; then ok "HA API up"; else bad "HA API: '$MSG'"; fi + +W=$(ha_api /api/states/input_boolean.pfv_watchdog_auto_restart 2>/dev/null | jq -r .state) +if [ "$W" = "on" ] || [ "$W" = "off" ]; then ok "watchdog present ($W)"; else bad "watchdog switch state: $W"; fi + +STALE=$(ha_api /api/states 2>/dev/null | jq -r ' + def epoch: sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601; + [ .[] + | select(.entity_id | test("tsys1_ups_status|tsys1_triplite_status|pfv_tsys[0-9]_cpu_temperature")) + | select(.last_updated != null) + | {e: .entity_id, age: ((now - (.last_updated | epoch)) / 60)} ] + | map(select(.age > 20)) + | if length > 0 then (map(.e) | join(", ")) else "0" end' 2>/dev/null) +if [ "$STALE" = "0" ]; then ok "critical sensors fresh"; else bad "stale critical sensors: ${STALE:-query-failed}"; fi + +REL=$(ha_api /api/states/sensor.pfv_deploy_gitea_release_head 2>/dev/null) +RELSTATE=$(echo "$REL" | jq -r '.state[:7]' 2>/dev/null) +RELAGE=$(echo "$REL" | jq -r ' + def epoch: sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601; + ((now - (.last_updated | epoch)) / 60) | floor' 2>/dev/null) +if [ -n "$RELSTATE" ] && [ "${RELAGE:-999}" -le 30 ]; then + ok "deploy sensor fresh (release ${RELSTATE}, ${RELAGE} min)" +else + bad "deploy sensor stale/missing (${RELSTATE:-?} @ ${RELAGE:-?} min)" +fi + +echo "=== Pipeline / CI ===" +chk "gitea reachable" "curl -sS -o /dev/null https://git.knownelement.com/api/v1/version" +chk "runner container up" "docker ps --format '{{.Names}}' | grep -q ukrrs-pfv-gitea-runner" + +cd /home/reachableceo/projects/KNEL/pfv-bms 2>/dev/null && { + D=$(git rev-parse --short=7 dev 2>/dev/null) + R=$(git rev-parse --short=7 origin/release 2>/dev/null) + if [ -n "$D" ] && [ "$D" = "$R" ]; then ok "dev == release (${D})"; else + echo " NOTE dev ${D:-?} != release ${R:-?} (pending promotion)" + fi + cd - >/dev/null || true +} + +echo "=== Infra ===" +chk "DNS pfv-bms.knel.net" "getent hosts pfv-bms.knel.net" +chk "DNS git.knownelement.com" "getent hosts git.knownelement.com" +chk "gitea version API" "curl -sS https://git.knownelement.com/api/v1/version" + +NUTCHK=$(VM_IP=pfv-tsys1.knel.net VM_USER=root timeout 45 bash tests/remote.sh vm \ + 'upsc tsys1_ups@localhost battery.charge >/dev/null 2>&1 && upsc tsys1_triplite@localhost battery.charge >/dev/null 2>&1 && echo NUT-OK' 2>/dev/null | grep -c NUT-OK) +if [ "${NUTCHK:-0}" -ge 1 ]; then + ok "NUT: both UPSes responding on tsys1" +else + bad "NUT check on tsys1" +fi + +# shellcheck disable=SC2016 # remote awk must see literal $5 on the far side +DISKCHK=$(VM_IP=pfv-bms-lan.knel.net VM_USER=root VM_PORT=22222 timeout 45 bash tests/remote.sh vm \ + 'df -m /mnt/data | awk "NR==2 {if (\$5+0 < 80) print \"DISK-OK\"}"' 2>/dev/null | grep -c DISK-OK) +if [ "${DISKCHK:-0}" -ge 1 ]; then + ok "pfv-bms data disk <80% used" +else + bad "pfv-bms data disk >=80% or unreachable" +fi + +echo +echo "=== SWEEP SUMMARY: $PASS pass, $FAIL fail ===" +if [ -n "$FAILED_ITEMS" ]; then + echo "FAILED ITEMS:" + echo "$FAILED_ITEMS" +fi +exit "$FAIL"