diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..6112725 --- /dev/null +++ b/STATUS.md @@ -0,0 +1,18 @@ +# STATUS — point-in-time agent state (sparse; history lives in Redmine) + +Updated: 2026-09-06 ~07:10 CDT · dev @ aa8d788 · release @ 83b32d5 + +| ticket | tier | state | item | +|---|---|---|---| +| #811 | — | ⛔ | NeedsInput: merge PR #3, PVE tokens (#790), beta onboarding click, rotations | +| #344 | prod | 🔄 | release PR #3 open, CI green, review gate passed (FAIL→fixed→dispositioned) | +| #790 | dev | ✅ | shutdown chain DRY RUN, level-based re-polls; exec wiring waits on PVE tokens | +| #628 | docs | ✅ | restore runbook + baseline backup 34a05d70 off-box on ultix | +| #781 | beta | 🔄 | VM 53110 @ 192.168.3.240, HA up, owner created; onboarding form + config sync left | +| #813 | — | ⏳ | CI check_config job parked — needs UI job log, recipe in ticket | +| #345 | prod | 🔄 | rest_command verified registered; swap-back = founder re-test call | + +## Inbox +- 2026-09-06: tsys6 trend false-positive fix — watch trend sensors 24h post-deploy +- 2026-09-06: stlpc-artroom dead all morning — policy ask in #811 item 8 +- 2026-09-06: runner container got external SIGTERM 11:35 UTC, restarted 11:37 — OAM check diff --git a/automations.yaml b/automations.yaml index 6facb31..5f54ce0 100644 --- a/automations.yaml +++ b/automations.yaml @@ -68,15 +68,36 @@ mode: single - id: pfv_plant_ups_runtime_low alias: PFV plant - UPS runtime low while on battery - description: On battery and runtime burned below 10 min — the graceful - shutdown window is closing. VMs before hypervisors per #455. + description: >- + On battery and runtime burned below 10 min — the graceful shutdown + window is closing. VMs before hypervisors per #455 ordering. Covers + BOTH UPSes (APC + Tripp Lite — the Tripp Lite gap closed 2026-09-06); + the status check maps each runtime sensor to its own UPS status. triggers: - trigger: numeric_state - entity_id: sensor.pfv_ups_runtime_minutes + entity_id: + - sensor.pfv_ups_runtime_minutes + - sensor.pfv_tripp_lite_runtime_minutes below: 10 + # 5-min level re-check: numeric_state alone fires only on the threshold + # CROSSING — a blip at that minute (or an HA restart mid-outage) would + # silence the page for the whole discharge. Level-based polling turns + # this into an escalation ladder while the outage persists. + - trigger: time_pattern + minutes: /5 conditions: - condition: template - value_template: "{{ states('sensor.tsys1_ups_status') not in ['Online', 'unknown', 'unavailable'] }}" + value_template: >- + {% set ns = namespace(hit=false) %} + {% for rt, st in [ + ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'), + ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %} + {% if is_number(states(rt)) and (states(rt) | float) < 10 + and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %} + {% set ns.hit = true %} + {% endif %} + {% endfor %} + {{ ns.hit }} actions: - action: notify.send_message target: @@ -86,7 +107,7 @@ data: title: UPS RUNTIME LOW url: /lovelace-pfv-server-room/ups - message: On battery {{ states('sensor.pfv_ups_runtime_minutes') }} min left — start graceful shutdown now (#455 ordering). + message: '{{ state_attr(trigger.entity_id, ''friendly_name'') }}: {{ states(trigger.entity_id) }} min left — start graceful shutdown now (#455 ordering).' mode: single - id: pfv_plant_ups_power_restored alias: PFV plant - utility power restored @@ -190,31 +211,55 @@ mode: single - id: pfv_plant_sensor_watchdog alias: PFV plant - sensor watchdog (unavailable sensors) - description: HA is the environmental brain (#344) — a silent sensor is an outage. - Re-alert suppressed by 4h cooldown via last_notified input_text if present. + description: > + HA is the environmental brain (#344) — a silent sensor is an outage. + Fleet-wide sweep (every pfv_* sensor, broader than the 5-min critical + watchdog). PUSHES ONLY when sensors are actually dead, and re-pages a + still-dead fleet at most every 4h — the unconditional 30-min page was + retired 2026-09-06 after live-fire alert fatigue. All-OK cycles log + quietly; the Kuma dead-man heartbeat (#792) owns liveness signaling. + mode: single + max_exceeded: silent triggers: - trigger: time_pattern minutes: /30 actions: - - action: notify.send_message - target: - entity_id: - - notify.ultix_sidecar_new - - notify.allthes_ipad - data: - title: SENSOR WATCHDOG - url: /lovelace-pfv-server-room/overview - message: >- - {% set fleet = states.sensor | selectattr('entity_id', 'match', 'sensor\.pfv_') - | list %} - {% set dead = fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list %} - {% if dead | length > 0 %} + - variables: + fleet: "{{ states.sensor | selectattr('entity_id', 'match', 'sensor\\.pfv_') | list }}" + dead: "{{ fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list }}" + # 4h cooldown: page again only if the last page is older than 4h + # (or we never paged). Fresh HA start (no restore) counts as never. + # as_timestamp math — aware/naive datetime subtraction throws in Jinja. + notify_gate: >- + {{ states('input_datetime.pfv_plant_watchdog_last_notify') in ['unknown', 'unavailable', 'none'] + or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_plant_watchdog_last_notify')) > 14400 }} + - if: + - condition: template + value_template: "{{ dead | length > 0 and notify_gate }}" + then: + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: SENSOR WATCHDOG + url: /lovelace-pfv-server-room/overview + message: >- {{ dead | length }} of {{ fleet | length }} fleet sensors silent: {{ dead | map(attribute='entity_id') | join(', ') }} - {% else %} - all {{ fleet | length }} fleet sensors OK - {% endif %} - mode: single + - action: input_datetime.set_datetime + target: + entity_id: input_datetime.pfv_plant_watchdog_last_notify + data: + datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}" + else: + - action: logbook.log + data: + name: Plant watchdog + message: >- + Sweep: all {{ fleet | length }} fleet sensors answering + ({{ dead | length }} silent, cooldown {{ 'active' if not notify_gate else 'n/a' }}). - id: pfv_plant_temp_rising_fast alias: PFV plant - temperature rising fast (rate-of-change) description: Catches dead AC minutes before the 104F threshold trips. diff --git a/configuration.yaml b/configuration.yaml index 55b8f62..74f702f 100644 --- a/configuration.yaml +++ b/configuration.yaml @@ -56,3 +56,12 @@ input_boolean: pfv_watchdog_auto_restart: name: Watchdog auto core restart icon: mdi:restart-alert + +# 30-min plant watchdog cooldown (#344, 2026-09-06): re-pages a dead sensor +# at most every 4h instead of every cycle. Set by the automation on notify. +input_datetime: + pfv_plant_watchdog_last_notify: + name: Plant watchdog last notify + has_date: true + has_time: true + icon: mdi:bell-cancel diff --git a/docs/restore-runbook.md b/docs/restore-runbook.md new file mode 100644 index 0000000..afcac6f --- /dev/null +++ b/docs/restore-runbook.md @@ -0,0 +1,78 @@ +# pfv-bms restore runbook (#628) + +Three restore layers exist. Work **inside-out**: usually only layer 1 is +needed. Restore drills run on the **dev lane only** — never on hosts +running prod VMs (standing rule after the 2026-09-03 PBS drill that +I/O-starved prod HA, #684 / t/298). + +## Layer 0 — what is NOT in git + +| Artifact | Where it lives | Covered by | +|----------|----------------|------------| +| `secrets.yaml` (tokens, webhook ids, unlock URL) | on-box `/config/secrets.yaml` | Layer 1 + 2 backups | +| `.storage/` (dashboards-in-storage, registry, auth) | on-box `/config/.storage/` | Layer 1 + 2 backups | +| Zigbee/HomeKit/BT pairings | on-box | Layer 1 + 2 backups | + +Git (this repo, `release` branch) is the source of truth for +configuration **YAML only**. A fresh box restored from git still needs a +`secrets.yaml` re-provisioned from the keeper's records (~/.creds notes ++ badge/webhook ids) before HA will start. + +## Layer 1 — HA-level backup (config + settings, small, fast) + +Taken on-box via the `ha` CLI; produces a self-contained `.tar`: + +```bash +ssh -p 22222 root@pfv-bms.knel.net 'ha backups new --name "pfv-bms-$(date +%F)"' +ssh -p 22222 root@pfv-bms.knel.net 'ha backups list' # find slug +ssh -p 22222 root@pfv-bms.knel.net 'ha backups info ' # verify size/state +# OFF-BOX COPY (the on-box copy dies with the VM disk): +scp -P 22222 'root@pfv-bms.knel.net:/mnt/data/supervisor/backup/.tar' /backup/destination/ +``` + +HAOS automatic backups are CONFIRMED LIVE (daily "Automatic backup" +entries observed 2026-09-05/06) — layer 1 has a baseline; the off-box +copy remains the manual step. + +Restore (same or fresh HAOS box): + +```bash +scp -P 22222 backup.tar root@pfv-bms.knel.net:/mnt/data/supervisor/backup/ +ssh -p 22222 root@pfv-bms.knel.net 'ha backups restore --type full' +``` + +Baseline on record: slug `34a05d70`, name +`pfv-bms-baseline-2026-09-06-post-790`, 788 MB compressed, taken after +the alert-hygiene + #790 batches (dev `c4ebb8a`). Off-box copy: +`~/projects/KNEL/PhysicalPlant/pfv-bms-backups/` on ultix-streaming +(sha256 6ff2fa6bd5f3dd2f2a47…). + +## Layer 2 — VM-level backup (prod PBS, nightly) + +- All prod VMs — **including VM 100 (pfv-bms) on pfv-tsys1** — back up + nightly to prod PBS, staggered 21:00–04:30 Central, throttled + 60/30 MiB/s + job bwlimit 50 MiB/s (founder ruling t/321). +- Restore: PVE web UI or `qmrestore` — **target a scratch VM on the dev + lane** (verify the restore boots there), never restore in place over + the live VM without a Kuma maintenance window. +- Gap (open, OAM lane): pfv-tsys5 has no PBS job yet — beta/test VMs on + tsys5 are unprotected; fine for beta, not for anything promoted. + +## Layer 3 — config-from-git (last resort, fresh HAOS) + +1. Flash/install HAOS, complete onboarding, join Tailscale. +2. `git clone git.knownelement.com:KNEL/KNELBMS` → copy tree to `/config` + at the `release` HEAD the fleet expects. +3. Re-provision `secrets.yaml` (Layer 0 table) — from keeper records. +4. `ha core check` before starting; then watch the first boot: the + post-deploy validation automation pages if criticals fail to + materialize (#778). +5. Verify the trail: deploy sensor matches `release` HEAD, Kuma + heartbeat green, doorman roster sensors populated. + +## Related + +- Deploy pipeline + gate: README "Deployment pipeline"; `ha core check` + on-box before any release push. +- Incident history: #684 (PBS drill), #780 (tsys1 storage), t/298 + (Sep 3 wedge), t/320 (reload vs restart deploy semantics). diff --git a/packages/doorman.yaml b/packages/doorman.yaml index d6f87db..7e04b8b 100644 --- a/packages/doorman.yaml +++ b/packages/doorman.yaml @@ -84,11 +84,14 @@ input_boolean: shell_command: # Unlock dispatch via curl — swapped from rest_command 2026-09-04: the - # rest_command integration silently fails to register on this box - # (404 on /api/services/rest_command despite valid config + secret; - # root-cause pending in #345/#741). curl is present in the HA core - # container. Token lives in the URL secret — process-list exposure - # accepted (isolated container). + # rest_command integration silently failed to register on this box + # (404 on /api/services/rest_command despite valid config + secret). + # NOT reproducible: 2026-09-06 the integration + doorman_unlock service + # are registered again (suspect: transient state after a reload-only + # deploy window; rest_command now also exposes a reload service). + # curl stays as the verified-live dispatch path — swapping back to + # rest_command (drops the token from the process list) is gated on one + # armed badge re-test (founder call, questions-v1). doorman_unlock: "curl -sS -m 10 '{{ doorman_unlock_url }}'" rest_command: diff --git a/packages/plant_snmp.yaml b/packages/plant_snmp.yaml index 8f852ed..5ed139f 100644 --- a/packages/plant_snmp.yaml +++ b/packages/plant_snmp.yaml @@ -543,26 +543,56 @@ sensor: unique_id: pfv_garage_pdu_energy # --- Rate-of-change early warning (#682): +10°F/15min = AC trouble [#344] --- +# DRAC sensors quantize to whole °C and FLAP ~1.8°F between polls; raw-fed +# trend sensors false-tripped TEMP RISING FAST (2026-09-06 05:02 CDT, tsys6). +# Trend inputs are 5-min time-weighted moving averages — real climbs survive +# the smoothing, quantization flapping does not. +filter: + - platform: time_simple_moving_average + name: "PFV tsys1 rack temperature smoothed" + entity_id: sensor.pfv_tsys1_rack_temperature + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys1_rack_temperature_smoothed + - platform: time_simple_moving_average + name: "PFV tsys6 DRAC ambient smoothed" + entity_id: sensor.pfv_tsys6_drac_ambient + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys6_drac_ambient_smoothed + - platform: time_simple_moving_average + name: "PFV tsys7 DRAC inlet smoothed" + entity_id: sensor.pfv_tsys7_drac_inlet + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys7_drac_inlet_smoothed + - platform: time_simple_moving_average + name: "PFV garage temperature smoothed" + entity_id: sensor.pfv_garage_temperature + window: "00:05:00" + precision: 2 + unique_id: pfv_garage_temperature_smoothed + binary_sensor: - platform: trend sensors: server_room_temp_rising: - entity_id: sensor.pfv_tsys1_rack_temperature + entity_id: sensor.pfv_tsys1_rack_temperature_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_server_room_rising tsys6_ambient_rising: - entity_id: sensor.pfv_tsys6_drac_ambient + entity_id: sensor.pfv_tsys6_drac_ambient_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_tsys6_ambient_rising tsys7_inlet_rising: - entity_id: sensor.pfv_tsys7_drac_inlet + entity_id: sensor.pfv_tsys7_drac_inlet_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_tsys7_inlet_rising garage_temp_rising: - entity_id: sensor.pfv_garage_temperature + entity_id: sensor.pfv_garage_temperature_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_garage_rising diff --git a/packages/ups_shutdown.yaml b/packages/ups_shutdown.yaml new file mode 100644 index 0000000..b4ca765 --- /dev/null +++ b/packages/ups_shutdown.yaml @@ -0,0 +1,149 @@ +# ============================================================================ +# UPS graceful-shutdown chain (#790) — the #455 ordering, automated +# ============================================================================ +# DESIGN (unchanged from the 2026-09-05 session spec): +# DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the +# exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the +# VMs — the brain stays alive as long as possible). Execution (real +# qmshutdown / PVE API calls) is double-gated: +# 1. input_boolean.pfv_ups_shutdown_armed — founder arm, default OFF +# 2. input_boolean.pfv_ups_shutdown_dry_run — default ON +# Arming for the Sept 11-12 live test needs, in order: founder review of +# the VM ordering below (edit this file via PR), PVE API tokens provisioned +# into on-box secrets.yaml (NeedsInput ticket), then a supervised live test. +# +# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts): +# tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01, +# 104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode +# tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605 +# tsys7: 701-711, 902 | tsys9: 904, 905 +# ============================================================================ + +input_boolean: + # NOTE: `initial:` wins over restored state at EVERY HA start — a deploy + # mid-test silently reverts these to the safe defaults (armed OFF, + # dry-run ON). Safe direction, but re-arm after any restart during the + # supervised Sept 11-12 live test. + pfv_ups_shutdown_armed: + name: UPS shutdown chain ARMED + description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch. + icon: mdi:power-standby + initial: false + pfv_ups_shutdown_dry_run: + name: UPS shutdown DRY RUN + description: ON (default) = log/page the plan only, touch nothing. + icon: mdi:numeric-1-box-outline + initial: true + +input_datetime: + pfv_ups_shutdown_last_page: + name: UPS shutdown last plan page + has_date: true + has_time: true + icon: mdi:clock-alert-outline + +automation: + - id: pfv_ups_shutdown_plan + alias: PFV UPS shutdown - plan (dry-run by default) + description: >- + On battery with the shutdown window opening (runtime < 20 min or + charge < 50%), log + page the #455 ordered plan. Re-pages at most + every 6h per outage. When armed AND dry-run OFF (founder + supervised + test only), the plan page says EXECUTE — real dispatch wiring lands + with PVE API tokens (#790 NeedsInput); until then nothing but text + can be emitted by this automation. + mode: single + max_exceeded: silent + triggers: + - trigger: numeric_state + entity_id: + - sensor.pfv_ups_runtime_minutes + - sensor.pfv_tripp_lite_runtime_minutes + below: 20 + - trigger: numeric_state + entity_id: + - sensor.tsys1_ups_battery_charge + - sensor.tsys1_triplite_battery_charge + below: 50 + # 5-min level re-check — same edge-vs-level reasoning as the + # runtime-low alert: never let a threshold-crossing blip silence + # the plan page for an entire discharge. + - trigger: time_pattern + minutes: /5 + conditions: + - condition: template + value_template: >- + {% set ns = namespace(hit=false) %} + {% for rt, st in [ + ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'), + ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %} + {% if is_number(states(rt)) and (states(rt) | float) < 20 + and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %} + {% set ns.hit = true %} + {% endif %} + {% endfor %} + {% for ch, st in [ + ('sensor.tsys1_ups_battery_charge', 'sensor.tsys1_ups_status'), + ('sensor.tsys1_triplite_battery_charge', 'sensor.tsys1_triplite_status')] %} + {% if is_number(states(ch)) and (states(ch) | float) < 50 + and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %} + {% set ns.hit = true %} + {% endif %} + {% endfor %} + {{ ns.hit }} + # Status 'unavailable' suppresses (anti-noise); UPS COMMS LOST covers + # that failure mode separately after 5 min. + - condition: template + value_template: >- + {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none'] + or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }} + actions: + - variables: + execute_mode: >- + {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on') + and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }} + plan: >- + UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors): + 1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion, + 53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox + 2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313 + (drain + shutdown; cnodes 102/603/705 after workers) + 3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711, + 902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02 + 4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca + (ca LAST — signing path for everything above) + 5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now + 6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge); + tsys1, tsys3, tsys4, tsys5, tsys9 idle out on UPS exhaustion + (OptiPlex/Precision, no BMC) + mode_note: >- + {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode + else 'MODE: DRY RUN (plan only — nothing will be shut down)' }} + - action: logbook.log + data: + name: UPS shutdown chain + message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}" + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" + url: /lovelace-pfv-server-room/ups + message: >- + {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }} + min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min. + Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA → + pfv-bms (100) last → hypervisors. Full plan in the logbook. + - action: persistent_notification.create + data: + title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" + message: "{{ plan }}" + # Suppression stamp set LAST — a failed notify must not suppress the + # next 5-min re-page cycle. + - action: input_datetime.set_datetime + target: + entity_id: input_datetime.pfv_ups_shutdown_last_page + data: + datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}" diff --git a/tests/fixtures/secrets-stub.yaml b/tests/fixtures/secrets-stub.yaml new file mode 100644 index 0000000..4061fe5 --- /dev/null +++ b/tests/fixtures/secrets-stub.yaml @@ -0,0 +1,7 @@ +# CI-only stub for `homeassistant check_config` (see .gitea/workflows/ci.yml). +# NEVER copied to the live box. Values are syntactically-valid placeholders. +deploy_webhook_id: ci-stub-deploy-webhook +doorman_scan_webhook_id: ci-stub-scan-webhook +doorman_unlock_url: http://127.0.0.1:1/unlock-stub +gitea_auth_header: token ci-stub-gitea-token +kuma_push_url: http://127.0.0.1:1/push?status=up&msg=ci-stub diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh index db7b60a..0e5d40a 100644 --- a/tests/test_sdlc.sh +++ b/tests/test_sdlc.sh @@ -47,5 +47,46 @@ echo "=== secrets ===" t "no private keys in tree" \ "! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ." +echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ===" +# The 30-min sweep used to page every cycle even when the fleet was all-OK +# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate. +t "plant watchdog notify gated (dead-only + cooldown input)" \ + "grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml" + +# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed +# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT). +t "trend sensors fed from smoothed sources" \ + "grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'" + +# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's +# shutdown window was unmonitored. +t "runtime-low covers Tripp Lite" \ + "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'" +t "runtime-low status check maps per-UPS" \ + "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q \"sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status'\"" + +echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ===" +t "shutdown chain package present, dry-run default ON" \ + "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'" +t "armed switch default OFF" \ + "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'" +# Host coverage must hold in the PLAN BODY, not the header comment — +# scope the grep to the plan template variable. +PLAN_OK=1 +for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do + # full-read grep (not -q): the script runs with pipefail — -q's early + # exit SIGPIPEs sed and flips the pipeline status to 141 + sed -n '/plan: >-/,/mode_note:/p' packages/ups_shutdown.yaml | grep "$h" >/dev/null || PLAN_OK=0 +done +t "plan covers all seven hypervisors (plan body)" "[ \$PLAN_OK -eq 1 ]" +t "pfv-bms VM 100 ordered last of the VMs (#455)" \ + "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml" + +echo "=== level-based re-checks (review finding: edge-only triggers miss pages) ===" +t "runtime-low re-polls every 5 min" \ + "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'minutes: /5'" +t "shutdown chain re-polls every 5 min" \ + "grep -A30 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml | grep -q 'minutes: /5'" + echo "=== SDLC SUITE: $FAIL failures ===" exit "$FAIL"