Merge pull request 'Release: alert hygiene batch + UPS shutdown chain (dry-run) + restore runbook' (#3) from dev into release

Reviewed-on: #3

STATUS.md

@@ -0,0 +1,18 @@
+# STATUS — point-in-time agent state (sparse; history lives in Redmine)
+
+Updated: 2026-09-06 ~07:10 CDT · dev @ aa8d788 · release @ 83b32d5
+
+| ticket | tier | state | item |
+|---|---|---|---|
+| #811 | — | ⛔ | NeedsInput: merge PR #3, PVE tokens (#790), beta onboarding click, rotations |
+| #344 | prod | 🔄 | release PR #3 open, CI green, review gate passed (FAIL→fixed→dispositioned) |
+| #790 | dev | ✅ | shutdown chain DRY RUN, level-based re-polls; exec wiring waits on PVE tokens |
+| #628 | docs | ✅ | restore runbook + baseline backup 34a05d70 off-box on ultix |
+| #781 | beta | 🔄 | VM 53110 @ 192.168.3.240, HA up, owner created; onboarding form + config sync left |
+| #813 | — | ⏳ | CI check_config job parked — needs UI job log, recipe in ticket |
+| #345 | prod | 🔄 | rest_command verified registered; swap-back = founder re-test call |
+
+## Inbox
+- 2026-09-06: tsys6 trend false-positive fix — watch trend sensors 24h post-deploy
+- 2026-09-06: stlpc-artroom dead all morning — policy ask in #811 item 8
+- 2026-09-06: runner container got external SIGTERM 11:35 UTC, restarted 11:37 — OAM check

automations.yaml

@@ -68,15 +68,36 @@
   mode: single
 - id: pfv_plant_ups_runtime_low
   alias: PFV plant - UPS runtime low while on battery
-  description: On battery and runtime burned below 10 min — the graceful
+  description: >-
-    shutdown window is closing. VMs before hypervisors per #455.
+    On battery and runtime burned below 10 min — the graceful shutdown
+    window is closing. VMs before hypervisors per #455 ordering. Covers
+    BOTH UPSes (APC + Tripp Lite — the Tripp Lite gap closed 2026-09-06);
+    the status check maps each runtime sensor to its own UPS status.
   triggers:
   - trigger: numeric_state
-    entity_id: sensor.pfv_ups_runtime_minutes
+    entity_id:
+    - sensor.pfv_ups_runtime_minutes
+    - sensor.pfv_tripp_lite_runtime_minutes
     below: 10
+  # 5-min level re-check: numeric_state alone fires only on the threshold
+  # CROSSING — a blip at that minute (or an HA restart mid-outage) would
+  # silence the page for the whole discharge. Level-based polling turns
+  # this into an escalation ladder while the outage persists.
+  - trigger: time_pattern
+    minutes: /5
   conditions:
   - condition: template
-    value_template: "{{ states('sensor.tsys1_ups_status') not in ['Online', 'unknown', 'unavailable'] }}"
+    value_template: >-
+      {% set ns = namespace(hit=false) %}
+      {% for rt, st in [
+           ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'),
+           ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %}
+        {% if is_number(states(rt)) and (states(rt) | float) < 10
+              and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+          {% set ns.hit = true %}
+        {% endif %}
+      {% endfor %}
+      {{ ns.hit }}
   actions:
   - action: notify.send_message
     target:
@@ -86,7 +107,7 @@
     data:
       title: UPS RUNTIME LOW
       url: /lovelace-pfv-server-room/ups
-      message: On battery {{ states('sensor.pfv_ups_runtime_minutes') }} min left — start graceful shutdown now (#455 ordering).
+      message: '{{ state_attr(trigger.entity_id, ''friendly_name'') }}: {{ states(trigger.entity_id) }} min left — start graceful shutdown now (#455 ordering).'
   mode: single
 - id: pfv_plant_ups_power_restored
   alias: PFV plant - utility power restored
@@ -190,31 +211,55 @@
   mode: single
 - id: pfv_plant_sensor_watchdog
   alias: PFV plant - sensor watchdog (unavailable sensors)
-  description: HA is the environmental brain (#344) — a silent sensor is an outage.
+  description: >
-    Re-alert suppressed by 4h cooldown via last_notified input_text if present.
+    HA is the environmental brain (#344) — a silent sensor is an outage.
+    Fleet-wide sweep (every pfv_* sensor, broader than the 5-min critical
+    watchdog). PUSHES ONLY when sensors are actually dead, and re-pages a
+    still-dead fleet at most every 4h — the unconditional 30-min page was
+    retired 2026-09-06 after live-fire alert fatigue. All-OK cycles log
+    quietly; the Kuma dead-man heartbeat (#792) owns liveness signaling.
+  mode: single
+  max_exceeded: silent
   triggers:
   - trigger: time_pattern
     minutes: /30
   actions:
-  - action: notify.send_message
+  - variables:
-    target:
+      fleet: "{{ states.sensor | selectattr('entity_id', 'match', 'sensor\\.pfv_') | list }}"
-      entity_id:
+      dead: "{{ fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list }}"
-      - notify.ultix_sidecar_new
+      # 4h cooldown: page again only if the last page is older than 4h
-      - notify.allthes_ipad
+      # (or we never paged). Fresh HA start (no restore) counts as never.
-    data:
+      # as_timestamp math — aware/naive datetime subtraction throws in Jinja.
-      title: SENSOR WATCHDOG
+      notify_gate: >-
-      url: /lovelace-pfv-server-room/overview
+        {{ states('input_datetime.pfv_plant_watchdog_last_notify') in ['unknown', 'unavailable', 'none']
-      message: >-
+           or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_plant_watchdog_last_notify')) > 14400 }}
-        {% set fleet = states.sensor | selectattr('entity_id', 'match', 'sensor\.pfv_')
+  - if:
-          | list %}
+    - condition: template
-        {% set dead = fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list %}
+      value_template: "{{ dead | length > 0 and notify_gate }}"
-        {% if dead | length > 0 %}
+    then:
+    - action: notify.send_message
+      target:
+        entity_id:
+        - notify.ultix_sidecar_new
+        - notify.allthes_ipad
+      data:
+        title: SENSOR WATCHDOG
+        url: /lovelace-pfv-server-room/overview
+        message: >-
           {{ dead | length }} of {{ fleet | length }} fleet sensors silent:
           {{ dead | map(attribute='entity_id') | join(', ') }}
-        {% else %}
+    - action: input_datetime.set_datetime
-          all {{ fleet | length }} fleet sensors OK
+      target:
-        {% endif %}
+        entity_id: input_datetime.pfv_plant_watchdog_last_notify
-  mode: single
+      data:
+        datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
+    else:
+    - action: logbook.log
+      data:
+        name: Plant watchdog
+        message: >-
+          Sweep: all {{ fleet | length }} fleet sensors answering
+          ({{ dead | length }} silent, cooldown {{ 'active' if not notify_gate else 'n/a' }}).
 - id: pfv_plant_temp_rising_fast
   alias: PFV plant - temperature rising fast (rate-of-change)
   description: Catches dead AC minutes before the 104F threshold trips.

configuration.yaml

@@ -56,3 +56,12 @@ input_boolean:
   pfv_watchdog_auto_restart:
     name: Watchdog auto core restart
     icon: mdi:restart-alert
+
+# 30-min plant watchdog cooldown (#344, 2026-09-06): re-pages a dead sensor
+# at most every 4h instead of every cycle. Set by the automation on notify.
+input_datetime:
+  pfv_plant_watchdog_last_notify:
+    name: Plant watchdog last notify
+    has_date: true
+    has_time: true
+    icon: mdi:bell-cancel

docs/restore-runbook.md

@@ -0,0 +1,78 @@
+# pfv-bms restore runbook (#628)
+
+Three restore layers exist. Work **inside-out**: usually only layer 1 is
+needed. Restore drills run on the **dev lane only** — never on hosts
+running prod VMs (standing rule after the 2026-09-03 PBS drill that
+I/O-starved prod HA, #684 / t/298).
+
+## Layer 0 — what is NOT in git
+
+| Artifact | Where it lives | Covered by |
+|----------|----------------|------------|
+| `secrets.yaml` (tokens, webhook ids, unlock URL) | on-box `/config/secrets.yaml` | Layer 1 + 2 backups |
+| `.storage/` (dashboards-in-storage, registry, auth) | on-box `/config/.storage/` | Layer 1 + 2 backups |
+| Zigbee/HomeKit/BT pairings | on-box | Layer 1 + 2 backups |
+
+Git (this repo, `release` branch) is the source of truth for
+configuration **YAML only**. A fresh box restored from git still needs a
+`secrets.yaml` re-provisioned from the keeper's records (~/.creds notes
++ badge/webhook ids) before HA will start.
+
+## Layer 1 — HA-level backup (config + settings, small, fast)
+
+Taken on-box via the `ha` CLI; produces a self-contained `.tar`:
+
+```bash
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups new --name "pfv-bms-$(date +%F)"'
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups list'          # find slug
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups info <slug>'   # verify size/state
+# OFF-BOX COPY (the on-box copy dies with the VM disk):
+scp -P 22222 'root@pfv-bms.knel.net:/mnt/data/supervisor/backup/<slug>.tar' /backup/destination/
+```
+
+HAOS automatic backups are CONFIRMED LIVE (daily "Automatic backup"
+entries observed 2026-09-05/06) — layer 1 has a baseline; the off-box
+copy remains the manual step.
+
+Restore (same or fresh HAOS box):
+
+```bash
+scp -P 22222 backup.tar root@pfv-bms.knel.net:/mnt/data/supervisor/backup/
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups restore <slug> --type full'
+```
+
+Baseline on record: slug `34a05d70`, name
+`pfv-bms-baseline-2026-09-06-post-790`, 788 MB compressed, taken after
+the alert-hygiene + #790 batches (dev `c4ebb8a`). Off-box copy:
+`~/projects/KNEL/PhysicalPlant/pfv-bms-backups/` on ultix-streaming
+(sha256 6ff2fa6bd5f3dd2f2a47…).
+
+## Layer 2 — VM-level backup (prod PBS, nightly)
+
+- All prod VMs — **including VM 100 (pfv-bms) on pfv-tsys1** — back up
+  nightly to prod PBS, staggered 21:00–04:30 Central, throttled
+  60/30 MiB/s + job bwlimit 50 MiB/s (founder ruling t/321).
+- Restore: PVE web UI or `qmrestore` — **target a scratch VM on the dev
+  lane** (verify the restore boots there), never restore in place over
+  the live VM without a Kuma maintenance window.
+- Gap (open, OAM lane): pfv-tsys5 has no PBS job yet — beta/test VMs on
+  tsys5 are unprotected; fine for beta, not for anything promoted.
+
+## Layer 3 — config-from-git (last resort, fresh HAOS)
+
+1. Flash/install HAOS, complete onboarding, join Tailscale.
+2. `git clone git.knownelement.com:KNEL/KNELBMS` → copy tree to `/config`
+   at the `release` HEAD the fleet expects.
+3. Re-provision `secrets.yaml` (Layer 0 table) — from keeper records.
+4. `ha core check` before starting; then watch the first boot: the
+   post-deploy validation automation pages if criticals fail to
+   materialize (#778).
+5. Verify the trail: deploy sensor matches `release` HEAD, Kuma
+   heartbeat green, doorman roster sensors populated.
+
+## Related
+
+- Deploy pipeline + gate: README "Deployment pipeline"; `ha core check`
+  on-box before any release push.
+- Incident history: #684 (PBS drill), #780 (tsys1 storage), t/298
+  (Sep 3 wedge), t/320 (reload vs restart deploy semantics).

packages/doorman.yaml

@@ -84,11 +84,14 @@ input_boolean:
 
 shell_command:
   # Unlock dispatch via curl — swapped from rest_command 2026-09-04: the
-  # rest_command integration silently fails to register on this box
+  # rest_command integration silently failed to register on this box
-  # (404 on /api/services/rest_command despite valid config + secret;
+  # (404 on /api/services/rest_command despite valid config + secret).
-  # root-cause pending in #345/#741). curl is present in the HA core
+  # NOT reproducible: 2026-09-06 the integration + doorman_unlock service
-  # container. Token lives in the URL secret — process-list exposure
+  # are registered again (suspect: transient state after a reload-only
-  # accepted (isolated container).
+  # deploy window; rest_command now also exposes a reload service).
+  # curl stays as the verified-live dispatch path — swapping back to
+  # rest_command (drops the token from the process list) is gated on one
+  # armed badge re-test (founder call, questions-v1).
   doorman_unlock: "curl -sS -m 10 '{{ doorman_unlock_url }}'"
 
 rest_command:

packages/plant_snmp.yaml

@@ -543,26 +543,56 @@ sensor:
     unique_id: pfv_garage_pdu_energy
 
 # --- Rate-of-change early warning (#682): +10°F/15min = AC trouble [#344] ---
+# DRAC sensors quantize to whole °C and FLAP ~1.8°F between polls; raw-fed
+# trend sensors false-tripped TEMP RISING FAST (2026-09-06 05:02 CDT, tsys6).
+# Trend inputs are 5-min time-weighted moving averages — real climbs survive
+# the smoothing, quantization flapping does not.
+filter:
+  - platform: time_simple_moving_average
+    name: "PFV tsys1 rack temperature smoothed"
+    entity_id: sensor.pfv_tsys1_rack_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys1_rack_temperature_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys6 DRAC ambient smoothed"
+    entity_id: sensor.pfv_tsys6_drac_ambient
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys6_drac_ambient_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys7 DRAC inlet smoothed"
+    entity_id: sensor.pfv_tsys7_drac_inlet
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys7_drac_inlet_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV garage temperature smoothed"
+    entity_id: sensor.pfv_garage_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_garage_temperature_smoothed
+
 binary_sensor:
   - platform: trend
     sensors:
       server_room_temp_rising:
-        entity_id: sensor.pfv_tsys1_rack_temperature
+        entity_id: sensor.pfv_tsys1_rack_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_server_room_rising
       tsys6_ambient_rising:
-        entity_id: sensor.pfv_tsys6_drac_ambient
+        entity_id: sensor.pfv_tsys6_drac_ambient_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys6_ambient_rising
       tsys7_inlet_rising:
-        entity_id: sensor.pfv_tsys7_drac_inlet
+        entity_id: sensor.pfv_tsys7_drac_inlet_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys7_inlet_rising
       garage_temp_rising:
-        entity_id: sensor.pfv_garage_temperature
+        entity_id: sensor.pfv_garage_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_garage_rising

packages/ups_shutdown.yaml

@@ -0,0 +1,149 @@
+# ============================================================================
+# UPS graceful-shutdown chain (#790) — the #455 ordering, automated
+# ============================================================================
+# DESIGN (unchanged from the 2026-09-05 session spec):
+#   DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the
+#   exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the
+#   VMs — the brain stays alive as long as possible). Execution (real
+#   qmshutdown / PVE API calls) is double-gated:
+#     1. input_boolean.pfv_ups_shutdown_armed   — founder arm, default OFF
+#     2. input_boolean.pfv_ups_shutdown_dry_run — default ON
+#   Arming for the Sept 11-12 live test needs, in order: founder review of
+#   the VM ordering below (edit this file via PR), PVE API tokens provisioned
+#   into on-box secrets.yaml (NeedsInput ticket), then a supervised live test.
+#
+# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts):
+#   tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01,
+#          104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode
+#   tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605
+#   tsys7: 701-711, 902 | tsys9: 904, 905
+# ============================================================================
+
+input_boolean:
+  # NOTE: `initial:` wins over restored state at EVERY HA start — a deploy
+  # mid-test silently reverts these to the safe defaults (armed OFF,
+  # dry-run ON). Safe direction, but re-arm after any restart during the
+  # supervised Sept 11-12 live test.
+  pfv_ups_shutdown_armed:
+    name: UPS shutdown chain ARMED
+    description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch.
+    icon: mdi:power-standby
+    initial: false
+  pfv_ups_shutdown_dry_run:
+    name: UPS shutdown DRY RUN
+    description: ON (default) = log/page the plan only, touch nothing.
+    icon: mdi:numeric-1-box-outline
+    initial: true
+
+input_datetime:
+  pfv_ups_shutdown_last_page:
+    name: UPS shutdown last plan page
+    has_date: true
+    has_time: true
+    icon: mdi:clock-alert-outline
+
+automation:
+  - id: pfv_ups_shutdown_plan
+    alias: PFV UPS shutdown - plan (dry-run by default)
+    description: >-
+      On battery with the shutdown window opening (runtime < 20 min or
+      charge < 50%), log + page the #455 ordered plan. Re-pages at most
+      every 6h per outage. When armed AND dry-run OFF (founder + supervised
+      test only), the plan page says EXECUTE — real dispatch wiring lands
+      with PVE API tokens (#790 NeedsInput); until then nothing but text
+      can be emitted by this automation.
+    mode: single
+    max_exceeded: silent
+    triggers:
+      - trigger: numeric_state
+        entity_id:
+          - sensor.pfv_ups_runtime_minutes
+          - sensor.pfv_tripp_lite_runtime_minutes
+        below: 20
+      - trigger: numeric_state
+        entity_id:
+          - sensor.tsys1_ups_battery_charge
+          - sensor.tsys1_triplite_battery_charge
+        below: 50
+      # 5-min level re-check — same edge-vs-level reasoning as the
+      # runtime-low alert: never let a threshold-crossing blip silence
+      # the plan page for an entire discharge.
+      - trigger: time_pattern
+        minutes: /5
+    conditions:
+      - condition: template
+        value_template: >-
+          {% set ns = namespace(hit=false) %}
+          {% for rt, st in [
+               ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'),
+               ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %}
+            {% if is_number(states(rt)) and (states(rt) | float) < 20
+                  and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+              {% set ns.hit = true %}
+            {% endif %}
+          {% endfor %}
+          {% for ch, st in [
+               ('sensor.tsys1_ups_battery_charge', 'sensor.tsys1_ups_status'),
+               ('sensor.tsys1_triplite_battery_charge', 'sensor.tsys1_triplite_status')] %}
+            {% if is_number(states(ch)) and (states(ch) | float) < 50
+                  and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+              {% set ns.hit = true %}
+            {% endif %}
+          {% endfor %}
+          {{ ns.hit }}
+      # Status 'unavailable' suppresses (anti-noise); UPS COMMS LOST covers
+      # that failure mode separately after 5 min.
+      - condition: template
+        value_template: >-
+          {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none']
+             or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }}
+    actions:
+      - variables:
+          execute_mode: >-
+            {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on')
+               and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }}
+          plan: >-
+            UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors):
+            1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion,
+               53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox
+            2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313
+               (drain + shutdown; cnodes 102/603/705 after workers)
+            3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711,
+               902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02
+            4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca
+               (ca LAST — signing path for everything above)
+            5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now
+            6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge);
+               tsys1, tsys3, tsys4, tsys5, tsys9 idle out on UPS exhaustion
+               (OptiPlex/Precision, no BMC)
+          mode_note: >-
+            {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode
+               else 'MODE: DRY RUN (plan only — nothing will be shut down)' }}
+      - action: logbook.log
+        data:
+          name: UPS shutdown chain
+          message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}"
+      - action: notify.send_message
+        target:
+          entity_id:
+            - notify.ultix_sidecar_new
+            - notify.allthes_ipad
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          url: /lovelace-pfv-server-room/ups
+          message: >-
+            {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }}
+            min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min.
+            Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA →
+            pfv-bms (100) last → hypervisors. Full plan in the logbook.
+      - action: persistent_notification.create
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          message: "{{ plan }}"
+      # Suppression stamp set LAST — a failed notify must not suppress the
+      # next 5-min re-page cycle.
+      - action: input_datetime.set_datetime
+        target:
+          entity_id: input_datetime.pfv_ups_shutdown_last_page
+        data:
+          datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"

tests/fixtures/secrets-stub.yaml

@@ -0,0 +1,7 @@
+# CI-only stub for `homeassistant check_config` (see .gitea/workflows/ci.yml).
+# NEVER copied to the live box. Values are syntactically-valid placeholders.
+deploy_webhook_id: ci-stub-deploy-webhook
+doorman_scan_webhook_id: ci-stub-scan-webhook
+doorman_unlock_url: http://127.0.0.1:1/unlock-stub
+gitea_auth_header: token ci-stub-gitea-token
+kuma_push_url: http://127.0.0.1:1/push?status=up&msg=ci-stub

tests/test_sdlc.sh

@@ -47,5 +47,46 @@ echo "=== secrets ==="
 t "no private keys in tree" \
   "! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ."
 
+echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ==="
+# The 30-min sweep used to page every cycle even when the fleet was all-OK
+# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate.
+t "plant watchdog notify gated (dead-only + cooldown input)" \
+  "grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml"
+
+# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed
+# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT).
+t "trend sensors fed from smoothed sources" \
+  "grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'"
+
+# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's
+# shutdown window was unmonitored.
+t "runtime-low covers Tripp Lite" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'"
+t "runtime-low status check maps per-UPS" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q \"sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status'\""
+
+echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ==="
+t "shutdown chain package present, dry-run default ON" \
+  "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'"
+t "armed switch default OFF" \
+  "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'"
+# Host coverage must hold in the PLAN BODY, not the header comment —
+# scope the grep to the plan template variable.
+PLAN_OK=1
+for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do
+  # full-read grep (not -q): the script runs with pipefail — -q's early
+  # exit SIGPIPEs sed and flips the pipeline status to 141
+  sed -n '/plan: >-/,/mode_note:/p' packages/ups_shutdown.yaml | grep "$h" >/dev/null || PLAN_OK=0
+done
+t "plan covers all seven hypervisors (plan body)" "[ \$PLAN_OK -eq 1 ]"
+t "pfv-bms VM 100 ordered last of the VMs (#455)" \
+  "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml"
+
+echo "=== level-based re-checks (review finding: edge-only triggers miss pages) ==="
+t "runtime-low re-polls every 5 min" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'minutes: /5'"
+t "shutdown chain re-polls every 5 min" \
+  "grep -A30 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml | grep -q 'minutes: /5'"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"