fix(plant): alert hygiene — watchdog gate, trend smoothing, UPS [#344]

Three live-fire defects: unconditional 30-min watchdog page, tsys6 flap
false-positive, Tripp Lite runtime-low gap. TDD: 4 new regression tests.
On-box ha core check passed before commit.

Meat + verification: https://projects.knownelement.com/issues/344#note-4966

automations.yaml

@@ -68,15 +68,25 @@
   mode: single
 - id: pfv_plant_ups_runtime_low
   alias: PFV plant - UPS runtime low while on battery
-  description: On battery and runtime burned below 10 min — the graceful
+  description: >-
-    shutdown window is closing. VMs before hypervisors per #455.
+    On battery and runtime burned below 10 min — the graceful shutdown
+    window is closing. VMs before hypervisors per #455 ordering. Covers
+    BOTH UPSes (APC + Tripp Lite — the Tripp Lite gap closed 2026-09-06);
+    the status check maps each runtime sensor to its own UPS status.
   triggers:
   - trigger: numeric_state
-    entity_id: sensor.pfv_ups_runtime_minutes
+    entity_id:
+    - sensor.pfv_ups_runtime_minutes
+    - sensor.pfv_tripp_lite_runtime_minutes
     below: 10
   conditions:
   - condition: template
-    value_template: "{{ states('sensor.tsys1_ups_status') not in ['Online', 'unknown', 'unavailable'] }}"
+    value_template: >-
+      {% set status_map = {
+           'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
+           'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status'} %}
+      {{ states(status_map.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
+         not in ['Online', 'OL', 'unknown', 'unavailable'] }}
   actions:
   - action: notify.send_message
     target:
@@ -86,7 +96,7 @@
     data:
       title: UPS RUNTIME LOW
       url: /lovelace-pfv-server-room/ups
-      message: On battery {{ states('sensor.pfv_ups_runtime_minutes') }} min left — start graceful shutdown now (#455 ordering).
+      message: '{{ state_attr(trigger.entity_id, ''friendly_name'') }}: {{ states(trigger.entity_id) }} min left — start graceful shutdown now (#455 ordering).'
   mode: single
 - id: pfv_plant_ups_power_restored
   alias: PFV plant - utility power restored
@@ -190,12 +200,32 @@
   mode: single
 - id: pfv_plant_sensor_watchdog
   alias: PFV plant - sensor watchdog (unavailable sensors)
-  description: HA is the environmental brain (#344) — a silent sensor is an outage.
+  description: >
-    Re-alert suppressed by 4h cooldown via last_notified input_text if present.
+    HA is the environmental brain (#344) — a silent sensor is an outage.
+    Fleet-wide sweep (every pfv_* sensor, broader than the 5-min critical
+    watchdog). PUSHES ONLY when sensors are actually dead, and re-pages a
+    still-dead fleet at most every 4h — the unconditional 30-min page was
+    retired 2026-09-06 after live-fire alert fatigue. All-OK cycles log
+    quietly; the Kuma dead-man heartbeat (#792) owns liveness signaling.
+  mode: single
+  max_exceeded: silent
   triggers:
   - trigger: time_pattern
     minutes: /30
   actions:
+  - variables:
+      fleet: "{{ states.sensor | selectattr('entity_id', 'match', 'sensor\\.pfv_') | list }}"
+      dead: "{{ fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list }}"
+      # 4h cooldown: page again only if the last page is older than 4h
+      # (or we never paged). Fresh HA start (no restore) counts as never.
+      # as_timestamp math — aware/naive datetime subtraction throws in Jinja.
+      notify_gate: >-
+        {{ states('input_datetime.pfv_plant_watchdog_last_notify') in ['unknown', 'unavailable', 'none']
+           or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_plant_watchdog_last_notify')) > 14400 }}
+  - if:
+    - condition: template
+      value_template: "{{ dead | length > 0 and notify_gate }}"
+    then:
     - action: notify.send_message
       target:
         entity_id:
@@ -205,16 +235,20 @@
         title: SENSOR WATCHDOG
         url: /lovelace-pfv-server-room/overview
         message: >-
-        {% set fleet = states.sensor | selectattr('entity_id', 'match', 'sensor\.pfv_')
-          | list %}
-        {% set dead = fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list %}
-        {% if dead | length > 0 %}
           {{ dead | length }} of {{ fleet | length }} fleet sensors silent:
           {{ dead | map(attribute='entity_id') | join(', ') }}
-        {% else %}
+    - action: input_datetime.set_datetime
-          all {{ fleet | length }} fleet sensors OK
+      target:
-        {% endif %}
+        entity_id: input_datetime.pfv_plant_watchdog_last_notify
-  mode: single
+      data:
+        datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
+    else:
+    - action: logbook.log
+      data:
+        name: Plant watchdog
+        message: >-
+          Sweep: all {{ fleet | length }} fleet sensors answering
+          ({{ dead | length }} silent, cooldown {{ 'active' if not notify_gate else 'n/a' }}).
 - id: pfv_plant_temp_rising_fast
   alias: PFV plant - temperature rising fast (rate-of-change)
   description: Catches dead AC minutes before the 104F threshold trips.

configuration.yaml

@@ -56,3 +56,12 @@ input_boolean:
   pfv_watchdog_auto_restart:
     name: Watchdog auto core restart
     icon: mdi:restart-alert
+
+# 30-min plant watchdog cooldown (#344, 2026-09-06): re-pages a dead sensor
+# at most every 4h instead of every cycle. Set by the automation on notify.
+input_datetime:
+  pfv_plant_watchdog_last_notify:
+    name: Plant watchdog last notify
+    has_date: true
+    has_time: true
+    icon: mdi:bell-cancel

packages/doorman.yaml

@@ -84,11 +84,14 @@ input_boolean:
 
 shell_command:
   # Unlock dispatch via curl — swapped from rest_command 2026-09-04: the
-  # rest_command integration silently fails to register on this box
+  # rest_command integration silently failed to register on this box
-  # (404 on /api/services/rest_command despite valid config + secret;
+  # (404 on /api/services/rest_command despite valid config + secret).
-  # root-cause pending in #345/#741). curl is present in the HA core
+  # NOT reproducible: 2026-09-06 the integration + doorman_unlock service
-  # container. Token lives in the URL secret — process-list exposure
+  # are registered again (suspect: transient state after a reload-only
-  # accepted (isolated container).
+  # deploy window; rest_command now also exposes a reload service).
+  # curl stays as the verified-live dispatch path — swapping back to
+  # rest_command (drops the token from the process list) is gated on one
+  # armed badge re-test (founder call, questions-v1).
   doorman_unlock: "curl -sS -m 10 '{{ doorman_unlock_url }}'"
 
 rest_command:

packages/plant_snmp.yaml

@@ -543,26 +543,56 @@ sensor:
     unique_id: pfv_garage_pdu_energy
 
 # --- Rate-of-change early warning (#682): +10°F/15min = AC trouble [#344] ---
+# DRAC sensors quantize to whole °C and FLAP ~1.8°F between polls; raw-fed
+# trend sensors false-tripped TEMP RISING FAST (2026-09-06 05:02 CDT, tsys6).
+# Trend inputs are 5-min time-weighted moving averages — real climbs survive
+# the smoothing, quantization flapping does not.
+filter:
+  - platform: time_simple_moving_average
+    name: "PFV tsys1 rack temperature smoothed"
+    entity_id: sensor.pfv_tsys1_rack_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys1_rack_temperature_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys6 DRAC ambient smoothed"
+    entity_id: sensor.pfv_tsys6_drac_ambient
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys6_drac_ambient_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys7 DRAC inlet smoothed"
+    entity_id: sensor.pfv_tsys7_drac_inlet
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys7_drac_inlet_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV garage temperature smoothed"
+    entity_id: sensor.pfv_garage_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_garage_temperature_smoothed
+
 binary_sensor:
   - platform: trend
     sensors:
       server_room_temp_rising:
-        entity_id: sensor.pfv_tsys1_rack_temperature
+        entity_id: sensor.pfv_tsys1_rack_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_server_room_rising
       tsys6_ambient_rising:
-        entity_id: sensor.pfv_tsys6_drac_ambient
+        entity_id: sensor.pfv_tsys6_drac_ambient_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys6_ambient_rising
       tsys7_inlet_rising:
-        entity_id: sensor.pfv_tsys7_drac_inlet
+        entity_id: sensor.pfv_tsys7_drac_inlet_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys7_inlet_rising
       garage_temp_rising:
-        entity_id: sensor.pfv_garage_temperature
+        entity_id: sensor.pfv_garage_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_garage_rising

tests/test_sdlc.sh

@@ -47,5 +47,23 @@ echo "=== secrets ==="
 t "no private keys in tree" \
   "! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ."
 
+echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ==="
+# The 30-min sweep used to page every cycle even when the fleet was all-OK
+# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate.
+t "plant watchdog notify gated (dead-only + cooldown input)" \
+  "grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml"
+
+# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed
+# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT).
+t "trend sensors fed from smoothed sources" \
+  "grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'"
+
+# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's
+# shutdown window was unmonitored.
+t "runtime-low covers Tripp Lite" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'"
+t "runtime-low status check maps per-UPS" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'status_map'"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"