From 7e13612752cd079da050d7f6968bc3529ff28e39 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Sun, 6 Sep 2026 05:24:58 -0500 Subject: [PATCH] =?UTF-8?q?fix(plant):=20alert=20hygiene=20=E2=80=94=20wat?= =?UTF-8?q?chdog=20gate,=20trend=20smoothing,=20UPS=20[#344]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three live-fire defects: unconditional 30-min watchdog page, tsys6 flap false-positive, Tripp Lite runtime-low gap. TDD: 4 new regression tests. On-box ha core check passed before commit. Meat + verification: https://projects.knownelement.com/issues/344#note-4966 --- automations.yaml | 82 ++++++++++++++++++++++++++++------------ configuration.yaml | 9 +++++ packages/doorman.yaml | 13 ++++--- packages/plant_snmp.yaml | 38 +++++++++++++++++-- tests/test_sdlc.sh | 18 +++++++++ 5 files changed, 127 insertions(+), 33 deletions(-) diff --git a/automations.yaml b/automations.yaml index 6facb31..b11658c 100644 --- a/automations.yaml +++ b/automations.yaml @@ -68,15 +68,25 @@ mode: single - id: pfv_plant_ups_runtime_low alias: PFV plant - UPS runtime low while on battery - description: On battery and runtime burned below 10 min — the graceful - shutdown window is closing. VMs before hypervisors per #455. + description: >- + On battery and runtime burned below 10 min — the graceful shutdown + window is closing. VMs before hypervisors per #455 ordering. Covers + BOTH UPSes (APC + Tripp Lite — the Tripp Lite gap closed 2026-09-06); + the status check maps each runtime sensor to its own UPS status. triggers: - trigger: numeric_state - entity_id: sensor.pfv_ups_runtime_minutes + entity_id: + - sensor.pfv_ups_runtime_minutes + - sensor.pfv_tripp_lite_runtime_minutes below: 10 conditions: - condition: template - value_template: "{{ states('sensor.tsys1_ups_status') not in ['Online', 'unknown', 'unavailable'] }}" + value_template: >- + {% set status_map = { + 'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status', + 'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status'} %} + {{ states(status_map.get(trigger.entity_id, 'sensor.tsys1_ups_status')) + not in ['Online', 'OL', 'unknown', 'unavailable'] }} actions: - action: notify.send_message target: @@ -86,7 +96,7 @@ data: title: UPS RUNTIME LOW url: /lovelace-pfv-server-room/ups - message: On battery {{ states('sensor.pfv_ups_runtime_minutes') }} min left — start graceful shutdown now (#455 ordering). + message: '{{ state_attr(trigger.entity_id, ''friendly_name'') }}: {{ states(trigger.entity_id) }} min left — start graceful shutdown now (#455 ordering).' mode: single - id: pfv_plant_ups_power_restored alias: PFV plant - utility power restored @@ -190,31 +200,55 @@ mode: single - id: pfv_plant_sensor_watchdog alias: PFV plant - sensor watchdog (unavailable sensors) - description: HA is the environmental brain (#344) — a silent sensor is an outage. - Re-alert suppressed by 4h cooldown via last_notified input_text if present. + description: > + HA is the environmental brain (#344) — a silent sensor is an outage. + Fleet-wide sweep (every pfv_* sensor, broader than the 5-min critical + watchdog). PUSHES ONLY when sensors are actually dead, and re-pages a + still-dead fleet at most every 4h — the unconditional 30-min page was + retired 2026-09-06 after live-fire alert fatigue. All-OK cycles log + quietly; the Kuma dead-man heartbeat (#792) owns liveness signaling. + mode: single + max_exceeded: silent triggers: - trigger: time_pattern minutes: /30 actions: - - action: notify.send_message - target: - entity_id: - - notify.ultix_sidecar_new - - notify.allthes_ipad - data: - title: SENSOR WATCHDOG - url: /lovelace-pfv-server-room/overview - message: >- - {% set fleet = states.sensor | selectattr('entity_id', 'match', 'sensor\.pfv_') - | list %} - {% set dead = fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list %} - {% if dead | length > 0 %} + - variables: + fleet: "{{ states.sensor | selectattr('entity_id', 'match', 'sensor\\.pfv_') | list }}" + dead: "{{ fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list }}" + # 4h cooldown: page again only if the last page is older than 4h + # (or we never paged). Fresh HA start (no restore) counts as never. + # as_timestamp math — aware/naive datetime subtraction throws in Jinja. + notify_gate: >- + {{ states('input_datetime.pfv_plant_watchdog_last_notify') in ['unknown', 'unavailable', 'none'] + or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_plant_watchdog_last_notify')) > 14400 }} + - if: + - condition: template + value_template: "{{ dead | length > 0 and notify_gate }}" + then: + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: SENSOR WATCHDOG + url: /lovelace-pfv-server-room/overview + message: >- {{ dead | length }} of {{ fleet | length }} fleet sensors silent: {{ dead | map(attribute='entity_id') | join(', ') }} - {% else %} - all {{ fleet | length }} fleet sensors OK - {% endif %} - mode: single + - action: input_datetime.set_datetime + target: + entity_id: input_datetime.pfv_plant_watchdog_last_notify + data: + datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}" + else: + - action: logbook.log + data: + name: Plant watchdog + message: >- + Sweep: all {{ fleet | length }} fleet sensors answering + ({{ dead | length }} silent, cooldown {{ 'active' if not notify_gate else 'n/a' }}). - id: pfv_plant_temp_rising_fast alias: PFV plant - temperature rising fast (rate-of-change) description: Catches dead AC minutes before the 104F threshold trips. diff --git a/configuration.yaml b/configuration.yaml index 55b8f62..74f702f 100644 --- a/configuration.yaml +++ b/configuration.yaml @@ -56,3 +56,12 @@ input_boolean: pfv_watchdog_auto_restart: name: Watchdog auto core restart icon: mdi:restart-alert + +# 30-min plant watchdog cooldown (#344, 2026-09-06): re-pages a dead sensor +# at most every 4h instead of every cycle. Set by the automation on notify. +input_datetime: + pfv_plant_watchdog_last_notify: + name: Plant watchdog last notify + has_date: true + has_time: true + icon: mdi:bell-cancel diff --git a/packages/doorman.yaml b/packages/doorman.yaml index d6f87db..7e04b8b 100644 --- a/packages/doorman.yaml +++ b/packages/doorman.yaml @@ -84,11 +84,14 @@ input_boolean: shell_command: # Unlock dispatch via curl — swapped from rest_command 2026-09-04: the - # rest_command integration silently fails to register on this box - # (404 on /api/services/rest_command despite valid config + secret; - # root-cause pending in #345/#741). curl is present in the HA core - # container. Token lives in the URL secret — process-list exposure - # accepted (isolated container). + # rest_command integration silently failed to register on this box + # (404 on /api/services/rest_command despite valid config + secret). + # NOT reproducible: 2026-09-06 the integration + doorman_unlock service + # are registered again (suspect: transient state after a reload-only + # deploy window; rest_command now also exposes a reload service). + # curl stays as the verified-live dispatch path — swapping back to + # rest_command (drops the token from the process list) is gated on one + # armed badge re-test (founder call, questions-v1). doorman_unlock: "curl -sS -m 10 '{{ doorman_unlock_url }}'" rest_command: diff --git a/packages/plant_snmp.yaml b/packages/plant_snmp.yaml index 8f852ed..5ed139f 100644 --- a/packages/plant_snmp.yaml +++ b/packages/plant_snmp.yaml @@ -543,26 +543,56 @@ sensor: unique_id: pfv_garage_pdu_energy # --- Rate-of-change early warning (#682): +10°F/15min = AC trouble [#344] --- +# DRAC sensors quantize to whole °C and FLAP ~1.8°F between polls; raw-fed +# trend sensors false-tripped TEMP RISING FAST (2026-09-06 05:02 CDT, tsys6). +# Trend inputs are 5-min time-weighted moving averages — real climbs survive +# the smoothing, quantization flapping does not. +filter: + - platform: time_simple_moving_average + name: "PFV tsys1 rack temperature smoothed" + entity_id: sensor.pfv_tsys1_rack_temperature + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys1_rack_temperature_smoothed + - platform: time_simple_moving_average + name: "PFV tsys6 DRAC ambient smoothed" + entity_id: sensor.pfv_tsys6_drac_ambient + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys6_drac_ambient_smoothed + - platform: time_simple_moving_average + name: "PFV tsys7 DRAC inlet smoothed" + entity_id: sensor.pfv_tsys7_drac_inlet + window: "00:05:00" + precision: 2 + unique_id: pfv_tsys7_drac_inlet_smoothed + - platform: time_simple_moving_average + name: "PFV garage temperature smoothed" + entity_id: sensor.pfv_garage_temperature + window: "00:05:00" + precision: 2 + unique_id: pfv_garage_temperature_smoothed + binary_sensor: - platform: trend sensors: server_room_temp_rising: - entity_id: sensor.pfv_tsys1_rack_temperature + entity_id: sensor.pfv_tsys1_rack_temperature_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_server_room_rising tsys6_ambient_rising: - entity_id: sensor.pfv_tsys6_drac_ambient + entity_id: sensor.pfv_tsys6_drac_ambient_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_tsys6_ambient_rising tsys7_inlet_rising: - entity_id: sensor.pfv_tsys7_drac_inlet + entity_id: sensor.pfv_tsys7_drac_inlet_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_tsys7_inlet_rising garage_temp_rising: - entity_id: sensor.pfv_garage_temperature + entity_id: sensor.pfv_garage_temperature_smoothed sample_duration: 900 min_gradient: 0.011 unique_id: trend_garage_rising diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh index db7b60a..9ece3bf 100644 --- a/tests/test_sdlc.sh +++ b/tests/test_sdlc.sh @@ -47,5 +47,23 @@ echo "=== secrets ===" t "no private keys in tree" \ "! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ." +echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ===" +# The 30-min sweep used to page every cycle even when the fleet was all-OK +# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate. +t "plant watchdog notify gated (dead-only + cooldown input)" \ + "grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml" + +# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed +# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT). +t "trend sensors fed from smoothed sources" \ + "grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'" + +# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's +# shutdown window was unmonitored. +t "runtime-low covers Tripp Lite" \ + "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'" +t "runtime-low status check maps per-UPS" \ + "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'status_map'" + echo "=== SDLC SUITE: $FAIL failures ===" exit "$FAIL"