diff --git a/automations.yaml b/automations.yaml index 7e5bc64..affeb7a 100644 --- a/automations.yaml +++ b/automations.yaml @@ -526,3 +526,113 @@ back to {{ states(trigger.entity_id) }}°F and holding — was {{ trigger.from_state.state }}°F when the alert cleared (tier fire line {{ fire }}°F). mode: single +- id: pfv_watchdog_stale_entities + alias: PFV watchdog - stale entity auto-heal + description: > + Detects silent poller wedges (2026-09-04: snmp tsys9 went 12h + undetected; NUT APC 3h) and auto-heals config-entry platforms via + homeassistant.reload_config_entry — no restart needed. Escalates to + notify when reload does not recover. Auto core restart ONLY while + input_boolean.pfv_watchdog_auto_restart is ON and entities stay + stale — restart costs ~1 min downtime and re-registers webhooks + (#345, #789). v1 notify repeats each cycle while stale (visible + alarm); throttle in v2. + mode: single + max_exceeded: silent + triggers: + - trigger: time_pattern + minutes: /5 + variables: + checks: + - [sensor.pfv_tsys1_rack_temperature, 20] + - [sensor.pfv_tsys1_drive_temperature, 30] + - [sensor.pfv_tsys3_cpu_temperature, 20] + - [sensor.pfv_tsys4_ambient_temperature, 20] + - [sensor.pfv_tsys6_cpu_temperature, 20] + - [sensor.pfv_tsys6_drac_ambient, 30] + - [sensor.pfv_tsys7_cpu_temperature, 20] + - [sensor.pfv_tsys7_drac_inlet, 30] + - [sensor.pfv_tsys9_cpu_temperature, 20] + - [sensor.pfv_tsys9_drive_temperature, 30] + - [sensor.tsys1_ups_status, 15] + - [sensor.tsys1_triplite_status, 15] + actions: + - variables: + stale_list: >- + {% set ns = namespace(stale=[]) %} + {% for pair in checks %} + {% set s = states[pair[0]] %} + {% if s is not none and s.last_updated is defined + and (now() - s.last_updated).total_seconds() > pair[1] * 60 %} + {% set ns.stale = ns.stale + [pair[0]] %} + {% endif %} + {% endfor %} + {{ ns.stale }} + - condition: template + value_template: "{{ stale_list | length > 0 }}" + - repeat: + for_each: "{{ stale_list }}" + sequence: + - action: homeassistant.reload_config_entry + data: + entity_id: "{{ repeat.item }}" + - delay: + hours: 0 + minutes: 2 + seconds: 0 + - variables: + still_stale: >- + {% set ns = namespace(stale=[]) %} + {% for pair in checks %} + {% set s = states[pair[0]] %} + {% if s is not none and s.last_updated is defined + and (now() - s.last_updated).total_seconds() > pair[1] * 60 %} + {% set ns.stale = ns.stale + [pair[0]] %} + {% endif %} + {% endfor %} + {{ ns.stale }} + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: WATCHDOG + clickAction: /lovelace-pfv-server-room/temps + url: /lovelace-pfv-server-room/temps + message: >- + Stale after auto-reload: {{ still_stale | join(', ') }}. + Auto-restart is {{ 'ON' if is_state('input_boolean.pfv_watchdog_auto_restart','on') + else 'OFF (arm it on the watchdog switch if you want unattended restarts)' }}. + - if: + - condition: template + value_template: "{{ is_state('input_boolean.pfv_watchdog_auto_restart','on') }}" + then: + - delay: + hours: 0 + minutes: 30 + seconds: 0 + - variables: + hard_stale: >- + {% set ns = namespace(stale=[]) %} + {% for pair in checks %} + {% set s = states[pair[0]] %} + {% if s is not none and s.last_updated is defined + and (now() - s.last_updated).total_seconds() > pair[1] * 60 %} + {% set ns.stale = ns.stale + [pair[0]] %} + {% endif %} + {% endfor %} + {{ ns.stale }} + - if: + - condition: template + value_template: "{{ hard_stale | length > 0 }}" + then: + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: WATCHDOG — restarting core + message: "Still stale after reload + 30 min: {{ hard_stale | join(', ') }}. Restarting HA core now." + - action: homeassistant.restart diff --git a/configuration.yaml b/configuration.yaml index 9403948..1977491 100644 --- a/configuration.yaml +++ b/configuration.yaml @@ -34,3 +34,10 @@ lovelace: title: STLP filename: dashboards/stlp.yaml icon: mdi:home + +# Watchdog arm switch (#789): auto core restart on unresolved staleness. +# Default OFF — reload_config_entry heals config-entry wedges without it. +input_boolean: + pfv_watchdog_auto_restart: + name: Watchdog auto core restart + icon: mdi:restart-alert