feat(watchdog): stale-entity auto-heal — reload, notify, guarded restart [#789]

5-min staleness checks on 12 critical entities; reload_config_entry auto-heal (no restart); notify on failure; auto-restart behind input_boolean arm (default OFF). https://projects.knownelement.com/issues/789
2026-09-04 10:55:00 -05:00
parent 523a56bd52
commit 4f54635b33
2 changed files with 117 additions and 0 deletions
@@ -526,3 +526,113 @@
        back to {{ states(trigger.entity_id) }}°F and holding — was
        {{ trigger.from_state.state }}°F when the alert cleared (tier fire line {{ fire }}°F).
  mode: single
+- id: pfv_watchdog_stale_entities
+  alias: PFV watchdog - stale entity auto-heal
+  description: >
+    Detects silent poller wedges (2026-09-04: snmp tsys9 went 12h
+    undetected; NUT APC 3h) and auto-heals config-entry platforms via
+    homeassistant.reload_config_entry — no restart needed. Escalates to
+    notify when reload does not recover. Auto core restart ONLY while
+    input_boolean.pfv_watchdog_auto_restart is ON and entities stay
+    stale — restart costs ~1 min downtime and re-registers webhooks
+    (#345, #789). v1 notify repeats each cycle while stale (visible
+    alarm); throttle in v2.
+  mode: single
+  max_exceeded: silent
+  triggers:
+  - trigger: time_pattern
+    minutes: /5
+  variables:
+    checks:
+    - [sensor.pfv_tsys1_rack_temperature, 20]
+    - [sensor.pfv_tsys1_drive_temperature, 30]
+    - [sensor.pfv_tsys3_cpu_temperature, 20]
+    - [sensor.pfv_tsys4_ambient_temperature, 20]
+    - [sensor.pfv_tsys6_cpu_temperature, 20]
+    - [sensor.pfv_tsys6_drac_ambient, 30]
+    - [sensor.pfv_tsys7_cpu_temperature, 20]
+    - [sensor.pfv_tsys7_drac_inlet, 30]
+    - [sensor.pfv_tsys9_cpu_temperature, 20]
+    - [sensor.pfv_tsys9_drive_temperature, 30]
+    - [sensor.tsys1_ups_status, 15]
+    - [sensor.tsys1_triplite_status, 15]
+  actions:
+  - variables:
+      stale_list: >-
+        {% set ns = namespace(stale=[]) %}
+        {% for pair in checks %}
+          {% set s = states[pair[0]] %}
+          {% if s is not none and s.last_updated is defined
+                and (now() - s.last_updated).total_seconds() > pair[1] * 60 %}
+            {% set ns.stale = ns.stale + [pair[0]] %}
+          {% endif %}
+        {% endfor %}
+        {{ ns.stale }}
+  - condition: template
+    value_template: "{{ stale_list | length > 0 }}"
+  - repeat:
+      for_each: "{{ stale_list }}"
+      sequence:
+      - action: homeassistant.reload_config_entry
+        data:
+          entity_id: "{{ repeat.item }}"
+  - delay:
+      hours: 0
+      minutes: 2
+      seconds: 0
+  - variables:
+      still_stale: >-
+        {% set ns = namespace(stale=[]) %}
+        {% for pair in checks %}
+          {% set s = states[pair[0]] %}
+          {% if s is not none and s.last_updated is defined
+                and (now() - s.last_updated).total_seconds() > pair[1] * 60 %}
+            {% set ns.stale = ns.stale + [pair[0]] %}
+          {% endif %}
+        {% endfor %}
+        {{ ns.stale }}
+  - action: notify.send_message
+    target:
+      entity_id:
+      - notify.ultix_sidecar_new
+      - notify.allthes_ipad
+    data:
+      title: WATCHDOG
+      clickAction: /lovelace-pfv-server-room/temps
+      url: /lovelace-pfv-server-room/temps
+      message: >-
+        Stale after auto-reload: {{ still_stale | join(', ') }}.
+        Auto-restart is {{ 'ON' if is_state('input_boolean.pfv_watchdog_auto_restart','on')
+        else 'OFF (arm it on the watchdog switch if you want unattended restarts)' }}.
+  - if:
+    - condition: template
+      value_template: "{{ is_state('input_boolean.pfv_watchdog_auto_restart','on') }}"
+    then:
+    - delay:
+        hours: 0
+        minutes: 30
+        seconds: 0
+    - variables:
+        hard_stale: >-
+          {% set ns = namespace(stale=[]) %}
+          {% for pair in checks %}
+            {% set s = states[pair[0]] %}
+            {% if s is not none and s.last_updated is defined
+                  and (now() - s.last_updated).total_seconds() > pair[1] * 60 %}
+              {% set ns.stale = ns.stale + [pair[0]] %}
+            {% endif %}
+          {% endfor %}
+          {{ ns.stale }}
+    - if:
+      - condition: template
+        value_template: "{{ hard_stale | length > 0 }}"
+      then:
+      - action: notify.send_message
+        target:
+          entity_id:
+          - notify.ultix_sidecar_new
+          - notify.allthes_ipad
+        data:
+          title: WATCHDOG — restarting core
+          message: "Still stale after reload + 30 min: {{ hard_stale | join(', ') }}. Restarting HA core now."
+      - action: homeassistant.restart