# ============================================================================ # UPS graceful-shutdown chain (#790) — the #455 ordering, automated # ============================================================================ # DESIGN (unchanged from the 2026-09-05 session spec): # DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the # exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the # VMs — the brain stays alive as long as possible). Execution (real # qmshutdown / PVE API calls) is double-gated: # 1. input_boolean.pfv_ups_shutdown_armed — founder arm, default OFF # 2. input_boolean.pfv_ups_shutdown_dry_run — default ON # Arming for the Sept 11-12 live test needs, in order: founder review of # the VM ordering below (edit this file via PR), PVE API tokens provisioned # into on-box secrets.yaml (NeedsInput ticket), then a supervised live test. # # PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts): # tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01, # 104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode # tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605 # tsys7: 701-711, 902 | tsys9: 904, 905 # ============================================================================ input_boolean: # NOTE: `initial:` wins over restored state at EVERY HA start — a deploy # mid-test silently reverts these to the safe defaults (armed OFF, # dry-run ON). Safe direction, but re-arm after any restart during the # supervised Sept 11-12 live test. pfv_ups_shutdown_armed: name: UPS shutdown chain ARMED description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch. icon: mdi:power-standby initial: false pfv_ups_shutdown_dry_run: name: UPS shutdown DRY RUN description: ON (default) = log/page the plan only, touch nothing. icon: mdi:numeric-1-box-outline initial: true input_datetime: pfv_ups_shutdown_last_page: name: UPS shutdown last plan page has_date: true has_time: true icon: mdi:clock-alert-outline automation: - id: pfv_ups_shutdown_plan alias: PFV UPS shutdown - plan (dry-run by default) description: >- On battery with the shutdown window opening (runtime < 20 min or charge < 50%), log + page the #455 ordered plan. Re-pages at most every 6h per outage. When armed AND dry-run OFF (founder + supervised test only), the plan page says EXECUTE — real dispatch wiring lands with PVE API tokens (#790 NeedsInput); until then nothing but text can be emitted by this automation. mode: single max_exceeded: silent triggers: - trigger: numeric_state entity_id: - sensor.pfv_ups_runtime_minutes - sensor.pfv_tripp_lite_runtime_minutes below: 20 - trigger: numeric_state entity_id: - sensor.tsys1_ups_battery_charge - sensor.tsys1_triplite_battery_charge below: 50 # 5-min level re-check — same edge-vs-level reasoning as the # runtime-low alert: never let a threshold-crossing blip silence # the plan page for an entire discharge. - trigger: time_pattern minutes: /5 conditions: - condition: template value_template: >- {% set ns = namespace(hit=false) %} {% for rt, st in [ ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'), ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %} {% if is_number(states(rt)) and (states(rt) | float) < 20 and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %} {% set ns.hit = true %} {% endif %} {% endfor %} {% for ch, st in [ ('sensor.tsys1_ups_battery_charge', 'sensor.tsys1_ups_status'), ('sensor.tsys1_triplite_battery_charge', 'sensor.tsys1_triplite_status')] %} {% if is_number(states(ch)) and (states(ch) | float) < 50 and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %} {% set ns.hit = true %} {% endif %} {% endfor %} {{ ns.hit }} # Status 'unavailable' suppresses (anti-noise); UPS COMMS LOST covers # that failure mode separately after 5 min. - condition: template value_template: >- {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none'] or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }} actions: - variables: execute_mode: >- {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on') and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }} plan: >- UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors): 1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion, 53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox 2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313 (drain + shutdown; cnodes 102/603/705 after workers) 3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711, 902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02 4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca (ca LAST — signing path for everything above) 5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now 6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge); tsys1, tsys3, tsys4, tsys5, tsys9 idle out on UPS exhaustion (OptiPlex/Precision, no BMC) mode_note: >- {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode else 'MODE: DRY RUN (plan only — nothing will be shut down)' }} - action: logbook.log data: name: UPS shutdown chain message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}" - action: notify.send_message target: entity_id: - notify.ultix_sidecar_new - notify.allthes_ipad data: title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" url: /lovelace-pfv-server-room/ups message: >- {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }} min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min. Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA → pfv-bms (100) last → hypervisors. Full plan in the logbook. - action: persistent_notification.create data: title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" message: "{{ plan }}" # Suppression stamp set LAST — a failed notify must not suppress the # next 5-min re-page cycle. - action: input_datetime.set_datetime target: entity_id: input_datetime.pfv_ups_shutdown_last_page data: datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"