diff --git a/packages/ups_shutdown.yaml b/packages/ups_shutdown.yaml new file mode 100644 index 0000000..29202e6 --- /dev/null +++ b/packages/ups_shutdown.yaml @@ -0,0 +1,124 @@ +# ============================================================================ +# UPS graceful-shutdown chain (#790) — the #455 ordering, automated +# ============================================================================ +# DESIGN (unchanged from the 2026-09-05 session spec): +# DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the +# exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the +# VMs — the brain stays alive as long as possible). Execution (real +# qmshutdown / PVE API calls) is double-gated: +# 1. input_boolean.pfv_ups_shutdown_armed — founder arm, default OFF +# 2. input_boolean.pfv_ups_shutdown_dry_run — default ON +# Arming for the Sept 11-12 live test needs, in order: founder review of +# the VM ordering below (edit this file via PR), PVE API tokens provisioned +# into on-box secrets.yaml (NeedsInput ticket), then a supervised live test. +# +# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts): +# tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01, +# 104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode +# tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605 +# tsys7: 701-711, 902 | tsys9: 904, 905 +# ============================================================================ + +input_boolean: + pfv_ups_shutdown_armed: + name: UPS shutdown chain ARMED + description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch. + icon: mdi:power-standby + initial: false + pfv_ups_shutdown_dry_run: + name: UPS shutdown DRY RUN + description: ON (default) = log/page the plan only, touch nothing. + icon: mdi:numeric-1-box-outline + initial: true + +input_datetime: + pfv_ups_shutdown_last_page: + name: UPS shutdown last plan page + has_date: true + has_time: true + icon: mdi:clock-alert-outline + +automation: + - id: pfv_ups_shutdown_plan + alias: PFV UPS shutdown - plan (dry-run by default) + description: >- + On battery with the shutdown window opening (runtime < 20 min or + charge < 50%), log + page the #455 ordered plan. Re-pages at most + every 6h per outage. When armed AND dry-run OFF (founder + supervised + test only), the plan page says EXECUTE — real dispatch wiring lands + with PVE API tokens (#790 NeedsInput); until then nothing but text + can be emitted by this automation. + mode: single + max_exceeded: silent + triggers: + - trigger: numeric_state + entity_id: + - sensor.pfv_ups_runtime_minutes + - sensor.pfv_tripp_lite_runtime_minutes + below: 20 + - trigger: numeric_state + entity_id: + - sensor.tsys1_ups_battery_charge + - sensor.tsys1_triplite_battery_charge + below: 50 + conditions: + - condition: template + value_template: >- + {% set sm = { + 'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status', + 'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status', + 'sensor.tsys1_ups_battery_charge': 'sensor.tsys1_ups_status', + 'sensor.tsys1_triplite_battery_charge': 'sensor.tsys1_triplite_status'} %} + {{ states(sm.get(trigger.entity_id, 'sensor.tsys1_ups_status')) + not in ['Online', 'OL', 'unknown', 'unavailable'] }} + - condition: template + value_template: >- + {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none'] + or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }} + actions: + - action: input_datetime.set_datetime + target: + entity_id: input_datetime.pfv_ups_shutdown_last_page + data: + datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}" + - variables: + execute_mode: >- + {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on') + and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }} + plan: >- + UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors): + 1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion, + 53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox + 2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313 + (drain + shutdown; cnodes 102/603/705 after workers) + 3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711, + 902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02 + 4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca + (ca LAST — signing path for everything above) + 5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now + 6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge); + tsys1/3/4/5/9 idle out on UPS exhaustion (OptiPlex/Precision) + mode_note: >- + {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode + else 'MODE: DRY RUN (plan only — nothing will be shut down)' }} + - action: logbook.log + data: + name: UPS shutdown chain + message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}" + - action: notify.send_message + target: + entity_id: + - notify.ultix_sidecar_new + - notify.allthes_ipad + data: + title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" + url: /lovelace-pfv-server-room/ups + message: >- + {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }} + min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min. + Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA → + pfv-bms (100) last → hypervisors. Full plan in the logbook. + - action: persistent_notification.create + data: + title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}" + message: "{{ plan }}" diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh index 9ece3bf..fe7aee5 100644 --- a/tests/test_sdlc.sh +++ b/tests/test_sdlc.sh @@ -65,5 +65,15 @@ t "runtime-low covers Tripp Lite" \ t "runtime-low status check maps per-UPS" \ "sed -n '/id: pfv_plant_ups_runtime_low/,/^ mode:/p' automations.yaml | grep -q 'status_map'" +echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ===" +t "shutdown chain package present, dry-run default ON" \ + "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'" +t "armed switch default OFF" \ + "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'" +t "plan covers all seven hypervisors" \ + "for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do grep -q \$h packages/ups_shutdown.yaml || exit 1; done" +t "pfv-bms VM 100 ordered last of the VMs (#455)" \ + "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml" + echo "=== SDLC SUITE: $FAIL failures ===" exit "$FAIL"