2026-09-06 12:21:42 +00:00
2 changed files with 134 additions and 0 deletions
@@ -0,0 +1,124 @@
+# ============================================================================
+# UPS graceful-shutdown chain (#790) — the #455 ordering, automated
+# ============================================================================
+# DESIGN (unchanged from the 2026-09-05 session spec):
+#   DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the
+#   exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the
+#   VMs — the brain stays alive as long as possible). Execution (real
+#   qmshutdown / PVE API calls) is double-gated:
+#     1. input_boolean.pfv_ups_shutdown_armed   — founder arm, default OFF
+#     2. input_boolean.pfv_ups_shutdown_dry_run — default ON
+#   Arming for the Sept 11-12 live test needs, in order: founder review of
+#   the VM ordering below (edit this file via PR), PVE API tokens provisioned
+#   into on-box secrets.yaml (NeedsInput ticket), then a supervised live test.
+#
+# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts):
+#   tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01,
+#          104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode
+#   tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605
+#   tsys7: 701-711, 902 | tsys9: 904, 905
+# ============================================================================
+
+input_boolean:
+  pfv_ups_shutdown_armed:
+    name: UPS shutdown chain ARMED
+    description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch.
+    icon: mdi:power-standby
+    initial: false
+  pfv_ups_shutdown_dry_run:
+    name: UPS shutdown DRY RUN
+    description: ON (default) = log/page the plan only, touch nothing.
+    icon: mdi:numeric-1-box-outline
+    initial: true
+
+input_datetime:
+  pfv_ups_shutdown_last_page:
+    name: UPS shutdown last plan page
+    has_date: true
+    has_time: true
+    icon: mdi:clock-alert-outline
+
+automation:
+  - id: pfv_ups_shutdown_plan
+    alias: PFV UPS shutdown - plan (dry-run by default)
+    description: >-
+      On battery with the shutdown window opening (runtime < 20 min or
+      charge < 50%), log + page the #455 ordered plan. Re-pages at most
+      every 6h per outage. When armed AND dry-run OFF (founder + supervised
+      test only), the plan page says EXECUTE — real dispatch wiring lands
+      with PVE API tokens (#790 NeedsInput); until then nothing but text
+      can be emitted by this automation.
+    mode: single
+    max_exceeded: silent
+    triggers:
+      - trigger: numeric_state
+        entity_id:
+          - sensor.pfv_ups_runtime_minutes
+          - sensor.pfv_tripp_lite_runtime_minutes
+        below: 20
+      - trigger: numeric_state
+        entity_id:
+          - sensor.tsys1_ups_battery_charge
+          - sensor.tsys1_triplite_battery_charge
+        below: 50
+    conditions:
+      - condition: template
+        value_template: >-
+          {% set sm = {
+               'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
+               'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status',
+               'sensor.tsys1_ups_battery_charge': 'sensor.tsys1_ups_status',
+               'sensor.tsys1_triplite_battery_charge': 'sensor.tsys1_triplite_status'} %}
+          {{ states(sm.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
+             not in ['Online', 'OL', 'unknown', 'unavailable'] }}
+      - condition: template
+        value_template: >-
+          {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none']
+             or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }}
+    actions:
+      - action: input_datetime.set_datetime
+        target:
+          entity_id: input_datetime.pfv_ups_shutdown_last_page
+        data:
+          datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
+      - variables:
+          execute_mode: >-
+            {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on')
+               and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }}
+          plan: >-
+            UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors):
+            1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion,
+               53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox
+            2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313
+               (drain + shutdown; cnodes 102/603/705 after workers)
+            3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711,
+               902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02
+            4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca
+               (ca LAST — signing path for everything above)
+            5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now
+            6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge);
+               tsys1/3/4/5/9 idle out on UPS exhaustion (OptiPlex/Precision)
+          mode_note: >-
+            {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode
+               else 'MODE: DRY RUN (plan only — nothing will be shut down)' }}
+      - action: logbook.log
+        data:
+          name: UPS shutdown chain
+          message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}"
+      - action: notify.send_message
+        target:
+          entity_id:
+            - notify.ultix_sidecar_new
+            - notify.allthes_ipad
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          url: /lovelace-pfv-server-room/ups
+          message: >-
+            {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }}
+            min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min.
+            Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA →
+            pfv-bms (100) last → hypervisors. Full plan in the logbook.
+      - action: persistent_notification.create
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          message: "{{ plan }}"
@@ -65,5 +65,15 @@ t "runtime-low covers Tripp Lite" \
 t "runtime-low status check maps per-UPS" \
  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'status_map'"

+echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ==="
+t "shutdown chain package present, dry-run default ON" \
+  "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'"
+t "armed switch default OFF" \
+  "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'"
+t "plan covers all seven hypervisors" \
+  "for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do grep -q \$h packages/ups_shutdown.yaml || exit 1; done"
+t "pfv-bms VM 100 ordered last of the VMs (#455)" \
+  "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"