vptechops
f4d96b13f0
ci / lint (push) Successful in 39s
Top-level 'filter:' cannot merge from packages (expected dict); the booleans rejected 'description'. Both found live in the 12:38 deploy: package booleans + smoothed sensors failed to materialize. Meat: https://projects.knownelement.com/issues/344#comment
151 lines
7.2 KiB
YAML
151 lines
7.2 KiB
YAML
# ============================================================================
|
|
# UPS graceful-shutdown chain (#790) — the #455 ordering, automated
|
|
# ============================================================================
|
|
# DESIGN (unchanged from the 2026-09-05 session spec):
|
|
# DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the
|
|
# exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the
|
|
# VMs — the brain stays alive as long as possible). Execution (real
|
|
# qmshutdown / PVE API calls) is double-gated:
|
|
# 1. input_boolean.pfv_ups_shutdown_armed — founder arm, default OFF
|
|
# 2. input_boolean.pfv_ups_shutdown_dry_run — default ON
|
|
# Arming for the Sept 11-12 live test needs, in order: founder review of
|
|
# the VM ordering below (edit this file via PR), PVE API tokens provisioned
|
|
# into on-box secrets.yaml (NeedsInput ticket), then a supervised live test.
|
|
#
|
|
# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts):
|
|
# tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01,
|
|
# 104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode
|
|
# tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605
|
|
# tsys7: 701-711, 902 | tsys9: 904, 905
|
|
# ============================================================================
|
|
|
|
input_boolean:
|
|
# NOTE: `initial:` wins over restored state at EVERY HA start — a deploy
|
|
# mid-test silently reverts these to the safe defaults (armed OFF,
|
|
# dry-run ON). Safe direction, but re-arm after any restart during the
|
|
# supervised Sept 11-12 live test.
|
|
# (no description: keys — input_boolean rejects them, 2026-09-06)
|
|
pfv_ups_shutdown_armed:
|
|
# Founder arm. ON + dry-run OFF = real shutdown dispatch.
|
|
name: UPS shutdown chain ARMED
|
|
icon: mdi:power-standby
|
|
initial: false
|
|
pfv_ups_shutdown_dry_run:
|
|
# ON (default) = log/page the plan only, touch nothing.
|
|
name: UPS shutdown DRY RUN
|
|
icon: mdi:numeric-1-box-outline
|
|
initial: true
|
|
|
|
input_datetime:
|
|
pfv_ups_shutdown_last_page:
|
|
name: UPS shutdown last plan page
|
|
has_date: true
|
|
has_time: true
|
|
icon: mdi:clock-alert-outline
|
|
|
|
automation:
|
|
- id: pfv_ups_shutdown_plan
|
|
alias: PFV UPS shutdown - plan (dry-run by default)
|
|
description: >-
|
|
On battery with the shutdown window opening (runtime < 20 min or
|
|
charge < 50%), log + page the #455 ordered plan. Re-pages at most
|
|
every 6h per outage. When armed AND dry-run OFF (founder + supervised
|
|
test only), the plan page says EXECUTE — real dispatch wiring lands
|
|
with PVE API tokens (#790 NeedsInput); until then nothing but text
|
|
can be emitted by this automation.
|
|
mode: single
|
|
max_exceeded: silent
|
|
triggers:
|
|
- trigger: numeric_state
|
|
entity_id:
|
|
- sensor.pfv_ups_runtime_minutes
|
|
- sensor.pfv_tripp_lite_runtime_minutes
|
|
below: 20
|
|
- trigger: numeric_state
|
|
entity_id:
|
|
- sensor.tsys1_ups_battery_charge
|
|
- sensor.tsys1_triplite_battery_charge
|
|
below: 50
|
|
# 5-min level re-check — same edge-vs-level reasoning as the
|
|
# runtime-low alert: never let a threshold-crossing blip silence
|
|
# the plan page for an entire discharge.
|
|
- trigger: time_pattern
|
|
minutes: /5
|
|
conditions:
|
|
- condition: template
|
|
value_template: >-
|
|
{% set ns = namespace(hit=false) %}
|
|
{% for rt, st in [
|
|
('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'),
|
|
('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %}
|
|
{% if is_number(states(rt)) and (states(rt) | float) < 20
|
|
and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
|
|
{% set ns.hit = true %}
|
|
{% endif %}
|
|
{% endfor %}
|
|
{% for ch, st in [
|
|
('sensor.tsys1_ups_battery_charge', 'sensor.tsys1_ups_status'),
|
|
('sensor.tsys1_triplite_battery_charge', 'sensor.tsys1_triplite_status')] %}
|
|
{% if is_number(states(ch)) and (states(ch) | float) < 50
|
|
and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
|
|
{% set ns.hit = true %}
|
|
{% endif %}
|
|
{% endfor %}
|
|
{{ ns.hit }}
|
|
# Status 'unavailable' suppresses (anti-noise); UPS COMMS LOST covers
|
|
# that failure mode separately after 5 min.
|
|
- condition: template
|
|
value_template: >-
|
|
{{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none']
|
|
or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }}
|
|
actions:
|
|
- variables:
|
|
execute_mode: >-
|
|
{{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on')
|
|
and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }}
|
|
plan: >-
|
|
UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors):
|
|
1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion,
|
|
53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox
|
|
2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313
|
|
(drain + shutdown; cnodes 102/603/705 after workers)
|
|
3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711,
|
|
902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02
|
|
4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca
|
|
(ca LAST — signing path for everything above)
|
|
5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now
|
|
6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge);
|
|
tsys1, tsys3, tsys4, tsys5, tsys9 idle out on UPS exhaustion
|
|
(OptiPlex/Precision, no BMC)
|
|
mode_note: >-
|
|
{{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode
|
|
else 'MODE: DRY RUN (plan only — nothing will be shut down)' }}
|
|
- action: logbook.log
|
|
data:
|
|
name: UPS shutdown chain
|
|
message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}"
|
|
- action: notify.send_message
|
|
target:
|
|
entity_id:
|
|
- notify.ultix_sidecar_new
|
|
- notify.allthes_ipad
|
|
data:
|
|
title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
|
|
url: /lovelace-pfv-server-room/ups
|
|
message: >-
|
|
{{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }}
|
|
min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min.
|
|
Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA →
|
|
pfv-bms (100) last → hypervisors. Full plan in the logbook.
|
|
- action: persistent_notification.create
|
|
data:
|
|
title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
|
|
message: "{{ plan }}"
|
|
# Suppression stamp set LAST — a failed notify must not suppress the
|
|
# next 5-min re-page cycle.
|
|
- action: input_datetime.set_datetime
|
|
target:
|
|
entity_id: input_datetime.pfv_ups_shutdown_last_page
|
|
data:
|
|
datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
|