From 7e13612752cd079da050d7f6968bc3529ff28e39 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Sun, 6 Sep 2026 05:24:58 -0500
Subject: [PATCH 01/11] =?UTF-8?q?fix(plant):=20alert=20hygiene=20=E2=80=94?=
 =?UTF-8?q?=20watchdog=20gate,=20trend=20smoothing,=20UPS=20[#344]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three live-fire defects: unconditional 30-min watchdog page, tsys6 flap
false-positive, Tripp Lite runtime-low gap. TDD: 4 new regression tests.
On-box ha core check passed before commit.

Meat + verification: https://projects.knownelement.com/issues/344#note-4966
---
 automations.yaml         | 82 ++++++++++++++++++++++++++++------------
 configuration.yaml       |  9 +++++
 packages/doorman.yaml    | 13 ++++---
 packages/plant_snmp.yaml | 38 +++++++++++++++++--
 tests/test_sdlc.sh       | 18 +++++++++
 5 files changed, 127 insertions(+), 33 deletions(-)

diff --git a/automations.yaml b/automations.yaml
index 6facb31..b11658c 100644
--- a/automations.yaml
+++ b/automations.yaml
@@ -68,15 +68,25 @@
   mode: single
 - id: pfv_plant_ups_runtime_low
   alias: PFV plant - UPS runtime low while on battery
-  description: On battery and runtime burned below 10 min — the graceful
-    shutdown window is closing. VMs before hypervisors per #455.
+  description: >-
+    On battery and runtime burned below 10 min — the graceful shutdown
+    window is closing. VMs before hypervisors per #455 ordering. Covers
+    BOTH UPSes (APC + Tripp Lite — the Tripp Lite gap closed 2026-09-06);
+    the status check maps each runtime sensor to its own UPS status.
   triggers:
   - trigger: numeric_state
-    entity_id: sensor.pfv_ups_runtime_minutes
+    entity_id:
+    - sensor.pfv_ups_runtime_minutes
+    - sensor.pfv_tripp_lite_runtime_minutes
     below: 10
   conditions:
   - condition: template
-    value_template: "{{ states('sensor.tsys1_ups_status') not in ['Online', 'unknown', 'unavailable'] }}"
+    value_template: >-
+      {% set status_map = {
+           'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
+           'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status'} %}
+      {{ states(status_map.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
+         not in ['Online', 'OL', 'unknown', 'unavailable'] }}
   actions:
   - action: notify.send_message
     target:
@@ -86,7 +96,7 @@
     data:
       title: UPS RUNTIME LOW
       url: /lovelace-pfv-server-room/ups
-      message: On battery {{ states('sensor.pfv_ups_runtime_minutes') }} min left — start graceful shutdown now (#455 ordering).
+      message: '{{ state_attr(trigger.entity_id, ''friendly_name'') }}: {{ states(trigger.entity_id) }} min left — start graceful shutdown now (#455 ordering).'
   mode: single
 - id: pfv_plant_ups_power_restored
   alias: PFV plant - utility power restored
@@ -190,31 +200,55 @@
   mode: single
 - id: pfv_plant_sensor_watchdog
   alias: PFV plant - sensor watchdog (unavailable sensors)
-  description: HA is the environmental brain (#344) — a silent sensor is an outage.
-    Re-alert suppressed by 4h cooldown via last_notified input_text if present.
+  description: >
+    HA is the environmental brain (#344) — a silent sensor is an outage.
+    Fleet-wide sweep (every pfv_* sensor, broader than the 5-min critical
+    watchdog). PUSHES ONLY when sensors are actually dead, and re-pages a
+    still-dead fleet at most every 4h — the unconditional 30-min page was
+    retired 2026-09-06 after live-fire alert fatigue. All-OK cycles log
+    quietly; the Kuma dead-man heartbeat (#792) owns liveness signaling.
+  mode: single
+  max_exceeded: silent
   triggers:
   - trigger: time_pattern
     minutes: /30
   actions:
-  - action: notify.send_message
-    target:
-      entity_id:
-      - notify.ultix_sidecar_new
-      - notify.allthes_ipad
-    data:
-      title: SENSOR WATCHDOG
-      url: /lovelace-pfv-server-room/overview
-      message: >-
-        {% set fleet = states.sensor | selectattr('entity_id', 'match', 'sensor\.pfv_')
-          | list %}
-        {% set dead = fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list %}
-        {% if dead | length > 0 %}
+  - variables:
+      fleet: "{{ states.sensor | selectattr('entity_id', 'match', 'sensor\\.pfv_') | list }}"
+      dead: "{{ fleet | selectattr('state', 'in', ['unavailable', 'unknown']) | list }}"
+      # 4h cooldown: page again only if the last page is older than 4h
+      # (or we never paged). Fresh HA start (no restore) counts as never.
+      # as_timestamp math — aware/naive datetime subtraction throws in Jinja.
+      notify_gate: >-
+        {{ states('input_datetime.pfv_plant_watchdog_last_notify') in ['unknown', 'unavailable', 'none']
+           or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_plant_watchdog_last_notify')) > 14400 }}
+  - if:
+    - condition: template
+      value_template: "{{ dead | length > 0 and notify_gate }}"
+    then:
+    - action: notify.send_message
+      target:
+        entity_id:
+        - notify.ultix_sidecar_new
+        - notify.allthes_ipad
+      data:
+        title: SENSOR WATCHDOG
+        url: /lovelace-pfv-server-room/overview
+        message: >-
           {{ dead | length }} of {{ fleet | length }} fleet sensors silent:
           {{ dead | map(attribute='entity_id') | join(', ') }}
-        {% else %}
-          all {{ fleet | length }} fleet sensors OK
-        {% endif %}
-  mode: single
+    - action: input_datetime.set_datetime
+      target:
+        entity_id: input_datetime.pfv_plant_watchdog_last_notify
+      data:
+        datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
+    else:
+    - action: logbook.log
+      data:
+        name: Plant watchdog
+        message: >-
+          Sweep: all {{ fleet | length }} fleet sensors answering
+          ({{ dead | length }} silent, cooldown {{ 'active' if not notify_gate else 'n/a' }}).
 - id: pfv_plant_temp_rising_fast
   alias: PFV plant - temperature rising fast (rate-of-change)
   description: Catches dead AC minutes before the 104F threshold trips.
diff --git a/configuration.yaml b/configuration.yaml
index 55b8f62..74f702f 100644
--- a/configuration.yaml
+++ b/configuration.yaml
@@ -56,3 +56,12 @@ input_boolean:
   pfv_watchdog_auto_restart:
     name: Watchdog auto core restart
     icon: mdi:restart-alert
+
+# 30-min plant watchdog cooldown (#344, 2026-09-06): re-pages a dead sensor
+# at most every 4h instead of every cycle. Set by the automation on notify.
+input_datetime:
+  pfv_plant_watchdog_last_notify:
+    name: Plant watchdog last notify
+    has_date: true
+    has_time: true
+    icon: mdi:bell-cancel
diff --git a/packages/doorman.yaml b/packages/doorman.yaml
index d6f87db..7e04b8b 100644
--- a/packages/doorman.yaml
+++ b/packages/doorman.yaml
@@ -84,11 +84,14 @@ input_boolean:
 
 shell_command:
   # Unlock dispatch via curl — swapped from rest_command 2026-09-04: the
-  # rest_command integration silently fails to register on this box
-  # (404 on /api/services/rest_command despite valid config + secret;
-  # root-cause pending in #345/#741). curl is present in the HA core
-  # container. Token lives in the URL secret — process-list exposure
-  # accepted (isolated container).
+  # rest_command integration silently failed to register on this box
+  # (404 on /api/services/rest_command despite valid config + secret).
+  # NOT reproducible: 2026-09-06 the integration + doorman_unlock service
+  # are registered again (suspect: transient state after a reload-only
+  # deploy window; rest_command now also exposes a reload service).
+  # curl stays as the verified-live dispatch path — swapping back to
+  # rest_command (drops the token from the process list) is gated on one
+  # armed badge re-test (founder call, questions-v1).
   doorman_unlock: "curl -sS -m 10 '{{ doorman_unlock_url }}'"
 
 rest_command:
diff --git a/packages/plant_snmp.yaml b/packages/plant_snmp.yaml
index 8f852ed..5ed139f 100644
--- a/packages/plant_snmp.yaml
+++ b/packages/plant_snmp.yaml
@@ -543,26 +543,56 @@ sensor:
     unique_id: pfv_garage_pdu_energy
 
 # --- Rate-of-change early warning (#682): +10°F/15min = AC trouble [#344] ---
+# DRAC sensors quantize to whole °C and FLAP ~1.8°F between polls; raw-fed
+# trend sensors false-tripped TEMP RISING FAST (2026-09-06 05:02 CDT, tsys6).
+# Trend inputs are 5-min time-weighted moving averages — real climbs survive
+# the smoothing, quantization flapping does not.
+filter:
+  - platform: time_simple_moving_average
+    name: "PFV tsys1 rack temperature smoothed"
+    entity_id: sensor.pfv_tsys1_rack_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys1_rack_temperature_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys6 DRAC ambient smoothed"
+    entity_id: sensor.pfv_tsys6_drac_ambient
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys6_drac_ambient_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV tsys7 DRAC inlet smoothed"
+    entity_id: sensor.pfv_tsys7_drac_inlet
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_tsys7_drac_inlet_smoothed
+  - platform: time_simple_moving_average
+    name: "PFV garage temperature smoothed"
+    entity_id: sensor.pfv_garage_temperature
+    window: "00:05:00"
+    precision: 2
+    unique_id: pfv_garage_temperature_smoothed
+
 binary_sensor:
   - platform: trend
     sensors:
       server_room_temp_rising:
-        entity_id: sensor.pfv_tsys1_rack_temperature
+        entity_id: sensor.pfv_tsys1_rack_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_server_room_rising
       tsys6_ambient_rising:
-        entity_id: sensor.pfv_tsys6_drac_ambient
+        entity_id: sensor.pfv_tsys6_drac_ambient_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys6_ambient_rising
       tsys7_inlet_rising:
-        entity_id: sensor.pfv_tsys7_drac_inlet
+        entity_id: sensor.pfv_tsys7_drac_inlet_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_tsys7_inlet_rising
       garage_temp_rising:
-        entity_id: sensor.pfv_garage_temperature
+        entity_id: sensor.pfv_garage_temperature_smoothed
         sample_duration: 900
         min_gradient: 0.011
         unique_id: trend_garage_rising
diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh
index db7b60a..9ece3bf 100644
--- a/tests/test_sdlc.sh
+++ b/tests/test_sdlc.sh
@@ -47,5 +47,23 @@ echo "=== secrets ==="
 t "no private keys in tree" \
   "! grep -rInE 'BEGIN (RSA |OPENSSH |EC |DSA )?PRIVATE KEY' --exclude-dir=.git --exclude-dir=docs ."
 
+echo "=== alert hygiene: push only when something is wrong (2026-09-06 live-fire) ==="
+# The 30-min sweep used to page every cycle even when the fleet was all-OK
+# (48 pages/day of noise). Notify must sit behind the dead>0 + 4h-cooldown gate.
+t "plant watchdog notify gated (dead-only + cooldown input)" \
+  "grep -q 'pfv_plant_watchdog_last_notify' configuration.yaml && grep -q 'notify_gate' automations.yaml"
+
+# tsys6 DRAC ambient quantizes to whole degrees C and flaps ~2F; the raw-fed
+# trend sensor tripped TEMP RISING FAST with zero real climb (2026-09-06 05:02 CDT).
+t "trend sensors fed from smoothed sources" \
+  "grep -q 'time_simple_moving_average' packages/plant_snmp.yaml && sed -n '/^binary_sensor:/,/^utility_meter:/p' packages/plant_snmp.yaml | grep -q 'entity_id: sensor.pfv_tsys6_drac_ambient_smoothed'"
+
+# Tripp Lite had runtime sensors but no runtime-low alert — half the plant's
+# shutdown window was unmonitored.
+t "runtime-low covers Tripp Lite" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'"
+t "runtime-low status check maps per-UPS" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'status_map'"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"

From c4ebb8aae7f7c0eb4eae056ac3139ffcd937e524 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Sun, 6 Sep 2026 05:36:15 -0500
Subject: [PATCH 02/11] =?UTF-8?q?feat(ups):=20#455=20shutdown=20chain=20?=
 =?UTF-8?q?=E2=80=94=20dry-run=20plan=20pages,=20dual-gate=20arm=20[#790]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Plan data from live qm inventory (7 hosts). Execution wiring deferred
until PVE API tokens + founder ordering review. 4 regression tests.

Meat: https://projects.knownelement.com/issues/790#note-4970
---
 packages/ups_shutdown.yaml | 124 +++++++++++++++++++++++++++++++++++++
 tests/test_sdlc.sh         |  10 +++
 2 files changed, 134 insertions(+)
 create mode 100644 packages/ups_shutdown.yaml

diff --git a/packages/ups_shutdown.yaml b/packages/ups_shutdown.yaml
new file mode 100644
index 0000000..29202e6
--- /dev/null
+++ b/packages/ups_shutdown.yaml
@@ -0,0 +1,124 @@
+# ============================================================================
+# UPS graceful-shutdown chain (#790) — the #455 ordering, automated
+# ============================================================================
+# DESIGN (unchanged from the 2026-09-05 session spec):
+#   DRY_RUN is the DEFAULT and the shipped mode. The chain LOGS + PAGES the
+#   exact shutdown plan (#455: VMs before hypervisors, pfv-bms last of the
+#   VMs — the brain stays alive as long as possible). Execution (real
+#   qmshutdown / PVE API calls) is double-gated:
+#     1. input_boolean.pfv_ups_shutdown_armed   — founder arm, default OFF
+#     2. input_boolean.pfv_ups_shutdown_dry_run — default ON
+#   Arming for the Sept 11-12 live test needs, in order: founder review of
+#   the VM ordering below (edit this file via PR), PVE API tokens provisioned
+#   into on-box secrets.yaml (NeedsInput ticket), then a supervised live test.
+#
+# PLAN DATA — captured from LIVE inventory 2026-09-06 (qm list, all hosts):
+#   tsys1: 100 pfv-bms, 101 tsys-ca, 102 k8s-cnode1, 103 netinfra-01,
+#          104 librenms, 105 pve-datacenter | tsys3: 313 k8s-wnode
+#   tsys4: 400 PBS | tsys5: test/preprod bed (5xxx) | tsys6: 600-605
+#   tsys7: 701-711, 902 | tsys9: 904, 905
+# ============================================================================
+
+input_boolean:
+  pfv_ups_shutdown_armed:
+    name: UPS shutdown chain ARMED
+    description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch.
+    icon: mdi:power-standby
+    initial: false
+  pfv_ups_shutdown_dry_run:
+    name: UPS shutdown DRY RUN
+    description: ON (default) = log/page the plan only, touch nothing.
+    icon: mdi:numeric-1-box-outline
+    initial: true
+
+input_datetime:
+  pfv_ups_shutdown_last_page:
+    name: UPS shutdown last plan page
+    has_date: true
+    has_time: true
+    icon: mdi:clock-alert-outline
+
+automation:
+  - id: pfv_ups_shutdown_plan
+    alias: PFV UPS shutdown - plan (dry-run by default)
+    description: >-
+      On battery with the shutdown window opening (runtime < 20 min or
+      charge < 50%), log + page the #455 ordered plan. Re-pages at most
+      every 6h per outage. When armed AND dry-run OFF (founder + supervised
+      test only), the plan page says EXECUTE — real dispatch wiring lands
+      with PVE API tokens (#790 NeedsInput); until then nothing but text
+      can be emitted by this automation.
+    mode: single
+    max_exceeded: silent
+    triggers:
+      - trigger: numeric_state
+        entity_id:
+          - sensor.pfv_ups_runtime_minutes
+          - sensor.pfv_tripp_lite_runtime_minutes
+        below: 20
+      - trigger: numeric_state
+        entity_id:
+          - sensor.tsys1_ups_battery_charge
+          - sensor.tsys1_triplite_battery_charge
+        below: 50
+    conditions:
+      - condition: template
+        value_template: >-
+          {% set sm = {
+               'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
+               'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status',
+               'sensor.tsys1_ups_battery_charge': 'sensor.tsys1_ups_status',
+               'sensor.tsys1_triplite_battery_charge': 'sensor.tsys1_triplite_status'} %}
+          {{ states(sm.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
+             not in ['Online', 'OL', 'unknown', 'unavailable'] }}
+      - condition: template
+        value_template: >-
+          {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none']
+             or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }}
+    actions:
+      - action: input_datetime.set_datetime
+        target:
+          entity_id: input_datetime.pfv_ups_shutdown_last_page
+        data:
+          datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
+      - variables:
+          execute_mode: >-
+            {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on')
+               and not is_state('input_boolean.pfv_ups_shutdown_dry_run', 'on') }}
+          plan: >-
+            UPS ON BATTERY — #455 shutdown plan (VMs before hypervisors):
+            1. tsys5 test bed: qmshutdown 5105 awx, 5101 siem, 51017 sec onion,
+               53100-53102 preprod, 5108-5109, 513 sectestbed k8s, 5000 sandbox
+            2. k8s workers: tsys9 905, tsys7 701, tsys6 601, tsys3 313
+               (drain + shutdown; cnodes 102/603/705 after workers)
+            3. LOB/infra VMs: tsys6 600 awx, 602/605, tsys7 702 uisp, 703, 706-711,
+               902 ucs-2, 604/711 mailgw (after queue drain), tsys9 904 netinfra-02
+            4. tsys1: 104 librenms, 105 pve-dc, 103 netinfra-01, 101 tsys-ca
+               (ca LAST — signing path for everything above)
+            5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now
+            6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge);
+               tsys1/3/4/5/9 idle out on UPS exhaustion (OptiPlex/Precision)
+          mode_note: >-
+            {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode
+               else 'MODE: DRY RUN (plan only — nothing will be shut down)' }}
+      - action: logbook.log
+        data:
+          name: UPS shutdown chain
+          message: "{{ mode_note }} Plan: {{ plan | truncate(250) }}"
+      - action: notify.send_message
+        target:
+          entity_id:
+            - notify.ultix_sidecar_new
+            - notify.allthes_ipad
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          url: /lovelace-pfv-server-room/ups
+          message: >-
+            {{ mode_note }} Runtime APC {{ states('sensor.pfv_ups_runtime_minutes') }}
+            min / TrippLite {{ states('sensor.pfv_tripp_lite_runtime_minutes') }} min.
+            Order: test bed → k8s workers → LOB VMs → tsys1 infra → CA →
+            pfv-bms (100) last → hypervisors. Full plan in the logbook.
+      - action: persistent_notification.create
+        data:
+          title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
+          message: "{{ plan }}"
diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh
index 9ece3bf..fe7aee5 100644
--- a/tests/test_sdlc.sh
+++ b/tests/test_sdlc.sh
@@ -65,5 +65,15 @@ t "runtime-low covers Tripp Lite" \
 t "runtime-low status check maps per-UPS" \
   "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'status_map'"
 
+echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ==="
+t "shutdown chain package present, dry-run default ON" \
+  "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'"
+t "armed switch default OFF" \
+  "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'"
+t "plan covers all seven hypervisors" \
+  "for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do grep -q \$h packages/ups_shutdown.yaml || exit 1; done"
+t "pfv-bms VM 100 ordered last of the VMs (#455)" \
+  "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"

From 78a92582637606f6144576c5082e56d257b924e8 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Sun, 6 Sep 2026 05:42:45 -0500
Subject: [PATCH 03/11] docs: restore runbook (#628) + sparse STATUS.md [#628
 #344]

Baseline backup 34a05d70 pulled off-box. Runbook covers 3 layers:
HA-level, PBS VM-level (dev-lane drill rule), fresh-HAOS-from-git.

Meat: https://projects.knownelement.com/issues/790#note-4970
---
 STATUS.md               | 16 +++++++++
 docs/restore-runbook.md | 78 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 STATUS.md
 create mode 100644 docs/restore-runbook.md

diff --git a/STATUS.md b/STATUS.md
new file mode 100644
index 0000000..675cf60
--- /dev/null
+++ b/STATUS.md
@@ -0,0 +1,16 @@
+# STATUS — point-in-time agent state (sparse; history lives in Redmine)
+
+Updated: 2026-09-06 ~06:00 CDT · dev @ c4ebb8a · release @ 83b32d5
+
+| ticket | tier | state | item |
+|---|---|---|---|
+| #344 | prod | 🔄 | alert hygiene batch (7e13612) + #790 chain (c4ebb8a) on dev; release PR pending founder |
+| #790 | dev | ✅ | UPS shutdown chain landed, DRY RUN mode; exec wiring needs PVE tokens (NeedsInput) |
+| #628 | docs | ✅ | restore runbook + 788MB baseline backup (slug 34a05d70, off-box on ultix) |
+| #781 | infra | ⏳ | beta HA VM attempt in progress (HAOS on pfv-tsys5) |
+| #345 | prod | 🔄 | rest_command verified registered again; swap-back = founder re-test call |
+
+## Inbox
+- 2026-09-06: tsys6 ambient flap false-positive fixed via 5-min MA smoothing — watch trend sensors for 24h post-deploy
+- 2026-09-06: stlpc-artroom host down (raw+semantic sensors dead) — policy question in NeedsInput
+- gitea→HA deploy webhook still blocked (SSRF allowlist); sha-watch path is the pipeline
diff --git a/docs/restore-runbook.md b/docs/restore-runbook.md
new file mode 100644
index 0000000..afcac6f
--- /dev/null
+++ b/docs/restore-runbook.md
@@ -0,0 +1,78 @@
+# pfv-bms restore runbook (#628)
+
+Three restore layers exist. Work **inside-out**: usually only layer 1 is
+needed. Restore drills run on the **dev lane only** — never on hosts
+running prod VMs (standing rule after the 2026-09-03 PBS drill that
+I/O-starved prod HA, #684 / t/298).
+
+## Layer 0 — what is NOT in git
+
+| Artifact | Where it lives | Covered by |
+|----------|----------------|------------|
+| `secrets.yaml` (tokens, webhook ids, unlock URL) | on-box `/config/secrets.yaml` | Layer 1 + 2 backups |
+| `.storage/` (dashboards-in-storage, registry, auth) | on-box `/config/.storage/` | Layer 1 + 2 backups |
+| Zigbee/HomeKit/BT pairings | on-box | Layer 1 + 2 backups |
+
+Git (this repo, `release` branch) is the source of truth for
+configuration **YAML only**. A fresh box restored from git still needs a
+`secrets.yaml` re-provisioned from the keeper's records (~/.creds notes
++ badge/webhook ids) before HA will start.
+
+## Layer 1 — HA-level backup (config + settings, small, fast)
+
+Taken on-box via the `ha` CLI; produces a self-contained `.tar`:
+
+```bash
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups new --name "pfv-bms-$(date +%F)"'
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups list'          # find slug
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups info <slug>'   # verify size/state
+# OFF-BOX COPY (the on-box copy dies with the VM disk):
+scp -P 22222 'root@pfv-bms.knel.net:/mnt/data/supervisor/backup/<slug>.tar' /backup/destination/
+```
+
+HAOS automatic backups are CONFIRMED LIVE (daily "Automatic backup"
+entries observed 2026-09-05/06) — layer 1 has a baseline; the off-box
+copy remains the manual step.
+
+Restore (same or fresh HAOS box):
+
+```bash
+scp -P 22222 backup.tar root@pfv-bms.knel.net:/mnt/data/supervisor/backup/
+ssh -p 22222 root@pfv-bms.knel.net 'ha backups restore <slug> --type full'
+```
+
+Baseline on record: slug `34a05d70`, name
+`pfv-bms-baseline-2026-09-06-post-790`, 788 MB compressed, taken after
+the alert-hygiene + #790 batches (dev `c4ebb8a`). Off-box copy:
+`~/projects/KNEL/PhysicalPlant/pfv-bms-backups/` on ultix-streaming
+(sha256 6ff2fa6bd5f3dd2f2a47…).
+
+## Layer 2 — VM-level backup (prod PBS, nightly)
+
+- All prod VMs — **including VM 100 (pfv-bms) on pfv-tsys1** — back up
+  nightly to prod PBS, staggered 21:00–04:30 Central, throttled
+  60/30 MiB/s + job bwlimit 50 MiB/s (founder ruling t/321).
+- Restore: PVE web UI or `qmrestore` — **target a scratch VM on the dev
+  lane** (verify the restore boots there), never restore in place over
+  the live VM without a Kuma maintenance window.
+- Gap (open, OAM lane): pfv-tsys5 has no PBS job yet — beta/test VMs on
+  tsys5 are unprotected; fine for beta, not for anything promoted.
+
+## Layer 3 — config-from-git (last resort, fresh HAOS)
+
+1. Flash/install HAOS, complete onboarding, join Tailscale.
+2. `git clone git.knownelement.com:KNEL/KNELBMS` → copy tree to `/config`
+   at the `release` HEAD the fleet expects.
+3. Re-provision `secrets.yaml` (Layer 0 table) — from keeper records.
+4. `ha core check` before starting; then watch the first boot: the
+   post-deploy validation automation pages if criticals fail to
+   materialize (#778).
+5. Verify the trail: deploy sensor matches `release` HEAD, Kuma
+   heartbeat green, doorman roster sensors populated.
+
+## Related
+
+- Deploy pipeline + gate: README "Deployment pipeline"; `ha core check`
+  on-box before any release push.
+- Incident history: #684 (PBS drill), #780 (tsys1 storage), t/298
+  (Sep 3 wedge), t/320 (reload vs restart deploy semantics).

From bcbdeb74a2cdba89953945809b61a66209ea15c1 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Sun, 6 Sep 2026 06:01:51 -0500
Subject: [PATCH 04/11] =?UTF-8?q?docs:=20STATUS=20refresh=20=E2=80=94=20re?=
 =?UTF-8?q?lease=20PR=20#2,=20beta=20VM=2053110,=20asks=20in=20#811=20[#34?=
 =?UTF-8?q?4=20#781]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 STATUS.md | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/STATUS.md b/STATUS.md
index 675cf60..888f429 100644
--- a/STATUS.md
+++ b/STATUS.md
@@ -1,16 +1,18 @@
 # STATUS — point-in-time agent state (sparse; history lives in Redmine)
 
-Updated: 2026-09-06 ~06:00 CDT · dev @ c4ebb8a · release @ 83b32d5
+Updated: 2026-09-06 ~06:20 CDT · dev @ 78a9258 · release @ 83b32d5
 
 | ticket | tier | state | item |
 |---|---|---|---|
-| #344 | prod | 🔄 | alert hygiene batch (7e13612) + #790 chain (c4ebb8a) on dev; release PR pending founder |
-| #790 | dev | ✅ | UPS shutdown chain landed, DRY RUN mode; exec wiring needs PVE tokens (NeedsInput) |
-| #628 | docs | ✅ | restore runbook + 788MB baseline backup (slug 34a05d70, off-box on ultix) |
-| #781 | infra | ⏳ | beta HA VM attempt in progress (HAOS on pfv-tsys5) |
-| #345 | prod | 🔄 | rest_command verified registered again; swap-back = founder re-test call |
+| #811 | — | ⛔ | NeedsInput: founder asks (PR #2 merge, PVE tokens, beta, rotations) |
+| #344 | prod | 🔄 | release PR #2 open+mergeable (7e13612 c4ebb8a 78a9258); founder merge = deploy |
+| #790 | dev | ✅ | UPS shutdown chain in DRY RUN; exec wiring waits on PVE tokens |
+| #628 | docs | ✅ | restore runbook + baseline backup 34a05d70 off-box on ultix |
+| #781 | beta | 🔄 | VM 53110 pfv-bms-beta boots HAOS (sata0 fix); needs IPv4 + onboarding |
+| #345 | prod | 🔄 | rest_command verified registered; swap-back = founder re-test call |
 
 ## Inbox
-- 2026-09-06: tsys6 ambient flap false-positive fixed via 5-min MA smoothing — watch trend sensors for 24h post-deploy
-- 2026-09-06: stlpc-artroom host down (raw+semantic sensors dead) — policy question in NeedsInput
-- gitea→HA deploy webhook still blocked (SSRF allowlist); sha-watch path is the pipeline
+- 2026-09-06: tsys6 trend false-positive fix — watch trend sensors 24h post-deploy
+- 2026-09-06: stlpc-artroom dead all morning — policy ask in #811 item 8
+- 2026-09-06: GLPI 11 API needs OAuth (user_token dead) — CR for PR #2 filed manually
+- beta VM: MAC BC:24:11:7A:40:55 needs DHCP reservation on 192.168.3.0/24

From 0c44be9925eaca2197c9a433e06ccb85590f661d Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 06:29:34 -0500
Subject: [PATCH 05/11] =?UTF-8?q?ci:=20real=20HA=20load=20validation=20?=
 =?UTF-8?q?=E2=80=94=20check=5Fconfig=20in=20pinned=20container=20[#778]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Boots HA 2026.8.3 against the repo config with CI-only stub secrets;
catches bad Jinja/unknown keys/schema drift yamllint cannot see.

Meat: https://projects.knownelement.com/issues/778
---
 .gitea/workflows/ci.yml          | 19 +++++++++++++++++++
 tests/fixtures/secrets-stub.yaml |  7 +++++++
 2 files changed, 26 insertions(+)
 create mode 100644 tests/fixtures/secrets-stub.yaml

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index 08e2cc6..29d85a1 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -26,3 +26,22 @@ jobs:
         run: |
           chmod +x tests/test_sdlc.sh
           bash tests/test_sdlc.sh
+
+  # Real load validation: boots Home Assistant against the repo config and
+  # proves HA can parse + set up every integration (yamllint only proves the
+  # YAML parses; this catches unknown keys, bad Jinja, schema drift).
+  # Pinned to the version the box runs. Stub secrets are CI-only fixtures
+  # (tests/fixtures/secrets-stub.yaml) — never present on the live box.
+  config-check:
+    runs-on: ultix
+    container:
+      image: homeassistant/home-assistant:2026.8.3
+    steps:
+      - uses: actions/checkout@v4
+      - name: homeassistant check_config
+        run: |
+          cp tests/fixtures/secrets-stub.yaml secrets.yaml
+          python3 -m homeassistant --script check_config --config .
+          rc=$?
+          rm -f secrets.yaml home-assistant_v2.db home-assistant.log*
+          exit $rc
diff --git a/tests/fixtures/secrets-stub.yaml b/tests/fixtures/secrets-stub.yaml
new file mode 100644
index 0000000..4061fe5
--- /dev/null
+++ b/tests/fixtures/secrets-stub.yaml
@@ -0,0 +1,7 @@
+# CI-only stub for `homeassistant check_config` (see .gitea/workflows/ci.yml).
+# NEVER copied to the live box. Values are syntactically-valid placeholders.
+deploy_webhook_id: ci-stub-deploy-webhook
+doorman_scan_webhook_id: ci-stub-scan-webhook
+doorman_unlock_url: http://127.0.0.1:1/unlock-stub
+gitea_auth_header: token ci-stub-gitea-token
+kuma_push_url: http://127.0.0.1:1/push?status=up&msg=ci-stub

From b06ef7dbf06a3aeffb3c2dd7625cb435327d2c28 Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 06:43:44 -0500
Subject: [PATCH 06/11] ci: config-check base -> python:3.13 (HA py floor +
 glibc for runner node)

---
 .gitea/workflows/ci.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index 29d85a1..ed13d1d 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -30,14 +30,18 @@ jobs:
   # Real load validation: boots Home Assistant against the repo config and
   # proves HA can parse + set up every integration (yamllint only proves the
   # YAML parses; this catches unknown keys, bad Jinja, schema drift).
-  # Pinned to the version the box runs. Stub secrets are CI-only fixtures
-  # (tests/fixtures/secrets-stub.yaml) — never present on the live box.
+  # Pinned to the HA version the box runs; HA 2026.8 needs python 3.13 and a
+  # glibc debian base (the runner mounts a glibc node for checkout — alpine
+  # cannot exec it, first CI run proved that in 12 seconds).
+  # Stub secrets are CI-only fixtures — never present on the live box.
   config-check:
     runs-on: ultix
     container:
-      image: homeassistant/home-assistant:2026.8.3
+      image: python:3.13-bookworm
     steps:
       - uses: actions/checkout@v4
+      - name: install homeassistant 2026.8.3
+        run: pip install --quiet homeassistant==2026.8.3
       - name: homeassistant check_config
         run: |
           cp tests/fixtures/secrets-stub.yaml secrets.yaml

From d18e4674849f0ffc0d48d2aa1ae21136a8265888 Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 06:49:26 -0500
Subject: [PATCH 07/11] =?UTF-8?q?fix(plant):=20review-gate=20fixes=20?=
 =?UTF-8?q?=E2=80=94=20level-based=20UPS=20re-polls,=20test=20hardening=20?=
 =?UTF-8?q?[#790=20#344]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adversarial review verdict FAIL -> addressed: runtime-low + shutdown
chain re-poll /5 min (edge-crossing could miss a page mid-outage);
suppression stamp set after notify; plan body names all 7 hosts
explicitly; test fixes (pipefail SIGPIPE, comment-only assertion).
Reviewer P1 'tripp_lite slug' refuted with live entity registry
evidence (entity_id derives from name; sensor exists, 8f48730 aligned).
On-box ha core check passed with the changed files.

Review thread: https://git.knownelement.com/KNEL/KNELBMS/pulls/3
---
 automations.yaml           | 21 ++++++++++++----
 packages/ups_shutdown.yaml | 51 ++++++++++++++++++++++++++++----------
 tests/test_sdlc.sh         | 19 +++++++++++---
 3 files changed, 70 insertions(+), 21 deletions(-)

diff --git a/automations.yaml b/automations.yaml
index b11658c..5f54ce0 100644
--- a/automations.yaml
+++ b/automations.yaml
@@ -79,14 +79,25 @@
     - sensor.pfv_ups_runtime_minutes
     - sensor.pfv_tripp_lite_runtime_minutes
     below: 10
+  # 5-min level re-check: numeric_state alone fires only on the threshold
+  # CROSSING — a blip at that minute (or an HA restart mid-outage) would
+  # silence the page for the whole discharge. Level-based polling turns
+  # this into an escalation ladder while the outage persists.
+  - trigger: time_pattern
+    minutes: /5
   conditions:
   - condition: template
     value_template: >-
-      {% set status_map = {
-           'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
-           'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status'} %}
-      {{ states(status_map.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
-         not in ['Online', 'OL', 'unknown', 'unavailable'] }}
+      {% set ns = namespace(hit=false) %}
+      {% for rt, st in [
+           ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'),
+           ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %}
+        {% if is_number(states(rt)) and (states(rt) | float) < 10
+              and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+          {% set ns.hit = true %}
+        {% endif %}
+      {% endfor %}
+      {{ ns.hit }}
   actions:
   - action: notify.send_message
     target:
diff --git a/packages/ups_shutdown.yaml b/packages/ups_shutdown.yaml
index 29202e6..b4ca765 100644
--- a/packages/ups_shutdown.yaml
+++ b/packages/ups_shutdown.yaml
@@ -20,6 +20,10 @@
 # ============================================================================
 
 input_boolean:
+  # NOTE: `initial:` wins over restored state at EVERY HA start — a deploy
+  # mid-test silently reverts these to the safe defaults (armed OFF,
+  # dry-run ON). Safe direction, but re-arm after any restart during the
+  # supervised Sept 11-12 live test.
   pfv_ups_shutdown_armed:
     name: UPS shutdown chain ARMED
     description: Founder-level arm. ON + dry-run OFF = real shutdown dispatch.
@@ -61,26 +65,39 @@ automation:
           - sensor.tsys1_ups_battery_charge
           - sensor.tsys1_triplite_battery_charge
         below: 50
+      # 5-min level re-check — same edge-vs-level reasoning as the
+      # runtime-low alert: never let a threshold-crossing blip silence
+      # the plan page for an entire discharge.
+      - trigger: time_pattern
+        minutes: /5
     conditions:
       - condition: template
         value_template: >-
-          {% set sm = {
-               'sensor.pfv_ups_runtime_minutes': 'sensor.tsys1_ups_status',
-               'sensor.pfv_tripp_lite_runtime_minutes': 'sensor.tsys1_triplite_status',
-               'sensor.tsys1_ups_battery_charge': 'sensor.tsys1_ups_status',
-               'sensor.tsys1_triplite_battery_charge': 'sensor.tsys1_triplite_status'} %}
-          {{ states(sm.get(trigger.entity_id, 'sensor.tsys1_ups_status'))
-             not in ['Online', 'OL', 'unknown', 'unavailable'] }}
+          {% set ns = namespace(hit=false) %}
+          {% for rt, st in [
+               ('sensor.pfv_ups_runtime_minutes', 'sensor.tsys1_ups_status'),
+               ('sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status')] %}
+            {% if is_number(states(rt)) and (states(rt) | float) < 20
+                  and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+              {% set ns.hit = true %}
+            {% endif %}
+          {% endfor %}
+          {% for ch, st in [
+               ('sensor.tsys1_ups_battery_charge', 'sensor.tsys1_ups_status'),
+               ('sensor.tsys1_triplite_battery_charge', 'sensor.tsys1_triplite_status')] %}
+            {% if is_number(states(ch)) and (states(ch) | float) < 50
+                  and states(st) not in ['Online', 'OL', 'unknown', 'unavailable'] %}
+              {% set ns.hit = true %}
+            {% endif %}
+          {% endfor %}
+          {{ ns.hit }}
+      # Status 'unavailable' suppresses (anti-noise); UPS COMMS LOST covers
+      # that failure mode separately after 5 min.
       - condition: template
         value_template: >-
           {{ states('input_datetime.pfv_ups_shutdown_last_page') in ['unknown', 'unavailable', 'none']
              or as_timestamp(now()) - as_timestamp(states('input_datetime.pfv_ups_shutdown_last_page')) > 21600 }}
     actions:
-      - action: input_datetime.set_datetime
-        target:
-          entity_id: input_datetime.pfv_ups_shutdown_last_page
-        data:
-          datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
       - variables:
           execute_mode: >-
             {{ is_state('input_boolean.pfv_ups_shutdown_armed', 'on')
@@ -97,7 +114,8 @@ automation:
                (ca LAST — signing path for everything above)
             5. pfv-bms VM 100 LAST of the VMs — the brain stays up until now
             6. Hypervisors: iDRAC graceful off tsys6/tsys7 (PowerEdge);
-               tsys1/3/4/5/9 idle out on UPS exhaustion (OptiPlex/Precision)
+               tsys1, tsys3, tsys4, tsys5, tsys9 idle out on UPS exhaustion
+               (OptiPlex/Precision, no BMC)
           mode_note: >-
             {{ 'MODE: EXECUTE (armed, live dispatch)' if execute_mode
                else 'MODE: DRY RUN (plan only — nothing will be shut down)' }}
@@ -122,3 +140,10 @@ automation:
         data:
           title: "{{ 'UPS SHUTDOWN EXECUTING' if execute_mode else 'UPS SHUTDOWN PLAN (dry run)' }}"
           message: "{{ plan }}"
+      # Suppression stamp set LAST — a failed notify must not suppress the
+      # next 5-min re-page cycle.
+      - action: input_datetime.set_datetime
+        target:
+          entity_id: input_datetime.pfv_ups_shutdown_last_page
+        data:
+          datetime: "{{ now().strftime('%Y-%m-%d %H:%M:%S') }}"
diff --git a/tests/test_sdlc.sh b/tests/test_sdlc.sh
index fe7aee5..0e5d40a 100644
--- a/tests/test_sdlc.sh
+++ b/tests/test_sdlc.sh
@@ -63,17 +63,30 @@ t "trend sensors fed from smoothed sources" \
 t "runtime-low covers Tripp Lite" \
   "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'sensor.pfv_tripp_lite_runtime_minutes'"
 t "runtime-low status check maps per-UPS" \
-  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'status_map'"
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q \"sensor.pfv_tripp_lite_runtime_minutes', 'sensor.tsys1_triplite_status'\""
 
 echo "=== UPS shutdown chain (#790 — dry-run is the shipped mode) ==="
 t "shutdown chain package present, dry-run default ON" \
   "grep -q 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml && grep -A4 'pfv_ups_shutdown_dry_run:' packages/ups_shutdown.yaml | grep -q 'initial: true'"
 t "armed switch default OFF" \
   "grep -A4 'pfv_ups_shutdown_armed:' packages/ups_shutdown.yaml | grep -q 'initial: false'"
-t "plan covers all seven hypervisors" \
-  "for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do grep -q \$h packages/ups_shutdown.yaml || exit 1; done"
+# Host coverage must hold in the PLAN BODY, not the header comment —
+# scope the grep to the plan template variable.
+PLAN_OK=1
+for h in tsys1 tsys3 tsys4 tsys5 tsys6 tsys7 tsys9; do
+  # full-read grep (not -q): the script runs with pipefail — -q's early
+  # exit SIGPIPEs sed and flips the pipeline status to 141
+  sed -n '/plan: >-/,/mode_note:/p' packages/ups_shutdown.yaml | grep "$h" >/dev/null || PLAN_OK=0
+done
+t "plan covers all seven hypervisors (plan body)" "[ \$PLAN_OK -eq 1 ]"
 t "pfv-bms VM 100 ordered last of the VMs (#455)" \
   "grep -q 'pfv-bms VM 100 LAST' packages/ups_shutdown.yaml"
 
+echo "=== level-based re-checks (review finding: edge-only triggers miss pages) ==="
+t "runtime-low re-polls every 5 min" \
+  "sed -n '/id: pfv_plant_ups_runtime_low/,/^  mode:/p' automations.yaml | grep -q 'minutes: /5'"
+t "shutdown chain re-polls every 5 min" \
+  "grep -A30 'id: pfv_ups_shutdown_plan' packages/ups_shutdown.yaml | grep -q 'minutes: /5'"
+
 echo "=== SDLC SUITE: $FAIL failures ==="
 exit "$FAIL"

From efe314f34e598716086f1dbcfbb4d94b20e26b9f Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 06:57:05 -0500
Subject: [PATCH 08/11] =?UTF-8?q?ci:=20config-check=20py3.14=20=E2=80=94?=
 =?UTF-8?q?=20HA=202026.8.3=20requires-python=20>=3D3.14.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitea/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index ed13d1d..30e9132 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -30,14 +30,14 @@ jobs:
   # Real load validation: boots Home Assistant against the repo config and
   # proves HA can parse + set up every integration (yamllint only proves the
   # YAML parses; this catches unknown keys, bad Jinja, schema drift).
-  # Pinned to the HA version the box runs; HA 2026.8 needs python 3.13 and a
-  # glibc debian base (the runner mounts a glibc node for checkout — alpine
-  # cannot exec it, first CI run proved that in 12 seconds).
+  # Pinned to the HA version the box runs. HA 2026.8 requires python
+  # >=3.14.2 (pip refuses older), and the base must be glibc debian — the
+  # runner mounts a glibc node for checkout, alpine cannot exec it.
   # Stub secrets are CI-only fixtures — never present on the live box.
   config-check:
     runs-on: ultix
     container:
-      image: python:3.13-bookworm
+      image: python:3.14-bookworm
     steps:
       - uses: actions/checkout@v4
       - name: install homeassistant 2026.8.3

From 55ff572463ee8834e7fe1eb62416477babcd0166 Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 07:03:46 -0500
Subject: [PATCH 09/11] =?UTF-8?q?ci:=20config-check=20clones=20via=20job?=
 =?UTF-8?q?=20token=20=E2=80=94=20no=20node=20for=20actions/checkout?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitea/workflows/ci.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index 30e9132..92658ef 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -30,16 +30,21 @@ jobs:
   # Real load validation: boots Home Assistant against the repo config and
   # proves HA can parse + set up every integration (yamllint only proves the
   # YAML parses; this catches unknown keys, bad Jinja, schema drift).
-  # Pinned to the HA version the box runs. HA 2026.8 requires python
-  # >=3.14.2 (pip refuses older), and the base must be glibc debian — the
-  # runner mounts a glibc node for checkout, alpine cannot exec it.
-  # Stub secrets are CI-only fixtures — never present on the live box.
+  # Pinned to the HA version the box runs (needs python >=3.14).
+  # actions/checkout is NOT used: it needs node in the job container, and
+  # python images have none (lint works because node:20 ships node).
+  # Clone with the job's auto token instead. Stub secrets are CI-only
+  # fixtures — never present on the live box.
   config-check:
     runs-on: ultix
     container:
       image: python:3.14-bookworm
     steps:
-      - uses: actions/checkout@v4
+      - name: clone
+        run: |
+          test -n "$GITHUB_TOKEN" || { echo "GITHUB_TOKEN missing"; exit 1; }
+          git clone --depth 1 "https://x-access-token:${GITHUB_TOKEN}@git.knownelement.com/${GITHUB_REPOSITORY}.git" .
+          git log -1 --oneline
       - name: install homeassistant 2026.8.3
         run: pip install --quiet homeassistant==2026.8.3
       - name: homeassistant check_config

From aa8d78825a707b914e2162e3f8041a6a5f839c92 Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 07:15:14 -0500
Subject: [PATCH 10/11] =?UTF-8?q?ci:=20park=20config-check=20job=20?=
 =?UTF-8?q?=E2=80=94=20runner-side=20debugging=20needed=20[#778]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Local check_config passes (validated); CI job fails in the runner env.
Three causes already fixed (alpine-musl node mount, py3.13 floor,
checkout needs node); 4s failure needs UI job logs. Full trail in the
follow-up ticket. On-box 'ha core check' remains the release gate.
---
 .gitea/workflows/ci.yml | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index 92658ef..08e2cc6 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -26,31 +26,3 @@ jobs:
         run: |
           chmod +x tests/test_sdlc.sh
           bash tests/test_sdlc.sh
-
-  # Real load validation: boots Home Assistant against the repo config and
-  # proves HA can parse + set up every integration (yamllint only proves the
-  # YAML parses; this catches unknown keys, bad Jinja, schema drift).
-  # Pinned to the HA version the box runs (needs python >=3.14).
-  # actions/checkout is NOT used: it needs node in the job container, and
-  # python images have none (lint works because node:20 ships node).
-  # Clone with the job's auto token instead. Stub secrets are CI-only
-  # fixtures — never present on the live box.
-  config-check:
-    runs-on: ultix
-    container:
-      image: python:3.14-bookworm
-    steps:
-      - name: clone
-        run: |
-          test -n "$GITHUB_TOKEN" || { echo "GITHUB_TOKEN missing"; exit 1; }
-          git clone --depth 1 "https://x-access-token:${GITHUB_TOKEN}@git.knownelement.com/${GITHUB_REPOSITORY}.git" .
-          git log -1 --oneline
-      - name: install homeassistant 2026.8.3
-        run: pip install --quiet homeassistant==2026.8.3
-      - name: homeassistant check_config
-        run: |
-          cp tests/fixtures/secrets-stub.yaml secrets.yaml
-          python3 -m homeassistant --script check_config --config .
-          rc=$?
-          rm -f secrets.yaml home-assistant_v2.db home-assistant.log*
-          exit $rc

From 27f6967e1aefc0b2868c6f94325ba942528f3536 Mon Sep 17 00:00:00 2001
From: vptechops <tsgstaff-coo-vptechops@turnsys.com>
Date: Sun, 6 Sep 2026 07:18:27 -0500
Subject: [PATCH 11/11] =?UTF-8?q?docs:=20STATUS=20=E2=80=94=20review=20gat?=
 =?UTF-8?q?e=20passed,=20CI=20parked=20to=20#813,=20beta=20at=20192.168.3.?=
 =?UTF-8?q?240=20[#344=20#781=20#813]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 STATUS.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/STATUS.md b/STATUS.md
index 888f429..6112725 100644
--- a/STATUS.md
+++ b/STATUS.md
@@ -1,18 +1,18 @@
 # STATUS — point-in-time agent state (sparse; history lives in Redmine)
 
-Updated: 2026-09-06 ~06:20 CDT · dev @ 78a9258 · release @ 83b32d5
+Updated: 2026-09-06 ~07:10 CDT · dev @ aa8d788 · release @ 83b32d5
 
 | ticket | tier | state | item |
 |---|---|---|---|
-| #811 | — | ⛔ | NeedsInput: founder asks (PR #2 merge, PVE tokens, beta, rotations) |
-| #344 | prod | 🔄 | release PR #2 open+mergeable (7e13612 c4ebb8a 78a9258); founder merge = deploy |
-| #790 | dev | ✅ | UPS shutdown chain in DRY RUN; exec wiring waits on PVE tokens |
+| #811 | — | ⛔ | NeedsInput: merge PR #3, PVE tokens (#790), beta onboarding click, rotations |
+| #344 | prod | 🔄 | release PR #3 open, CI green, review gate passed (FAIL→fixed→dispositioned) |
+| #790 | dev | ✅ | shutdown chain DRY RUN, level-based re-polls; exec wiring waits on PVE tokens |
 | #628 | docs | ✅ | restore runbook + baseline backup 34a05d70 off-box on ultix |
-| #781 | beta | 🔄 | VM 53110 pfv-bms-beta boots HAOS (sata0 fix); needs IPv4 + onboarding |
+| #781 | beta | 🔄 | VM 53110 @ 192.168.3.240, HA up, owner created; onboarding form + config sync left |
+| #813 | — | ⏳ | CI check_config job parked — needs UI job log, recipe in ticket |
 | #345 | prod | 🔄 | rest_command verified registered; swap-back = founder re-test call |
 
 ## Inbox
 - 2026-09-06: tsys6 trend false-positive fix — watch trend sensors 24h post-deploy
 - 2026-09-06: stlpc-artroom dead all morning — policy ask in #811 item 8
-- 2026-09-06: GLPI 11 API needs OAuth (user_token dead) — CR for PR #2 filed manually
-- beta VM: MAC BC:24:11:7A:40:55 needs DHCP reservation on 192.168.3.0/24
+- 2026-09-06: runner container got external SIGTERM 11:35 UTC, restarted 11:37 — OAM check