From 8ba0ae1d6fb22b88d5f985f0e8ec7433683e7563 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Thu, 3 Sep 2026 21:57:15 -0500
Subject: [PATCH] =?UTF-8?q?feat(cm):=20pve-snapshot.sh=20=E2=80=94=20snaps?=
 =?UTF-8?q?hot=20lifecycle=20for=20change=20management=20[#769]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

create/list/rollback/clean with cm-<ticket>-<label>-<ts> naming,
protected-VM regex (pfv-bms/netinfra/PMG/PBS), Kuma-window gate on
rollback, retention cleaner, local audit log. Verified end-to-end on
tsys5 against throwaway VM 59999 (created, rolled back, destroyed).

https://projects.knownelement.com/issues/769#note-4152
---
 scripts/pve-snapshot.sh | 145 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100755 scripts/pve-snapshot.sh

diff --git a/scripts/pve-snapshot.sh b/scripts/pve-snapshot.sh
new file mode 100755
index 0000000..981f58e
--- /dev/null
+++ b/scripts/pve-snapshot.sh
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# pve-snapshot.sh — change-management snapshot lifecycle for PVE VMs [#769]
+#
+# Supports the sectestbed loop: deploy playbook -> check -> adapt -> repeat,
+# with a rollback point at every iteration. Snapshots are named
+#   cm-<ticket>-<label>-<YYYYMMDD-HHMMSS>
+# so they sort chronologically and are auditable per ticket.
+#
+# Usage:
+#   pve-snapshot.sh create <vmid> <label> <ticket#> [--desc "text"]
+#   pve-snapshot.sh list   <vmid>
+#   pve-snapshot.sh rollback <vmid> <snapshot-name> [--yes] [--no-start]
+#   pve-snapshot.sh clean  <vmid> [--keep 5] [--min-age-days 14] [--yes]
+#
+# Environment:
+#   PROX_HOST      target PVE host DNS name (default pfv-tsys5 — the
+#                  non-prod host where sectestbed + preprod live)
+#   PROTECTED_RE   regex of VM names that are NEVER touched without
+#                  --force-protected (default covers the founder's
+#                  golden rules: pfv-bms, netinfra, PMG, PBS)
+#
+# Golden rules honored (Charles, 2026-09-03):
+#   - pfv-bms and netinfra-01/02 are protected by default (name regex).
+#   - NEVER a hypervisor reboot; this tool only snapshots/rolls back VMs.
+#   - rollback STOPS the VM: an Uptime Kuma maintenance window is required
+#     for any monitored VM — the tool prints the reminder and refuses
+#     rollback on monitored-looking names without --yes.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REMOTE="${REMOTE:-$SCRIPT_DIR/../tests/remote.sh}"
+PROX_HOST="${PROX_HOST:-pfv-tsys5}"
+PROTECTED_RE="${PROTECTED_RE:-(^|-)bms|netinfra|mailgw|pbs|-pmg|pfv-bms}"
+LOG_FILE="${LOG_FILE:-$SCRIPT_DIR/../.crush/cm-snapshots.log}"
+
+log() { printf '%s\n' "$*" >&2; }
+die() { log "FAIL: $*"; exit 1; }
+
+[ -e "$REMOTE" ] || die "SSH chokepoint not found at $REMOTE"
+
+prox() { timeout "${PROX_TIMEOUT:-120}" env PROX_HOST="$PROX_HOST" bash "$REMOTE" prox "$1"; }
+
+vm_name() { prox "qm list" | awk -v id="$1" '$1==id {print $2}'; }
+
+guard_protected() {
+    local vmid="$1" name force
+    name="$(vm_name "$vmid")"
+    [ -n "$name" ] || die "VMID $vmid not found on $PROX_HOST"
+    if printf '%s' "$name" | grep -Eq "$PROTECTED_RE"; then
+        [ "${3:-}" = "force" ] || die "$name matches PROTECTED_RE ($PROTECTED_RE); pass --force-protected if you REALLY mean it (founder golden rules)"
+    fi
+    log "$2: $name (VMID $vmid) on $PROX_HOST"
+}
+
+nowstamp() { date +%Y%m%d-%H%M%S; }
+
+log_action() {
+    mkdir -p "$(dirname "$LOG_FILE")"
+    printf '%s host=%s %s\n' "$(date -Is)" "$PROX_HOST" "$*" >> "$LOG_FILE"
+}
+
+cmd="${1:-}"; [ -n "$cmd" ] || die "usage: pve-snapshot.sh {create|list|rollback|clean} ... (see header)"
+shift
+
+case "$cmd" in
+create)
+    vmid="${1:?vmid}"; label="${2:?label}"; ticket="${3:?ticket#}"; shift 3
+    desc="cm ticket=#$ticket date=$(date -Is) host=$PROX_HOST"
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --desc) desc="$desc $2"; shift 2 ;;
+            --force-protected) guard_force="force"; shift ;;
+            *) die "unknown option $1" ;;
+        esac
+    done
+    guard_protected "$vmid" "create" "${guard_force:-}"
+    snap="cm-${ticket}-${label}-$(nowstamp)"
+    prox "qm snapshot $vmid '$snap' --description \"$desc\""
+    log_action "create vmid=$vmid snap=$snap ticket=$ticket"
+    log "created $snap on $vmid"
+    ;;
+list)
+    vmid="${1:?vmid}"
+    prox "qm listsnapshot $vmid"
+    ;;
+rollback)
+    vmid="${1:?vmid}"; snap="${2:?snapshot-name}"; shift 2
+    yes=""; nostart=""; force=""
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --yes) yes="yes"; shift ;;
+            --no-start) nostart="yes"; shift ;;
+            --force-protected) force="force"; shift ;;
+            *) die "unknown option $1" ;;
+        esac
+    done
+    guard_protected "$vmid" "rollback" "$force"
+    [ -n "$yes" ] || die "rollback STOPS the VM. Set an Uptime Kuma maintenance window for monitored VMs, then pass --yes"
+    log "rolling back $vmid to $snap (VM will be stopped)"
+    prox "qm rollback $vmid '$snap'"
+    log_action "rollback vmid=$vmid snap=$snap"
+    if [ -z "$nostart" ]; then
+        log "starting $vmid"
+        prox "qm start $vmid"
+    fi
+    log "rollback complete — re-verify service health from an independent vantage before moving on"
+    ;;
+clean)
+    vmid="${1:?vmid}"; shift
+    keep=5; minage=14; yes=""; force=""
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --keep) keep="$2"; shift 2 ;;
+            --min-age-days) minage="$2"; shift 2 ;;
+            --yes) yes="yes"; shift ;;
+            --force-protected) force="force"; shift ;;
+            *) die "unknown option $1" ;;
+        esac
+    done
+    guard_protected "$vmid" "clean" "$force"
+    [ -n "$yes" ] || die "dry-run only without --yes"
+    cutoff=$(( $(date +%s) - minage*86400 ))
+    mapfile -t snaps < <(prox "qm listsnapshot $vmid" \
+        | awk '/cm-/ {print $2}' \
+        | sed -E 's/.*-([0-9]{8})-[0-9]{6}$/\1/' \
+        | paste -d: - <(prox "qm listsnapshot $vmid" | awk '/cm-/ {print $2}') \
+        | while IFS=: read -r d s; do
+            ts="${d:0:4}-${d:4:2}-${d:6:2} 00:00:00"
+            [ "$(date -d "$ts" +%s)" -lt "$cutoff" ] && printf '%s\n' "$s"
+          done)
+    n="${#snaps[@]}"
+    if [ "$n" -le "$keep" ]; then
+        log "nothing to clean ($n old snapshots <= keep=$keep)"
+        exit 0
+    fi
+    for s in "${snaps[@]:0:$((n-keep))}"; do
+        log "deleting old snapshot $s"
+        prox "qm delsnapshot $vmid '$s'"
+        log_action "clean vmid=$vmid snap=$s"
+    done
+    ;;
+*)
+    die "unknown command $cmd"
+    ;;
+esac