PFVCluster/scripts/pve-snapshot.sh

#!/usr/bin/env bash
# pve-snapshot.sh — change-management snapshot lifecycle for PVE VMs [#769]
#
# Supports the sectestbed loop: deploy playbook -> check -> adapt -> repeat,
# with a rollback point at every iteration. Snapshots are named
#   cm-<ticket>-<label>-<YYYYMMDD-HHMMSS>
# so they sort chronologically and are auditable per ticket.
#
# Usage:
#   pve-snapshot.sh create <vmid> <label> <ticket#> [--desc "text"]
#   pve-snapshot.sh list   <vmid>
#   pve-snapshot.sh rollback <vmid> <snapshot-name> [--yes] [--no-start]
#   pve-snapshot.sh clean  <vmid> [--keep 5] [--min-age-days 14] [--yes]
#
# Environment:
#   PROX_HOST      target PVE host DNS name (default pfv-tsys5 — the
#                  non-prod host where sectestbed + preprod live)
#   PROTECTED_RE   regex of VM names that are NEVER touched without
#                  --force-protected (default covers the founder's
#                  golden rules: pfv-bms, netinfra, PMG, PBS)
#
# Golden rules honored (Charles, 2026-09-03):
#   - pfv-bms and netinfra-01/02 are protected by default (name regex).
#   - NEVER a hypervisor reboot; this tool only snapshots/rolls back VMs.
#   - rollback STOPS the VM: an Uptime Kuma maintenance window is required
#     for any monitored VM — the tool prints the reminder and refuses
#     rollback on monitored-looking names without --yes.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REMOTE="${REMOTE:-$SCRIPT_DIR/../tests/remote.sh}"
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
PROTECTED_RE="${PROTECTED_RE:-(^|-)bms|netinfra|mailgw|pbs|-pmg|pfv-bms}"
LOG_FILE="${LOG_FILE:-$SCRIPT_DIR/../.crush/cm-snapshots.log}"

log() { printf '%s\n' "$*" >&2; }
die() { log "FAIL: $*"; exit 1; }

[ -e "$REMOTE" ] || die "SSH chokepoint not found at $REMOTE"

prox() { timeout "${PROX_TIMEOUT:-120}" env PROX_HOST="$PROX_HOST" bash "$REMOTE" prox "$1"; }

vm_name() { prox "qm list" | awk -v id="$1" '$1==id {print $2}'; }

guard_protected() {
    local vmid="$1" name force
    name="$(vm_name "$vmid")"
    [ -n "$name" ] || die "VMID $vmid not found on $PROX_HOST"
    if printf '%s' "$name" | grep -Eq "$PROTECTED_RE"; then
        [ "${3:-}" = "force" ] || die "$name matches PROTECTED_RE ($PROTECTED_RE); pass --force-protected if you REALLY mean it (founder golden rules)"
    fi
    log "$2: $name (VMID $vmid) on $PROX_HOST"
}

nowstamp() { date +%Y%m%d-%H%M%S; }

log_action() {
    mkdir -p "$(dirname "$LOG_FILE")"
    printf '%s host=%s %s\n' "$(date -Is)" "$PROX_HOST" "$*" >> "$LOG_FILE"
}

cmd="${1:-}"; [ -n "$cmd" ] || die "usage: pve-snapshot.sh {create|list|rollback|clean} ... (see header)"
shift

case "$cmd" in
create)
    vmid="${1:?vmid}"; label="${2:?label}"; ticket="${3:?ticket#}"; shift 3
    desc="cm ticket=#$ticket date=$(date -Is) host=$PROX_HOST"
    while [ $# -gt 0 ]; do
        case "$1" in
            --desc) desc="$desc $2"; shift 2 ;;
            --force-protected) guard_force="force"; shift ;;
            *) die "unknown option $1" ;;
        esac
    done
    guard_protected "$vmid" "create" "${guard_force:-}"
    snap="cm-${ticket}-${label}-$(nowstamp)"
    prox "qm snapshot $vmid '$snap' --description \"$desc\""
    log_action "create vmid=$vmid snap=$snap ticket=$ticket"
    log "created $snap on $vmid"
    ;;
list)
    vmid="${1:?vmid}"
    prox "qm listsnapshot $vmid"
    ;;
rollback)
    vmid="${1:?vmid}"; snap="${2:?snapshot-name}"; shift 2
    yes=""; nostart=""; force=""
    while [ $# -gt 0 ]; do
        case "$1" in
            --yes) yes="yes"; shift ;;
            --no-start) nostart="yes"; shift ;;
            --force-protected) force="force"; shift ;;
            *) die "unknown option $1" ;;
        esac
    done
    guard_protected "$vmid" "rollback" "$force"
    [ -n "$yes" ] || die "rollback STOPS the VM. Set an Uptime Kuma maintenance window for monitored VMs, then pass --yes"
    # OQ5 approved (questions-v10): auto-open a Kuma window when creds exist.
    if [ -f "$HOME/.creds/uptime-kuma.env" ]; then
        win="$(timeout 240 docker run --rm -v "$SCRIPT_DIR/..:/app" -w /app \
            --env-file "$HOME/.creds/uptime-kuma.env" -e TZ=America/Chicago \
            python:3.12-slim sh -c 'pip install -q --no-cache-dir "python-socketio[client]" websocket-client >/dev/null 2>&1; python3 scripts/kuma-maintenance.py open --title "cm rollback vmid='"$vmid"'" --minutes 30 --desc "pve-snapshot.sh '"$snap"'"' 2>/dev/null \
            | grep -o 'id=[0-9]*' | head -1 | cut -d= -f2 || true)"
        if [ -n "$win" ]; then
            log "Kuma maintenance window id=$win opened (30m)"
        else
            log "WARN: could not open Kuma window (creds present) — create it manually before proceeding"
        fi
    fi
    log "rolling back $vmid to $snap (VM will be stopped)"
    prox "qm rollback $vmid '$snap'"
    log_action "rollback vmid=$vmid snap=$snap"
    if [ -z "$nostart" ]; then
        log "starting $vmid"
        prox "qm start $vmid"
    fi
    log "rollback complete — re-verify service health from an independent vantage before moving on"
    ;;
clean)
    vmid="${1:?vmid}"; shift
    keep=5; minage=14; yes=""; force=""
    while [ $# -gt 0 ]; do
        case "$1" in
            --keep) keep="$2"; shift 2 ;;
            --min-age-days) minage="$2"; shift 2 ;;
            --yes) yes="yes"; shift ;;
            --force-protected) force="force"; shift ;;
            *) die "unknown option $1" ;;
        esac
    done
    guard_protected "$vmid" "clean" "$force"
    [ -n "$yes" ] || die "dry-run only without --yes"
    cutoff=$(( $(date +%s) - minage*86400 ))
    mapfile -t snaps < <(prox "qm listsnapshot $vmid" \
        | awk '/cm-/ {print $2}' \
        | sed -E 's/.*-([0-9]{8})-[0-9]{6}$/\1/' \
        | paste -d: - <(prox "qm listsnapshot $vmid" | awk '/cm-/ {print $2}') \
        | while IFS=: read -r d s; do
            ts="${d:0:4}-${d:4:2}-${d:6:2} 00:00:00"
            [ "$(date -d "$ts" +%s)" -lt "$cutoff" ] && printf '%s\n' "$s"
          done)
    n="${#snaps[@]}"
    if [ "$n" -le "$keep" ]; then
        log "nothing to clean ($n old snapshots <= keep=$keep)"
        exit 0
    fi
    for s in "${snaps[@]:0:$((n-keep))}"; do
        log "deleting old snapshot $s"
        prox "qm delsnapshot $vmid '$s'"
        log_action "clean vmid=$vmid snap=$s"
    done
    ;;
*)
    die "unknown command $cmd"
    ;;
esac