feat(cm): pve-snapshot.sh — snapshot lifecycle for change management [#769]
create/list/rollback/clean with cm-<ticket>-<label>-<ts> naming, protected-VM regex (pfv-bms/netinfra/PMG/PBS), Kuma-window gate on rollback, retention cleaner, local audit log. Verified end-to-end on tsys5 against throwaway VM 59999 (created, rolled back, destroyed). https://projects.knownelement.com/issues/769#note-4152
This commit is contained in:
Executable
+145
@@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# pve-snapshot.sh — change-management snapshot lifecycle for PVE VMs [#769]
|
||||||
|
#
|
||||||
|
# Supports the sectestbed loop: deploy playbook -> check -> adapt -> repeat,
|
||||||
|
# with a rollback point at every iteration. Snapshots are named
|
||||||
|
# cm-<ticket>-<label>-<YYYYMMDD-HHMMSS>
|
||||||
|
# so they sort chronologically and are auditable per ticket.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# pve-snapshot.sh create <vmid> <label> <ticket#> [--desc "text"]
|
||||||
|
# pve-snapshot.sh list <vmid>
|
||||||
|
# pve-snapshot.sh rollback <vmid> <snapshot-name> [--yes] [--no-start]
|
||||||
|
# pve-snapshot.sh clean <vmid> [--keep 5] [--min-age-days 14] [--yes]
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# PROX_HOST target PVE host DNS name (default pfv-tsys5 — the
|
||||||
|
# non-prod host where sectestbed + preprod live)
|
||||||
|
# PROTECTED_RE regex of VM names that are NEVER touched without
|
||||||
|
# --force-protected (default covers the founder's
|
||||||
|
# golden rules: pfv-bms, netinfra, PMG, PBS)
|
||||||
|
#
|
||||||
|
# Golden rules honored (Charles, 2026-09-03):
|
||||||
|
# - pfv-bms and netinfra-01/02 are protected by default (name regex).
|
||||||
|
# - NEVER a hypervisor reboot; this tool only snapshots/rolls back VMs.
|
||||||
|
# - rollback STOPS the VM: an Uptime Kuma maintenance window is required
|
||||||
|
# for any monitored VM — the tool prints the reminder and refuses
|
||||||
|
# rollback on monitored-looking names without --yes.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REMOTE="${REMOTE:-$SCRIPT_DIR/../tests/remote.sh}"
|
||||||
|
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
||||||
|
PROTECTED_RE="${PROTECTED_RE:-(^|-)bms|netinfra|mailgw|pbs|-pmg|pfv-bms}"
|
||||||
|
LOG_FILE="${LOG_FILE:-$SCRIPT_DIR/../.crush/cm-snapshots.log}"
|
||||||
|
|
||||||
|
log() { printf '%s\n' "$*" >&2; }
|
||||||
|
die() { log "FAIL: $*"; exit 1; }
|
||||||
|
|
||||||
|
[ -e "$REMOTE" ] || die "SSH chokepoint not found at $REMOTE"
|
||||||
|
|
||||||
|
prox() { timeout "${PROX_TIMEOUT:-120}" env PROX_HOST="$PROX_HOST" bash "$REMOTE" prox "$1"; }
|
||||||
|
|
||||||
|
vm_name() { prox "qm list" | awk -v id="$1" '$1==id {print $2}'; }
|
||||||
|
|
||||||
|
guard_protected() {
|
||||||
|
local vmid="$1" name force
|
||||||
|
name="$(vm_name "$vmid")"
|
||||||
|
[ -n "$name" ] || die "VMID $vmid not found on $PROX_HOST"
|
||||||
|
if printf '%s' "$name" | grep -Eq "$PROTECTED_RE"; then
|
||||||
|
[ "${3:-}" = "force" ] || die "$name matches PROTECTED_RE ($PROTECTED_RE); pass --force-protected if you REALLY mean it (founder golden rules)"
|
||||||
|
fi
|
||||||
|
log "$2: $name (VMID $vmid) on $PROX_HOST"
|
||||||
|
}
|
||||||
|
|
||||||
|
nowstamp() { date +%Y%m%d-%H%M%S; }
|
||||||
|
|
||||||
|
log_action() {
|
||||||
|
mkdir -p "$(dirname "$LOG_FILE")"
|
||||||
|
printf '%s host=%s %s\n' "$(date -Is)" "$PROX_HOST" "$*" >> "$LOG_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd="${1:-}"; [ -n "$cmd" ] || die "usage: pve-snapshot.sh {create|list|rollback|clean} ... (see header)"
|
||||||
|
shift
|
||||||
|
|
||||||
|
case "$cmd" in
|
||||||
|
create)
|
||||||
|
vmid="${1:?vmid}"; label="${2:?label}"; ticket="${3:?ticket#}"; shift 3
|
||||||
|
desc="cm ticket=#$ticket date=$(date -Is) host=$PROX_HOST"
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--desc) desc="$desc $2"; shift 2 ;;
|
||||||
|
--force-protected) guard_force="force"; shift ;;
|
||||||
|
*) die "unknown option $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
guard_protected "$vmid" "create" "${guard_force:-}"
|
||||||
|
snap="cm-${ticket}-${label}-$(nowstamp)"
|
||||||
|
prox "qm snapshot $vmid '$snap' --description \"$desc\""
|
||||||
|
log_action "create vmid=$vmid snap=$snap ticket=$ticket"
|
||||||
|
log "created $snap on $vmid"
|
||||||
|
;;
|
||||||
|
list)
|
||||||
|
vmid="${1:?vmid}"
|
||||||
|
prox "qm listsnapshot $vmid"
|
||||||
|
;;
|
||||||
|
rollback)
|
||||||
|
vmid="${1:?vmid}"; snap="${2:?snapshot-name}"; shift 2
|
||||||
|
yes=""; nostart=""; force=""
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--yes) yes="yes"; shift ;;
|
||||||
|
--no-start) nostart="yes"; shift ;;
|
||||||
|
--force-protected) force="force"; shift ;;
|
||||||
|
*) die "unknown option $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
guard_protected "$vmid" "rollback" "$force"
|
||||||
|
[ -n "$yes" ] || die "rollback STOPS the VM. Set an Uptime Kuma maintenance window for monitored VMs, then pass --yes"
|
||||||
|
log "rolling back $vmid to $snap (VM will be stopped)"
|
||||||
|
prox "qm rollback $vmid '$snap'"
|
||||||
|
log_action "rollback vmid=$vmid snap=$snap"
|
||||||
|
if [ -z "$nostart" ]; then
|
||||||
|
log "starting $vmid"
|
||||||
|
prox "qm start $vmid"
|
||||||
|
fi
|
||||||
|
log "rollback complete — re-verify service health from an independent vantage before moving on"
|
||||||
|
;;
|
||||||
|
clean)
|
||||||
|
vmid="${1:?vmid}"; shift
|
||||||
|
keep=5; minage=14; yes=""; force=""
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--keep) keep="$2"; shift 2 ;;
|
||||||
|
--min-age-days) minage="$2"; shift 2 ;;
|
||||||
|
--yes) yes="yes"; shift ;;
|
||||||
|
--force-protected) force="force"; shift ;;
|
||||||
|
*) die "unknown option $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
guard_protected "$vmid" "clean" "$force"
|
||||||
|
[ -n "$yes" ] || die "dry-run only without --yes"
|
||||||
|
cutoff=$(( $(date +%s) - minage*86400 ))
|
||||||
|
mapfile -t snaps < <(prox "qm listsnapshot $vmid" \
|
||||||
|
| awk '/cm-/ {print $2}' \
|
||||||
|
| sed -E 's/.*-([0-9]{8})-[0-9]{6}$/\1/' \
|
||||||
|
| paste -d: - <(prox "qm listsnapshot $vmid" | awk '/cm-/ {print $2}') \
|
||||||
|
| while IFS=: read -r d s; do
|
||||||
|
ts="${d:0:4}-${d:4:2}-${d:6:2} 00:00:00"
|
||||||
|
[ "$(date -d "$ts" +%s)" -lt "$cutoff" ] && printf '%s\n' "$s"
|
||||||
|
done)
|
||||||
|
n="${#snaps[@]}"
|
||||||
|
if [ "$n" -le "$keep" ]; then
|
||||||
|
log "nothing to clean ($n old snapshots <= keep=$keep)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
for s in "${snaps[@]:0:$((n-keep))}"; do
|
||||||
|
log "deleting old snapshot $s"
|
||||||
|
prox "qm delsnapshot $vmid '$s'"
|
||||||
|
log_action "clean vmid=$vmid snap=$s"
|
||||||
|
done
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "unknown command $cmd"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user