mrcharles
83e4d7e8ce
Add an end-to-end validation loop for the sandbox VM and the single
SSH/SCP chokepoint it depends on:
- Project-Tests/remote.sh: the only place ssh/scp is invoked. Provides
prox / vm / vmroot / *-file / *-copy modes. Centralizes host/user/key
config and keeps remote access auditable and reusable.
- Project-Tests/vm-validation.sh: drives a Proxmox VM through
snapshot -> deploy -> validate with one-command rollback. Deployment is
GIT-BASED: the VM clones/pulls the public repo itself, exactly as a
fresh server would, so results are identical regardless of who runs it
(no reliance on a local working copy or rsync). Resolves the absolute
repo path on the VM before sudo to avoid the '~' -> root's home trap.
- logs/.gitignore: ignore generated validation/test logs (was a no-op
`!.gitignore` with no ignore rule; logs would have been committed).
Also fixes a `help`-branch typo (`${BASH_SOURCE[0]}`) and adds
PROX_USER (defaults to root) since the bare Proxmox hostname defaulted
to the wrong SSH user.
🤖 Generated with [Crush](https://github.com/charmassociates/crush)
Assisted-by: GLM-5 via Crush <crush@charm.land>
270 lines
10 KiB
Bash
270 lines
10 KiB
Bash
#!/usr/bin/bash
|
|
#
|
|
# vm-validation.sh
|
|
#
|
|
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
|
|
#
|
|
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
|
|
# one-command rollback. It is designed to be re-run after code fixes are pushed.
|
|
#
|
|
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
|
|
# itself, exactly as a real fresh server would — so the result is identical no
|
|
# matter who runs this script (no reliance on a local working copy or rsync).
|
|
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
|
|
#
|
|
# USAGE:
|
|
# # Discover the numeric VMID on Proxmox:
|
|
# ./Project-Tests/vm-validation.sh find-vmid
|
|
#
|
|
# # Full loop (snapshot + deploy + validate), auto-rollback on failure:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh all
|
|
#
|
|
# # Individual steps:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh validate
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
|
|
#
|
|
# # Clean re-deploy from scratch (delete + re-clone on VM):
|
|
# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
|
|
#
|
|
# CONFIG (override via env, all have sensible defaults):
|
|
# PROX_HOST Proxmox node hostname (default: pfv-tsys5)
|
|
# PROX_USER SSH user on Proxmox (default: root)
|
|
# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox)
|
|
# VM_IP VM IP for SSH (default: 192.168.3.50)
|
|
# VM_USER SSH user on the VM (default: localuser)
|
|
# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid)
|
|
# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
|
|
# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild)
|
|
# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy)
|
|
# CLEAN_CLONE if set, delete + re-clone on VM (default: unset)
|
|
#
|
|
set -uo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
|
PROX_USER="${PROX_USER:-root}"
|
|
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
|
|
VM_IP="${VM_IP:-192.168.3.50}"
|
|
VM_USER="${VM_USER:-localuser}"
|
|
VM_ID="${VM_ID:-}"
|
|
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
|
|
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
|
|
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_SRC="$(cd "$HERE/.." && pwd)"
|
|
REMOTE="$HERE/remote.sh"
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
|
|
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
|
|
die() { log "ERROR: $*"; exit 1; }
|
|
|
|
# All remote access funnels through remote.sh.
|
|
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
|
|
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
|
|
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
|
|
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
|
|
|
|
require_vm_id() {
|
|
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
|
|
}
|
|
|
|
wait_for_vm_ssh() {
|
|
log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
|
|
for i in $(seq 1 60); do
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH is up (after ${i} tries)."
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
die "VM did not become SSH-reachable within 5 minutes."
|
|
}
|
|
|
|
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
|
|
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
|
|
resolve_remote_repo() {
|
|
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
cmd_find_vmid() {
|
|
log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
|
|
prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
|
|
| { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
|
|
log "Set VM_ID=<number> env var based on the row above."
|
|
}
|
|
|
|
cmd_snapshot() {
|
|
require_vm_id
|
|
log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
|
|
prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Snapshot creation failed."
|
|
echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
|
|
log "Snapshot '$SNAP_NAME' recorded as rollback target."
|
|
}
|
|
|
|
cmd_rollback() {
|
|
require_vm_id
|
|
local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
|
|
[[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
|
|
log "Rolling back VMID $VM_ID to snapshot '$target'..."
|
|
# Proxmox rollback requires the VM to be stopped.
|
|
prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
sleep 5
|
|
prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Rollback command failed."
|
|
log "Starting VMID $VM_ID..."
|
|
prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
wait_for_vm_ssh
|
|
log "Rollback complete."
|
|
}
|
|
|
|
# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
|
|
bootstrap_git_on_vm() {
|
|
log "Ensuring git is present on the VM..."
|
|
vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
|
|
vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
}
|
|
|
|
# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
|
|
sync_repo_on_vm() {
|
|
bootstrap_git_on_vm
|
|
if [[ -n "${CLEAN_CLONE:-}" ]]; then
|
|
log "CLEAN_CLONE set: removing existing clone on VM."
|
|
vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
fi
|
|
log "Ensuring repo is cloned/pulled on the VM from:"
|
|
log " $REPO_URL"
|
|
vm "
|
|
set -e
|
|
if [ -d ~/${REMOTE_REPO}/.git ]; then
|
|
cd ~/${REMOTE_REPO}
|
|
git fetch --all --prune
|
|
git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
|
|
git clean -xfd
|
|
else
|
|
git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
|
|
cd ~/${REMOTE_REPO}
|
|
fi
|
|
git log --oneline -1
|
|
" 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
|
|
log "Repo ready on VM."
|
|
}
|
|
|
|
# The remote setup runner: a self-contained script we ship to the VM so the
|
|
# sudo'd setup runs from a known-good absolute path with full logging. Using a
|
|
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
|
|
deploy_runner_script() {
|
|
cat <<RUNNER
|
|
#!/usr/bin/bash
|
|
# remote-setup-runner.sh (generated by vm-validation.sh)
|
|
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
|
|
set -uo pipefail
|
|
# Ensure a sane TERM so the framework's tput-based color helpers work when run
|
|
# over a non-interactive SSH session (which has no TTY/TERM by default).
|
|
export TERM="\${TERM:-linux}"
|
|
REPO_ABS="\${1:?repo abs path required}"
|
|
REMOTE_LOG="/tmp/knel-setup.log"
|
|
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
|
|
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
|
|
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
|
|
rc=\${PIPESTATUS[0]}
|
|
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
|
|
exit \$rc
|
|
RUNNER
|
|
}
|
|
|
|
cmd_deploy() {
|
|
require_vm_id
|
|
sync_repo_on_vm
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
log "Repo absolute path on VM: $repo_abs"
|
|
|
|
# Ship the runner script and execute it as root via sudo, passing abs path.
|
|
local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
|
|
deploy_runner_script > "$runner_local"
|
|
vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
|
|
bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."
|
|
|
|
log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
|
|
# Resolve abs runner path the same way (no ~ under sudo).
|
|
local runner_abs
|
|
runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
|
|
vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
|
|
# Fetch the remote log for full fidelity (strip ANSI color codes).
|
|
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
|
|
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
|
|
|
|
local rc_marker
|
|
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
|
|
log "Setup run finished. Marker: ${rc_marker:-unknown}"
|
|
|
|
if [[ "${rc_marker:-}" != "rc=0" ]]; then
|
|
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
|
|
return 1
|
|
fi
|
|
log "Setup completed successfully."
|
|
}
|
|
|
|
cmd_validate() {
|
|
require_vm_id
|
|
log "Running post-deploy validation suite on the VM..."
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
|
|
}
|
|
|
|
cmd_all() {
|
|
require_vm_id
|
|
log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
|
|
cmd_snapshot
|
|
if cmd_deploy && cmd_validate; then
|
|
log "=== ALL GREEN ==="
|
|
return 0
|
|
fi
|
|
log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
|
|
cmd_rollback "$SNAP_NAME"
|
|
log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
subcmd="${1:-}"
|
|
case "$subcmd" in
|
|
find-vmid) cmd_find_vmid ;;
|
|
snapshot) cmd_snapshot ;;
|
|
deploy) cmd_deploy ;;
|
|
validate) cmd_validate ;;
|
|
rollback) cmd_rollback "${2:-}" ;;
|
|
all) cmd_all ;;
|
|
""|-h|--help|help)
|
|
sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
|
|
exit 0
|
|
;;
|
|
*) die "Unknown command '$subcmd'. Run '$0 help'." ;;
|
|
esac
|