test(validation): add reproducible git-based VM validation harness
Add an end-to-end validation loop for the sandbox VM and the single
SSH/SCP chokepoint it depends on:
- Project-Tests/remote.sh: the only place ssh/scp is invoked. Provides
prox / vm / vmroot / *-file / *-copy modes. Centralizes host/user/key
config and keeps remote access auditable and reusable.
- Project-Tests/vm-validation.sh: drives a Proxmox VM through
snapshot -> deploy -> validate with one-command rollback. Deployment is
GIT-BASED: the VM clones/pulls the public repo itself, exactly as a
fresh server would, so results are identical regardless of who runs it
(no reliance on a local working copy or rsync). Resolves the absolute
repo path on the VM before sudo to avoid the '~' -> root's home trap.
- logs/.gitignore: ignore generated validation/test logs (was a no-op
`!.gitignore` with no ignore rule; logs would have been committed).
Also fixes a `help`-branch typo (`${BASH_SOURCE[0]}`) and adds
PROX_USER (defaults to root) since the bare Proxmox hostname defaulted
to the wrong SSH user.
🤖 Generated with [Crush](https://github.com/charmassociates/crush)
Assisted-by: GLM-5 via Crush <crush@charm.land>
This commit is contained in:
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/bash
|
||||
#
|
||||
# vm-validation.sh
|
||||
#
|
||||
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
|
||||
#
|
||||
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
|
||||
# one-command rollback. It is designed to be re-run after code fixes are pushed.
|
||||
#
|
||||
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
|
||||
# itself, exactly as a real fresh server would — so the result is identical no
|
||||
# matter who runs this script (no reliance on a local working copy or rsync).
|
||||
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
|
||||
#
|
||||
# USAGE:
|
||||
# # Discover the numeric VMID on Proxmox:
|
||||
# ./Project-Tests/vm-validation.sh find-vmid
|
||||
#
|
||||
# # Full loop (snapshot + deploy + validate), auto-rollback on failure:
|
||||
# VM_ID=6000 ./Project-Tests/vm-validation.sh all
|
||||
#
|
||||
# # Individual steps:
|
||||
# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
|
||||
# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
|
||||
# VM_ID=6000 ./Project-Tests/vm-validation.sh validate
|
||||
# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
|
||||
#
|
||||
# # Clean re-deploy from scratch (delete + re-clone on VM):
|
||||
# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
|
||||
#
|
||||
# CONFIG (override via env, all have sensible defaults):
|
||||
# PROX_HOST Proxmox node hostname (default: pfv-tsys5)
|
||||
# PROX_USER SSH user on Proxmox (default: root)
|
||||
# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox)
|
||||
# VM_IP VM IP for SSH (default: 192.168.3.50)
|
||||
# VM_USER SSH user on the VM (default: localuser)
|
||||
# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid)
|
||||
# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
|
||||
# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild)
|
||||
# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy)
|
||||
# CLEAN_CLONE if set, delete + re-clone on VM (default: unset)
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
||||
PROX_USER="${PROX_USER:-root}"
|
||||
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
|
||||
VM_IP="${VM_IP:-192.168.3.50}"
|
||||
VM_USER="${VM_USER:-localuser}"
|
||||
VM_ID="${VM_ID:-}"
|
||||
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
|
||||
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
|
||||
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_SRC="$(cd "$HERE/.." && pwd)"
|
||||
REMOTE="$HERE/remote.sh"
|
||||
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
|
||||
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
# All remote access funnels through remote.sh.
|
||||
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
|
||||
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
|
||||
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
|
||||
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
|
||||
|
||||
require_vm_id() {
|
||||
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
|
||||
}
|
||||
|
||||
wait_for_vm_ssh() {
|
||||
log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
|
||||
for i in $(seq 1 60); do
|
||||
if vm 'true' >/dev/null 2>&1; then
|
||||
log "SSH is up (after ${i} tries)."
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
die "VM did not become SSH-reachable within 5 minutes."
|
||||
}
|
||||
|
||||
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
|
||||
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
|
||||
resolve_remote_repo() {
|
||||
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
cmd_find_vmid() {
|
||||
log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
|
||||
prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
|
||||
| { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
|
||||
log "Set VM_ID=<number> env var based on the row above."
|
||||
}
|
||||
|
||||
cmd_snapshot() {
|
||||
require_vm_id
|
||||
log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
|
||||
prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
|
||||
|| die "Snapshot creation failed."
|
||||
echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
|
||||
log "Snapshot '$SNAP_NAME' recorded as rollback target."
|
||||
}
|
||||
|
||||
cmd_rollback() {
|
||||
require_vm_id
|
||||
local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
|
||||
[[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
|
||||
log "Rolling back VMID $VM_ID to snapshot '$target'..."
|
||||
# Proxmox rollback requires the VM to be stopped.
|
||||
prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
sleep 5
|
||||
prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
|
||||
|| die "Rollback command failed."
|
||||
log "Starting VMID $VM_ID..."
|
||||
prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
wait_for_vm_ssh
|
||||
log "Rollback complete."
|
||||
}
|
||||
|
||||
# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
|
||||
bootstrap_git_on_vm() {
|
||||
log "Ensuring git is present on the VM..."
|
||||
vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
|
||||
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
|
||||
vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
}
|
||||
|
||||
# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
|
||||
sync_repo_on_vm() {
|
||||
bootstrap_git_on_vm
|
||||
if [[ -n "${CLEAN_CLONE:-}" ]]; then
|
||||
log "CLEAN_CLONE set: removing existing clone on VM."
|
||||
vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
fi
|
||||
log "Ensuring repo is cloned/pulled on the VM from:"
|
||||
log " $REPO_URL"
|
||||
vm "
|
||||
set -e
|
||||
if [ -d ~/${REMOTE_REPO}/.git ]; then
|
||||
cd ~/${REMOTE_REPO}
|
||||
git fetch --all --prune
|
||||
git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
|
||||
git clean -xfd
|
||||
else
|
||||
git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
|
||||
cd ~/${REMOTE_REPO}
|
||||
fi
|
||||
git log --oneline -1
|
||||
" 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
|
||||
log "Repo ready on VM."
|
||||
}
|
||||
|
||||
# The remote setup runner: a self-contained script we ship to the VM so the
|
||||
# sudo'd setup runs from a known-good absolute path with full logging. Using a
|
||||
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
|
||||
deploy_runner_script() {
|
||||
cat <<RUNNER
|
||||
#!/usr/bin/bash
|
||||
# remote-setup-runner.sh (generated by vm-validation.sh)
|
||||
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
|
||||
set -uo pipefail
|
||||
# Ensure a sane TERM so the framework's tput-based color helpers work when run
|
||||
# over a non-interactive SSH session (which has no TTY/TERM by default).
|
||||
export TERM="\${TERM:-linux}"
|
||||
REPO_ABS="\${1:?repo abs path required}"
|
||||
REMOTE_LOG="/tmp/knel-setup.log"
|
||||
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
|
||||
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
|
||||
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
|
||||
rc=\${PIPESTATUS[0]}
|
||||
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
|
||||
exit \$rc
|
||||
RUNNER
|
||||
}
|
||||
|
||||
cmd_deploy() {
|
||||
require_vm_id
|
||||
sync_repo_on_vm
|
||||
local repo_abs
|
||||
repo_abs="$(resolve_remote_repo)"
|
||||
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
||||
log "Repo absolute path on VM: $repo_abs"
|
||||
|
||||
# Ship the runner script and execute it as root via sudo, passing abs path.
|
||||
local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
|
||||
deploy_runner_script > "$runner_local"
|
||||
vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
|
||||
bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
|
||||
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."
|
||||
|
||||
log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
|
||||
# Resolve abs runner path the same way (no ~ under sudo).
|
||||
local runner_abs
|
||||
runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
|
||||
vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
|
||||
# Fetch the remote log for full fidelity (strip ANSI color codes).
|
||||
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
|
||||
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
|
||||
|
||||
local rc_marker
|
||||
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
|
||||
log "Setup run finished. Marker: ${rc_marker:-unknown}"
|
||||
|
||||
if [[ "${rc_marker:-}" != "rc=0" ]]; then
|
||||
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
|
||||
return 1
|
||||
fi
|
||||
log "Setup completed successfully."
|
||||
}
|
||||
|
||||
cmd_validate() {
|
||||
require_vm_id
|
||||
log "Running post-deploy validation suite on the VM..."
|
||||
local repo_abs
|
||||
repo_abs="$(resolve_remote_repo)"
|
||||
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
||||
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
|
||||
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
|
||||
}
|
||||
|
||||
cmd_all() {
|
||||
require_vm_id
|
||||
log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
|
||||
cmd_snapshot
|
||||
if cmd_deploy && cmd_validate; then
|
||||
log "=== ALL GREEN ==="
|
||||
return 0
|
||||
fi
|
||||
log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
|
||||
cmd_rollback "$SNAP_NAME"
|
||||
log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
subcmd="${1:-}"
|
||||
case "$subcmd" in
|
||||
find-vmid) cmd_find_vmid ;;
|
||||
snapshot) cmd_snapshot ;;
|
||||
deploy) cmd_deploy ;;
|
||||
validate) cmd_validate ;;
|
||||
rollback) cmd_rollback "${2:-}" ;;
|
||||
all) cmd_all ;;
|
||||
""|-h|--help|help)
|
||||
sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
|
||||
exit 0
|
||||
;;
|
||||
*) die "Unknown command '$subcmd'. Run '$0 help'." ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user