#!/usr/bin/bash # # vm-validation.sh # # End-to-end validation driver for KNELServerBuild on a sandbox VM. # # This script drives a Proxmox VM through: snapshot -> deploy -> validate, with # one-command rollback. It is designed to be re-run after code fixes are pushed. # # DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo # itself, exactly as a real fresh server would — so the result is identical no # matter who runs this script (no reliance on a local working copy or rsync). # All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here. # # USAGE: # # Discover the numeric VMID on Proxmox: # ./Project-Tests/vm-validation.sh find-vmid # # # Full loop (snapshot + deploy + validate), auto-rollback on failure: # VM_ID=6000 ./Project-Tests/vm-validation.sh all # # # Individual steps: # VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot # VM_ID=6000 ./Project-Tests/vm-validation.sh deploy # VM_ID=6000 ./Project-Tests/vm-validation.sh validate # VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name] # # # Clean re-deploy from scratch (delete + re-clone on VM): # VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy # # CONFIG (override via env, all have sensible defaults): # PROX_HOST Proxmox node hostname (default: pfv-tsys5) # PROX_USER SSH user on Proxmox (default: root) # VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox) # VM_IP VM IP for SSH (default: 192.168.3.50) # VM_USER SSH user on the VM (default: localuser) # VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid) # REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git) # REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild) # SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy) # CLEAN_CLONE if set, delete + re-clone on VM (default: unset) # set -uo pipefail # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- PROX_HOST="${PROX_HOST:-pfv-tsys5}" PROX_USER="${PROX_USER:-root}" VM_NAME="${VM_NAME:-sectestbed-sandbox}" VM_IP="${VM_IP:-192.168.3.50}" VM_USER="${VM_USER:-localuser}" VM_ID="${VM_ID:-}" REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}" REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}" SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}" ACCESS_PUBKEY="${ACCESS_PUBKEY:-$HOME/.ssh/id_ed25519.pub}" # Re-inject the validation pubkey after each deploy (secharden-ssh replaces # authorized_keys with the managed production key set, locking out the # bootstrap/dev key). Set RESTORE_ACCESS=0 to disable. RESTORE_ACCESS="${RESTORE_ACCESS:-1}" HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_SRC="$(cd "$HERE/.." && pwd)" REMOTE="$HERE/remote.sh" STAMP="$(date +%Y%m%d-%H%M%S)" SNAP_NAME="${SNAP_PREFIX}-${STAMP}" LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation" mkdir -p "$LOCAL_LOG_DIR" LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; } die() { log "ERROR: $*"; exit 1; } # All remote access funnels through remote.sh. vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox require_vm_id() { [[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid" } wait_for_vm_ssh() { log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..." for i in $(seq 1 60); do if vm 'true' >/dev/null 2>&1; then log "SSH is up (after ${i} tries)." return 0 fi sleep 5 done die "VM did not become SSH-reachable within 5 minutes." } # Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo # it. Using an absolute path avoids the '~' -> root's home trap under sudo. resolve_remote_repo() { vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" } # Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF # BAND via the Proxmox guest agent (qm guest exec runs as root inside the VM # and does not depend on SSH). This is necessary because secharden-ssh replaces # authorized_keys with the managed production key set, which would otherwise # lock out the bootstrap key used to drive validation. No-op if SSH still works. restore_vm_access() { [[ "$RESTORE_ACCESS" = "1" ]] || { log "RESTORE_ACCESS=0; skipping access restore."; return 0; } [[ -f "$ACCESS_PUBKEY" ]] || { log "WARN: ACCESS_PUBKEY not found ($ACCESS_PUBKEY); cannot restore access."; return 0; } if vm 'true' >/dev/null 2>&1; then log "SSH access already works; no need to restore." return 0 fi log "SSH access lost (expected after secharden-ssh). Restoring via Proxmox guest agent..." local payload_b64 # Leading newline guards against the managed authorized_keys lacking a # trailing newline (which would otherwise concatenate two keys into one). payload_b64="$(printf '\n%s' "$(cat "$ACCESS_PUBKEY")" | base64 -w0)" prox "qm guest exec $VM_ID -- /bin/sh -c 'echo $payload_b64 | base64 -d >> /home/${VM_USER}/.ssh/authorized_keys'" \ >/dev/null 2>&1 || { log "WARN: guest-agent key append failed."; return 0; } prox "qm guest exec $VM_ID -- /bin/sh -c 'chown ${VM_USER}:${VM_USER} /home/${VM_USER}/.ssh/authorized_keys; chmod 600 /home/${VM_USER}/.ssh/authorized_keys'" \ >/dev/null 2>&1 || true if vm 'true' >/dev/null 2>&1; then log "Access restored." else log "WARN: access still not working after restore. Check the deployed sshd_config." fi } # --------------------------------------------------------------------------- # Commands # --------------------------------------------------------------------------- cmd_find_vmid() { log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':" prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \ | { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; } log "Set VM_ID= env var based on the row above." } cmd_snapshot() { require_vm_id log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..." prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \ || die "Snapshot creation failed." echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot" log "Snapshot '$SNAP_NAME' recorded as rollback target." } cmd_rollback() { require_vm_id local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}" [[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk." log "Rolling back VMID $VM_ID to snapshot '$target'..." # Proxmox rollback requires the VM to be stopped. prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true sleep 5 prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \ || die "Rollback command failed." log "Starting VMID $VM_ID..." prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true wait_for_vm_ssh log "Rollback complete." } # Ensure the VM has git + ca-certificates (fresh-server bootstrap). bootstrap_git_on_vm() { log "Ensuring git is present on the VM..." vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \ 2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM." vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true } # Clone or pull the repo on the VM. Returns absolute path on stdout (via log). sync_repo_on_vm() { bootstrap_git_on_vm if [[ -n "${CLEAN_CLONE:-}" ]]; then log "CLEAN_CLONE set: removing existing clone on VM." vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true fi log "Ensuring repo is cloned/pulled on the VM from:" log " $REPO_URL" vm " set -e if [ -d ~/${REMOTE_REPO}/.git ]; then cd ~/${REMOTE_REPO} git fetch --all --prune git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main git clean -xfd else git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO} cd ~/${REMOTE_REPO} fi git log --oneline -1 " 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM." log "Repo ready on VM." } # The remote setup runner: a self-contained script we ship to the VM so the # sudo'd setup runs from a known-good absolute path with full logging. Using a # file avoids nested-quote hell across local -> ssh -> sudo -> bash -c. deploy_runner_script() { cat <&1 | tee -a "\$REMOTE_LOG" rc=\${PIPESTATUS[0]} echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG" exit \$rc RUNNER } cmd_deploy() { require_vm_id sync_repo_on_vm local repo_abs repo_abs="$(resolve_remote_repo)" [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM." log "Repo absolute path on VM: $repo_abs" # Ship the runner script and execute it as root via sudo, passing abs path. local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh" deploy_runner_script > "$runner_local" vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG" bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \ 2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script." log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..." # Resolve abs runner path the same way (no ~ under sudo). local runner_abs runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh" vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true # secharden-ssh (run near the end of setup) replaces authorized_keys with the # managed production key set, locking out the bootstrap key. Restore the # validation key out-of-band BEFORE we try to fetch the log over SSH. restore_vm_access # Fetch the remote log for full fidelity (strip ANSI color codes). vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \ > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true local rc_marker rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true) log "Setup run finished. Marker: ${rc_marker:-unknown}" if [[ "${rc_marker:-}" != "rc=0" ]]; then log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log" return 1 fi log "Setup completed successfully." } cmd_validate() { require_vm_id log "Running post-deploy validation suite on the VM..." local repo_abs repo_abs="$(resolve_remote_repo)" [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM." vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true log "Validation run finished. Inspect output above / in $LOCAL_LOG." } cmd_all() { require_vm_id log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ===" cmd_snapshot if cmd_deploy && cmd_validate; then log "=== ALL GREEN ===" return 0 fi log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ===" cmd_rollback "$SNAP_NAME" log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate" return 1 } # --------------------------------------------------------------------------- # Dispatch # --------------------------------------------------------------------------- subcmd="${1:-}" case "$subcmd" in find-vmid) cmd_find_vmid ;; snapshot) cmd_snapshot ;; deploy) cmd_deploy ;; validate) cmd_validate ;; rollback) cmd_rollback "${2:-}" ;; all) cmd_all ;; ""|-h|--help|help) sed -n '2,49p' "${BASH_SOURCE[0]}" >&2 exit 0 ;; *) die "Unknown command '$subcmd'. Run '$0 help'." ;; esac