mrcharles
6d77775bd6
secharden-ssh intentionally replaces authorized_keys with the managed production key set, which locks out the bootstrap/dev key the validation harness uses to drive the VM. After the first deploy that reaches SSH hardening, the harness could no longer connect to fetch logs or run the test suite, breaking the iteration loop. Add restore_vm_access(): after each deploy, if SSH is unreachable, it re-injects the validation pubkey OUT OF BAND via the Proxmox guest agent (qm guest exec runs as root inside the VM and does not depend on SSH). The injected payload is prefixed with a newline to avoid key concatenation when the managed file lacks a trailing newline. Config: ACCESS_PUBKEY (default ~/.ssh/id_ed25519.pub), RESTORE_ACCESS=1. Disable with RESTORE_ACCESS=0. 🤖 Generated with [Crush](https://github.com/charmassociates/crush) Assisted-by: GLM-5 via Crush <crush@charm.land>
308 lines
13 KiB
Bash
Executable File
308 lines
13 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
#
|
|
# vm-validation.sh
|
|
#
|
|
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
|
|
#
|
|
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
|
|
# one-command rollback. It is designed to be re-run after code fixes are pushed.
|
|
#
|
|
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
|
|
# itself, exactly as a real fresh server would — so the result is identical no
|
|
# matter who runs this script (no reliance on a local working copy or rsync).
|
|
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
|
|
#
|
|
# USAGE:
|
|
# # Discover the numeric VMID on Proxmox:
|
|
# ./Project-Tests/vm-validation.sh find-vmid
|
|
#
|
|
# # Full loop (snapshot + deploy + validate), auto-rollback on failure:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh all
|
|
#
|
|
# # Individual steps:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh validate
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
|
|
#
|
|
# # Clean re-deploy from scratch (delete + re-clone on VM):
|
|
# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
|
|
#
|
|
# CONFIG (override via env, all have sensible defaults):
|
|
# PROX_HOST Proxmox node hostname (default: pfv-tsys5)
|
|
# PROX_USER SSH user on Proxmox (default: root)
|
|
# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox)
|
|
# VM_IP VM IP for SSH (default: 192.168.3.50)
|
|
# VM_USER SSH user on the VM (default: localuser)
|
|
# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid)
|
|
# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
|
|
# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild)
|
|
# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy)
|
|
# CLEAN_CLONE if set, delete + re-clone on VM (default: unset)
|
|
#
|
|
set -uo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
|
PROX_USER="${PROX_USER:-root}"
|
|
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
|
|
VM_IP="${VM_IP:-192.168.3.50}"
|
|
VM_USER="${VM_USER:-localuser}"
|
|
VM_ID="${VM_ID:-}"
|
|
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
|
|
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
|
|
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
|
|
ACCESS_PUBKEY="${ACCESS_PUBKEY:-$HOME/.ssh/id_ed25519.pub}"
|
|
# Re-inject the validation pubkey after each deploy (secharden-ssh replaces
|
|
# authorized_keys with the managed production key set, locking out the
|
|
# bootstrap/dev key). Set RESTORE_ACCESS=0 to disable.
|
|
RESTORE_ACCESS="${RESTORE_ACCESS:-1}"
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_SRC="$(cd "$HERE/.." && pwd)"
|
|
REMOTE="$HERE/remote.sh"
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
|
|
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
|
|
die() { log "ERROR: $*"; exit 1; }
|
|
|
|
# All remote access funnels through remote.sh.
|
|
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
|
|
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
|
|
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
|
|
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
|
|
|
|
require_vm_id() {
|
|
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
|
|
}
|
|
|
|
wait_for_vm_ssh() {
|
|
log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
|
|
for i in $(seq 1 60); do
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH is up (after ${i} tries)."
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
die "VM did not become SSH-reachable within 5 minutes."
|
|
}
|
|
|
|
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
|
|
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
|
|
resolve_remote_repo() {
|
|
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
|
|
}
|
|
|
|
# Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF
|
|
# BAND via the Proxmox guest agent (qm guest exec runs as root inside the VM
|
|
# and does not depend on SSH). This is necessary because secharden-ssh replaces
|
|
# authorized_keys with the managed production key set, which would otherwise
|
|
# lock out the bootstrap key used to drive validation. No-op if SSH still works.
|
|
restore_vm_access() {
|
|
[[ "$RESTORE_ACCESS" = "1" ]] || { log "RESTORE_ACCESS=0; skipping access restore."; return 0; }
|
|
[[ -f "$ACCESS_PUBKEY" ]] || { log "WARN: ACCESS_PUBKEY not found ($ACCESS_PUBKEY); cannot restore access."; return 0; }
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH access already works; no need to restore."
|
|
return 0
|
|
fi
|
|
log "SSH access lost (expected after secharden-ssh). Restoring via Proxmox guest agent..."
|
|
local payload_b64
|
|
# Leading newline guards against the managed authorized_keys lacking a
|
|
# trailing newline (which would otherwise concatenate two keys into one).
|
|
payload_b64="$(printf '\n%s' "$(cat "$ACCESS_PUBKEY")" | base64 -w0)"
|
|
prox "qm guest exec $VM_ID -- /bin/sh -c 'echo $payload_b64 | base64 -d >> /home/${VM_USER}/.ssh/authorized_keys'" \
|
|
>/dev/null 2>&1 || { log "WARN: guest-agent key append failed."; return 0; }
|
|
prox "qm guest exec $VM_ID -- /bin/sh -c 'chown ${VM_USER}:${VM_USER} /home/${VM_USER}/.ssh/authorized_keys; chmod 600 /home/${VM_USER}/.ssh/authorized_keys'" \
|
|
>/dev/null 2>&1 || true
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "Access restored."
|
|
else
|
|
log "WARN: access still not working after restore. Check the deployed sshd_config."
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
cmd_find_vmid() {
|
|
log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
|
|
prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
|
|
| { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
|
|
log "Set VM_ID=<number> env var based on the row above."
|
|
}
|
|
|
|
cmd_snapshot() {
|
|
require_vm_id
|
|
log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
|
|
prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Snapshot creation failed."
|
|
echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
|
|
log "Snapshot '$SNAP_NAME' recorded as rollback target."
|
|
}
|
|
|
|
cmd_rollback() {
|
|
require_vm_id
|
|
local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
|
|
[[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
|
|
log "Rolling back VMID $VM_ID to snapshot '$target'..."
|
|
# Proxmox rollback requires the VM to be stopped.
|
|
prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
sleep 5
|
|
prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Rollback command failed."
|
|
log "Starting VMID $VM_ID..."
|
|
prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
wait_for_vm_ssh
|
|
log "Rollback complete."
|
|
}
|
|
|
|
# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
|
|
bootstrap_git_on_vm() {
|
|
log "Ensuring git is present on the VM..."
|
|
vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
|
|
vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
}
|
|
|
|
# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
|
|
sync_repo_on_vm() {
|
|
bootstrap_git_on_vm
|
|
if [[ -n "${CLEAN_CLONE:-}" ]]; then
|
|
log "CLEAN_CLONE set: removing existing clone on VM."
|
|
vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
fi
|
|
log "Ensuring repo is cloned/pulled on the VM from:"
|
|
log " $REPO_URL"
|
|
vm "
|
|
set -e
|
|
if [ -d ~/${REMOTE_REPO}/.git ]; then
|
|
cd ~/${REMOTE_REPO}
|
|
git fetch --all --prune
|
|
git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
|
|
git clean -xfd
|
|
else
|
|
git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
|
|
cd ~/${REMOTE_REPO}
|
|
fi
|
|
git log --oneline -1
|
|
" 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
|
|
log "Repo ready on VM."
|
|
}
|
|
|
|
# The remote setup runner: a self-contained script we ship to the VM so the
|
|
# sudo'd setup runs from a known-good absolute path with full logging. Using a
|
|
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
|
|
deploy_runner_script() {
|
|
cat <<RUNNER
|
|
#!/usr/bin/bash
|
|
# remote-setup-runner.sh (generated by vm-validation.sh)
|
|
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
|
|
set -uo pipefail
|
|
# Ensure a sane TERM so the framework's tput-based color helpers work when run
|
|
# over a non-interactive SSH session (which has no TTY/TERM by default).
|
|
export TERM="\${TERM:-linux}"
|
|
REPO_ABS="\${1:?repo abs path required}"
|
|
REMOTE_LOG="/tmp/knel-setup.log"
|
|
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
|
|
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
|
|
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
|
|
rc=\${PIPESTATUS[0]}
|
|
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
|
|
exit \$rc
|
|
RUNNER
|
|
}
|
|
|
|
cmd_deploy() {
|
|
require_vm_id
|
|
sync_repo_on_vm
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
log "Repo absolute path on VM: $repo_abs"
|
|
|
|
# Ship the runner script and execute it as root via sudo, passing abs path.
|
|
local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
|
|
deploy_runner_script > "$runner_local"
|
|
vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
|
|
bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."
|
|
|
|
log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
|
|
# Resolve abs runner path the same way (no ~ under sudo).
|
|
local runner_abs
|
|
runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
|
|
vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
|
|
# secharden-ssh (run near the end of setup) replaces authorized_keys with the
|
|
# managed production key set, locking out the bootstrap key. Restore the
|
|
# validation key out-of-band BEFORE we try to fetch the log over SSH.
|
|
restore_vm_access
|
|
|
|
# Fetch the remote log for full fidelity (strip ANSI color codes).
|
|
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
|
|
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
|
|
|
|
local rc_marker
|
|
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
|
|
log "Setup run finished. Marker: ${rc_marker:-unknown}"
|
|
|
|
if [[ "${rc_marker:-}" != "rc=0" ]]; then
|
|
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
|
|
return 1
|
|
fi
|
|
log "Setup completed successfully."
|
|
}
|
|
|
|
cmd_validate() {
|
|
require_vm_id
|
|
log "Running post-deploy validation suite on the VM..."
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
|
|
}
|
|
|
|
cmd_all() {
|
|
require_vm_id
|
|
log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
|
|
cmd_snapshot
|
|
if cmd_deploy && cmd_validate; then
|
|
log "=== ALL GREEN ==="
|
|
return 0
|
|
fi
|
|
log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
|
|
cmd_rollback "$SNAP_NAME"
|
|
log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
subcmd="${1:-}"
|
|
case "$subcmd" in
|
|
find-vmid) cmd_find_vmid ;;
|
|
snapshot) cmd_snapshot ;;
|
|
deploy) cmd_deploy ;;
|
|
validate) cmd_validate ;;
|
|
rollback) cmd_rollback "${2:-}" ;;
|
|
all) cmd_all ;;
|
|
""|-h|--help|help)
|
|
sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
|
|
exit 0
|
|
;;
|
|
*) die "Unknown command '$subcmd'. Run '$0 help'." ;;
|
|
esac
|