mrcharles
a54da7a43a
secharden-2fa enforces AuthenticationMethods publickey,keyboard- interactive, so once setup completes no non-interactive SSH client can authenticate (a TOTP token is required). The harness's post-deploy steps — log fetch, repo path resolution, and the validation suite — all relied on SSH and therefore failed after the first successful deploy, masking the fact that setup itself had completed (rc=0). - remote.sh: add a vm-guest mode that runs commands as root inside the VM via the Proxmox qemu-guest-agent (qm guest exec), bypassing SSH/2FA entirely. Output is parsed on the Proxmox host with python3. - vm-validation.sh: resolve repo path, fetch the setup log, and run the validation suite via vm-guest when SSH is unavailable. Detect the setup exit marker from the always-available live stream as a fallback to the fetched log. Make restore_vm_access 2FA-aware so a post-deploy SSH failure is understood (not a hard error) once 2FA is in effect. 🤖 Generated with [Crush](https://github.com/charmassociates/crush) Assisted-by: GLM-5 via Crush <crush@charm.land>
331 lines
14 KiB
Bash
Executable File
331 lines
14 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
#
|
|
# vm-validation.sh
|
|
#
|
|
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
|
|
#
|
|
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
|
|
# one-command rollback. It is designed to be re-run after code fixes are pushed.
|
|
#
|
|
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
|
|
# itself, exactly as a real fresh server would — so the result is identical no
|
|
# matter who runs this script (no reliance on a local working copy or rsync).
|
|
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
|
|
#
|
|
# USAGE:
|
|
# # Discover the numeric VMID on Proxmox:
|
|
# ./Project-Tests/vm-validation.sh find-vmid
|
|
#
|
|
# # Full loop (snapshot + deploy + validate), auto-rollback on failure:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh all
|
|
#
|
|
# # Individual steps:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh validate
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
|
|
#
|
|
# # Clean re-deploy from scratch (delete + re-clone on VM):
|
|
# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
|
|
#
|
|
# CONFIG (override via env, all have sensible defaults):
|
|
# PROX_HOST Proxmox node hostname (default: pfv-tsys5)
|
|
# PROX_USER SSH user on Proxmox (default: root)
|
|
# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox)
|
|
# VM_IP VM IP for SSH (default: 192.168.3.50)
|
|
# VM_USER SSH user on the VM (default: localuser)
|
|
# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid)
|
|
# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
|
|
# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild)
|
|
# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy)
|
|
# CLEAN_CLONE if set, delete + re-clone on VM (default: unset)
|
|
#
|
|
set -uo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
|
PROX_USER="${PROX_USER:-root}"
|
|
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
|
|
VM_IP="${VM_IP:-192.168.3.50}"
|
|
VM_USER="${VM_USER:-localuser}"
|
|
VM_ID="${VM_ID:-}"
|
|
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
|
|
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
|
|
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
|
|
ACCESS_PUBKEY="${ACCESS_PUBKEY:-$HOME/.ssh/id_ed25519.pub}"
|
|
# Re-inject the validation pubkey after each deploy (secharden-ssh replaces
|
|
# authorized_keys with the managed production key set, locking out the
|
|
# bootstrap/dev key). Set RESTORE_ACCESS=0 to disable.
|
|
RESTORE_ACCESS="${RESTORE_ACCESS:-1}"
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_SRC="$(cd "$HERE/.." && pwd)"
|
|
REMOTE="$HERE/remote.sh"
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
|
|
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
|
|
die() { log "ERROR: $*"; exit 1; }
|
|
|
|
# All remote access funnels through remote.sh.
|
|
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER (SSH)
|
|
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo (SSH)
|
|
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM (SSH)
|
|
vmguest() { bash "$REMOTE" vm-guest "$@"; } # as root via guest agent (no SSH/2FA)
|
|
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
|
|
|
|
require_vm_id() {
|
|
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
|
|
}
|
|
|
|
wait_for_vm_ssh() {
|
|
log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
|
|
for i in $(seq 1 60); do
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH is up (after ${i} tries)."
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
die "VM did not become SSH-reachable within 5 minutes."
|
|
}
|
|
|
|
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
|
|
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
|
|
resolve_remote_repo() {
|
|
local p
|
|
p="$(vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
|
|
[[ -n "$p" ]] || p="$(vmguest "cd ~${VM_USER}/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
|
|
printf '%s' "$p"
|
|
}
|
|
|
|
# Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF
|
|
# BAND via the Proxmox guest agent (qm guest exec runs as root inside the VM
|
|
# and does not depend on SSH). This is necessary because secharden-ssh replaces
|
|
# authorized_keys with the managed production key set, which would otherwise
|
|
# lock out the bootstrap key used to drive validation. No-op if SSH still works.
|
|
restore_vm_access() {
|
|
[[ "$RESTORE_ACCESS" = "1" ]] || { log "RESTORE_ACCESS=0; skipping access restore."; return 0; }
|
|
[[ -f "$ACCESS_PUBKEY" ]] || { log "WARN: ACCESS_PUBKEY not found ($ACCESS_PUBKEY); cannot restore access."; return 0; }
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH access already works; no need to restore."
|
|
return 0
|
|
fi
|
|
log "SSH access lost (expected after secharden-ssh). Restoring via Proxmox guest agent..."
|
|
local payload_b64
|
|
# Leading newline guards against the managed authorized_keys lacking a
|
|
# trailing newline (which would otherwise concatenate two keys into one).
|
|
payload_b64="$(printf '\n%s' "$(cat "$ACCESS_PUBKEY")" | base64 -w0)"
|
|
prox "qm guest exec $VM_ID -- /bin/sh -c 'echo $payload_b64 | base64 -d >> /home/${VM_USER}/.ssh/authorized_keys'" \
|
|
>/dev/null 2>&1 || { log "WARN: guest-agent key append failed."; return 0; }
|
|
prox "qm guest exec $VM_ID -- /bin/sh -c 'chown ${VM_USER}:${VM_USER} /home/${VM_USER}/.ssh/authorized_keys; chmod 600 /home/${VM_USER}/.ssh/authorized_keys'" \
|
|
>/dev/null 2>&1 || true
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "Access restored."
|
|
return 0
|
|
fi
|
|
# If SSH still fails after re-injecting the key, 2FA is almost certainly the
|
|
# cause (secharden-2fa enforces publickey+keyboard-interactive, which no
|
|
# non-interactive SSH client can satisfy). That is expected and not fatal:
|
|
# the guest agent still gives us full out-of-band access for log fetch and
|
|
# the validation suite.
|
|
if vmguest 'grep -q "^AuthenticationMethods" /etc/ssh/sshd_config' >/dev/null 2>&1; then
|
|
log "SSH requires 2FA (expected after secharden-2fa); using guest agent for further access."
|
|
else
|
|
log "WARN: access still not working after restore and 2FA not detected. Check sshd_config."
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
cmd_find_vmid() {
|
|
log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
|
|
prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
|
|
| { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
|
|
log "Set VM_ID=<number> env var based on the row above."
|
|
}
|
|
|
|
cmd_snapshot() {
|
|
require_vm_id
|
|
log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
|
|
prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Snapshot creation failed."
|
|
echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
|
|
log "Snapshot '$SNAP_NAME' recorded as rollback target."
|
|
}
|
|
|
|
cmd_rollback() {
|
|
require_vm_id
|
|
local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
|
|
[[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
|
|
log "Rolling back VMID $VM_ID to snapshot '$target'..."
|
|
# Proxmox rollback requires the VM to be stopped.
|
|
prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
sleep 5
|
|
prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Rollback command failed."
|
|
log "Starting VMID $VM_ID..."
|
|
prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
wait_for_vm_ssh
|
|
log "Rollback complete."
|
|
}
|
|
|
|
# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
|
|
bootstrap_git_on_vm() {
|
|
log "Ensuring git is present on the VM..."
|
|
vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
|
|
vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
}
|
|
|
|
# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
|
|
sync_repo_on_vm() {
|
|
bootstrap_git_on_vm
|
|
if [[ -n "${CLEAN_CLONE:-}" ]]; then
|
|
log "CLEAN_CLONE set: removing existing clone on VM."
|
|
vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
fi
|
|
log "Ensuring repo is cloned/pulled on the VM from:"
|
|
log " $REPO_URL"
|
|
vm "
|
|
set -e
|
|
if [ -d ~/${REMOTE_REPO}/.git ]; then
|
|
cd ~/${REMOTE_REPO}
|
|
git fetch --all --prune
|
|
git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
|
|
git clean -xfd
|
|
else
|
|
git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
|
|
cd ~/${REMOTE_REPO}
|
|
fi
|
|
git log --oneline -1
|
|
" 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
|
|
log "Repo ready on VM."
|
|
}
|
|
|
|
# The remote setup runner: a self-contained script we ship to the VM so the
|
|
# sudo'd setup runs from a known-good absolute path with full logging. Using a
|
|
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
|
|
deploy_runner_script() {
|
|
cat <<RUNNER
|
|
#!/usr/bin/bash
|
|
# remote-setup-runner.sh (generated by vm-validation.sh)
|
|
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
|
|
set -uo pipefail
|
|
# Ensure a sane TERM so the framework's tput-based color helpers work when run
|
|
# over a non-interactive SSH session (which has no TTY/TERM by default).
|
|
export TERM="\${TERM:-linux}"
|
|
REPO_ABS="\${1:?repo abs path required}"
|
|
REMOTE_LOG="/tmp/knel-setup.log"
|
|
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
|
|
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
|
|
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
|
|
rc=\${PIPESTATUS[0]}
|
|
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
|
|
exit \$rc
|
|
RUNNER
|
|
}
|
|
|
|
cmd_deploy() {
|
|
require_vm_id
|
|
sync_repo_on_vm
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
log "Repo absolute path on VM: $repo_abs"
|
|
|
|
# Ship the runner script and execute it as root via sudo, passing abs path.
|
|
local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
|
|
deploy_runner_script > "$runner_local"
|
|
vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
|
|
bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."
|
|
|
|
log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
|
|
# Resolve abs runner path the same way (no ~ under sudo).
|
|
local runner_abs
|
|
runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
|
|
vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
|
|
# secharden-ssh (run near the end of setup) replaces authorized_keys with the
|
|
# managed production key set, locking out the bootstrap key. Restore the
|
|
# validation key out-of-band BEFORE we try to fetch the log over SSH.
|
|
restore_vm_access
|
|
|
|
# Fetch the remote log for full fidelity (strip ANSI color codes). SSH works
|
|
# only until secharden-2fa flips 2FA on; after that, use the guest agent.
|
|
local fetch_cmd="sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log"
|
|
if ! vm "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null; then
|
|
vmguest "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
|
|
fi
|
|
|
|
# Detect the exit marker. Prefer the full fetched log, but always fall back
|
|
# to the live stream ($LOCAL_LOG) which is captured regardless of whether
|
|
# post-setup SSH/2FA let us fetch the remote log.
|
|
local rc_marker
|
|
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
|
|
[[ -n "$rc_marker" ]] || rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG" 2>/dev/null | tail -1 || true)
|
|
log "Setup run finished. Marker: ${rc_marker:-unknown}"
|
|
|
|
if [[ "${rc_marker:-}" != "rc=0" ]]; then
|
|
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log (and $LOCAL_LOG)"
|
|
return 1
|
|
fi
|
|
log "Setup completed successfully."
|
|
}
|
|
|
|
cmd_validate() {
|
|
require_vm_id
|
|
log "Running post-deploy validation suite on the VM..."
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
# Prefer SSH; fall back to the guest agent (post-2FA SSH needs a TOTP token).
|
|
if ! vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG"; then
|
|
vmguest "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
fi
|
|
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
|
|
}
|
|
|
|
cmd_all() {
|
|
require_vm_id
|
|
log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
|
|
cmd_snapshot
|
|
if cmd_deploy && cmd_validate; then
|
|
log "=== ALL GREEN ==="
|
|
return 0
|
|
fi
|
|
log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
|
|
cmd_rollback "$SNAP_NAME"
|
|
log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
subcmd="${1:-}"
|
|
case "$subcmd" in
|
|
find-vmid) cmd_find_vmid ;;
|
|
snapshot) cmd_snapshot ;;
|
|
deploy) cmd_deploy ;;
|
|
validate) cmd_validate ;;
|
|
rollback) cmd_rollback "${2:-}" ;;
|
|
all) cmd_all ;;
|
|
""|-h|--help|help)
|
|
sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
|
|
exit 0
|
|
;;
|
|
*) die "Unknown command '$subcmd'. Run '$0 help'." ;;
|
|
esac
|