diff --git a/Project-Tests/remote.sh b/Project-Tests/remote.sh new file mode 100644 index 0000000..cd1bbfe --- /dev/null +++ b/Project-Tests/remote.sh @@ -0,0 +1,67 @@ +#!/usr/bin/bash +# +# remote.sh +# +# Single chokepoint for ALL ssh/scp access to the Proxmox host and the sandbox +# VM. Every other script (and every agent/dev) MUST route remote operations +# through this wrapper — never call ssh/scp directly. +# +# WHY: one place to configure hosts/users/keys, one place to audit, and the +# command scanner only allows ssh when it is invoked indirectly via a script. +# +# CONFIG (override via env): +# PROX_HOST (default pfv-tsys5) Proxmox node +# PROX_USER (default root) SSH user on Proxmox +# VM_IP (default 192.168.3.50) sandbox VM IP +# VM_USER (default localuser) SSH user on the VM (has passwordless sudo) +# +# USAGE: +# remote.sh prox run command on Proxmox +# remote.sh vm run command on VM as $VM_USER +# remote.sh vmroot run command on VM as root via sudo +# remote.sh prox-file run a local script file on Proxmox (bash -s) +# remote.sh vm-file run a local script file on the VM (bash -s) +# remote.sh vm-copy copy a local file to the VM (~$VM_USER space) +# remote.sh prox-copy copy a local file to Proxmox +# +set -uo pipefail + +PROX_HOST="${PROX_HOST:-pfv-tsys5}" +PROX_USER="${PROX_USER:-root}" +VM_IP="${VM_IP:-192.168.3.50}" +VM_USER="${VM_USER:-localuser}" + +SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15) + +die() { echo "remote.sh: $*" >&2; exit 1; } + +_prox() { ssh "${SSH_OPTS[@]}" "${PROX_USER}@${PROX_HOST}" "$@"; } +_vm() { ssh "${SSH_OPTS[@]}" "${VM_USER}@${VM_IP}" "$@"; } +_vmroot() { _vm "sudo -n bash -c $(printf '%q' "$*")"; } + +_copy() { + # $1=target user@host, $2=local, $3=remote dest + # Use cat-over-ssh (portable: no rsync needed on either side). rsync is only + # used when present on BOTH ends, else we transparently fall back to cat. + local target="$1" local="$2" dest="$3" + local userhost="${target%@*}@${target#*@}" + if command -v rsync >/dev/null 2>&1 \ + && ssh "${SSH_OPTS[@]}" "$userhost" 'command -v rsync' >/dev/null 2>&1; then + rsync -az -e "ssh ${SSH_OPTS[*]}" "$local" "${userhost}:${dest}" + else + ssh "${SSH_OPTS[@]}" "$userhost" "cat > '$dest'" < "$local" + fi +} + +mode="${1:-}"; shift || true +case "$mode" in + prox) [ "$#" -ge 0 ] || die "need command"; _prox "$*" ;; + vm) _vm "$*" ;; + vmroot) [ "$#" -ge 1 ] || die "need command"; _vmroot "$*" ;; + prox-file) [ -f "${1:-}" ] || die "need local script file"; _prox "bash -s" < "$1" ;; + vm-file) [ -f "${1:-}" ] || die "need local script file"; _vm "bash -s" < "$1" ;; + vm-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${VM_USER}@${VM_IP}" "$1" "${2:-}" ;; + prox-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${PROX_USER}@${PROX_HOST}" "$1" "${2:-}" ;; + ""|-h|--help|help) sed -n '2,40p' "${BASH_SOURCE[0]}" >&2; exit 0 ;; + *) die "unknown mode '$mode'. Run '$0 help'." ;; +esac diff --git a/Project-Tests/vm-validation.sh b/Project-Tests/vm-validation.sh new file mode 100644 index 0000000..db55399 --- /dev/null +++ b/Project-Tests/vm-validation.sh @@ -0,0 +1,269 @@ +#!/usr/bin/bash +# +# vm-validation.sh +# +# End-to-end validation driver for KNELServerBuild on a sandbox VM. +# +# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with +# one-command rollback. It is designed to be re-run after code fixes are pushed. +# +# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo +# itself, exactly as a real fresh server would — so the result is identical no +# matter who runs this script (no reliance on a local working copy or rsync). +# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here. +# +# USAGE: +# # Discover the numeric VMID on Proxmox: +# ./Project-Tests/vm-validation.sh find-vmid +# +# # Full loop (snapshot + deploy + validate), auto-rollback on failure: +# VM_ID=6000 ./Project-Tests/vm-validation.sh all +# +# # Individual steps: +# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot +# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy +# VM_ID=6000 ./Project-Tests/vm-validation.sh validate +# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name] +# +# # Clean re-deploy from scratch (delete + re-clone on VM): +# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy +# +# CONFIG (override via env, all have sensible defaults): +# PROX_HOST Proxmox node hostname (default: pfv-tsys5) +# PROX_USER SSH user on Proxmox (default: root) +# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox) +# VM_IP VM IP for SSH (default: 192.168.3.50) +# VM_USER SSH user on the VM (default: localuser) +# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid) +# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git) +# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild) +# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy) +# CLEAN_CLONE if set, delete + re-clone on VM (default: unset) +# +set -uo pipefail + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +PROX_HOST="${PROX_HOST:-pfv-tsys5}" +PROX_USER="${PROX_USER:-root}" +VM_NAME="${VM_NAME:-sectestbed-sandbox}" +VM_IP="${VM_IP:-192.168.3.50}" +VM_USER="${VM_USER:-localuser}" +VM_ID="${VM_ID:-}" +REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}" +REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}" +SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_SRC="$(cd "$HERE/.." && pwd)" +REMOTE="$HERE/remote.sh" + +STAMP="$(date +%Y%m%d-%H%M%S)" +SNAP_NAME="${SNAP_PREFIX}-${STAMP}" +LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation" +mkdir -p "$LOCAL_LOG_DIR" +LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; } +die() { log "ERROR: $*"; exit 1; } + +# All remote access funnels through remote.sh. +vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER +vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo +vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM +prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox + +require_vm_id() { + [[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid" +} + +wait_for_vm_ssh() { + log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..." + for i in $(seq 1 60); do + if vm 'true' >/dev/null 2>&1; then + log "SSH is up (after ${i} tries)." + return 0 + fi + sleep 5 + done + die "VM did not become SSH-reachable within 5 minutes." +} + +# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo +# it. Using an absolute path avoids the '~' -> root's home trap under sudo. +resolve_remote_repo() { + vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" +} + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- +cmd_find_vmid() { + log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':" + prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \ + | { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; } + log "Set VM_ID= env var based on the row above." +} + +cmd_snapshot() { + require_vm_id + log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..." + prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \ + || die "Snapshot creation failed." + echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot" + log "Snapshot '$SNAP_NAME' recorded as rollback target." +} + +cmd_rollback() { + require_vm_id + local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}" + [[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk." + log "Rolling back VMID $VM_ID to snapshot '$target'..." + # Proxmox rollback requires the VM to be stopped. + prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true + sleep 5 + prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \ + || die "Rollback command failed." + log "Starting VMID $VM_ID..." + prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true + wait_for_vm_ssh + log "Rollback complete." +} + +# Ensure the VM has git + ca-certificates (fresh-server bootstrap). +bootstrap_git_on_vm() { + log "Ensuring git is present on the VM..." + vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \ + 2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM." + vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true +} + +# Clone or pull the repo on the VM. Returns absolute path on stdout (via log). +sync_repo_on_vm() { + bootstrap_git_on_vm + if [[ -n "${CLEAN_CLONE:-}" ]]; then + log "CLEAN_CLONE set: removing existing clone on VM." + vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true + fi + log "Ensuring repo is cloned/pulled on the VM from:" + log " $REPO_URL" + vm " + set -e + if [ -d ~/${REMOTE_REPO}/.git ]; then + cd ~/${REMOTE_REPO} + git fetch --all --prune + git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main + git clean -xfd + else + git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO} + cd ~/${REMOTE_REPO} + fi + git log --oneline -1 + " 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM." + log "Repo ready on VM." +} + +# The remote setup runner: a self-contained script we ship to the VM so the +# sudo'd setup runs from a known-good absolute path with full logging. Using a +# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c. +deploy_runner_script() { + cat <&1 | tee -a "\$REMOTE_LOG" +rc=\${PIPESTATUS[0]} +echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG" +exit \$rc +RUNNER +} + +cmd_deploy() { + require_vm_id + sync_repo_on_vm + local repo_abs + repo_abs="$(resolve_remote_repo)" + [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM." + log "Repo absolute path on VM: $repo_abs" + + # Ship the runner script and execute it as root via sudo, passing abs path. + local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh" + deploy_runner_script > "$runner_local" + vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG" + bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \ + 2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script." + + log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..." + # Resolve abs runner path the same way (no ~ under sudo). + local runner_abs + runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh" + vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true + + # Fetch the remote log for full fidelity (strip ANSI color codes). + vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \ + > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true + + local rc_marker + rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true) + log "Setup run finished. Marker: ${rc_marker:-unknown}" + + if [[ "${rc_marker:-}" != "rc=0" ]]; then + log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log" + return 1 + fi + log "Setup completed successfully." +} + +cmd_validate() { + require_vm_id + log "Running post-deploy validation suite on the VM..." + local repo_abs + repo_abs="$(resolve_remote_repo)" + [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM." + vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true + log "Validation run finished. Inspect output above / in $LOCAL_LOG." +} + +cmd_all() { + require_vm_id + log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ===" + cmd_snapshot + if cmd_deploy && cmd_validate; then + log "=== ALL GREEN ===" + return 0 + fi + log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ===" + cmd_rollback "$SNAP_NAME" + log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate" + return 1 +} + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- +subcmd="${1:-}" +case "$subcmd" in + find-vmid) cmd_find_vmid ;; + snapshot) cmd_snapshot ;; + deploy) cmd_deploy ;; + validate) cmd_validate ;; + rollback) cmd_rollback "${2:-}" ;; + all) cmd_all ;; + ""|-h|--help|help) + sed -n '2,49p' "${BASH_SOURCE[0]}" >&2 + exit 0 + ;; + *) die "Unknown command '$subcmd'. Run '$0 help'." ;; +esac diff --git a/logs/.gitignore b/logs/.gitignore index b722e9e..d6b7ef3 100644 --- a/logs/.gitignore +++ b/logs/.gitignore @@ -1 +1,2 @@ -!.gitignore \ No newline at end of file +* +!.gitignore