mrcharles
163ef9de16
The KexAlgorithms line in ssh-audit-hardening.conf began with a space. In sshd_config a leading whitespace marks a line continuation, so the entire directive was absorbed as arguments to the (non-existent) previous directive. The effective kexalgorithms collapsed to only the two trailing GSSAPI entries (gss-curve25519-sha256-, gss-group16-sha512-), which no normal OpenSSH client can negotiate. Result: after secharden-ssh deployed this file, every SSH connection to the host died in [preauth] with no usable key exchange algorithm. sshd -t still returned 0, so the breakage was completely silent. Drop the leading space so the directive is parsed as intended. This restores normal client compatibility while keeping the hardened algorithm set. 🤖 Generated with [Crush](https://github.com/charmassociates/crush) Assisted-by: GLM-5 via Crush <crush@charm.land>
270 lines
10 KiB
Bash
Executable File
270 lines
10 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
#
|
|
# vm-validation.sh
|
|
#
|
|
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
|
|
#
|
|
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
|
|
# one-command rollback. It is designed to be re-run after code fixes are pushed.
|
|
#
|
|
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
|
|
# itself, exactly as a real fresh server would — so the result is identical no
|
|
# matter who runs this script (no reliance on a local working copy or rsync).
|
|
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
|
|
#
|
|
# USAGE:
|
|
# # Discover the numeric VMID on Proxmox:
|
|
# ./Project-Tests/vm-validation.sh find-vmid
|
|
#
|
|
# # Full loop (snapshot + deploy + validate), auto-rollback on failure:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh all
|
|
#
|
|
# # Individual steps:
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh validate
|
|
# VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
|
|
#
|
|
# # Clean re-deploy from scratch (delete + re-clone on VM):
|
|
# VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
|
|
#
|
|
# CONFIG (override via env, all have sensible defaults):
|
|
# PROX_HOST Proxmox node hostname (default: pfv-tsys5)
|
|
# PROX_USER SSH user on Proxmox (default: root)
|
|
# VM_NAME VM name for VMID lookup/logging (default: sectestbed-sandbox)
|
|
# VM_IP VM IP for SSH (default: 192.168.3.50)
|
|
# VM_USER SSH user on the VM (default: localuser)
|
|
# VM_ID Numeric VMID on Proxmox (REQUIRED except for find-vmid)
|
|
# REPO_URL git URL the VM clones (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
|
|
# REMOTE_REPO clone dir under ~$VM_USER (default: KNELServerBuild)
|
|
# SNAP_PREFIX snapshot name prefix (default: pre-knel-deploy)
|
|
# CLEAN_CLONE if set, delete + re-clone on VM (default: unset)
|
|
#
|
|
set -uo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
|
|
PROX_USER="${PROX_USER:-root}"
|
|
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
|
|
VM_IP="${VM_IP:-192.168.3.50}"
|
|
VM_USER="${VM_USER:-localuser}"
|
|
VM_ID="${VM_ID:-}"
|
|
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
|
|
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
|
|
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_SRC="$(cd "$HERE/.." && pwd)"
|
|
REMOTE="$HERE/remote.sh"
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
|
|
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
|
|
die() { log "ERROR: $*"; exit 1; }
|
|
|
|
# All remote access funnels through remote.sh.
|
|
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
|
|
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
|
|
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
|
|
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
|
|
|
|
require_vm_id() {
|
|
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
|
|
}
|
|
|
|
wait_for_vm_ssh() {
|
|
log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
|
|
for i in $(seq 1 60); do
|
|
if vm 'true' >/dev/null 2>&1; then
|
|
log "SSH is up (after ${i} tries)."
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
die "VM did not become SSH-reachable within 5 minutes."
|
|
}
|
|
|
|
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
|
|
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
|
|
resolve_remote_repo() {
|
|
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
cmd_find_vmid() {
|
|
log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
|
|
prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
|
|
| { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
|
|
log "Set VM_ID=<number> env var based on the row above."
|
|
}
|
|
|
|
cmd_snapshot() {
|
|
require_vm_id
|
|
log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
|
|
prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Snapshot creation failed."
|
|
echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
|
|
log "Snapshot '$SNAP_NAME' recorded as rollback target."
|
|
}
|
|
|
|
cmd_rollback() {
|
|
require_vm_id
|
|
local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
|
|
[[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
|
|
log "Rolling back VMID $VM_ID to snapshot '$target'..."
|
|
# Proxmox rollback requires the VM to be stopped.
|
|
prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
sleep 5
|
|
prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
|
|
|| die "Rollback command failed."
|
|
log "Starting VMID $VM_ID..."
|
|
prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
wait_for_vm_ssh
|
|
log "Rollback complete."
|
|
}
|
|
|
|
# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
|
|
bootstrap_git_on_vm() {
|
|
log "Ensuring git is present on the VM..."
|
|
vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
|
|
vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
}
|
|
|
|
# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
|
|
sync_repo_on_vm() {
|
|
bootstrap_git_on_vm
|
|
if [[ -n "${CLEAN_CLONE:-}" ]]; then
|
|
log "CLEAN_CLONE set: removing existing clone on VM."
|
|
vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
fi
|
|
log "Ensuring repo is cloned/pulled on the VM from:"
|
|
log " $REPO_URL"
|
|
vm "
|
|
set -e
|
|
if [ -d ~/${REMOTE_REPO}/.git ]; then
|
|
cd ~/${REMOTE_REPO}
|
|
git fetch --all --prune
|
|
git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
|
|
git clean -xfd
|
|
else
|
|
git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
|
|
cd ~/${REMOTE_REPO}
|
|
fi
|
|
git log --oneline -1
|
|
" 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
|
|
log "Repo ready on VM."
|
|
}
|
|
|
|
# The remote setup runner: a self-contained script we ship to the VM so the
|
|
# sudo'd setup runs from a known-good absolute path with full logging. Using a
|
|
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
|
|
deploy_runner_script() {
|
|
cat <<RUNNER
|
|
#!/usr/bin/bash
|
|
# remote-setup-runner.sh (generated by vm-validation.sh)
|
|
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
|
|
set -uo pipefail
|
|
# Ensure a sane TERM so the framework's tput-based color helpers work when run
|
|
# over a non-interactive SSH session (which has no TTY/TERM by default).
|
|
export TERM="\${TERM:-linux}"
|
|
REPO_ABS="\${1:?repo abs path required}"
|
|
REMOTE_LOG="/tmp/knel-setup.log"
|
|
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
|
|
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
|
|
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
|
|
rc=\${PIPESTATUS[0]}
|
|
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
|
|
exit \$rc
|
|
RUNNER
|
|
}
|
|
|
|
cmd_deploy() {
|
|
require_vm_id
|
|
sync_repo_on_vm
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
log "Repo absolute path on VM: $repo_abs"
|
|
|
|
# Ship the runner script and execute it as root via sudo, passing abs path.
|
|
local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
|
|
deploy_runner_script > "$runner_local"
|
|
vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
|
|
bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
|
|
2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."
|
|
|
|
log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
|
|
# Resolve abs runner path the same way (no ~ under sudo).
|
|
local runner_abs
|
|
runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
|
|
vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
|
|
# Fetch the remote log for full fidelity (strip ANSI color codes).
|
|
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
|
|
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
|
|
|
|
local rc_marker
|
|
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
|
|
log "Setup run finished. Marker: ${rc_marker:-unknown}"
|
|
|
|
if [[ "${rc_marker:-}" != "rc=0" ]]; then
|
|
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
|
|
return 1
|
|
fi
|
|
log "Setup completed successfully."
|
|
}
|
|
|
|
cmd_validate() {
|
|
require_vm_id
|
|
log "Running post-deploy validation suite on the VM..."
|
|
local repo_abs
|
|
repo_abs="$(resolve_remote_repo)"
|
|
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
|
|
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
|
|
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
|
|
}
|
|
|
|
cmd_all() {
|
|
require_vm_id
|
|
log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
|
|
cmd_snapshot
|
|
if cmd_deploy && cmd_validate; then
|
|
log "=== ALL GREEN ==="
|
|
return 0
|
|
fi
|
|
log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
|
|
cmd_rollback "$SNAP_NAME"
|
|
log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
subcmd="${1:-}"
|
|
case "$subcmd" in
|
|
find-vmid) cmd_find_vmid ;;
|
|
snapshot) cmd_snapshot ;;
|
|
deploy) cmd_deploy ;;
|
|
validate) cmd_validate ;;
|
|
rollback) cmd_rollback "${2:-}" ;;
|
|
all) cmd_all ;;
|
|
""|-h|--help|help)
|
|
sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
|
|
exit 0
|
|
;;
|
|
*) die "Unknown command '$subcmd'. Run '$0 help'." ;;
|
|
esac
|