From a54da7a43ade3a5bb671349be4f50aab000d2648 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Mon, 27 Jul 2026 10:42:45 -0500 Subject: [PATCH] test(validation): route post-deploy ops through guest agent for 2FA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit secharden-2fa enforces AuthenticationMethods publickey,keyboard- interactive, so once setup completes no non-interactive SSH client can authenticate (a TOTP token is required). The harness's post-deploy steps — log fetch, repo path resolution, and the validation suite — all relied on SSH and therefore failed after the first successful deploy, masking the fact that setup itself had completed (rc=0). - remote.sh: add a vm-guest mode that runs commands as root inside the VM via the Proxmox qemu-guest-agent (qm guest exec), bypassing SSH/2FA entirely. Output is parsed on the Proxmox host with python3. - vm-validation.sh: resolve repo path, fetch the setup log, and run the validation suite via vm-guest when SSH is unavailable. Detect the setup exit marker from the always-available live stream as a fallback to the fetched log. Make restore_vm_access 2FA-aware so a post-deploy SSH failure is understood (not a hard error) once 2FA is in effect. 🤖 Generated with [Crush](https://github.com/charmassociates/crush) Assisted-by: GLM-5 via Crush --- Project-Tests/remote.sh | 30 +++++++++++++++++++++++ Project-Tests/vm-validation.sh | 45 +++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/Project-Tests/remote.sh b/Project-Tests/remote.sh index cd1bbfe..a5819cd 100755 --- a/Project-Tests/remote.sh +++ b/Project-Tests/remote.sh @@ -30,6 +30,8 @@ PROX_HOST="${PROX_HOST:-pfv-tsys5}" PROX_USER="${PROX_USER:-root}" VM_IP="${VM_IP:-192.168.3.50}" VM_USER="${VM_USER:-localuser}" +VM_ID="${VM_ID:-}" +GUEST_TIMEOUT="${GUEST_TIMEOUT:-900}" SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15) @@ -53,6 +55,33 @@ _copy() { fi } +# Out-of-band VM access via the Proxmox qemu-guest-agent. This runs commands +# as root inside the VM and does NOT depend on SSH, so it works even after +# secharden-ssh replaces authorized_keys and secharden-2fa enforces +# publickey+keyboard-interactive (which blocks non-interactive SSH). +GUEST_PARSER="/root/.knel-guest-parse.py" +GUEST_PARSER_SRC="import sys, json +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(3) +sys.stdout.write(d.get('out-data', '') or '') +sys.stderr.write(d.get('err-data', '') or '') +ec = d.get('exitcode', 1) +sys.exit(ec if ec is not None else 1)" + +_ensure_guest_parser() { + if _prox "test -f '$GUEST_PARSER'" >/dev/null 2>&1; then return 0; fi + printf '%s\n' "$GUEST_PARSER_SRC" | _prox "cat > '$GUEST_PARSER'" >/dev/null 2>&1 +} + +_vm_guest() { + [ -n "$VM_ID" ] || die "vm-guest requires VM_ID" + _ensure_guest_parser + local cmdb64; cmdb64="$(printf '%s' "$*" | base64 -w0)" + _prox "qm guest exec $VM_ID --timeout ${GUEST_TIMEOUT} -- /bin/sh -c 'echo $cmdb64 | base64 -d | /bin/sh' 2>/dev/null | python3 '$GUEST_PARSER'" +} + mode="${1:-}"; shift || true case "$mode" in prox) [ "$#" -ge 0 ] || die "need command"; _prox "$*" ;; @@ -62,6 +91,7 @@ case "$mode" in vm-file) [ -f "${1:-}" ] || die "need local script file"; _vm "bash -s" < "$1" ;; vm-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${VM_USER}@${VM_IP}" "$1" "${2:-}" ;; prox-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${PROX_USER}@${PROX_HOST}" "$1" "${2:-}" ;; + vm-guest) [ "$#" -ge 1 ] || die "need command"; _vm_guest "$*" ;; ""|-h|--help|help) sed -n '2,40p' "${BASH_SOURCE[0]}" >&2; exit 0 ;; *) die "unknown mode '$mode'. Run '$0 help'." ;; esac diff --git a/Project-Tests/vm-validation.sh b/Project-Tests/vm-validation.sh index b6d924e..ab08a35 100755 --- a/Project-Tests/vm-validation.sh +++ b/Project-Tests/vm-validation.sh @@ -77,10 +77,11 @@ log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; } die() { log "ERROR: $*"; exit 1; } # All remote access funnels through remote.sh. -vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER -vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo -vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM -prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox +vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER (SSH) +vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo (SSH) +vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM (SSH) +vmguest() { bash "$REMOTE" vm-guest "$@"; } # as root via guest agent (no SSH/2FA) +prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox require_vm_id() { [[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid" @@ -101,7 +102,10 @@ wait_for_vm_ssh() { # Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo # it. Using an absolute path avoids the '~' -> root's home trap under sudo. resolve_remote_repo() { - vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" + local p + p="$(vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)" + [[ -n "$p" ]] || p="$(vmguest "cd ~${VM_USER}/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)" + printf '%s' "$p" } # Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF @@ -127,8 +131,17 @@ restore_vm_access() { >/dev/null 2>&1 || true if vm 'true' >/dev/null 2>&1; then log "Access restored." + return 0 + fi + # If SSH still fails after re-injecting the key, 2FA is almost certainly the + # cause (secharden-2fa enforces publickey+keyboard-interactive, which no + # non-interactive SSH client can satisfy). That is expected and not fatal: + # the guest agent still gives us full out-of-band access for log fetch and + # the validation suite. + if vmguest 'grep -q "^AuthenticationMethods" /etc/ssh/sshd_config' >/dev/null 2>&1; then + log "SSH requires 2FA (expected after secharden-2fa); using guest agent for further access." else - log "WARN: access still not working after restore. Check the deployed sshd_config." + log "WARN: access still not working after restore and 2FA not detected. Check sshd_config." fi } @@ -249,16 +262,23 @@ cmd_deploy() { # validation key out-of-band BEFORE we try to fetch the log over SSH. restore_vm_access - # Fetch the remote log for full fidelity (strip ANSI color codes). - vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \ - > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true + # Fetch the remote log for full fidelity (strip ANSI color codes). SSH works + # only until secharden-2fa flips 2FA on; after that, use the guest agent. + local fetch_cmd="sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" + if ! vm "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null; then + vmguest "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true + fi + # Detect the exit marker. Prefer the full fetched log, but always fall back + # to the live stream ($LOCAL_LOG) which is captured regardless of whether + # post-setup SSH/2FA let us fetch the remote log. local rc_marker rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true) + [[ -n "$rc_marker" ]] || rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG" 2>/dev/null | tail -1 || true) log "Setup run finished. Marker: ${rc_marker:-unknown}" if [[ "${rc_marker:-}" != "rc=0" ]]; then - log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log" + log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log (and $LOCAL_LOG)" return 1 fi log "Setup completed successfully." @@ -270,7 +290,10 @@ cmd_validate() { local repo_abs repo_abs="$(resolve_remote_repo)" [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM." - vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true + # Prefer SSH; fall back to the guest agent (post-2FA SSH needs a TOTP token). + if ! vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG"; then + vmguest "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true + fi log "Validation run finished. Inspect output above / in $LOCAL_LOG." }