KNEL/KNELServerBuild
2
0
Fork 0
Code Pull Requests Actions Packages Releases 1 Activity

test(validation): route post-deploy ops through guest agent for 2FA

Browse Source 
secharden-2fa enforces AuthenticationMethods publickey,keyboard-
interactive, so once setup completes no non-interactive SSH client can
authenticate (a TOTP token is required). The harness's post-deploy steps
— log fetch, repo path resolution, and the validation suite — all relied
on SSH and therefore failed after the first successful deploy, masking
the fact that setup itself had completed (rc=0).

- remote.sh: add a vm-guest mode that runs commands as root inside the
  VM via the Proxmox qemu-guest-agent (qm guest exec), bypassing SSH/2FA
  entirely. Output is parsed on the Proxmox host with python3.
- vm-validation.sh: resolve repo path, fetch the setup log, and run the
  validation suite via vm-guest when SSH is unavailable. Detect the
  setup exit marker from the always-available live stream as a fallback
  to the fetched log. Make restore_vm_access 2FA-aware so a post-deploy
  SSH failure is understood (not a hard error) once 2FA is in effect.

🤖 Generated with [Crush](https://github.com/charmassociates/crush)

Assisted-by: GLM-5 via Crush <crush@charm.land>
This commit is contained in:
mrcharles
2026-07-27 10:42:45 -05:00
parent 40dfda47f2
commit a54da7a43a
2 changed files with 64 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Project-Tests/vm-validation.sh
+34 -11
View File
@@ -77,10 +77,11 @@ log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
die() { log "ERROR: $*"; exit 1; }
# All remote access funnels through remote.sh.
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER (SSH)
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo (SSH)
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM (SSH)
vmguest() { bash "$REMOTE" vm-guest "$@"; } # as root via guest agent (no SSH/2FA)
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
require_vm_id() {
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
@@ -101,7 +102,10 @@ wait_for_vm_ssh() {
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
resolve_remote_repo() {
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
local p
p="$(vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
[[ -n "$p" ]] || p="$(vmguest "cd ~${VM_USER}/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
printf '%s' "$p"
}
# Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF
@@ -127,8 +131,17 @@ restore_vm_access() {
>/dev/null 2>&1 || true
if vm 'true' >/dev/null 2>&1; then
log "Access restored."
return 0
fi
# If SSH still fails after re-injecting the key, 2FA is almost certainly the
# cause (secharden-2fa enforces publickey+keyboard-interactive, which no
# non-interactive SSH client can satisfy). That is expected and not fatal:
# the guest agent still gives us full out-of-band access for log fetch and
# the validation suite.
if vmguest 'grep -q "^AuthenticationMethods" /etc/ssh/sshd_config' >/dev/null 2>&1; then
log "SSH requires 2FA (expected after secharden-2fa); using guest agent for further access."
else
log "WARN: access still not working after restore. Check the deployed sshd_config."
log "WARN: access still not working after restore and 2FA not detected. Check sshd_config."
fi
}
@@ -249,16 +262,23 @@ cmd_deploy() {
# validation key out-of-band BEFORE we try to fetch the log over SSH.
restore_vm_access
# Fetch the remote log for full fidelity (strip ANSI color codes).
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
# Fetch the remote log for full fidelity (strip ANSI color codes). SSH works
# only until secharden-2fa flips 2FA on; after that, use the guest agent.
local fetch_cmd="sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log"
if ! vm "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null; then
vmguest "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
fi
# Detect the exit marker. Prefer the full fetched log, but always fall back
# to the live stream ($LOCAL_LOG) which is captured regardless of whether
# post-setup SSH/2FA let us fetch the remote log.
local rc_marker
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
[[ -n "$rc_marker" ]] || rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG" 2>/dev/null | tail -1 || true)
log "Setup run finished. Marker: ${rc_marker:-unknown}"
if [[ "${rc_marker:-}" != "rc=0" ]]; then
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log (and $LOCAL_LOG)"
return 1
fi
log "Setup completed successfully."
@@ -270,7 +290,10 @@ cmd_validate() {
local repo_abs
repo_abs="$(resolve_remote_repo)"
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
# Prefer SSH; fall back to the guest agent (post-2FA SSH needs a TOTP token).
if ! vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG"; then
vmguest "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
fi
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
}