KNEL/KNELServerBuild
2
0
Fork 0
Code Pull Requests Actions Packages Releases 1 Activity

test(validation): route post-deploy ops through guest agent for 2FA

Browse Source 
secharden-2fa enforces AuthenticationMethods publickey,keyboard-
interactive, so once setup completes no non-interactive SSH client can
authenticate (a TOTP token is required). The harness's post-deploy steps
— log fetch, repo path resolution, and the validation suite — all relied
on SSH and therefore failed after the first successful deploy, masking
the fact that setup itself had completed (rc=0).

- remote.sh: add a vm-guest mode that runs commands as root inside the
  VM via the Proxmox qemu-guest-agent (qm guest exec), bypassing SSH/2FA
  entirely. Output is parsed on the Proxmox host with python3.
- vm-validation.sh: resolve repo path, fetch the setup log, and run the
  validation suite via vm-guest when SSH is unavailable. Detect the
  setup exit marker from the always-available live stream as a fallback
  to the fetched log. Make restore_vm_access 2FA-aware so a post-deploy
  SSH failure is understood (not a hard error) once 2FA is in effect.

🤖 Generated with [Crush](https://github.com/charmassociates/crush)

Assisted-by: GLM-5 via Crush <crush@charm.land>
This commit is contained in:
mrcharles
2026-07-27 10:42:45 -05:00
parent 40dfda47f2
commit a54da7a43a
2 changed files with 64 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Project-Tests/remote.sh
+30
View File
@@ -30,6 +30,8 @@ PROX_HOST="${PROX_HOST:-pfv-tsys5}"
PROX_USER="${PROX_USER:-root}"
VM_IP="${VM_IP:-192.168.3.50}"
VM_USER="${VM_USER:-localuser}"
VM_ID="${VM_ID:-}"
GUEST_TIMEOUT="${GUEST_TIMEOUT:-900}"
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
@@ -53,6 +55,33 @@ _copy() {
fi
}
# Out-of-band VM access via the Proxmox qemu-guest-agent. This runs commands
# as root inside the VM and does NOT depend on SSH, so it works even after
# secharden-ssh replaces authorized_keys and secharden-2fa enforces
# publickey+keyboard-interactive (which blocks non-interactive SSH).
GUEST_PARSER="/root/.knel-guest-parse.py"
GUEST_PARSER_SRC="import sys, json
try:
d = json.load(sys.stdin)
except Exception:
sys.exit(3)
sys.stdout.write(d.get('out-data', '') or '')
sys.stderr.write(d.get('err-data', '') or '')
ec = d.get('exitcode', 1)
sys.exit(ec if ec is not None else 1)"
_ensure_guest_parser() {
if _prox "test -f '$GUEST_PARSER'" >/dev/null 2>&1; then return 0; fi
printf '%s\n' "$GUEST_PARSER_SRC" | _prox "cat > '$GUEST_PARSER'" >/dev/null 2>&1
}
_vm_guest() {
[ -n "$VM_ID" ] || die "vm-guest requires VM_ID"
_ensure_guest_parser
local cmdb64; cmdb64="$(printf '%s' "$*" | base64 -w0)"
_prox "qm guest exec $VM_ID --timeout ${GUEST_TIMEOUT} -- /bin/sh -c 'echo $cmdb64 | base64 -d | /bin/sh' 2>/dev/null | python3 '$GUEST_PARSER'"
}
mode="${1:-}"; shift || true
case "$mode" in
prox) [ "$#" -ge 0 ] || die "need command"; _prox "$*" ;;
@@ -62,6 +91,7 @@ case "$mode" in
vm-file) [ -f "${1:-}" ] || die "need local script file"; _vm "bash -s" < "$1" ;;
vm-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${VM_USER}@${VM_IP}" "$1" "${2:-}" ;;
prox-copy) [ -f "${1:-}" ] || die "need local file"; _copy "${PROX_USER}@${PROX_HOST}" "$1" "${2:-}" ;;
vm-guest) [ "$#" -ge 1 ] || die "need command"; _vm_guest "$*" ;;
""|-h|--help|help) sed -n '2,40p' "${BASH_SOURCE[0]}" >&2; exit 0 ;;
*) die "unknown mode '$mode'. Run '$0 help'." ;;
esac
Project-Tests/vm-validation.sh
+34 -11
View File
@@ -77,10 +77,11 @@ log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
die() { log "ERROR: $*"; exit 1; }
# All remote access funnels through remote.sh.
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
vm() { bash "$REMOTE" vm "$@"; } # as $VM_USER (SSH)
vmroot() { bash "$REMOTE" vmroot "$@"; } # as root via sudo (SSH)
vmfile() { bash "$REMOTE" vm-file "$@"; } # run local script on VM (SSH)
vmguest() { bash "$REMOTE" vm-guest "$@"; } # as root via guest agent (no SSH/2FA)
prox() { bash "$REMOTE" prox "$@"; } # as $PROX_USER on Proxmox
require_vm_id() {
[[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
@@ -101,7 +102,10 @@ wait_for_vm_ssh() {
# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
resolve_remote_repo() {
vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd"
local p
p="$(vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
[[ -n "$p" ]] || p="$(vmguest "cd ~${VM_USER}/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
printf '%s' "$p"
}
# Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF
@@ -127,8 +131,17 @@ restore_vm_access() {
>/dev/null 2>&1 || true
if vm 'true' >/dev/null 2>&1; then
log "Access restored."
return 0
fi
# If SSH still fails after re-injecting the key, 2FA is almost certainly the
# cause (secharden-2fa enforces publickey+keyboard-interactive, which no
# non-interactive SSH client can satisfy). That is expected and not fatal:
# the guest agent still gives us full out-of-band access for log fetch and
# the validation suite.
if vmguest 'grep -q "^AuthenticationMethods" /etc/ssh/sshd_config' >/dev/null 2>&1; then
log "SSH requires 2FA (expected after secharden-2fa); using guest agent for further access."
else
log "WARN: access still not working after restore. Check the deployed sshd_config."
log "WARN: access still not working after restore and 2FA not detected. Check sshd_config."
fi
}
@@ -249,16 +262,23 @@ cmd_deploy() {
# validation key out-of-band BEFORE we try to fetch the log over SSH.
restore_vm_access
# Fetch the remote log for full fidelity (strip ANSI color codes).
vm "sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log" \
> "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
# Fetch the remote log for full fidelity (strip ANSI color codes). SSH works
# only until secharden-2fa flips 2FA on; after that, use the guest agent.
local fetch_cmd="sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log"
if ! vm "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null; then
vmguest "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
fi
# Detect the exit marker. Prefer the full fetched log, but always fall back
# to the live stream ($LOCAL_LOG) which is captured regardless of whether
# post-setup SSH/2FA let us fetch the remote log.
local rc_marker
rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
[[ -n "$rc_marker" ]] || rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG" 2>/dev/null | tail -1 || true)
log "Setup run finished. Marker: ${rc_marker:-unknown}"
if [[ "${rc_marker:-}" != "rc=0" ]]; then
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log"
log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log (and $LOCAL_LOG)"
return 1
fi
log "Setup completed successfully."
@@ -270,7 +290,10 @@ cmd_validate() {
local repo_abs
repo_abs="$(resolve_remote_repo)"
[[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
# Prefer SSH; fall back to the guest agent (post-2FA SSH needs a TOTP token).
if ! vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG"; then
vmguest "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
fi
log "Validation run finished. Inspect output above / in $LOCAL_LOG."
}