#!/usr/bin/bash
#
# vm-validation.sh
#
# End-to-end validation driver for KNELServerBuild on a sandbox VM.
#
# This script drives a Proxmox VM through: snapshot -> deploy -> validate, with
# one-command rollback. It is designed to be re-run after code fixes are pushed.
#
# DESIGN: deployment is GIT-BASED. The VM clones (or pulls) the public repo
# itself, exactly as a real fresh server would — so the result is identical no
# matter who runs this script (no reliance on a local working copy or rsync).
# All SSH/SCP access goes through Project-Tests/remote.sh; never call ssh here.
#
# USAGE:
#   # Discover the numeric VMID on Proxmox:
#   ./Project-Tests/vm-validation.sh find-vmid
#
#   # Full loop (snapshot + deploy + validate), auto-rollback on failure:
#   VM_ID=6000 ./Project-Tests/vm-validation.sh all
#
#   # Individual steps:
#   VM_ID=6000 ./Project-Tests/vm-validation.sh snapshot
#   VM_ID=6000 ./Project-Tests/vm-validation.sh deploy
#   VM_ID=6000 ./Project-Tests/vm-validation.sh validate
#   VM_ID=6000 ./Project-Tests/vm-validation.sh rollback [snapshot-name]
#
#   # Clean re-deploy from scratch (delete + re-clone on VM):
#   VM_ID=6000 CLEAN_CLONE=1 ./Project-Tests/vm-validation.sh deploy
#
# CONFIG (override via env, all have sensible defaults):
#   PROX_HOST    Proxmox node hostname            (default: pfv-tsys5)
#   PROX_USER    SSH user on Proxmox              (default: root)
#   VM_NAME      VM name for VMID lookup/logging  (default: sectestbed-sandbox)
#   VM_IP        VM IP for SSH                    (default: 192.168.3.50)
#   VM_USER      SSH user on the VM               (default: localuser)
#   VM_ID        Numeric VMID on Proxmox          (REQUIRED except for find-vmid)
#   REPO_URL     git URL the VM clones            (default: https://git.knownelement.com/KNEL/KNELServerBuild.git)
#   REMOTE_REPO  clone dir under ~$VM_USER        (default: KNELServerBuild)
#   SNAP_PREFIX  snapshot name prefix             (default: pre-knel-deploy)
#   CLEAN_CLONE  if set, delete + re-clone on VM  (default: unset)
#
set -uo pipefail

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
PROX_HOST="${PROX_HOST:-pfv-tsys5}"
PROX_USER="${PROX_USER:-root}"
VM_NAME="${VM_NAME:-sectestbed-sandbox}"
VM_IP="${VM_IP:-192.168.3.50}"
VM_USER="${VM_USER:-localuser}"
VM_ID="${VM_ID:-}"
REPO_URL="${REPO_URL:-https://git.knownelement.com/KNEL/KNELServerBuild.git}"
REMOTE_REPO="${REMOTE_REPO:-KNELServerBuild}"
SNAP_PREFIX="${SNAP_PREFIX:-pre-knel-deploy}"
ACCESS_PUBKEY="${ACCESS_PUBKEY:-$HOME/.ssh/id_ed25519.pub}"
# Re-inject the validation pubkey after each deploy (secharden-ssh replaces
# authorized_keys with the managed production key set, locking out the
# bootstrap/dev key). Set RESTORE_ACCESS=0 to disable.
RESTORE_ACCESS="${RESTORE_ACCESS:-1}"

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_SRC="$(cd "$HERE/.." && pwd)"
REMOTE="$HERE/remote.sh"

STAMP="$(date +%Y%m%d-%H%M%S)"
SNAP_NAME="${SNAP_PREFIX}-${STAMP}"
LOCAL_LOG_DIR="$REPO_SRC/logs/vm-validation"
mkdir -p "$LOCAL_LOG_DIR"
LOCAL_LOG="$LOCAL_LOG_DIR/run-${STAMP}.log"

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
log()  { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$LOCAL_LOG"; }
die()  { log "ERROR: $*"; exit 1; }

# All remote access funnels through remote.sh.
vm()      { bash "$REMOTE" vm "$@"; }           # as $VM_USER (SSH)
vmroot()  { bash "$REMOTE" vmroot "$@"; }       # as root via sudo (SSH)
vmfile()  { bash "$REMOTE" vm-file "$@"; }      # run local script on VM (SSH)
vmguest() { bash "$REMOTE" vm-guest "$@"; }     # as root via guest agent (no SSH/2FA)
prox()    { bash "$REMOTE" prox "$@"; }         # as $PROX_USER on Proxmox

require_vm_id() {
  [[ -n "$VM_ID" ]] || die "VM_ID is required for this command. Find it with: $0 find-vmid"
}

wait_for_vm_ssh() {
  log "Waiting for SSH on ${VM_USER}@${VM_IP} to come up..."
  for i in $(seq 1 60); do
    if vm 'true' >/dev/null 2>&1; then
      log "SSH is up (after ${i} tries)."
      return 0
    fi
    sleep 5
  done
  die "VM did not become SSH-reachable within 5 minutes."
}

# Resolve the ABSOLUTE path of the repo clone on the VM (as $VM_USER) and echo
# it. Using an absolute path avoids the '~' -> root's home trap under sudo.
resolve_remote_repo() {
  local p
  p="$(vm "cd ~/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
  [[ -n "$p" ]] || p="$(vmguest "cd ~${VM_USER}/${REMOTE_REPO} 2>/dev/null && pwd" 2>/dev/null)"
  printf '%s' "$p"
}

# Re-inject the validation pubkey into ~$VM_USER/.ssh/authorized_keys OUT OF
# BAND via the Proxmox guest agent (qm guest exec runs as root inside the VM
# and does not depend on SSH). This is necessary because secharden-ssh replaces
# authorized_keys with the managed production key set, which would otherwise
# lock out the bootstrap key used to drive validation. No-op if SSH still works.
restore_vm_access() {
  [[ "$RESTORE_ACCESS" = "1" ]] || { log "RESTORE_ACCESS=0; skipping access restore."; return 0; }
  [[ -f "$ACCESS_PUBKEY" ]] || { log "WARN: ACCESS_PUBKEY not found ($ACCESS_PUBKEY); cannot restore access."; return 0; }
  if vm 'true' >/dev/null 2>&1; then
    log "SSH access already works; no need to restore."
    return 0
  fi
  log "SSH access lost (expected after secharden-ssh). Restoring via Proxmox guest agent..."
  local payload_b64
  # Leading newline guards against the managed authorized_keys lacking a
  # trailing newline (which would otherwise concatenate two keys into one).
  payload_b64="$(printf '\n%s' "$(cat "$ACCESS_PUBKEY")" | base64 -w0)"
  prox "qm guest exec $VM_ID -- /bin/sh -c 'echo $payload_b64 | base64 -d >> /home/${VM_USER}/.ssh/authorized_keys'" \
    >/dev/null 2>&1 || { log "WARN: guest-agent key append failed."; return 0; }
  prox "qm guest exec $VM_ID -- /bin/sh -c 'chown ${VM_USER}:${VM_USER} /home/${VM_USER}/.ssh/authorized_keys; chmod 600 /home/${VM_USER}/.ssh/authorized_keys'" \
    >/dev/null 2>&1 || true
  if vm 'true' >/dev/null 2>&1; then
    log "Access restored."
    return 0
  fi
  # If SSH still fails after re-injecting the key, 2FA is almost certainly the
  # cause (secharden-2fa enforces publickey+keyboard-interactive, which no
  # non-interactive SSH client can satisfy). That is expected and not fatal:
  # the guest agent still gives us full out-of-band access for log fetch and
  # the validation suite.
  if vmguest 'grep -q "^AuthenticationMethods" /etc/ssh/sshd_config' >/dev/null 2>&1; then
    log "SSH requires 2FA (expected after secharden-2fa); using guest agent for further access."
  else
    log "WARN: access still not working after restore and 2FA not detected. Check sshd_config."
  fi
}

# ---------------------------------------------------------------------------
# Commands
# ---------------------------------------------------------------------------
cmd_find_vmid() {
  log "Listing VMs on Proxmox host '$PROX_HOST' matching '$VM_NAME':"
  prox 'qm list' 2>&1 | tee -a "$LOCAL_LOG" \
    | { IFS= read -r header; echo "$header"; grep -i "$VM_NAME" || true; }
  log "Set VM_ID=<number> env var based on the row above."
}

cmd_snapshot() {
  require_vm_id
  log "Creating snapshot '$SNAP_NAME' of VMID $VM_ID on $PROX_HOST..."
  prox "qm snapshot $VM_ID $SNAP_NAME --vmstate 1" 2>&1 | tee -a "$LOCAL_LOG" \
    || die "Snapshot creation failed."
  echo "$SNAP_NAME" > "$LOCAL_LOG_DIR/.last-snapshot"
  log "Snapshot '$SNAP_NAME' recorded as rollback target."
}

cmd_rollback() {
  require_vm_id
  local target="${1:-$(cat "$LOCAL_LOG_DIR/.last-snapshot" 2>/dev/null || true)}"
  [[ -n "$target" ]] || die "No snapshot name given and no .last-snapshot on disk."
  log "Rolling back VMID $VM_ID to snapshot '$target'..."
  # Proxmox rollback requires the VM to be stopped.
  prox "qm stop $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
  sleep 5
  prox "qm rollback $VM_ID $target" 2>&1 | tee -a "$LOCAL_LOG" \
    || die "Rollback command failed."
  log "Starting VMID $VM_ID..."
  prox "qm start $VM_ID" 2>&1 | tee -a "$LOCAL_LOG" || true
  wait_for_vm_ssh
  log "Rollback complete."
}

# Ensure the VM has git + ca-certificates (fresh-server bootstrap).
bootstrap_git_on_vm() {
  log "Ensuring git is present on the VM..."
  vm 'command -v git >/dev/null 2>&1 || sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install git ca-certificates' \
    2>&1 | tee -a "$LOCAL_LOG" || die "Failed to bootstrap git on VM."
  vm 'sudo -n DEBIAN_FRONTEND=noninteractive apt-get -y -qq install ca-certificates' 2>&1 | tee -a "$LOCAL_LOG" || true
}

# Clone or pull the repo on the VM. Returns absolute path on stdout (via log).
sync_repo_on_vm() {
  bootstrap_git_on_vm
  if [[ -n "${CLEAN_CLONE:-}" ]]; then
    log "CLEAN_CLONE set: removing existing clone on VM."
    vm "rm -rf ~/${REMOTE_REPO}" 2>&1 | tee -a "$LOCAL_LOG" || true
  fi
  log "Ensuring repo is cloned/pulled on the VM from:"
  log "  $REPO_URL"
  vm "
    set -e
    if [ -d ~/${REMOTE_REPO}/.git ]; then
      cd ~/${REMOTE_REPO}
      git fetch --all --prune
      git reset --hard origin/HEAD 2>/dev/null || git reset --hard origin/main
      git clean -xfd
    else
      git clone --filter=blob:none '$REPO_URL' ~/${REMOTE_REPO}
      cd ~/${REMOTE_REPO}
    fi
    git log --oneline -1
  " 2>&1 | tee -a "$LOCAL_LOG" || die "Repo sync failed on VM."
  log "Repo ready on VM."
}

# The remote setup runner: a self-contained script we ship to the VM so the
# sudo'd setup runs from a known-good absolute path with full logging. Using a
# file avoids nested-quote hell across local -> ssh -> sudo -> bash -c.
deploy_runner_script() {
  cat <<RUNNER
#!/usr/bin/bash
# remote-setup-runner.sh (generated by vm-validation.sh)
# Runs ProjectCode/SetupNewSystem.sh from the repo given by \$1, as root.
set -uo pipefail
# Ensure a sane TERM so the framework's tput-based color helpers work when run
# over a non-interactive SSH session (which has no TTY/TERM by default).
export TERM="\${TERM:-linux}"
REPO_ABS="\${1:?repo abs path required}"
REMOTE_LOG="/tmp/knel-setup.log"
echo "=== KNEL SetupNewSystem start: \$(date -Is) repo=\$REPO_ABS ===" | tee -a "\$REMOTE_LOG"
cd "\$REPO_ABS/ProjectCode" || { echo "FATAL: ProjectCode missing at \$REPO_ABS"; exit 2; }
bash SetupNewSystem.sh 2>&1 | tee -a "\$REMOTE_LOG"
rc=\${PIPESTATUS[0]}
echo "=== KNEL SetupNewSystem end: rc=\$rc \$(date -Is) ===" | tee -a "\$REMOTE_LOG"
exit \$rc
RUNNER
}

cmd_deploy() {
  require_vm_id
  sync_repo_on_vm
  local repo_abs
  repo_abs="$(resolve_remote_repo)"
  [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
  log "Repo absolute path on VM: $repo_abs"

  # Ship the runner script and execute it as root via sudo, passing abs path.
  local runner_local="$LOCAL_LOG_DIR/remote-setup-runner.sh"
  deploy_runner_script > "$runner_local"
  vm "mkdir -p ~/${REMOTE_REPO}/Project-Tests/.run" 2>&1 | tee -a "$LOCAL_LOG"
  bash "$REMOTE" vm-copy "$runner_local" "${REMOTE_REPO}/Project-Tests/.run/remote-setup-runner.sh" \
    2>&1 | tee -a "$LOCAL_LOG" || die "Failed to ship runner script."

  log "Running SetupNewSystem.sh on the VM as root (this takes several minutes)..."
  # Resolve abs runner path the same way (no ~ under sudo).
  local runner_abs
  runner_abs="$(vm "cd ~/${REMOTE_REPO}/Project-Tests/.run && pwd")/remote-setup-runner.sh"
  vmroot "bash '$runner_abs' '$repo_abs'" 2>&1 | tee -a "$LOCAL_LOG" || true

  # secharden-ssh (run near the end of setup) replaces authorized_keys with the
  # managed production key set, locking out the bootstrap key. Restore the
  # validation key out-of-band BEFORE we try to fetch the log over SSH.
  restore_vm_access

  # Fetch the remote log for full fidelity (strip ANSI color codes). SSH works
  # only until secharden-2fa flips 2FA on; after that, use the guest agent.
  local fetch_cmd="sed -r 's/\\x1B\\[[0-9;]*[mK]//g' /tmp/knel-setup.log 2>/dev/null || cat /tmp/knel-setup.log"
  if ! vm "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null; then
    vmguest "$fetch_cmd" > "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null || true
  fi

  # Detect the exit marker. Prefer the full fetched log, but always fall back
  # to the live stream ($LOCAL_LOG) which is captured regardless of whether
  # post-setup SSH/2FA let us fetch the remote log.
  local rc_marker
  rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG_DIR/setup-output-${STAMP}.log" 2>/dev/null | tail -1 || true)
  [[ -n "$rc_marker" ]] || rc_marker=$(grep -oE 'rc=[0-9]+' "$LOCAL_LOG" 2>/dev/null | tail -1 || true)
  log "Setup run finished. Marker: ${rc_marker:-unknown}"

  if [[ "${rc_marker:-}" != "rc=0" ]]; then
    log "Setup did NOT complete cleanly. See: $LOCAL_LOG_DIR/setup-output-${STAMP}.log (and $LOCAL_LOG)"
    return 1
  fi
  log "Setup completed successfully."
}

cmd_validate() {
  require_vm_id
  log "Running post-deploy validation suite on the VM..."
  local repo_abs
  repo_abs="$(resolve_remote_repo)"
  [[ -n "$repo_abs" ]] || die "Could not resolve absolute repo path on VM."
  # Prefer SSH; fall back to the guest agent (post-2FA SSH needs a TOTP token).
  if ! vmroot "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG"; then
    vmguest "cd '$repo_abs' && bash Project-Tests/run-tests.sh all" 2>&1 | tee -a "$LOCAL_LOG" || true
  fi
  log "Validation run finished. Inspect output above / in $LOCAL_LOG."
}

cmd_all() {
  require_vm_id
  log "=== FULL VALIDATION LOOP: $VM_NAME (VMID $VM_ID) ==="
  cmd_snapshot
  if cmd_deploy && cmd_validate; then
    log "=== ALL GREEN ==="
    return 0
  fi
  log "=== FAILURE — auto-rolling back to '$SNAP_NAME' ==="
  cmd_rollback "$SNAP_NAME"
  log "Rolled back. Fix and push, then re-run: VM_ID=$VM_ID $0 deploy && VM_ID=$VM_ID $0 validate"
  return 1
}

# ---------------------------------------------------------------------------
# Dispatch
# ---------------------------------------------------------------------------
subcmd="${1:-}"
case "$subcmd" in
  find-vmid) cmd_find_vmid ;;
  snapshot)  cmd_snapshot ;;
  deploy)    cmd_deploy ;;
  validate)  cmd_validate ;;
  rollback)  cmd_rollback "${2:-}" ;;
  all)       cmd_all ;;
  ""|-h|--help|help)
    sed -n '2,49p' "${BASH_SOURCE[0]}" >&2
    exit 0
    ;;
  *) die "Unknown command '$subcmd'. Run '$0 help'." ;;
esac
