PFVCluster/validate-vms.sh

#!/bin/bash
###############################################################################
# validate-vms.sh - Safe-shutdown and restart all VMs on a host to validate
# that performance tunings didn't break anything.
#
# For each VM:
#   1. qm shutdown <vmid> --timeout 120  (ACPI safe shutdown)
#   2. Wait for stopped state
#   3. qm start <vmid>
#   4. Wait for running state
#   5. Check qm agent responds (if agent enabled)
#
# Usage: bash validate-vms.sh <host> [host...]
###############################################################################
set -uo pipefail
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)

for HOST in "$@"; do
    echo "================================================================"
    echo "[$HOST] VM SAFE-SHUTDOWN/RESTART VALIDATION"
    echo "================================================================"
    echo ""

    if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
        echo "  UNREACHABLE — skipping"
        continue
    fi

    # Get list of running VMs
    VM_LIST=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"{print \$1}"')
    if [ -z "$VM_LIST" ]; then
        echo "  No running VMs — nothing to validate"
        continue
    fi

    VM_COUNT=$(echo "$VM_LIST" | wc -w)
    echo "  Found $VM_COUNT running VM(s): $(echo "$VM_LIST" | tr '\n' ' ')"
    echo ""

    # --- Phase 1: Safe shutdown all VMs ---
    echo "--- PHASE 1: Safe shutdown all VMs (120s timeout each) ---"
    for vmid in $VM_LIST; do
        name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
        echo -n "  [$vmid $name] shutting down... "
        ssh "${SSH_OPTS[@]}" "root@$HOST" "qm shutdown $vmid --timeout 120 --forceStop 1" 2>&1 | head -1
    done

    # Wait for all to stop (max 180s total)
    echo ""
    echo -n "  Waiting for all VMs to stop"
    WAIT_DEADLINE=$(( $(date +%s) + 180 ))
    while [ "$(date +%s)" -lt "$WAIT_DEADLINE" ]; do
        echo -n "."
        all_stopped=1
        for vmid in $VM_LIST; do
            status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
            if [ "$status" = "running" ]; then
                all_stopped=0
                break
            fi
        done
        [ "$all_stopped" = "1" ] && break
        sleep 5
    done
    echo " done"
    echo ""

    # Show stopped state
    echo "--- VM status after shutdown ---"
    ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
    echo ""

    # --- Phase 2: Start all VMs ---
    echo "--- PHASE 2: Start all VMs ---"
    for vmid in $VM_LIST; do
        name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
        echo -n "  [$vmid $name] starting... "
        start_output=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1)
        if [ $? -eq 0 ]; then
            echo "OK"
        else
            echo "FAILED: $start_output"
        fi
    done

    # Wait 20s for VMs to fully start
    echo ""
    echo "  Waiting 20s for VMs to boot..."
    sleep 20

    # --- Phase 3: Verify all VMs running ---
    echo ""
    echo "--- PHASE 3: Verification ---"
    ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
    echo ""

    # Check NFS mounts still healthy
    echo "--- NFS mount health ---"
    ssh "${SSH_OPTS[@]}" "root@$HOST" '
        mount_count=$(nfsstat -m 2>/dev/null | grep -c "^/mnt")
        conn_count=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
        echo "  NFS mounts: $mount_count"
        echo "  NFS TCP connections: $conn_count"
        if [ "$mount_count" -gt 0 ]; then
            echo "  First mount options:"
            nfsstat -m 2>/dev/null | head -2 | tail -1 | sed "s/^/    /"
        fi
    '
    echo ""

    # Check guest agent responsiveness (if agent enabled)
    echo "--- Guest agent check (VMs with agent:1) ---"
    for vmid in $VM_LIST; do
        agent=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | grep -c '^agent: 1'")
        if [ "$agent" = "1" ]; then
            name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
            echo -n "  [$vmid $name] agent ping... "
            result=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "timeout 10 qm agent $vmid ping 2>&1")
            if [ $? -eq 0 ]; then
                echo "OK"
            else
                echo "no response (VM may still be booting)"
            fi
        fi
    done

    echo ""
    echo "================================================================"
    echo "[$HOST] VALIDATION COMPLETE"
    echo "================================================================"
    echo ""
done