131 lines
4.7 KiB
Bash
Executable File
131 lines
4.7 KiB
Bash
Executable File
#!/bin/bash
|
|
###############################################################################
|
|
# validate-vms.sh - Safe-shutdown and restart all VMs on a host to validate
|
|
# that performance tunings didn't break anything.
|
|
#
|
|
# For each VM:
|
|
# 1. qm shutdown <vmid> --timeout 120 (ACPI safe shutdown)
|
|
# 2. Wait for stopped state
|
|
# 3. qm start <vmid>
|
|
# 4. Wait for running state
|
|
# 5. Check qm agent responds (if agent enabled)
|
|
#
|
|
# Usage: bash validate-vms.sh <host> [host...]
|
|
###############################################################################
|
|
set -uo pipefail
|
|
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
|
|
|
for HOST in "$@"; do
|
|
echo "================================================================"
|
|
echo "[$HOST] VM SAFE-SHUTDOWN/RESTART VALIDATION"
|
|
echo "================================================================"
|
|
echo ""
|
|
|
|
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
|
echo " UNREACHABLE — skipping"
|
|
continue
|
|
fi
|
|
|
|
# Get list of running VMs
|
|
VM_LIST=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"{print \$1}"')
|
|
if [ -z "$VM_LIST" ]; then
|
|
echo " No running VMs — nothing to validate"
|
|
continue
|
|
fi
|
|
|
|
VM_COUNT=$(echo "$VM_LIST" | wc -w)
|
|
echo " Found $VM_COUNT running VM(s): $(echo "$VM_LIST" | tr '\n' ' ')"
|
|
echo ""
|
|
|
|
# --- Phase 1: Safe shutdown all VMs ---
|
|
echo "--- PHASE 1: Safe shutdown all VMs (120s timeout each) ---"
|
|
for vmid in $VM_LIST; do
|
|
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
|
echo -n " [$vmid $name] shutting down... "
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm shutdown $vmid --timeout 120 --forceStop 1" 2>&1 | head -1
|
|
done
|
|
|
|
# Wait for all to stop (max 180s total)
|
|
echo ""
|
|
echo -n " Waiting for all VMs to stop"
|
|
WAIT_DEADLINE=$(( $(date +%s) + 180 ))
|
|
while [ "$(date +%s)" -lt "$WAIT_DEADLINE" ]; do
|
|
echo -n "."
|
|
all_stopped=1
|
|
for vmid in $VM_LIST; do
|
|
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
|
if [ "$status" = "running" ]; then
|
|
all_stopped=0
|
|
break
|
|
fi
|
|
done
|
|
[ "$all_stopped" = "1" ] && break
|
|
sleep 5
|
|
done
|
|
echo " done"
|
|
echo ""
|
|
|
|
# Show stopped state
|
|
echo "--- VM status after shutdown ---"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
|
echo ""
|
|
|
|
# --- Phase 2: Start all VMs ---
|
|
echo "--- PHASE 2: Start all VMs ---"
|
|
for vmid in $VM_LIST; do
|
|
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
|
echo -n " [$vmid $name] starting... "
|
|
if start_output=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1); then
|
|
echo "OK"
|
|
else
|
|
echo "FAILED: $start_output"
|
|
fi
|
|
done
|
|
|
|
# Wait 20s for VMs to fully start
|
|
echo ""
|
|
echo " Waiting 20s for VMs to boot..."
|
|
sleep 20
|
|
|
|
# --- Phase 3: Verify all VMs running ---
|
|
echo ""
|
|
echo "--- PHASE 3: Verification ---"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
|
echo ""
|
|
|
|
# Check NFS mounts still healthy
|
|
echo "--- NFS mount health ---"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
|
mount_count=$(nfsstat -m 2>/dev/null | grep -c "^/mnt")
|
|
conn_count=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
|
echo " NFS mounts: $mount_count"
|
|
echo " NFS TCP connections: $conn_count"
|
|
if [ "$mount_count" -gt 0 ]; then
|
|
echo " First mount options:"
|
|
nfsstat -m 2>/dev/null | head -2 | tail -1 | sed "s/^/ /"
|
|
fi
|
|
'
|
|
echo ""
|
|
|
|
# Check guest agent responsiveness (if agent enabled)
|
|
echo "--- Guest agent check (VMs with agent:1) ---"
|
|
for vmid in $VM_LIST; do
|
|
agent=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | grep -c '^agent: 1'")
|
|
if [ "$agent" = "1" ]; then
|
|
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
|
echo -n " [$vmid $name] agent ping... "
|
|
if ssh "${SSH_OPTS[@]}" "root@$HOST" "timeout 10 qm agent $vmid ping" >/dev/null 2>&1; then
|
|
echo "OK"
|
|
else
|
|
echo "no response (VM may still be booting)"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "================================================================"
|
|
echo "[$HOST] VALIDATION COMPLETE"
|
|
echo "================================================================"
|
|
echo ""
|
|
done
|