refactor: reorganize merged repo into clean directory structure
Reorganize the merged KNELServerBuild + PFVCluster repo:
provisioning/ server provisioning (was ProjectCode/ +
Project-Includes/ + Project-ConfigFiles/)
tests/ test suite (was Project-Tests/)
perf/ Proxmox perf scripts (was top-level *.sh + scripts/)
docs/ all documentation (was ProjectDocs/ + PROJECT.md +
K8S.md + TODO.md)
dns-cluster-setup/ Technitium DNS cluster (unchanged)
netinfra/ netinfra audit scripts (unchanged)
switches/ switch configs (unchanged)
vendor/ vendored KNELShellFramework (unchanged)
Update all internal path references from old directory names
(ProjectCode/, Project-Includes/, Project-Tests/) to the new ones
(provisioning/, tests/) across all scripts.
🤖 Generated with [Crush](https://github.com/charmassociates/crush)
Assisted-by: GLM-5 via Crush <crush@charm.land>
This commit is contained in:
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# validate-vms.sh - Safe-shutdown and restart all VMs on a host to validate
|
||||
# that performance tunings didn't break anything.
|
||||
#
|
||||
# For each VM:
|
||||
# 1. qm shutdown <vmid> --timeout 120 (ACPI safe shutdown)
|
||||
# 2. Wait for stopped state
|
||||
# 3. qm start <vmid>
|
||||
# 4. Wait for running state
|
||||
# 5. Check qm agent responds (if agent enabled)
|
||||
#
|
||||
# Usage: bash validate-vms.sh <host> [host...]
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for HOST in "$@"; do
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VM SAFE-SHUTDOWN/RESTART VALIDATION"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get list of running VMs
|
||||
VM_LIST=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"{print \$1}"')
|
||||
if [ -z "$VM_LIST" ]; then
|
||||
echo " No running VMs — nothing to validate"
|
||||
continue
|
||||
fi
|
||||
|
||||
VM_COUNT=$(echo "$VM_LIST" | wc -w)
|
||||
echo " Found $VM_COUNT running VM(s): $(echo "$VM_LIST" | tr '\n' ' ')"
|
||||
echo ""
|
||||
|
||||
# --- Phase 1: Safe shutdown all VMs ---
|
||||
echo "--- PHASE 1: Safe shutdown all VMs (120s timeout each) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] shutting down... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm shutdown $vmid --timeout 120 --forceStop 1" 2>&1 | head -1
|
||||
done
|
||||
|
||||
# Wait for all to stop (max 180s total)
|
||||
echo ""
|
||||
echo -n " Waiting for all VMs to stop"
|
||||
WAIT_DEADLINE=$(( $(date +%s) + 180 ))
|
||||
while [ "$(date +%s)" -lt "$WAIT_DEADLINE" ]; do
|
||||
echo -n "."
|
||||
all_stopped=1
|
||||
for vmid in $VM_LIST; do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" = "running" ]; then
|
||||
all_stopped=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ "$all_stopped" = "1" ] && break
|
||||
sleep 5
|
||||
done
|
||||
echo " done"
|
||||
echo ""
|
||||
|
||||
# Show stopped state
|
||||
echo "--- VM status after shutdown ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# --- Phase 2: Start all VMs ---
|
||||
echo "--- PHASE 2: Start all VMs ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] starting... "
|
||||
start_output=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1)
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "OK"
|
||||
else
|
||||
echo "FAILED: $start_output"
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait 20s for VMs to fully start
|
||||
echo ""
|
||||
echo " Waiting 20s for VMs to boot..."
|
||||
sleep 20
|
||||
|
||||
# --- Phase 3: Verify all VMs running ---
|
||||
echo ""
|
||||
echo "--- PHASE 3: Verification ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# Check NFS mounts still healthy
|
||||
echo "--- NFS mount health ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
mount_count=$(nfsstat -m 2>/dev/null | grep -c "^/mnt")
|
||||
conn_count=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
||||
echo " NFS mounts: $mount_count"
|
||||
echo " NFS TCP connections: $conn_count"
|
||||
if [ "$mount_count" -gt 0 ]; then
|
||||
echo " First mount options:"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1 | sed "s/^/ /"
|
||||
fi
|
||||
'
|
||||
echo ""
|
||||
|
||||
# Check guest agent responsiveness (if agent enabled)
|
||||
echo "--- Guest agent check (VMs with agent:1) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
agent=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | grep -c '^agent: 1'")
|
||||
if [ "$agent" = "1" ]; then
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] agent ping... "
|
||||
result=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "timeout 10 qm agent $vmid ping 2>&1")
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "OK"
|
||||
else
|
||||
echo "no response (VM may still be booting)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VALIDATION COMPLETE"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
done
|
||||
Reference in New Issue
Block a user