mrcharles
28e0b0c7a6
Fresh Proxmox fleet audit (2026-07-28) with current VM placements, RAM, CPU, and storage for all 7 reachable hosts. Written to docs/proxmox/AUDIT-2026-07-28.md — supersedes placement data in PROJECT.md sections 4-8. Key audit findings: - CRITICAL: UCS01/02 and netinfra01/02 HA pairs both still on tsys4 storage. tsys4 failure = DNS/DHCP/NTP + LDAP/AD fully dark. These migrations were the #1 recommendation from the previous audit and have not been done. - CRITICAL: 2 of 3 active k3s cnodes (cnode1 + cnode2) on tsys4 NFS. tsys4 failure = etcd quorum lost. - 59% of running VMs still on tsys4 storage (improved from 68%). - cnode VMIDs have changed since PROJECT.md was written (cnode1 is now VMID 906 on tsys9, cnode2 is VMID 705 on tsys7, etc.) Gardening fixes: - Removed duplicate fleet-audit.sh (check.sh + deploy-check.sh already exist for this purpose) - Fixed hardcoded path /home/reachableceo/projects/perfopt in 13 perf/ scripts to use BASH_SOURCE-derived relative paths (per AGENTS.md self-locating scripts convention) - Updated STATUS.md Known Issues with the two critical findings - Updated STATUS.md Pending with prioritized pre-k8s action items - Registered AUDIT-2026-07-28.md in docmap.md 💘 Generated with Crush Assisted-by: Crush:glm-5.2
129 lines
5.3 KiB
Bash
Executable File
129 lines
5.3 KiB
Bash
Executable File
#!/bin/bash
|
||
# finish-host.sh - applies ALL remaining changes to a host and verifies.
|
||
#
|
||
# Steps:
|
||
# 1. Start all VMs (triggers NFS lazy-mount)
|
||
# 2. Wait for NFS mounts to appear
|
||
# 3. Verify NFS nconnect=4 + noatime
|
||
# 4. Apply bond0 xmit_hash_policy=layer3+4
|
||
# 5. Full end-to-end verification
|
||
#
|
||
# Usage: bash finish-host.sh <host> [--apply]
|
||
# Default is dry-run (starts VMs + shows what bond change would do, but doesn't edit interfaces)
|
||
set -uo pipefail
|
||
|
||
HOST="${1:-}"
|
||
MODE="${2:-dryrun}"
|
||
[ "$MODE" = "--apply" ] && MODE="apply" || MODE="dryrun"
|
||
|
||
if [ -z "$HOST" ]; then
|
||
echo "Usage: $0 <host> [--apply]"
|
||
exit 1
|
||
fi
|
||
|
||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/scripts"
|
||
|
||
echo "==================================================================="
|
||
echo " finish-host.sh — $HOST (mode: $MODE)"
|
||
echo "==================================================================="
|
||
echo ""
|
||
|
||
# =========================================================================
|
||
# STEP 1: Start all VMs
|
||
# =========================================================================
|
||
echo "=== STEP 1: Start all VMs on $HOST ==="
|
||
# Get list of all VMs (not just stopped — start is idempotent)
|
||
vm_list=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1{print \$1}"')
|
||
for vmid in $vm_list; do
|
||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||
if [ "$status" != "running" ]; then
|
||
echo " Starting VM $vmid..."
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1 | sed 's/^/ /'
|
||
else
|
||
echo " VM $vmid already running"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Waiting 15s for VMs to boot and trigger NFS mounts..."
|
||
sleep 15
|
||
|
||
echo ""
|
||
echo "--- VM status after start ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||
|
||
# =========================================================================
|
||
# STEP 2: Verify NFS mounts came back with nconnect=4
|
||
# =========================================================================
|
||
echo ""
|
||
echo "=== STEP 2: Verify NFS mounts with nconnect=4 ==="
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null' | head -30
|
||
|
||
echo ""
|
||
echo "--- NFS TCP connections to :2049 ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||
conn_count=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||
echo " Total NFS TCP connections: $conn_count"
|
||
|
||
# =========================================================================
|
||
# STEP 3: Apply bond0 hash policy
|
||
# =========================================================================
|
||
echo ""
|
||
echo "=== STEP 3: Apply bond0 xmit_hash_policy=layer3+4 (mode: $MODE) ==="
|
||
|
||
# Upload the bond hash script
|
||
scp "${SSH_OPTS[@]}" "$SCRIPT_DIR/apply-bond-hash.sh" "root@$HOST:/root/apply-bond-hash.sh" >/dev/null 2>&1
|
||
|
||
if [ "$MODE" = "apply" ]; then
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh --apply' 2>&1
|
||
else
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh' 2>&1
|
||
fi
|
||
|
||
# =========================================================================
|
||
# STEP 4: Full verification
|
||
# =========================================================================
|
||
echo ""
|
||
echo "=== STEP 4: Full end-to-end verification ==="
|
||
echo ""
|
||
echo "--- Uptime ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
|
||
echo ""
|
||
echo "--- CPU governor ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
|
||
echo ""
|
||
echo "--- vm.swappiness ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
|
||
echo ""
|
||
echo "--- TCP BBR ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
|
||
echo ""
|
||
echo "--- tuned profile ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'tuned-adm active 2>/dev/null'
|
||
echo ""
|
||
echo "--- bond0 hash policy + LACP state ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 2>/dev/null | head -25'
|
||
echo ""
|
||
echo "--- NFS mount options (first 3 mounts) ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -24'
|
||
echo ""
|
||
echo "--- NFS TCP connections (expect 4 per server × 2 servers = 8) ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||
nfs_conns=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||
echo " Count: $nfs_conns"
|
||
echo ""
|
||
echo "--- VMs running ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||
echo ""
|
||
echo "--- Failed services ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
|
||
echo ""
|
||
echo "--- Network interfaces (speed/duplex/mtu) ---"
|
||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'for ifc in bond0 nic0 nic1 nic2 vmbr0 datanet; do [ -d "/sys/class/net/$ifc" ] && printf "%-12s speed=%-8s duplex=%-8s mtu=%s\n" "$ifc" "$(cat /sys/class/net/$ifc/speed 2>/dev/null)" "$(cat /sys/class/net/$ifc/duplex 2>/dev/null)" "$(cat /sys/class/net/$ifc/mtu 2>/dev/null)"; done'
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " COMPLETE — $HOST"
|
||
echo "==================================================================="
|