mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
170 lines
6.8 KiB
Bash
Executable File
170 lines
6.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# validate-fixes.sh - READ-ONLY validation of all applied tunings.
|
|
# Does NOT reboot, shutdown VMs, or modify anything.
|
|
set -uo pipefail
|
|
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys6 pfv-tsys7 pfv-tsys9)
|
|
|
|
echo "==================================================================="
|
|
echo " READ-ONLY VALIDATION — $(date)"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
for HOST in "${HOSTS[@]}"; do
|
|
echo "================================================================"
|
|
echo "[$HOST]"
|
|
echo "================================================================"
|
|
|
|
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
|
echo " UNREACHABLE"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
|
pass=0; fail=0
|
|
check() {
|
|
local label="$1" actual="$2" expected="$3"
|
|
if [ "$actual" = "$expected" ]; then
|
|
printf " [OK] %-30s %s\n" "$label" "$actual"
|
|
pass=$((pass+1))
|
|
else
|
|
printf " [FAIL] %-30s got=%s want=%s\n" "$label" "$actual" "$expected"
|
|
fail=$((fail+1))
|
|
fi
|
|
}
|
|
|
|
# 1. CPU governor
|
|
gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "n/a")
|
|
if [ "$gov" = "n/a" ]; then
|
|
printf " [SKIP] %-30s %s\n" "CPU governor" "(no cpufreq driver — OK for server BIOS)"
|
|
else
|
|
check "CPU governor" "$gov" "performance"
|
|
fi
|
|
|
|
# 2. vm.swappiness
|
|
swap=$(sysctl -n vm.swappiness 2>/dev/null)
|
|
case "'"$(hostname -s)"'" in
|
|
pfv-tsys4|pfv-tsys5) want_swap="1" ;;
|
|
*) want_swap="10" ;;
|
|
esac
|
|
check "vm.swappiness" "$swap" "$want_swap"
|
|
|
|
# 3. TCP congestion control
|
|
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
|
|
check "tcp_congestion_control" "$cc" "bbr"
|
|
|
|
# 4. default_qdisc (paired with BBR)
|
|
qd=$(sysctl -n net.core.default_qdisc 2>/dev/null)
|
|
check "net.core.default_qdisc" "$qd" "fq"
|
|
|
|
# 5. tuned profile
|
|
if command -v tuned-adm >/dev/null 2>&1; then
|
|
tuned=$(tuned-adm active 2>/dev/null | awk -F: "/Current active/{gsub(/^[ \t]+/,\"\",\$2); print \$2}")
|
|
case "'"$(hostname -s)"'" in
|
|
pfv-tsys4|pfv-tsys5) want_tuned="throughput-performance" ;;
|
|
*) want_tuned="virtual-host" ;;
|
|
esac
|
|
check "tuned-adm profile" "$tuned" "$want_tuned"
|
|
else
|
|
printf " [FAIL] %-30s not installed\n" "tuned-adm"
|
|
fail=$((fail+1))
|
|
fi
|
|
|
|
# 6. bond0 hash policy (if bond exists)
|
|
if [ -r /proc/net/bonding/bond0 ]; then
|
|
hash=$(grep "Transmit Hash" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
|
check "bond0 xmit_hash_policy" "$hash" "layer3+4"
|
|
ports=$(grep "Number of ports" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
|
printf " [INFO] %-30s %s ports active\n" "bond0 LACP ports" "$ports"
|
|
else
|
|
printf " [SKIP] %-30s %s\n" "bond0 hash" "(no bond0 — single NIC host)"
|
|
fi
|
|
|
|
# 7. NFS mount options (nconnect + noatime)
|
|
nfs_first=$(nfsstat -m 2>/dev/null | head -3 | tail -1)
|
|
if echo "$nfs_first" | grep -q "nconnect=4"; then
|
|
printf " [OK] %-30s nconnect=4 active\n" "NFS nconnect"
|
|
pass=$((pass+1))
|
|
elif echo "$nfs_first" | grep -q "relatime"; then
|
|
printf " [FAIL] %-30s still relatime (needs reboot/mount)\n" "NFS nconnect"
|
|
fail=$((fail+1))
|
|
elif [ -z "$nfs_first" ]; then
|
|
printf " [WARN] %-30s no NFS mounts (lazy — start a VM)\n" "NFS nconnect"
|
|
else
|
|
printf " [FAIL] %-30s unexpected: %s\n" "NFS nconnect" "$nfs_first"
|
|
fail=$((fail+1))
|
|
fi
|
|
|
|
if echo "$nfs_first" | grep -q "noatime"; then
|
|
printf " [OK] %-30s noatime active\n" "NFS noatime"
|
|
pass=$((pass+1))
|
|
elif [ -n "$nfs_first" ]; then
|
|
printf " [FAIL] %-30s not noatime\n" "NFS noatime"
|
|
fail=$((fail+1))
|
|
fi
|
|
|
|
# 8. NFS TCP connection count
|
|
nfs_conns=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
|
if [ "$nfs_conns" -ge 8 ]; then
|
|
printf " [OK] %-30s %s connections\n" "NFS TCP conns" "$nfs_conns"
|
|
pass=$((pass+1))
|
|
elif [ "$nfs_conns" -gt 0 ]; then
|
|
printf " [WARN] %-30s %s (expect 8 with nconnect=4)\n" "NFS TCP conns" "$nfs_conns"
|
|
else
|
|
printf " [WARN] %-30s 0 (lazy mounts — start a VM)\n" "NFS TCP conns"
|
|
fi
|
|
|
|
# 9. sysctl persistence
|
|
if [ -r /etc/sysctl.d/99-perfopt.conf ]; then
|
|
printf " [OK] %-30s /etc/sysctl.d/99-perfopt.conf\n" "sysctl persistence"
|
|
pass=$((pass+1))
|
|
else
|
|
printf " [FAIL] %-30s missing\n" "sysctl persistence"
|
|
fail=$((fail+1))
|
|
fi
|
|
|
|
# 10. Observability packages
|
|
for cmd in sar jq numactl nvme mtr bmon; do
|
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
|
printf " [FAIL] %-30s not installed\n" "obs: $cmd"
|
|
fail=$((fail+1))
|
|
fi
|
|
done
|
|
if command -v sar >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 && \
|
|
command -v numactl >/dev/null 2>&1 && command -v nvme >/dev/null 2>&1 && \
|
|
command -v mtr >/dev/null 2>&1 && command -v bmon >/dev/null 2>&1; then
|
|
printf " [OK] %-30s all installed\n" "observability packages"
|
|
pass=$((pass+1))
|
|
fi
|
|
|
|
# 11. Failed services
|
|
failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
|
|
if [ "$failed_count" = "0" ]; then
|
|
printf " [OK] %-30s none\n" "failed services"
|
|
pass=$((pass+1))
|
|
else
|
|
printf " [FAIL] %-30s %s failed:\n" "failed services" "$failed_count"
|
|
systemctl --failed --no-legend 2>/dev/null | sed "s/^/ /"
|
|
fail=$((fail+1))
|
|
fi
|
|
|
|
# 12. VM status (read-only — just report)
|
|
running_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"" | wc -l)
|
|
stopped_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3!=\"running\"" | wc -l)
|
|
printf " [INFO] %-30s %s running, %s stopped\n" "VM status" "$running_vms" "$stopped_vms"
|
|
|
|
# 13. uptime
|
|
printf " [INFO] %-30s %s\n" "uptime" "$(uptime | sed "s/.*up //" | sed "s/,.*//")"
|
|
|
|
echo ""
|
|
echo " RESULT: $pass passed, $fail failed"
|
|
'
|
|
echo ""
|
|
done
|
|
|
|
echo "==================================================================="
|
|
echo " SUMMARY"
|
|
echo "==================================================================="
|