mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
89 lines
3.3 KiB
Bash
Executable File
89 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# reboot-and-verify.sh - reboots a host and verifies NFS nconnect activates.
|
|
# Usage: bash reboot-and-verify.sh <host>
|
|
set -uo pipefail
|
|
HOST="$1"
|
|
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
|
|
|
echo "================================================================"
|
|
echo "[$HOST] PRE-REBOOT STATE"
|
|
echo "================================================================"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
|
echo "--- VMs ---"
|
|
qm list 2>/dev/null
|
|
echo "--- NFS TCP conns: $(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l) ---"
|
|
echo "--- NFS first mount: ---"
|
|
nfsstat -m 2>/dev/null | head -2 | tail -1
|
|
echo "--- uptime ---"
|
|
uptime
|
|
'
|
|
|
|
echo ""
|
|
echo "================================================================"
|
|
echo "[$HOST] ISSUING REBOOT"
|
|
echo "================================================================"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &'
|
|
echo "Reboot sent at $(date +%H:%M:%S)"
|
|
|
|
echo ""
|
|
echo "================================================================"
|
|
echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 min)"
|
|
echo "================================================================"
|
|
DEADLINE=$(( $(date +%s) + 600 ))
|
|
LAST_PRINT=0
|
|
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
|
now=$(date +%s)
|
|
if [ $((now - LAST_PRINT)) -ge 15 ]; then
|
|
printf ' [%s] waiting... (%ss elapsed)\n' "$(date +%H:%M:%S)" "$(( now - DEADLINE + 600 ))"
|
|
LAST_PRINT=$now
|
|
fi
|
|
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
|
# Verify uptime is actually low (host really rebooted, not still up)
|
|
up_mins=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/uptime | awk "{print int(\$1/60)}"')
|
|
if [ "${up_mins:-999}" -lt 5 ]; then
|
|
echo " [$(date +%H:%M:%S)] SSH back, uptime ${up_mins}min — real reboot confirmed"
|
|
break
|
|
fi
|
|
fi
|
|
sleep 10
|
|
done
|
|
|
|
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
|
echo "FAILED: $HOST not back after 10 minutes"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Waiting 30s for services to settle..."
|
|
sleep 30
|
|
|
|
echo ""
|
|
echo "================================================================"
|
|
echo "[$HOST] POST-REBOOT VERIFICATION"
|
|
echo "================================================================"
|
|
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
|
echo "--- uptime ---"
|
|
uptime
|
|
echo ""
|
|
echo "--- governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) ---"
|
|
echo "--- swappiness: $(sysctl -n vm.swappiness) ---"
|
|
echo "--- tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control) ---"
|
|
echo "--- tuned: $(tuned-adm active 2>/dev/null | grep Current) ---"
|
|
echo ""
|
|
echo "--- NFS first mount: ---"
|
|
nfsstat -m 2>/dev/null | head -2 | tail -1
|
|
echo ""
|
|
echo "--- NFS TCP conns (expect 8 with nconnect=4): ---"
|
|
ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l
|
|
echo ""
|
|
echo "--- VMs: ---"
|
|
qm list 2>/dev/null
|
|
echo ""
|
|
echo "--- Failed services: ---"
|
|
systemctl --failed --no-legend 2>/dev/null | head -5
|
|
echo "(empty = none)"
|
|
'
|
|
echo ""
|
|
echo "================================================================"
|
|
echo "[$HOST] DONE"
|
|
echo "================================================================"
|