prep for next ai session

2026-08-01 15:45:23 -05:00
parent 46c35106fb
commit 6244c1cc25
139 changed files with 16712 additions and 0 deletions
@@ -0,0 +1,88 @@
+#!/bin/bash
+# reboot-and-verify.sh - reboots a host and verifies NFS nconnect activates.
+# Usage: bash reboot-and-verify.sh <host>
+set -uo pipefail
+HOST="$1"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
+
+echo "================================================================"
+echo "[$HOST] PRE-REBOOT STATE"
+echo "================================================================"
+ssh "${SSH_OPTS[@]}" "root@$HOST" '
+    echo "--- VMs ---"
+    qm list 2>/dev/null
+    echo "--- NFS TCP conns: $(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l) ---"
+    echo "--- NFS first mount: ---"
+    nfsstat -m 2>/dev/null | head -2 | tail -1
+    echo "--- uptime ---"
+    uptime
+'
+
+echo ""
+echo "================================================================"
+echo "[$HOST] ISSUING REBOOT"
+echo "================================================================"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &'
+echo "Reboot sent at $(date +%H:%M:%S)"
+
+echo ""
+echo "================================================================"
+echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 min)"
+echo "================================================================"
+DEADLINE=$(( $(date +%s) + 600 ))
+LAST_PRINT=0
+while [ "$(date +%s)" -lt "$DEADLINE" ]; do
+    now=$(date +%s)
+    if [ $((now - LAST_PRINT)) -ge 15 ]; then
+        printf '  [%s] waiting... (%ss elapsed)\n' "$(date +%H:%M:%S)" "$(( now - DEADLINE + 600 ))"
+        LAST_PRINT=$now
+    fi
+    if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
+        # Verify uptime is actually low (host really rebooted, not still up)
+        up_mins=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/uptime | awk "{print int(\$1/60)}"')
+        if [ "${up_mins:-999}" -lt 5 ]; then
+            echo "  [$(date +%H:%M:%S)] SSH back, uptime ${up_mins}min — real reboot confirmed"
+            break
+        fi
+    fi
+    sleep 10
+done
+
+if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
+    echo "FAILED: $HOST not back after 10 minutes"
+    exit 1
+fi
+
+echo "Waiting 30s for services to settle..."
+sleep 30
+
+echo ""
+echo "================================================================"
+echo "[$HOST] POST-REBOOT VERIFICATION"
+echo "================================================================"
+ssh "${SSH_OPTS[@]}" "root@$HOST" '
+    echo "--- uptime ---"
+    uptime
+    echo ""
+    echo "--- governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) ---"
+    echo "--- swappiness: $(sysctl -n vm.swappiness) ---"
+    echo "--- tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control) ---"
+    echo "--- tuned: $(tuned-adm active 2>/dev/null | grep Current) ---"
+    echo ""
+    echo "--- NFS first mount: ---"
+    nfsstat -m 2>/dev/null | head -2 | tail -1
+    echo ""
+    echo "--- NFS TCP conns (expect 8 with nconnect=4): ---"
+    ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l
+    echo ""
+    echo "--- VMs: ---"
+    qm list 2>/dev/null
+    echo ""
+    echo "--- Failed services: ---"
+    systemctl --failed --no-legend 2>/dev/null | head -5
+    echo "(empty = none)"
+'
+echo ""
+echo "================================================================"
+echo "[$HOST] DONE"
+echo "================================================================"