prep for next ai session

2026-08-01 15:45:23 -05:00
parent 46c35106fb
commit 6244c1cc25
139 changed files with 16712 additions and 0 deletions
@@ -0,0 +1,58 @@
+#!/bin/bash
+# wait-for-host.sh - polls SSH until host is back, then runs verification.
+# Usage: bash wait-for-host.sh <host>
+set -uo pipefail
+HOST="$1"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
+
+DEADLINE=$(( $(date +%s) + 600 ))
+echo "Polling $HOST for SSH return (max 10 min)..."
+while [ "$(date +%s)" -lt "$DEADLINE" ]; do
+    if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
+        echo "[$(date +%H:%M:%S)] SSH is back!"
+        break
+    fi
+    sleep 10
+    echo "  [$(date +%H:%M:%S)] still down..."
+done
+
+if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
+    echo "FAILED: $HOST not back after 10 minutes"
+    exit 1
+fi
+
+echo "Waiting 30s for services to settle..."
+sleep 30
+
+echo ""
+echo "================================================================"
+echo "[$HOST] POST-REBOOT VERIFICATION"
+echo "================================================================"
+echo "--- Uptime (should be < 5 min) ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
+echo ""
+echo "--- TCP congestion control (expect bbr) ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
+echo ""
+echo "--- vm.swappiness ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
+echo ""
+echo "--- scaling_governor ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
+echo ""
+echo "--- NFS mount options (looking for nconnect=4 + noatime) ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -16'
+echo ""
+echo "--- NFS TCP connection count to :2049 (expect ~8 = 4 per server × 2 servers) ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l'
+echo ""
+echo "--- Running VMs ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
+echo ""
+echo "--- Failed services ---"
+ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
+echo ""
+echo "================================================================"
+echo "[$HOST] DONE"
+echo "================================================================"