prep for next ai session

2026-08-01 15:45:23 -05:00
parent 46c35106fb
commit 6244c1cc25
139 changed files with 16712 additions and 0 deletions
@@ -0,0 +1,89 @@
+#!/usr/bin/bash
+# shellcheck disable=SC2012,SC2001 # diagnostic script; ls -la listings and sed line-prefixing are intentional
+#
+# console/validate-conman.sh — verify conman can actually reach devices via
+# ser2net TCP ports and is capturing log output to files.
+#
+# This tests the real data path: conman → TCP 200X → ser2net → /dev/consoles/X → device
+#
+set -uo pipefail
+
+TS_IP=$(tailscale ip -4)
+LOGDIR="/var/log/conman"
+
+echo "============================================"
+echo "  Conman Data Path + Log Validation"
+echo "  Host: $(hostname)   TS IP: $TS_IP"
+echo "============================================"
+
+echo ""
+echo "--- 1. conman.conf log settings ---"
+grep -E "logdir|LOGDIR|^GLOBAL LOG" /etc/conman.conf 2>/dev/null | grep -v "^#" || echo "  (no explicit logdir — defaults to /var/log/conman)"
+echo "  Log dir: $LOGDIR"
+ls -la "$LOGDIR"/ 2>/dev/null | head -15 || echo "  ($LOGDIR does not exist yet)"
+
+echo ""
+echo "--- 2. CONSOLE entries: each has a log= directive? ---"
+# Extract the auto-generated block and check each CONSOLE line has log=
+sed -n '/BEGIN PFV CONSOLE/,/END PFV CONSOLE/p' /etc/conman.conf | grep "^CONSOLE" | while read -r line; do
+  name=$(echo "$line" | sed -n 's/.*name="\([^"]*\)".*/\1/p')
+  if echo "$line" | grep -q 'log='; then
+    logfile=$(echo "$line" | sed -n 's/.*log="\([^"]*\)".*/\1/p')
+    echo "  [OK] $name → log=$logfile"
+  else
+    echo "  [FAIL] $name has NO log= directive"
+  fi
+done
+
+echo ""
+echo "--- 3. Trigger log capture: connect to each console briefly ---"
+# conman -e changes the escape char. We use -j (join, read-only) with a timeout.
+# Actually, conman doesn't have a built-in "connect for N seconds" — but conmand
+# connects to each device ON STARTUP and keeps the connection open for logging.
+# The log files should already be created. Let's check timestamps.
+
+echo "  conmand connects to all consoles on startup. Checking if logs exist..."
+echo ""
+echo "--- 4. Log file inventory ---"
+for name in pfv-core-sw01 pfv-tor3-mgmt pfv-tor3-stor pfv-rrinfra-rtr pfv-r2-tor-top subodev-torsw pfv-r2-sw; do
+  logfile="$LOGDIR/${name}.log"
+  if [ -f "$logfile" ]; then
+    SIZE=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
+    MTIME=$(stat -c%y "$logfile" 2>/dev/null | cut -d. -f1)
+    echo "  [OK] $logfile ($SIZE bytes, modified $MTIME)"
+  else
+    echo "  [MISSING] $logfile — conmand may not be writing yet"
+  fi
+done
+
+echo ""
+echo "--- 5. conmand connection status (journal) ---"
+# conmand logs connection attempts/errors to syslog
+journalctl -u conmand --no-pager -n 50 2>/dev/null | grep -iE "connect|error|fail|console|refused|timeout" | tail -15 || echo "  (no relevant journal entries)"
+
+echo ""
+echo "--- 6. Verify ser2net is proxying data (telnet rfc2217) ---"
+echo "  Probing TCP $TS_IP:2007 for data..."
+RESPONSE=$(timeout 3 bash -c "printf '\r\r' | nc -w 2 $TS_IP 2007 2>/dev/null" | tr -cd '[:print:][:space:]' | head -5)
+if [ -n "$RESPONSE" ]; then
+  echo "  [OK] Data flowing through ser2net TCP 2007:"
+  echo "$RESPONSE" | sed 's/^/    /'
+else
+  echo "  (no immediate response — device may need more interaction)"
+fi
+
+echo ""
+echo "--- 7. Check if conmand has open connections to ser2net ports ---"
+CONMAND_PID=$(pgrep -x conmand 2>/dev/null || echo "")
+if [ -n "$CONMAND_PID" ]; then
+  echo "  conmand PID: $CONMAND_PID"
+  echo "  Open connections to ser2net (expect 7 to 100.x:200X):"
+  ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -oE "100\.[0-9.]+:200[0-9]" | sort | sed 's/^/    /'
+  COUNT=$(ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -c ":200")
+  echo "  Total conmand→ser2net connections: $COUNT (expect 7)"
+else
+  echo "  [FAIL] conmand not running"
+fi
+
+echo ""
+echo "============================================"