PFVCluster/console/validate-conman.sh

#!/usr/bin/bash
#
# console/validate-conman.sh — verify conman can actually reach devices via
# ser2net TCP ports and is capturing log output to files.
#
# This tests the real data path: conman → TCP 200X → ser2net → /dev/consoles/X → device
#
set -uo pipefail

TS_IP=$(tailscale ip -4)
LOGDIR="/var/log/conman"

echo "============================================"
echo "  Conman Data Path + Log Validation"
echo "  Host: $(hostname)   TS IP: $TS_IP"
echo "============================================"

echo ""
echo "--- 1. conman.conf log settings ---"
grep -E "logdir|LOGDIR|^GLOBAL LOG" /etc/conman.conf 2>/dev/null | grep -v "^#" || echo "  (no explicit logdir — defaults to /var/log/conman)"
echo "  Log dir: $LOGDIR"
ls -la "$LOGDIR"/ 2>/dev/null | head -15 || echo "  ($LOGDIR does not exist yet)"

echo ""
echo "--- 2. CONSOLE entries: each has a log= directive? ---"
# Extract the auto-generated block and check each CONSOLE line has log=
sed -n '/BEGIN PFV CONSOLE/,/END PFV CONSOLE/p' /etc/conman.conf | grep "^CONSOLE" | while read -r line; do
  name=$(echo "$line" | sed -n 's/.*name="\([^"]*\)".*/\1/p')
  if echo "$line" | grep -q 'log='; then
    logfile=$(echo "$line" | sed -n 's/.*log="\([^"]*\)".*/\1/p')
    echo "  [OK] $name → log=$logfile"
  else
    echo "  [FAIL] $name has NO log= directive"
  fi
done

echo ""
echo "--- 3. Trigger log capture: connect to each console briefly ---"
# conman -e changes the escape char. We use -j (join, read-only) with a timeout.
# Actually, conman doesn't have a built-in "connect for N seconds" — but conmand
# connects to each device ON STARTUP and keeps the connection open for logging.
# The log files should already be created. Let's check timestamps.

echo "  conmand connects to all consoles on startup. Checking if logs exist..."
echo ""
echo "--- 4. Log file inventory ---"
for name in pfv-core-sw01 pfv-tor3-mgmt pfv-tor3-stor pfv-rrinfra-rtr pfv-r2-tor-top subodev-torsw pfv-r2-sw; do
  logfile="$LOGDIR/${name}.log"
  if [ -f "$logfile" ]; then
    SIZE=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
    MTIME=$(stat -c%y "$logfile" 2>/dev/null | cut -d. -f1)
    echo "  [OK] $logfile ($SIZE bytes, modified $MTIME)"
  else
    echo "  [MISSING] $logfile — conmand may not be writing yet"
  fi
done

echo ""
echo "--- 5. conmand connection status (journal) ---"
# conmand logs connection attempts/errors to syslog
journalctl -u conmand --no-pager -n 50 2>/dev/null | grep -iE "connect|error|fail|console|refused|timeout" | tail -15 || echo "  (no relevant journal entries)"

echo ""
echo "--- 6. Verify ser2net is actually proxying data (check for byte flow) ---"
# Pick a known-active port (2007 = pfv-r2-sw, the old Dell with menu UI that responded)
echo "  Probing TCP $TS_IP:2007 for data..."
RESPONSE=$(timeout 3 bash -c "printf '\r\r' | nc -w 2 $TS_IP 2007 2>/dev/null" | tr -cd '[:print:][:space:]' | head -5)
if [ -n "$RESPONSE" ]; then
  echo "  [OK] Data flowing through ser2net TCP 2007:"
  echo "$RESPONSE" | sed 's/^/    /'
else
  echo "  (no immediate response — device may need more interaction)"
fi

echo ""
echo "--- 7. Check if conmand has open connections to ser2net ports ---"
CONMAND_PID=$(pgrep -x conmand 2>/dev/null || echo "")
if [ -n "$CONMAND_PID" ]; then
  echo "  conmand PID: $CONMAND_PID"
  echo "  Open connections to ser2net (expect 7 to 100.x:200X):"
  ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -oE "100\.[0-9.]+:200[0-9]" | sort | sed 's/^/    /'
  COUNT=$(ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -c ":200")
  echo "  Total conmand→ser2net connections: $COUNT (expect 7)"
else
  echo "  [FAIL] conmand not running"
fi

echo ""
echo "============================================"