mrcharles
fa0c77fb58
The ser2net layer between conman and the serial devices was causing
stair-stepping in terminal output. conman's dev="host:port" uses telnet
protocol, but ser2net's accepter was raw TCP (no telnet mode). The telnet
NVT state machine in conman was stripping bare CR characters from device
output — particularly from Dell switches that use old-style \n\r (LF+CR)
line endings instead of standard \r\n. In telnet, a bare \r not followed
by \n or NUL is non-compliant and gets dropped, leaving bare \n that
causes stair-stepping in the raw-mode terminal.
Fix: conman now opens serial devices directly via the stable udev
symlinks (/dev/consoles/<name>) with seropts, eliminating the telnet
layer entirely. ser2net is stopped and disabled but remains installed
for emergency TCP access (documented workflow: stop conmand, start
ser2net, use telnet, then reverse).
Architecture change:
Before: device → serial → ser2net (raw TCP) → conman (telnet NVT) → terminal
After: device → serial → conman (direct) → terminal
Verified: 7/7 serial devices held by conmand, 7/7 log files capturing,
all \r bytes preserved in terminal output (confirmed via PTY capture).
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
93 lines
3.5 KiB
Bash
93 lines
3.5 KiB
Bash
#!/usr/bin/bash
|
|
#
|
|
# console/validate-conman.sh — verify conman can actually reach devices via
|
|
# ser2net TCP ports and is capturing log output to files.
|
|
#
|
|
# This tests the real data path: conman → TCP 200X → ser2net → /dev/consoles/X → device
|
|
#
|
|
set -uo pipefail
|
|
|
|
TS_IP=$(tailscale ip -4)
|
|
LOGDIR="/var/log/conman"
|
|
|
|
echo "============================================"
|
|
echo " Conman Data Path + Log Validation"
|
|
echo " Host: $(hostname) TS IP: $TS_IP"
|
|
echo "============================================"
|
|
|
|
echo ""
|
|
echo "--- 1. conman.conf log settings ---"
|
|
grep -E "logdir|LOGDIR|^GLOBAL LOG" /etc/conman.conf 2>/dev/null | grep -v "^#" || echo " (no explicit logdir — defaults to /var/log/conman)"
|
|
echo " Log dir: $LOGDIR"
|
|
ls -la "$LOGDIR"/ 2>/dev/null | head -15 || echo " ($LOGDIR does not exist yet)"
|
|
|
|
echo ""
|
|
echo "--- 2. CONSOLE entries: each has a log= directive? ---"
|
|
# Extract the auto-generated block and check each CONSOLE line has log=
|
|
sed -n '/BEGIN PFV CONSOLE/,/END PFV CONSOLE/p' /etc/conman.conf | grep "^CONSOLE" | while read -r line; do
|
|
name=$(echo "$line" | sed -n 's/.*name="\([^"]*\)".*/\1/p')
|
|
if echo "$line" | grep -q 'log='; then
|
|
logfile=$(echo "$line" | sed -n 's/.*log="\([^"]*\)".*/\1/p')
|
|
echo " [OK] $name → log=$logfile"
|
|
else
|
|
echo " [FAIL] $name has NO log= directive"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- 3. Trigger log capture: connect to each console briefly ---"
|
|
# conman -e changes the escape char. We use -j (join, read-only) with a timeout.
|
|
# Actually, conman doesn't have a built-in "connect for N seconds" — but conmand
|
|
# connects to each device ON STARTUP and keeps the connection open for logging.
|
|
# The log files should already be created. Let's check timestamps.
|
|
|
|
echo " conmand connects to all consoles on startup. Checking if logs exist..."
|
|
echo ""
|
|
echo "--- 4. Log file inventory ---"
|
|
for name in pfv-core-sw01 pfv-tor3-mgmt pfv-tor3-stor pfv-rrinfra-rtr pfv-r2-tor-top subodev-torsw pfv-r2-sw; do
|
|
logfile="$LOGDIR/${name}.log"
|
|
if [ -f "$logfile" ]; then
|
|
SIZE=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
|
|
MTIME=$(stat -c%y "$logfile" 2>/dev/null | cut -d. -f1)
|
|
echo " [OK] $logfile ($SIZE bytes, modified $MTIME)"
|
|
else
|
|
echo " [MISSING] $logfile — conmand may not be writing yet"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- 5. conmand connection status (journal) ---"
|
|
# conmand logs connection attempts/errors to syslog
|
|
journalctl -u conmand --no-pager -n 50 2>/dev/null | grep -iE "connect|error|fail|console|refused|timeout" | tail -15 || echo " (no relevant journal entries)"
|
|
|
|
echo ""
|
|
echo "--- 6. Verify serial devices are open by conmand ---"
|
|
echo " Checking /dev/consoles/* are held by conmand..."
|
|
CONMAND_PID=$(pgrep -x conmand 2>/dev/null || echo "")
|
|
if [ -n "$CONMAND_PID" ]; then
|
|
echo " conmand PID: $CONMAND_PID"
|
|
for devlink in /dev/consoles/*; do
|
|
[ -e "$devlink" ] || continue
|
|
name=$(basename "$devlink")
|
|
if lsof "$devlink" 2>/dev/null | grep -q conmand; then
|
|
echo " [OK] $name ($devlink) opened by conmand"
|
|
else
|
|
echo " [WARN] $name ($devlink) not held by conmand"
|
|
fi
|
|
done
|
|
else
|
|
echo " [FAIL] conmand not running"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- 7. Verify ser2net is stopped (conman owns devices) ---"
|
|
if systemctl is-active --quiet ser2net 2>/dev/null; then
|
|
echo " [WARN] ser2net is running — will conflict with conmand for serial devices!"
|
|
echo " Fix: systemctl stop ser2net"
|
|
else
|
|
echo " [OK] ser2net is stopped (conman owns serial devices directly)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "============================================"
|