mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
128 lines
4.3 KiB
Bash
Executable File
128 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# sw-capture-remote.sh - orchestrate a serial capture from this workstation.
|
|
#
|
|
# Flow:
|
|
# 1. De-conflict: abort if any local ssh to pfv-tsys4 is in flight
|
|
# (other agent could be there).
|
|
# 2. Free the serial port: kill whatever holds /dev/ttyUSBx
|
|
# (typically a screen session). Targeted, not blanket.
|
|
# 3. scp driver + .cmds to pfv-tsys4.
|
|
# 4. Run driver over ssh, capture stderr to console.
|
|
# 5. scp the resulting log back to returned-logs/.
|
|
#
|
|
# Usage:
|
|
# sw-capture-remote.sh <switch-name> [device]
|
|
#
|
|
# <switch-name> e.g. pfv-core-sw01 (must have switches/<name>.cmds)
|
|
# [device] /dev/ttyUSBx on pfv-tsys4. Defaults per switch map below.
|
|
#
|
|
# Currently scoped to pfv-core-sw01 only (per user direction). The other
|
|
# two switches are deferred; their defaults are placeholders.
|
|
set -u
|
|
|
|
SWITCH=${1:-}
|
|
DEVICE=${2:-}
|
|
|
|
if [ -z "$SWITCH" ]; then
|
|
echo "Usage: $0 <switch-name> [device]" >&2
|
|
echo " e.g. $0 pfv-core-sw01 /dev/ttyUSB2" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Switch -> default device map (ttyUSB2 = core-sw01 confirmed by user).
|
|
case "$SWITCH" in
|
|
pfv-core-sw01)
|
|
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB2 ;;
|
|
pfv-r3-tor-mgmt)
|
|
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB0 # TENTATIVE - unconfirmed
|
|
if [ "${2:-}" = "" ]; then
|
|
echo "NOTE: pfv-r3-tor-mgmt device is tentative (/dev/ttyUSB0)." >&2
|
|
echo " Pass the device explicitly if different." >&2
|
|
fi ;;
|
|
pfv-r3-tor-stor)
|
|
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB1 # TENTATIVE - unconfirmed
|
|
if [ "${2:-}" = "" ]; then
|
|
echo "NOTE: pfv-r3-tor-stor device is tentative (/dev/ttyUSB1)." >&2
|
|
echo " Pass the device explicitly if different." >&2
|
|
fi ;;
|
|
*)
|
|
echo "unknown switch: $SWITCH" >&2; exit 2 ;;
|
|
esac
|
|
|
|
BAUD=9600
|
|
HOST=root@pfv-tsys4
|
|
HERE=/home/reachableceo/projects/perfopt
|
|
LOCAL_DRIVER=$HERE/scripts/sw-capture.py
|
|
LOCAL_CMDS=$HERE/switches/$SWITCH.cmds
|
|
LOCAL_LOG=$HERE/returned-logs/$SWITCH.log
|
|
REMOTE_DRIVER=/root/sw-capture.py
|
|
REMOTE_CMDS=/root/$SWITCH.cmds
|
|
REMOTE_LOG=/root/$SWITCH.log
|
|
|
|
[ -f "$LOCAL_DRIVER" ] || { echo "missing $LOCAL_DRIVER" >&2; exit 2; }
|
|
[ -f "$LOCAL_CMDS" ] || { echo "missing $LOCAL_CMDS" >&2; exit 2; }
|
|
|
|
ts() { date +%H:%M:%S; }
|
|
|
|
echo "[$(ts)] switch=$SWITCH device=$DEVICE baud=$BAUD host=$HOST"
|
|
|
|
# 1. De-conflict: any local ssh to pfv-tsys4 in flight?
|
|
echo "[$(ts)] checking for in-flight ssh to pfv-tsys4..."
|
|
if ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys4|scp.*pfv-tsys4' | grep -v grep >/tmp/.swcap.ps 2>&1; then
|
|
cat /tmp/.swcap.ps
|
|
echo "[$(ts)] ABORT: another ssh/scp to pfv-tsys4 is running (other agent?)." >&2
|
|
exit 1
|
|
fi
|
|
echo "[$(ts)] clear."
|
|
rm -f /tmp/.swcap.ps
|
|
|
|
# 2. Free the serial port: kill whatever holds $DEVICE.
|
|
echo "[$(ts)] freeing $DEVICE on $HOST (targeted; other screen sessions untouched)..."
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "$HOST" \
|
|
"fuser -v $DEVICE 2>&1 | tee /dev/stderr; \
|
|
fuser -k -TERM $DEVICE 2>/dev/null; sleep 1; \
|
|
if fuser $DEVICE 2>/dev/null; then \
|
|
echo 'still held after SIGTERM, escalating to SIGKILL'; \
|
|
fuser -k -KILL $DEVICE 2>/dev/null; sleep 1; \
|
|
fi; \
|
|
fuser $DEVICE 2>/dev/null && echo 'STILL HELD' || echo 'FREE'"
|
|
|
|
# Re-check; abort if still held.
|
|
HELD=$(ssh -o BatchMode=yes "$HOST" "fuser $DEVICE 2>/dev/null && echo HELD || echo FREE")
|
|
if [ "$HELD" = "HELD" ]; then
|
|
echo "[$(ts)] ABORT: $DEVICE still held on $HOST." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# 3. Copy driver + cmds.
|
|
echo "[$(ts)] copying driver + cmds to $HOST..."
|
|
scp -q "$LOCAL_DRIVER" "$HOST:$REMOTE_DRIVER"
|
|
scp -q "$LOCAL_CMDS" "$HOST:$REMOTE_CMDS"
|
|
|
|
# 4. Run the capture on pfv-tsys4. Stream stderr (progress) to console.
|
|
echo "[$(ts)] running capture..."
|
|
ssh -o BatchMode=yes -o ServerAliveInterval=10 "$HOST" \
|
|
"python3 $REMOTE_DRIVER \
|
|
--device $DEVICE --baud $BAUD \
|
|
--cmds $REMOTE_CMDS --log $REMOTE_LOG"
|
|
RC=$?
|
|
echo "[$(ts)] capture exit code: $RC"
|
|
|
|
# 5. Pull log back.
|
|
echo "[$(ts)] pulling log back to $LOCAL_LOG..."
|
|
mkdir -p "$(dirname "$LOCAL_LOG")"
|
|
scp -q "$HOST:$REMOTE_LOG" "$LOCAL_LOG"
|
|
if [ -f "$LOCAL_LOG" ]; then
|
|
SZ=$(wc -c < "$LOCAL_LOG")
|
|
echo "[$(ts)] OK: $LOCAL_LOG ($SZ bytes)"
|
|
echo "----- head -----"
|
|
head -30 "$LOCAL_LOG"
|
|
echo "----- tail -----"
|
|
tail -10 "$LOCAL_LOG"
|
|
else
|
|
echo "[$(ts)] ERROR: log not pulled back." >&2
|
|
exit 1
|
|
fi
|
|
|
|
exit $RC
|