mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
92 lines
3.3 KiB
Bash
92 lines
3.3 KiB
Bash
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# run-lacp-retrans-cause.sh (workstation wrapper)
|
|
#
|
|
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
|
# right order, scps logs back, prints them.
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
|
-o StrictHostKeyChecking=accept-new)
|
|
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
|
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
|
SCRIPT="lacp-retrans-cause.sh"
|
|
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
|
REMOTE_SCRIPT="/root/${SCRIPT}"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
|
|
echo "==================================================================="
|
|
echo " LACP retransmit-cause investigation"
|
|
echo " Receiver: $RECV Sender: $SEND"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
echo "--- preflight ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
ssh "${SSH[@]}" "root@$h" \
|
|
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
|
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
|
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
|
done
|
|
echo ""
|
|
|
|
echo "--- deploy ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
|
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
|
&& echo "deployed" || echo "FAILED"
|
|
done
|
|
echo ""
|
|
|
|
echo "--- cleanup stale iperf3 ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
|
done
|
|
echo ""
|
|
|
|
echo "--- launching receiver on $RECV (background) ---"
|
|
ssh "${SSH[@]}" "root@$RECV" \
|
|
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
|
sleep 5
|
|
echo ""
|
|
|
|
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
|
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
|
echo ""
|
|
sleep 3
|
|
|
|
echo "--- fetching logs ---"
|
|
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
|
src=""
|
|
case "$f" in
|
|
*receiver*) src="$RECV" ;;
|
|
*sender*) src="$SEND" ;;
|
|
esac
|
|
printf ' %-32s <- %s : ' "$f" "$src"
|
|
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
|
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
|
else
|
|
echo "MISSING"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
echo "==================================================================="
|
|
echo " RECEIVER LOG ($RECV)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " SENDER LOG ($SEND)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " Local copies in: $LOCAL_LOG_DIR/"
|
|
echo "==================================================================="
|