mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
113 lines
4.3 KiB
Bash
Executable File
113 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# run-lacp-rx-distribution.sh (workstation wrapper)
|
|
#
|
|
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
|
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
|
# scps both logs back to returned-logs/iperf/.
|
|
#
|
|
# Run from the workstation:
|
|
# bash scripts/run-lacp-rx-distribution.sh
|
|
#
|
|
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
|
-o StrictHostKeyChecking=accept-new)
|
|
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
|
SEND="pfv-tsys7" # sender (control: its TX split)
|
|
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
|
SCRIPT="lacp-rx-distribution.sh"
|
|
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
|
REMOTE_SCRIPT="/root/${SCRIPT}"
|
|
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
|
|
echo "==================================================================="
|
|
echo " LACP per-slave RX/TX distribution test"
|
|
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
|
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
ssh "${SSH[@]}" "root@$h" \
|
|
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
|
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
|
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
|
2>&1 | head -1
|
|
done
|
|
echo ""
|
|
|
|
# 1. Copy the script to both hosts + chmod
|
|
echo "--- deploy $SCRIPT to both hosts ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
|
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
|
&& echo "deployed + chmod +x" \
|
|
|| echo "DEPLOY FAILED"
|
|
done
|
|
echo ""
|
|
|
|
# 2. Kill any stale iperf3 on both hosts
|
|
echo "--- cleanup stale iperf3 ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
|
done
|
|
echo ""
|
|
|
|
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
|
echo "--- starting receiver on $RECV (background) ---"
|
|
ssh "${SSH[@]}" "root@$RECV" \
|
|
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
|
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
|
sleep 5
|
|
echo ""
|
|
|
|
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
|
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
|
ssh "${SSH[@]}" "root@$SEND" \
|
|
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
|
echo ""
|
|
|
|
# 5. Give receiver a moment to finish writing its log
|
|
sleep 3
|
|
|
|
# 6. Fetch logs back
|
|
echo "--- fetching logs ---"
|
|
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
|
src=""
|
|
case "$f" in
|
|
lacp-rx-receiver*) src="$RECV" ;;
|
|
lacp-rx-sender*) src="$SEND" ;;
|
|
esac
|
|
printf ' %-28s <- %s : ' "$f" "$src"
|
|
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
|
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
|
else
|
|
echo "MISSING"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
# 7. Show the receiver log (the decisive one)
|
|
echo "==================================================================="
|
|
echo " RECEIVER LOG ($RECV — the decisive side)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " SENDER LOG ($SEND — control)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " Local copies in: $LOCAL_LOG_DIR/"
|
|
echo "==================================================================="
|