mrcharles
28e0b0c7a6
Fresh Proxmox fleet audit (2026-07-28) with current VM placements, RAM, CPU, and storage for all 7 reachable hosts. Written to docs/proxmox/AUDIT-2026-07-28.md — supersedes placement data in PROJECT.md sections 4-8. Key audit findings: - CRITICAL: UCS01/02 and netinfra01/02 HA pairs both still on tsys4 storage. tsys4 failure = DNS/DHCP/NTP + LDAP/AD fully dark. These migrations were the #1 recommendation from the previous audit and have not been done. - CRITICAL: 2 of 3 active k3s cnodes (cnode1 + cnode2) on tsys4 NFS. tsys4 failure = etcd quorum lost. - 59% of running VMs still on tsys4 storage (improved from 68%). - cnode VMIDs have changed since PROJECT.md was written (cnode1 is now VMID 906 on tsys9, cnode2 is VMID 705 on tsys7, etc.) Gardening fixes: - Removed duplicate fleet-audit.sh (check.sh + deploy-check.sh already exist for this purpose) - Fixed hardcoded path /home/reachableceo/projects/perfopt in 13 perf/ scripts to use BASH_SOURCE-derived relative paths (per AGENTS.md self-locating scripts convention) - Updated STATUS.md Known Issues with the two critical findings - Updated STATUS.md Pending with prioritized pre-k8s action items - Registered AUDIT-2026-07-28.md in docmap.md 💘 Generated with Crush Assisted-by: Crush:glm-5.2
113 lines
4.3 KiB
Bash
Executable File
113 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# run-lacp-rx-distribution.sh (workstation wrapper)
|
|
#
|
|
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
|
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
|
# scps both logs back to returned-logs/iperf/.
|
|
#
|
|
# Run from the workstation:
|
|
# bash scripts/run-lacp-rx-distribution.sh
|
|
#
|
|
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
|
-o StrictHostKeyChecking=accept-new)
|
|
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
|
SEND="pfv-tsys7" # sender (control: its TX split)
|
|
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
|
SCRIPT="lacp-rx-distribution.sh"
|
|
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
|
REMOTE_SCRIPT="/root/${SCRIPT}"
|
|
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
|
|
echo "==================================================================="
|
|
echo " LACP per-slave RX/TX distribution test"
|
|
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
|
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
ssh "${SSH[@]}" "root@$h" \
|
|
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
|
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
|
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
|
2>&1 | head -1
|
|
done
|
|
echo ""
|
|
|
|
# 1. Copy the script to both hosts + chmod
|
|
echo "--- deploy $SCRIPT to both hosts ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
|
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
|
&& echo "deployed + chmod +x" \
|
|
|| echo "DEPLOY FAILED"
|
|
done
|
|
echo ""
|
|
|
|
# 2. Kill any stale iperf3 on both hosts
|
|
echo "--- cleanup stale iperf3 ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
|
done
|
|
echo ""
|
|
|
|
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
|
echo "--- starting receiver on $RECV (background) ---"
|
|
ssh "${SSH[@]}" "root@$RECV" \
|
|
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
|
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
|
sleep 5
|
|
echo ""
|
|
|
|
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
|
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
|
ssh "${SSH[@]}" "root@$SEND" \
|
|
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
|
echo ""
|
|
|
|
# 5. Give receiver a moment to finish writing its log
|
|
sleep 3
|
|
|
|
# 6. Fetch logs back
|
|
echo "--- fetching logs ---"
|
|
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
|
src=""
|
|
case "$f" in
|
|
lacp-rx-receiver*) src="$RECV" ;;
|
|
lacp-rx-sender*) src="$SEND" ;;
|
|
esac
|
|
printf ' %-28s <- %s : ' "$f" "$src"
|
|
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
|
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
|
else
|
|
echo "MISSING"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
# 7. Show the receiver log (the decisive one)
|
|
echo "==================================================================="
|
|
echo " RECEIVER LOG ($RECV — the decisive side)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " SENDER LOG ($SEND — control)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " Local copies in: $LOCAL_LOG_DIR/"
|
|
echo "==================================================================="
|