mrcharles
28e0b0c7a6
Fresh Proxmox fleet audit (2026-07-28) with current VM placements, RAM, CPU, and storage for all 7 reachable hosts. Written to docs/proxmox/AUDIT-2026-07-28.md — supersedes placement data in PROJECT.md sections 4-8. Key audit findings: - CRITICAL: UCS01/02 and netinfra01/02 HA pairs both still on tsys4 storage. tsys4 failure = DNS/DHCP/NTP + LDAP/AD fully dark. These migrations were the #1 recommendation from the previous audit and have not been done. - CRITICAL: 2 of 3 active k3s cnodes (cnode1 + cnode2) on tsys4 NFS. tsys4 failure = etcd quorum lost. - 59% of running VMs still on tsys4 storage (improved from 68%). - cnode VMIDs have changed since PROJECT.md was written (cnode1 is now VMID 906 on tsys9, cnode2 is VMID 705 on tsys7, etc.) Gardening fixes: - Removed duplicate fleet-audit.sh (check.sh + deploy-check.sh already exist for this purpose) - Fixed hardcoded path /home/reachableceo/projects/perfopt in 13 perf/ scripts to use BASH_SOURCE-derived relative paths (per AGENTS.md self-locating scripts convention) - Updated STATUS.md Known Issues with the two critical findings - Updated STATUS.md Pending with prioritized pre-k8s action items - Registered AUDIT-2026-07-28.md in docmap.md 💘 Generated with Crush Assisted-by: Crush:glm-5.2
92 lines
3.3 KiB
Bash
92 lines
3.3 KiB
Bash
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# run-lacp-retrans-cause.sh (workstation wrapper)
|
|
#
|
|
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
|
# right order, scps logs back, prints them.
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
|
-o StrictHostKeyChecking=accept-new)
|
|
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
|
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
|
SCRIPT="lacp-retrans-cause.sh"
|
|
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
|
REMOTE_SCRIPT="/root/${SCRIPT}"
|
|
mkdir -p "$LOCAL_LOG_DIR"
|
|
|
|
echo "==================================================================="
|
|
echo " LACP retransmit-cause investigation"
|
|
echo " Receiver: $RECV Sender: $SEND"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
echo "--- preflight ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
ssh "${SSH[@]}" "root@$h" \
|
|
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
|
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
|
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
|
done
|
|
echo ""
|
|
|
|
echo "--- deploy ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
printf ' %-12s ' "$h"
|
|
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
|
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
|
&& echo "deployed" || echo "FAILED"
|
|
done
|
|
echo ""
|
|
|
|
echo "--- cleanup stale iperf3 ---"
|
|
for h in "$RECV" "$SEND"; do
|
|
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
|
done
|
|
echo ""
|
|
|
|
echo "--- launching receiver on $RECV (background) ---"
|
|
ssh "${SSH[@]}" "root@$RECV" \
|
|
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
|
sleep 5
|
|
echo ""
|
|
|
|
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
|
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
|
echo ""
|
|
sleep 3
|
|
|
|
echo "--- fetching logs ---"
|
|
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
|
src=""
|
|
case "$f" in
|
|
*receiver*) src="$RECV" ;;
|
|
*sender*) src="$SEND" ;;
|
|
esac
|
|
printf ' %-32s <- %s : ' "$f" "$src"
|
|
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
|
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
|
else
|
|
echo "MISSING"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
echo "==================================================================="
|
|
echo " RECEIVER LOG ($RECV)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " SENDER LOG ($SEND)"
|
|
echo "==================================================================="
|
|
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " Local copies in: $LOCAL_LOG_DIR/"
|
|
echo "==================================================================="
|