mrcharles
21adb89d4e
Fresh Proxmox fleet audit (2026-07-28) with current VM placements, RAM, CPU, and storage for all 7 reachable hosts. Written to docs/proxmox/AUDIT-2026-07-28.md — supersedes placement data in PROJECT.md sections 4-8. Key audit findings: - CRITICAL: UCS01/02 and netinfra01/02 HA pairs both still on tsys4 storage. tsys4 failure = DNS/DHCP/NTP + LDAP/AD fully dark. These migrations were the #1 recommendation from the previous audit and have not been done. - CRITICAL: 2 of 3 active k3s cnodes (cnode1 + cnode2) on tsys4 NFS. tsys4 failure = etcd quorum lost. - 59% of running VMs still on tsys4 storage (improved from 68%). - cnode VMIDs have changed since PROJECT.md was written (cnode1 is now VMID 906 on tsys9, cnode2 is VMID 705 on tsys7, etc.) Gardening fixes: - Removed duplicate fleet-audit.sh (check.sh + deploy-check.sh already exist for this purpose) - Fixed hardcoded path /home/reachableceo/projects/perfopt in 13 perf/ scripts to use BASH_SOURCE-derived relative paths (per AGENTS.md self-locating scripts convention) - Updated STATUS.md Known Issues with the two critical findings - Updated STATUS.md Pending with prioritized pre-k8s action items - Registered AUDIT-2026-07-28.md in docmap.md 💘 Generated with Crush Assisted-by: Crush:glm-5.2
120 lines
4.3 KiB
Bash
Executable File
120 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
||
# iperf-tsys6-tsys7.sh - validate 2Gbps LACP between the two tuned hosts.
|
||
set -uo pipefail
|
||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||
LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/returned-logs/iperf"
|
||
mkdir -p "$LOG_DIR"
|
||
|
||
TSYS6="10.100.100.6"
|
||
TSYS7="10.100.100.7"
|
||
|
||
run_test() {
|
||
local client="$1" server="$2" server_ip="$3" label="$4" logfile="$5"
|
||
shift 4
|
||
local extra_args="$*"
|
||
|
||
echo -n " [$label] ... "
|
||
|
||
# Start server in one-shot mode
|
||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||
sleep 1
|
||
|
||
{
|
||
echo "=== iperf3: $label ==="
|
||
echo "Client: $client Server: $server ($server_ip)"
|
||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "Args: $extra_args"
|
||
echo ""
|
||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${server_ip} $extra_args" 2>&1
|
||
echo ""
|
||
echo "=== END ==="
|
||
} > "$logfile" 2>&1
|
||
|
||
# Extract result
|
||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||
if [ -n "$sum" ]; then
|
||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||
retrans=$(echo "$sum" | awk '{print $8}')
|
||
else
|
||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||
retrans=$(echo "$single" | awk '{print $9}')
|
||
fi
|
||
echo "${bitrate} (retrans: ${retrans:-0})"
|
||
}
|
||
|
||
echo "==================================================================="
|
||
echo " iperf3: tsys6 ↔ tsys7 (both have 2×1G LACP + layer3+4 hash)"
|
||
echo " Expectation: ~1.8-2.0 Gbps for 8-stream TCP"
|
||
echo "==================================================================="
|
||
echo ""
|
||
|
||
# Pre-flight: confirm bond state on both
|
||
echo "--- bond0 state on tsys6 ---"
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||
echo ""
|
||
echo "--- bond0 state on tsys7 ---"
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||
echo ""
|
||
|
||
echo "--- Running tests ---"
|
||
echo ""
|
||
|
||
# Test 1: tsys7 → tsys6, 8-stream TCP forward
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP 8-stream forward" \
|
||
"$LOG_DIR/tsys6-tsys7-01-tcp-8stream-forward.log" \
|
||
"-P 8 -t 30 -l 128k -O 2"
|
||
|
||
# Test 2: tsys6 → tsys7, 8-stream TCP forward (reverse direction)
|
||
run_test pfv-tsys6 pfv-tsys7 "$TSYS7" \
|
||
"tsys6→tsys7 TCP 8-stream forward" \
|
||
"$LOG_DIR/tsys6-tsys7-02-tcp-8stream-forward.log" \
|
||
"-P 8 -t 30 -l 128k -O 2"
|
||
|
||
# Test 3: tsys7 → tsys6, single stream (should be ~940 Mbps — single flow)
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP single-stream" \
|
||
"$LOG_DIR/tsys6-tsys7-03-tcp-single.log" \
|
||
"-t 20 -O 2"
|
||
|
||
# Test 4: tsys7 → tsys6, 4-stream (nconnect=4 mirrors this)
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP 4-stream" \
|
||
"$LOG_DIR/tsys6-tsys7-04-tcp-4stream.log" \
|
||
"-P 4 -t 30 -l 128k -O 2"
|
||
|
||
# Test 5: UDP saturation
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 UDP saturation" \
|
||
"$LOG_DIR/tsys6-tsys7-05-udp.log" \
|
||
"-u -b 3G -t 10 -l 8972"
|
||
|
||
# Cleanup
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " SUMMARY"
|
||
echo "==================================================================="
|
||
echo ""
|
||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||
for f in "$LOG_DIR"/tsys6-tsys7-*.log; do
|
||
[ -r "$f" ] || continue
|
||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||
if [ -n "$sum" ]; then
|
||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||
retrans=$(echo "$sum" | awk '{print $8}')
|
||
else
|
||
single=$(grep 'sender$' "$f" | tail -1)
|
||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||
retrans=$(echo "$single" | awk '{print $9}')
|
||
fi
|
||
printf "%-45s %15s %10s\n" "$label" "$bitrate" "${retrans:--}"
|
||
done
|
||
echo ""
|
||
echo "Expected: 8-stream ~1.8-2.0 Gbps, single-stream ~940 Mbps"
|