mrcharles
28e0b0c7a6
Fresh Proxmox fleet audit (2026-07-28) with current VM placements, RAM, CPU, and storage for all 7 reachable hosts. Written to docs/proxmox/AUDIT-2026-07-28.md — supersedes placement data in PROJECT.md sections 4-8. Key audit findings: - CRITICAL: UCS01/02 and netinfra01/02 HA pairs both still on tsys4 storage. tsys4 failure = DNS/DHCP/NTP + LDAP/AD fully dark. These migrations were the #1 recommendation from the previous audit and have not been done. - CRITICAL: 2 of 3 active k3s cnodes (cnode1 + cnode2) on tsys4 NFS. tsys4 failure = etcd quorum lost. - 59% of running VMs still on tsys4 storage (improved from 68%). - cnode VMIDs have changed since PROJECT.md was written (cnode1 is now VMID 906 on tsys9, cnode2 is VMID 705 on tsys7, etc.) Gardening fixes: - Removed duplicate fleet-audit.sh (check.sh + deploy-check.sh already exist for this purpose) - Fixed hardcoded path /home/reachableceo/projects/perfopt in 13 perf/ scripts to use BASH_SOURCE-derived relative paths (per AGENTS.md self-locating scripts convention) - Updated STATUS.md Known Issues with the two critical findings - Updated STATUS.md Pending with prioritized pre-k8s action items - Registered AUDIT-2026-07-28.md in docmap.md 💘 Generated with Crush Assisted-by: Crush:glm-5.2
185 lines
7.5 KiB
Bash
Executable File
185 lines
7.5 KiB
Bash
Executable File
#!/bin/bash
|
||
###############################################################################
|
||
# iperf-storage-tests.sh
|
||
#
|
||
# Installs iperf3 on all online hosts, then runs a matrix of storage-network
|
||
# throughput tests. Saves all output to returned-logs/iperf/.
|
||
#
|
||
# Test matrix (all over VLAN1000 storage network, 10.100.100.0/24):
|
||
# 1. tsys7 → tsys4 (USB cdc_ncm NIC) — the smoking gun
|
||
# 2. tsys7 → tsys5 (bond0, 1 active slave) — PCI NIC comparison
|
||
# 3. tsys7 → tsys6 (bond0, 2 active slaves) — working LACP baseline
|
||
# 4. Reverse: tsys4 → tsys7 (USB NIC TX direction)
|
||
# 5. Reverse: tsys5 → tsys7
|
||
#
|
||
# Each test: TCP 8-stream 30s forward + reverse + UDP saturation.
|
||
###############################################################################
|
||
set -uo pipefail
|
||
|
||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||
LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/returned-logs/iperf"
|
||
mkdir -p "$LOG_DIR"
|
||
|
||
ALL_HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||
|
||
# Storage network IPs
|
||
declare -A SIP
|
||
SIP[pfv-tsys1]="10.100.100.1"
|
||
SIP[pfv-tsys3]="10.100.100.3"
|
||
SIP[pfv-tsys4]="10.100.100.4"
|
||
SIP[pfv-tsys5]="10.100.100.5"
|
||
SIP[pfv-tsys6]="10.100.100.6"
|
||
SIP[pfv-tsys7]="10.100.100.7"
|
||
|
||
echo "==================================================================="
|
||
echo " STEP 1: Install iperf3 on all online hosts"
|
||
echo "==================================================================="
|
||
|
||
for host in "${ALL_HOSTS[@]}"; do
|
||
echo -n " [$host] "
|
||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||
echo "UNREACHABLE — skipping"
|
||
continue
|
||
fi
|
||
# Check if iperf3 already installed
|
||
if ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
|
||
echo "iperf3 already installed"
|
||
else
|
||
printf "installing... "
|
||
ssh "${SSH_OPTS[@]}" "root@$host" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " STEP 2: Kill any existing iperf3 processes everywhere"
|
||
echo "==================================================================="
|
||
for host in "${ALL_HOSTS[@]}"; do
|
||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
done
|
||
echo " Done."
|
||
|
||
# Helper: run an iperf3 test and save output
|
||
run_iperf() {
|
||
local client="$1" server="$2" direction="$3" label="$4" logfile="$5"
|
||
local client_ip="${SIP[$client]}" server_ip="${SIP[$server]}"
|
||
|
||
echo -n " [$client → $server] $label ... "
|
||
|
||
# Start server in one-shot mode (-1 means serve one client then exit)
|
||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||
sleep 1
|
||
|
||
# Run client
|
||
{
|
||
echo "=== iperf3: $label ==="
|
||
echo "Client: $client ($client_ip)"
|
||
echo "Server: $server ($server_ip)"
|
||
echo "Direction: $direction"
|
||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo ""
|
||
if [ "$direction" = "forward" ]; then
|
||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2" 2>&1
|
||
elif [ "$direction" = "reverse" ]; then
|
||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2 -R" 2>&1
|
||
elif [ "$direction" = "udp" ]; then
|
||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||
"iperf3 -c ${server_ip} -B ${client_ip} -u -b 2G -t 10 -l 8972" 2>&1
|
||
elif [ "$direction" = "single" ]; then
|
||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||
"iperf3 -c ${server_ip} -B ${client_ip} -t 20 -O 2" 2>&1
|
||
fi
|
||
echo ""
|
||
echo "=== END ==="
|
||
} > "$logfile" 2>&1
|
||
|
||
# Extract summary line
|
||
if grep -q "sender" "$logfile"; then
|
||
bitrate=$(grep "sender" "$logfile" | tail -1 | awk '{print $7, $8}')
|
||
echo "done: ${bitrate}"
|
||
else
|
||
echo "done (check log for details)"
|
||
fi
|
||
}
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " STEP 3: Run iperf3 test matrix"
|
||
echo "==================================================================="
|
||
echo ""
|
||
echo "All tests over VLAN1000 storage network (10.100.100.0/24)."
|
||
echo "TCP tests: 8 parallel streams, 30s, 128k blocks."
|
||
echo ""
|
||
|
||
# --- Test 1: tsys7 → tsys4 (USB cdc_ncm target) ---
|
||
echo "--- TEST 1: tsys7 → tsys4 (USB cdc_ncm NIC) ---"
|
||
run_iperf pfv-tsys7 pfv-tsys4 forward "TCP 8-stream forward (tsys7→tsys4 USB)" \
|
||
"$LOG_DIR/01-tsys7-to-tsys4-tcp-forward.log"
|
||
run_iperf pfv-tsys7 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys7)" \
|
||
"$LOG_DIR/02-tsys7-to-tsys4-tcp-reverse.log"
|
||
run_iperf pfv-tsys7 pfv-tsys4 single "TCP single-stream forward (tsys7→tsys4 USB)" \
|
||
"$LOG_DIR/03-tsys7-to-tsys4-tcp-single.log"
|
||
run_iperf pfv-tsys7 pfv-tsys4 udp "UDP saturation (tsys7→tsys4 USB)" \
|
||
"$LOG_DIR/04-tsys7-to-tsys4-udp.log"
|
||
|
||
echo ""
|
||
|
||
# --- Test 2: tsys7 → tsys5 (bond0, PCI NIC, 1 active slave) ---
|
||
echo "--- TEST 2: tsys7 → tsys5 (PCI NIC, broken bond - 1 slave) ---"
|
||
run_iperf pfv-tsys7 pfv-tsys5 forward "TCP 8-stream forward (tsys7→tsys5 PCI)" \
|
||
"$LOG_DIR/05-tsys7-to-tsys5-tcp-forward.log"
|
||
run_iperf pfv-tsys7 pfv-tsys5 reverse "TCP 8-stream reverse (tsys5 PCI→tsys7)" \
|
||
"$LOG_DIR/06-tsys7-to-tsys5-tcp-reverse.log"
|
||
run_iperf pfv-tsys7 pfv-tsys5 single "TCP single-stream forward (tsys7→tsys5 PCI)" \
|
||
"$LOG_DIR/07-tsys7-to-tsys5-tcp-single.log"
|
||
run_iperf pfv-tsys7 pfv-tsys5 udp "UDP saturation (tsys7→tsys5 PCI)" \
|
||
"$LOG_DIR/08-tsys7-to-tsys5-udp.log"
|
||
|
||
echo ""
|
||
|
||
# --- Test 3: tsys7 → tsys6 (working 2-slave LACP baseline, layer3+4) ---
|
||
echo "--- TEST 3: tsys7 → tsys6 (working 2×1G LACP baseline) ---"
|
||
run_iperf pfv-tsys7 pfv-tsys6 forward "TCP 8-stream forward (tsys7→tsys6 LACP)" \
|
||
"$LOG_DIR/09-tsys7-to-tsys6-tcp-forward.log"
|
||
run_iperf pfv-tsys7 pfv-tsys6 reverse "TCP 8-stream reverse (tsys6 LACP→tsys7)" \
|
||
"$LOG_DIR/10-tsys7-to-tsys6-tcp-reverse.log"
|
||
|
||
echo ""
|
||
|
||
# --- Test 4: tsys6 → tsys4 (pre-tuning baseline) ---
|
||
echo "--- TEST 4: tsys6 → tsys4 (baseline before tsys6 tuning) ---"
|
||
run_iperf pfv-tsys6 pfv-tsys4 forward "TCP 8-stream forward (tsys6→tsys4 USB)" \
|
||
"$LOG_DIR/11-tsys6-to-tsys4-tcp-forward.log"
|
||
run_iperf pfv-tsys6 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys6)" \
|
||
"$LOG_DIR/12-tsys6-to-tsys4-tcp-reverse.log"
|
||
|
||
echo ""
|
||
|
||
# --- Cleanup: kill iperf3 everywhere ---
|
||
echo "--- Cleanup ---"
|
||
for host in "${ALL_HOSTS[@]}"; do
|
||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
done
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " RESULTS SUMMARY"
|
||
echo "==================================================================="
|
||
echo ""
|
||
printf "%-45s %s\n" "TEST" "THROUGHPUT"
|
||
printf "%-45s %s\n" "----" "----------"
|
||
for f in "$LOG_DIR"/*.log; do
|
||
[ -r "$f" ] || continue
|
||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||
bitrate=$(grep -E "sender$" "$f" | tail -1 | awk '{print $7, $8}')
|
||
[ -z "$bitrate" ] && bitrate=$(grep -E "Mbits/sec|Gbits/sec" "$f" | tail -1 | grep -oE '[0-9.]+ [MG]bits/sec' | head -1)
|
||
[ -z "$bitrate" ] && bitrate="(see log)"
|
||
printf "%-45s %s\n" "$label" "$bitrate"
|
||
done
|
||
|
||
echo ""
|
||
echo "Full logs saved to: $LOG_DIR/"
|
||
echo "==================================================================="
|