mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
120 lines
4.2 KiB
Bash
Executable File
120 lines
4.2 KiB
Bash
Executable File
#!/bin/bash
|
||
# iperf-tsys6-tsys7.sh - validate 2Gbps LACP between the two tuned hosts.
|
||
set -uo pipefail
|
||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||
LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||
mkdir -p "$LOG_DIR"
|
||
|
||
TSYS6="10.100.100.6"
|
||
TSYS7="10.100.100.7"
|
||
|
||
run_test() {
|
||
local client="$1" server="$2" server_ip="$3" label="$4" logfile="$5"
|
||
shift 4
|
||
local extra_args="$*"
|
||
|
||
echo -n " [$label] ... "
|
||
|
||
# Start server in one-shot mode
|
||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||
sleep 1
|
||
|
||
{
|
||
echo "=== iperf3: $label ==="
|
||
echo "Client: $client Server: $server ($server_ip)"
|
||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "Args: $extra_args"
|
||
echo ""
|
||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${server_ip} $extra_args" 2>&1
|
||
echo ""
|
||
echo "=== END ==="
|
||
} > "$logfile" 2>&1
|
||
|
||
# Extract result
|
||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||
if [ -n "$sum" ]; then
|
||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||
retrans=$(echo "$sum" | awk '{print $8}')
|
||
else
|
||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||
retrans=$(echo "$single" | awk '{print $9}')
|
||
fi
|
||
echo "${bitrate} (retrans: ${retrans:-0})"
|
||
}
|
||
|
||
echo "==================================================================="
|
||
echo " iperf3: tsys6 ↔ tsys7 (both have 2×1G LACP + layer3+4 hash)"
|
||
echo " Expectation: ~1.8-2.0 Gbps for 8-stream TCP"
|
||
echo "==================================================================="
|
||
echo ""
|
||
|
||
# Pre-flight: confirm bond state on both
|
||
echo "--- bond0 state on tsys6 ---"
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||
echo ""
|
||
echo "--- bond0 state on tsys7 ---"
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||
echo ""
|
||
|
||
echo "--- Running tests ---"
|
||
echo ""
|
||
|
||
# Test 1: tsys7 → tsys6, 8-stream TCP forward
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP 8-stream forward" \
|
||
"$LOG_DIR/tsys6-tsys7-01-tcp-8stream-forward.log" \
|
||
"-P 8 -t 30 -l 128k -O 2"
|
||
|
||
# Test 2: tsys6 → tsys7, 8-stream TCP forward (reverse direction)
|
||
run_test pfv-tsys6 pfv-tsys7 "$TSYS7" \
|
||
"tsys6→tsys7 TCP 8-stream forward" \
|
||
"$LOG_DIR/tsys6-tsys7-02-tcp-8stream-forward.log" \
|
||
"-P 8 -t 30 -l 128k -O 2"
|
||
|
||
# Test 3: tsys7 → tsys6, single stream (should be ~940 Mbps — single flow)
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP single-stream" \
|
||
"$LOG_DIR/tsys6-tsys7-03-tcp-single.log" \
|
||
"-t 20 -O 2"
|
||
|
||
# Test 4: tsys7 → tsys6, 4-stream (nconnect=4 mirrors this)
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 TCP 4-stream" \
|
||
"$LOG_DIR/tsys6-tsys7-04-tcp-4stream.log" \
|
||
"-P 4 -t 30 -l 128k -O 2"
|
||
|
||
# Test 5: UDP saturation
|
||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||
"tsys7→tsys6 UDP saturation" \
|
||
"$LOG_DIR/tsys6-tsys7-05-udp.log" \
|
||
"-u -b 3G -t 10 -l 8972"
|
||
|
||
# Cleanup
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo " SUMMARY"
|
||
echo "==================================================================="
|
||
echo ""
|
||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||
for f in "$LOG_DIR"/tsys6-tsys7-*.log; do
|
||
[ -r "$f" ] || continue
|
||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||
if [ -n "$sum" ]; then
|
||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||
retrans=$(echo "$sum" | awk '{print $8}')
|
||
else
|
||
single=$(grep 'sender$' "$f" | tail -1)
|
||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||
retrans=$(echo "$single" | awk '{print $9}')
|
||
fi
|
||
printf "%-45s %15s %10s\n" "$label" "$bitrate" "${retrans:--}"
|
||
done
|
||
echo ""
|
||
echo "Expected: 8-stream ~1.8-2.0 Gbps, single-stream ~940 Mbps"
|