mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
218 lines
9.0 KiB
Bash
218 lines
9.0 KiB
Bash
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# lacp-retrans-cause.sh (HOST-NATIVE)
|
|
#
|
|
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
|
|
#
|
|
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
|
|
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
|
|
# saturation) or just unavoidable LACP reordering overhead?
|
|
#
|
|
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
|
|
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
|
|
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
|
|
# - /proc/net/softnet_stat drops -> ksoftirq starved
|
|
# - /proc/net/snmp TCP retranst -> TCP-level retrans
|
|
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
|
|
# coming from LACP reordering (out-of-order segments triggering fast
|
|
# retransmit), not packet loss — and are NOT fixable by tuning.
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
DURATION="${LACP_TEST_SECS:-12}"
|
|
STREAMS="${LACP_STREAMS:-8}"
|
|
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
|
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
|
LOG_DIR="/root"
|
|
BOND="bond0"
|
|
|
|
HOST="$(uname -n)"; HOST="${HOST%%.*}"
|
|
case "$HOST" in
|
|
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
|
|
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
|
|
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
|
|
esac
|
|
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
|
|
# ---- per-slave NIC counter snapshot -----------------------------------------
|
|
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
|
|
# plus all error-like counters. Falls back gracefully if a counter doesn't
|
|
# exist (different NIC drivers expose different names).
|
|
nic_snapshot() {
|
|
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
|
|
[ -n "$s" ] || continue
|
|
echo "[$s]"
|
|
# ethtool -S may exist; show only drop/error/retrans lines
|
|
if command -v ethtool >/dev/null 2>&1; then
|
|
ethtool -S "$s" 2>/dev/null \
|
|
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|
|
|| echo "(ethtool -S: no matching counters or unsupported)"
|
|
else
|
|
echo "(ethtool not installed)"
|
|
fi
|
|
# /sys counters (always available, driver-agnostic)
|
|
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
|
|
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
|
|
multicast collisions; do
|
|
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
|
|
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
|
|
done
|
|
done
|
|
}
|
|
|
|
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
|
|
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
|
|
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
|
|
# look identical in hex and decimal). Processed is informational only and we
|
|
# don't try to delta it (hex arithmetic is awk-version-dependent).
|
|
softnet_snapshot() {
|
|
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
|
|
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
|
|
}
|
|
|
|
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
|
|
tcp_snapshot() {
|
|
awk '/^Tcp:/{
|
|
if (!seen) {
|
|
seen=1
|
|
# /proc/net/snmp Tcp line 1 = names, line 2 = values
|
|
n=split($0, names, " ")
|
|
# re-find the values line
|
|
getline
|
|
vals=$0
|
|
split(vals, v, " ")
|
|
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
|
|
}
|
|
}' /proc/net/snmp
|
|
}
|
|
|
|
# ---- print deltas for selected counters ------------------------------------
|
|
# $1 = before file, $2 = after file
|
|
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
|
|
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
|
|
# integers (typically 0 on a healthy host), so +0 coercion is correct for
|
|
# the values we care about. We deliberately DON'T report `processed` deltas
|
|
# because hex arithmetic on large numbers is awk-version-dependent.
|
|
softnet_delta() {
|
|
awk '
|
|
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
|
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
|
|
next }
|
|
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
|
split($0, q, /[= ]+/);
|
|
dd = (q[5]+0) - drop[id]
|
|
sd = (q[7]+0) - sqz[id]
|
|
if (dd != 0 || sd != 0)
|
|
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
|
|
' "$1" "$2" | sort
|
|
}
|
|
|
|
# $1 = before file, $2 = after file, $3 = section label prefix
|
|
# File format: slave header line "[nic1]", then " counter_name = value" lines.
|
|
# Output only counters whose value changed.
|
|
nic_delta() {
|
|
awk '
|
|
FNR==NR { if ($0 ~ /^\[/) slave=$0
|
|
else if ($0 ~ /=/) {
|
|
# Everything before " = " is the counter name (trim spaces)
|
|
pos = index($0, "=")
|
|
name = substr($0, 1, pos-1)
|
|
gsub(/^ +| +$/, "", name)
|
|
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
|
before[slave SUBSEP name] = val
|
|
}
|
|
next }
|
|
{ if ($0 ~ /^\[/) slave=$0
|
|
else if ($0 ~ /=/) {
|
|
pos = index($0, "=")
|
|
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
|
|
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
|
b = before[slave SUBSEP name]+0
|
|
d = val+0 - b
|
|
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
|
|
} }
|
|
' "$1" "$2"
|
|
}
|
|
|
|
{
|
|
echo "=== LACP retransmit cause investigation ($ROLE) ==="
|
|
echo "Host: $HOST Role: $ROLE"
|
|
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "Bond: $BOND"
|
|
echo ""
|
|
echo "--- bond0 hash + driver ---"
|
|
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
|
|
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
|
|
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
|
|
speed=$(cat /sys/class/net/$s/speed 2>/dev/null)
|
|
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
|
|
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
|
|
done
|
|
echo ""
|
|
} | tee "$LOG"
|
|
|
|
echo "--- BEFORE snapshots ---"
|
|
nic_snapshot > "$TMP/nic_before"
|
|
softnet_snapshot > "$TMP/softnet_before"
|
|
tcp_snapshot > "$TMP/tcp_before"
|
|
|
|
if [ "$ROLE" = "receiver" ]; then
|
|
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
|
|
pkill -x iperf3 2>/dev/null; sleep 0.5
|
|
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
|
SRV_PID=$!
|
|
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
|
|
wait "$SRV_PID" 2>/dev/null
|
|
else
|
|
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
|
|
sleep 3
|
|
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
|
> "$TMP/iperf.out" 2>&1
|
|
fi
|
|
|
|
echo "--- AFTER snapshots ---"
|
|
nic_snapshot > "$TMP/nic_after"
|
|
softnet_snapshot > "$TMP/softnet_after"
|
|
tcp_snapshot > "$TMP/tcp_after"
|
|
|
|
{
|
|
echo ""
|
|
echo "--- iperf3 summary ---"
|
|
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
|
|
echo ""
|
|
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
|
|
# Show only the counters that changed
|
|
paste "$TMP/tcp_before" "$TMP/tcp_after" \
|
|
| awk '{
|
|
a=$3; b=$(NF)
|
|
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
|
|
}'
|
|
echo ""
|
|
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
|
|
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
|
|
echo ""
|
|
echo "--- NIC counter deltas (only non-zero) ---"
|
|
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
|
|
echo ""
|
|
echo "--- raw iperf3 tail (last 12 lines) ---"
|
|
tail -12 "$TMP/iperf.out"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " HOW TO READ THIS"
|
|
echo "==================================================================="
|
|
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
|
|
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
|
|
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
|
|
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
|
|
echo " check IRQ affinity, consider RPS."
|
|
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
|
|
echo " -> the retransmits are LACP reordering (out-of-order segments"
|
|
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
|
|
echo "==================================================================="
|
|
} | tee -a "$LOG"
|
|
|
|
echo ""
|
|
echo "Log: $LOG"
|