#!/usr/bin/env bash
###############################################################################
# lacp-retrans-cause.sh  (HOST-NATIVE)
#
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
#
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
# saturation) or just unavoidable LACP reordering overhead?
#
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
#   - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources   -> ring too small
#   - /proc/net/softnet_stat drops                              -> ksoftirq starved
#   - /proc/net/snmp TCP retranst                               -> TCP-level retrans
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
# coming from LACP reordering (out-of-order segments triggering fast
# retransmit), not packet loss — and are NOT fixable by tuning.
###############################################################################
set -uo pipefail

DURATION="${LACP_TEST_SECS:-12}"
STREAMS="${LACP_STREAMS:-8}"
PEER_RX_IP="10.100.100.6"   # tsys6 storage IP (receiver)
PEER_TX_IP="10.100.100.7"   # tsys7 storage IP (sender)
LOG_DIR="/root"
BOND="bond0"

HOST="$(uname -n)"; HOST="${HOST%%.*}"
case "$HOST" in
    *tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
    *tsys7) ROLE="sender";   LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
    *) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
esac

TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

# ---- per-slave NIC counter snapshot -----------------------------------------
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
# plus all error-like counters. Falls back gracefully if a counter doesn't
# exist (different NIC drivers expose different names).
nic_snapshot() {
    for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
        [ -n "$s" ] || continue
        echo "[$s]"
        # ethtool -S may exist; show only drop/error/retrans lines
        if command -v ethtool >/dev/null 2>&1; then
            ethtool -S "$s" 2>/dev/null \
                | grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
                || echo "(ethtool -S: no matching counters or unsupported)"
        else
            echo "(ethtool not installed)"
        fi
        # /sys counters (always available, driver-agnostic)
        for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
                 rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
                 multicast collisions; do
            v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
            [ -n "$v" ] && printf '  sysfs %-22s = %s\n' "$c" "$v"
        done
    done
}

# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
# look identical in hex and decimal). Processed is informational only and we
# don't try to delta it (hex arithmetic is awk-version-dependent).
softnet_snapshot() {
    awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
              NR-1, $1, $2, $3 }' /proc/net/softnet_stat
}

# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
tcp_snapshot() {
    awk '/^Tcp:/{
        if (!seen) {
            seen=1
            # /proc/net/snmp Tcp line 1 = names, line 2 = values
            n=split($0, names, " ")
            # re-find the values line
            getline
            vals=$0
            split(vals, v, " ")
            for (i=1;i<=n;i++) printf "  %-22s = %s\n", names[i], v[i]
        }
    }' /proc/net/snmp
}

# ---- print deltas for selected counters ------------------------------------
# $1 = before file, $2 = after file
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
# integers (typically 0 on a healthy host), so +0 coercion is correct for
# the values we care about. We deliberately DON'T report `processed` deltas
# because hex arithmetic on large numbers is awk-version-dependent.
softnet_delta() {
    awk '
        FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
                  split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
                  next }
        { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
              split($0, q, /[= ]+/);
              dd = (q[5]+0) - drop[id]
              sd = (q[7]+0) - sqz[id]
              if (dd != 0 || sd != 0)
                  printf "  %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
    ' "$1" "$2" | sort
}

# $1 = before file, $2 = after file, $3 = section label prefix
# File format: slave header line "[nic1]", then "  counter_name = value" lines.
# Output only counters whose value changed.
nic_delta() {
    awk '
        FNR==NR { if ($0 ~ /^\[/) slave=$0
                  else if ($0 ~ /=/) {
                      # Everything before " = " is the counter name (trim spaces)
                      pos = index($0, "=")
                      name = substr($0, 1, pos-1)
                      gsub(/^ +| +$/, "", name)
                      val  = substr($0, pos+1); gsub(/^ +| +$/, "", val)
                      before[slave SUBSEP name] = val
                  }
                  next }
        { if ($0 ~ /^\[/) slave=$0
          else if ($0 ~ /=/) {
              pos = index($0, "=")
              name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
              val  = substr($0, pos+1); gsub(/^ +| +$/, "", val)
              b = before[slave SUBSEP name]+0
              d = val+0 - b
              if (d != 0) printf "  %-8s %-30s %12s -> %12s  delta=%d\n", slave, name, b, val, d
          } }
    ' "$1" "$2"
}

{
echo "=== LACP retransmit cause investigation ($ROLE) ==="
echo "Host: $HOST   Role: $ROLE"
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Bond: $BOND"
echo ""
echo "--- bond0 hash + driver ---"
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
    drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
    speed=$(cat /sys/class/net/$s/speed 2>/dev/null)
    ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
    printf "  %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
done
echo ""
} | tee "$LOG"

echo "--- BEFORE snapshots ---"
nic_snapshot      > "$TMP/nic_before"
softnet_snapshot  > "$TMP/softnet_before"
tcp_snapshot      > "$TMP/tcp_before"

if [ "$ROLE" = "receiver" ]; then
    echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
    pkill -x iperf3 2>/dev/null; sleep 0.5
    nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
    SRV_PID=$!
    for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
    wait "$SRV_PID" 2>/dev/null
else
    echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
    sleep 3
    iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
        > "$TMP/iperf.out" 2>&1
fi

echo "--- AFTER snapshots ---"
nic_snapshot      > "$TMP/nic_after"
softnet_snapshot  > "$TMP/softnet_after"
tcp_snapshot      > "$TMP/tcp_after"

{
echo ""
echo "--- iperf3 summary ---"
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
echo ""
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
# Show only the counters that changed
paste "$TMP/tcp_before" "$TMP/tcp_after" \
    | awk '{
        a=$3; b=$(NF)
        if (a+0 != b+0) printf "  %-22s %12s -> %12s  delta=%d\n", $1, a, b, b-a
    }'
echo ""
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
echo ""
echo "--- NIC counter deltas (only non-zero) ---"
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
echo ""
echo "--- raw iperf3 tail (last 12 lines) ---"
tail -12 "$TMP/iperf.out"
echo ""
echo "==================================================================="
echo " HOW TO READ THIS"
echo "==================================================================="
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
echo "     -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
echo "     -> softirq starvation. Fix: increase net.core.netdev_budget,"
echo "        check IRQ affinity, consider RPS."
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
echo "     -> the retransmits are LACP reordering (out-of-order segments"
echo "        triggering fast retransmit), NOT packet loss. NOT fixable."
echo "==================================================================="
} | tee -a "$LOG"

echo ""
echo "Log: $LOG"