PFVCluster/scripts/lacp-rx-distribution.sh

#!/usr/bin/env bash
###############################################################################
# lacp-rx-distribution.sh  (HOST-NATIVE version)
#
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
#   - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
#     bond0 slave RX counters before+after, writes verdict + log.
#   - On tsys7 (sender):   snapshots local bond0 slave TX counters before+
#     after, runs iperf3 -c <peer> -P 8 -t 12.
#
# Each side writes only its own counters. No inter-host SSH required.
# iperf3 -s -1 (one-shot) handles client/server coordination.
#
# Output: /root/lacp-rx-{receiver,sender}.log
###############################################################################
set -uo pipefail

DURATION="${LACP_TEST_SECS:-12}"
STREAMS="${LACP_STREAMS:-8}"
PEER_RX_IP="10.100.100.6"   # tsys6 storage IP (receiver)
PEER_TX_IP="10.100.100.7"   # tsys7 storage IP (sender)
LOG_DIR="/root"
BOND="bond0"

# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
snapshot() {
    awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
        [ -n "$s" ] || continue
        rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
        tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
        printf '%s %s %s\n' "$s" "$rx" "$tx"
    done | sort
}

# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
analyze() {
    local label="$1" col="$2" before="$3" after="$4"
    join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
        {
            slave=$1
            b = (col==2 ? $2 : $3)+0
            a = (col==2 ? $4 : $5)+0
            d = a - b; if (d < 0) d = 0
            delta[slave]=d; bef[slave]=b; aft[slave]=a
            order[++n]=slave; total+=d
        }
        END {
            printf "\n--- %s (bytes) ---\n", lbl
            printf "  %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
            for (i=1;i<=n;i++){
                s=order[i]
                pct = (total>0 ? 100*delta[s]/total : 0)
                printf "  %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
            }
            printf "  %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
        }'
}

# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
dominant() {
    join "$2" "$3" | awk -v col="$1" '
        { b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
        END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
              printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
}

HOST="$(uname -n)"
HOST="${HOST%%.*}"
case "$HOST" in
    *tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
    *tsys7) ROLE="sender";   LOG="$LOG_DIR/lacp-rx-sender.log" ;;
    *) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
esac

TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

{
echo "=== LACP per-slave distribution ($ROLE) ==="
echo "Host: $HOST   Role: $ROLE"
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Bond: $BOND"
echo ""
echo "--- $BOND state ---"
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
    "/proc/net/bonding/$BOND" 2>&1
echo ""
} | tee "$LOG"

snapshot > "$TMP/before"

if [ "$ROLE" = "receiver" ]; then
    echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
    pkill -x iperf3 2>/dev/null; sleep 0.5
    nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
    SRV_PID=$!
    # Wait for the server to be ready (brief), then wait for it to exit
    # (one-shot server exits after serving one client).
    for _ in $(seq 1 60); do
        kill -0 "$SRV_PID" 2>/dev/null || break
        sleep 1
    done
    wait "$SRV_PID" 2>/dev/null
    echo "[receiver] iperf3 server finished."
else
    # sender: wait briefly for receiver to be listening, then run client.
    echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
    sleep 3
    iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
        > "$TMP/iperf.out" 2>&1
    echo "[sender] iperf3 client finished (exit=$?)."
fi

snapshot > "$TMP/after"

{
echo ""
echo "--- iperf3 output (raw tail) ---"
tail -25 "$TMP/iperf.out"
echo ""
if [ "$ROLE" = "receiver" ]; then
    analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
    dom="$(dominant 2 "$TMP/before" "$TMP/after")"
    pct=${dom%% *}; top=${dom##* }
    echo ""
    echo "==================================================================="
    if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
        echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
        echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
        echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
    else
        echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
        echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
    fi
    echo "==================================================================="
else
    analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
    dom="$(dominant 3 "$TMP/before" "$TMP/after")"
    pct=${dom%% *}; top=${dom##* }
    echo ""
    echo "==================================================================="
    if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
        echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
        echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
    else
        echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
    fi
    echo "==================================================================="
fi
} | tee -a "$LOG"

echo ""
echo "Log written: $LOG"