#!/bin/bash
###############################################################################
# iperf-storage-tests.sh
#
# Installs iperf3 on all online hosts, then runs a matrix of storage-network
# throughput tests. Saves all output to returned-logs/iperf/.
#
# Test matrix (all over VLAN1000 storage network, 10.100.100.0/24):
#   1. tsys7 → tsys4 (USB cdc_ncm NIC) — the smoking gun
#   2. tsys7 → tsys5 (bond0, 1 active slave) — PCI NIC comparison
#   3. tsys7 → tsys6 (bond0, 2 active slaves) — working LACP baseline
#   4. Reverse: tsys4 → tsys7 (USB NIC TX direction)
#   5. Reverse: tsys5 → tsys7
#
# Each test: TCP 8-stream 30s forward + reverse + UDP saturation.
###############################################################################
set -uo pipefail

SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
mkdir -p "$LOG_DIR"

ALL_HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)

# Storage network IPs
declare -A SIP
SIP[pfv-tsys1]="10.100.100.1"
SIP[pfv-tsys3]="10.100.100.3"
SIP[pfv-tsys4]="10.100.100.4"
SIP[pfv-tsys5]="10.100.100.5"
SIP[pfv-tsys6]="10.100.100.6"
SIP[pfv-tsys7]="10.100.100.7"

echo "==================================================================="
echo " STEP 1: Install iperf3 on all online hosts"
echo "==================================================================="

for host in "${ALL_HOSTS[@]}"; do
    echo -n "  [$host] "
    if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
        echo "UNREACHABLE — skipping"
        continue
    fi
    # Check if iperf3 already installed
    if ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
        echo "iperf3 already installed"
    else
        printf "installing... "
        ssh "${SSH_OPTS[@]}" "root@$host" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
    fi
done

echo ""
echo "==================================================================="
echo " STEP 2: Kill any existing iperf3 processes everywhere"
echo "==================================================================="
for host in "${ALL_HOSTS[@]}"; do
    ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
done
echo "  Done."

# Helper: run an iperf3 test and save output
run_iperf() {
    local client="$1" server="$2" direction="$3" label="$4" logfile="$5"
    local client_ip="${SIP[$client]}" server_ip="${SIP[$server]}"

    echo -n "  [$client → $server] $label ... "

    # Start server in one-shot mode (-1 means serve one client then exit)
    ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
    sleep 1

    # Run client
    {
        echo "=== iperf3: $label ==="
        echo "Client: $client ($client_ip)"
        echo "Server: $server ($server_ip)"
        echo "Direction: $direction"
        echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
        echo ""
        if [ "$direction" = "forward" ]; then
            ssh "${SSH_OPTS[@]}" "root@$client" \
                "iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2" 2>&1
        elif [ "$direction" = "reverse" ]; then
            ssh "${SSH_OPTS[@]}" "root@$client" \
                "iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2 -R" 2>&1
        elif [ "$direction" = "udp" ]; then
            ssh "${SSH_OPTS[@]}" "root@$client" \
                "iperf3 -c ${server_ip} -B ${client_ip} -u -b 2G -t 10 -l 8972" 2>&1
        elif [ "$direction" = "single" ]; then
            ssh "${SSH_OPTS[@]}" "root@$client" \
                "iperf3 -c ${server_ip} -B ${client_ip} -t 20 -O 2" 2>&1
        fi
        echo ""
        echo "=== END ==="
    } > "$logfile" 2>&1

    # Extract summary line
    if grep -q "sender" "$logfile"; then
        bitrate=$(grep "sender" "$logfile" | tail -1 | awk '{print $7, $8}')
        echo "done: ${bitrate}"
    else
        echo "done (check log for details)"
    fi
}

echo ""
echo "==================================================================="
echo " STEP 3: Run iperf3 test matrix"
echo "==================================================================="
echo ""
echo "All tests over VLAN1000 storage network (10.100.100.0/24)."
echo "TCP tests: 8 parallel streams, 30s, 128k blocks."
echo ""

# --- Test 1: tsys7 → tsys4 (USB cdc_ncm target) ---
echo "--- TEST 1: tsys7 → tsys4 (USB cdc_ncm NIC) ---"
run_iperf pfv-tsys7 pfv-tsys4 forward "TCP 8-stream forward (tsys7→tsys4 USB)" \
    "$LOG_DIR/01-tsys7-to-tsys4-tcp-forward.log"
run_iperf pfv-tsys7 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys7)" \
    "$LOG_DIR/02-tsys7-to-tsys4-tcp-reverse.log"
run_iperf pfv-tsys7 pfv-tsys4 single "TCP single-stream forward (tsys7→tsys4 USB)" \
    "$LOG_DIR/03-tsys7-to-tsys4-tcp-single.log"
run_iperf pfv-tsys7 pfv-tsys4 udp "UDP saturation (tsys7→tsys4 USB)" \
    "$LOG_DIR/04-tsys7-to-tsys4-udp.log"

echo ""

# --- Test 2: tsys7 → tsys5 (bond0, PCI NIC, 1 active slave) ---
echo "--- TEST 2: tsys7 → tsys5 (PCI NIC, broken bond - 1 slave) ---"
run_iperf pfv-tsys7 pfv-tsys5 forward "TCP 8-stream forward (tsys7→tsys5 PCI)" \
    "$LOG_DIR/05-tsys7-to-tsys5-tcp-forward.log"
run_iperf pfv-tsys7 pfv-tsys5 reverse "TCP 8-stream reverse (tsys5 PCI→tsys7)" \
    "$LOG_DIR/06-tsys7-to-tsys5-tcp-reverse.log"
run_iperf pfv-tsys7 pfv-tsys5 single "TCP single-stream forward (tsys7→tsys5 PCI)" \
    "$LOG_DIR/07-tsys7-to-tsys5-tcp-single.log"
run_iperf pfv-tsys7 pfv-tsys5 udp "UDP saturation (tsys7→tsys5 PCI)" \
    "$LOG_DIR/08-tsys7-to-tsys5-udp.log"

echo ""

# --- Test 3: tsys7 → tsys6 (working 2-slave LACP baseline, layer3+4) ---
echo "--- TEST 3: tsys7 → tsys6 (working 2×1G LACP baseline) ---"
run_iperf pfv-tsys7 pfv-tsys6 forward "TCP 8-stream forward (tsys7→tsys6 LACP)" \
    "$LOG_DIR/09-tsys7-to-tsys6-tcp-forward.log"
run_iperf pfv-tsys7 pfv-tsys6 reverse "TCP 8-stream reverse (tsys6 LACP→tsys7)" \
    "$LOG_DIR/10-tsys7-to-tsys6-tcp-reverse.log"

echo ""

# --- Test 4: tsys6 → tsys4 (pre-tuning baseline) ---
echo "--- TEST 4: tsys6 → tsys4 (baseline before tsys6 tuning) ---"
run_iperf pfv-tsys6 pfv-tsys4 forward "TCP 8-stream forward (tsys6→tsys4 USB)" \
    "$LOG_DIR/11-tsys6-to-tsys4-tcp-forward.log"
run_iperf pfv-tsys6 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys6)" \
    "$LOG_DIR/12-tsys6-to-tsys4-tcp-reverse.log"

echo ""

# --- Cleanup: kill iperf3 everywhere ---
echo "--- Cleanup ---"
for host in "${ALL_HOSTS[@]}"; do
    ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
done

echo ""
echo "==================================================================="
echo " RESULTS SUMMARY"
echo "==================================================================="
echo ""
printf "%-45s %s\n" "TEST" "THROUGHPUT"
printf "%-45s %s\n" "----" "----------"
for f in "$LOG_DIR"/*.log; do
    [ -r "$f" ] || continue
    label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
    bitrate=$(grep -E "sender$" "$f" | tail -1 | awk '{print $7, $8}')
    [ -z "$bitrate" ] && bitrate=$(grep -E "Mbits/sec|Gbits/sec" "$f" | tail -1 | grep -oE '[0-9.]+ [MG]bits/sec' | head -1)
    [ -z "$bitrate" ] && bitrate="(see log)"
    printf "%-45s %s\n" "$label" "$bitrate"
done

echo ""
echo "Full logs saved to: $LOG_DIR/"
echo "==================================================================="