#!/bin/bash
###############################################################################
# perf-matrix.sh — Any-to-any performance testing across the PFVCluster
#
# Tests three network planes:
#   A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network
#   B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x)
#   C. Storage I/O: dd read/write to NFS mounts
#
# Prerequisites:
#   - iperf3 installed on all hosts (systemd service: iperf3-server)
#   - iperf3 installed inside guest VMs
#   - SSH key + passwordless sudo on all guest VMs (remote.sh; sshd is the
#     only approved access channel — see AGENTS.md)
#
# Usage:
#   bash perf-matrix.sh                  # run all tests
#   bash perf-matrix.sh datanet          # host-to-host datanet only
#   bash perf-matrix.sh guests           # guest-to-guest datanet only
#   bash perf-matrix.sh storage          # NFS I/O only
#
# Environment:
#   REMOTE_SH   path to tests/remote.sh (auto-detected)
###############################################################################
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}"
LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs"
mkdir -p "$LOG_DIR"
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"

# --- Host datanet IPs (VLAN 1000) ---
declare -A DATANET_IP
DATANET_IP[pfv-tsys1]="10.100.100.1"
DATANET_IP[pfv-tsys3]="10.100.100.3"
DATANET_IP[pfv-tsys4]="10.100.100.4"
DATANET_IP[pfv-tsys5]="10.100.100.5"
DATANET_IP[pfv-tsys6]="10.100.100.6"
DATANET_IP[pfv-tsys7]="10.100.100.7"
DATANET_IP[pfv-tsys9]="10.100.100.9"

HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9"

# --- k8s/ultix VM targets (SSH for control; datanet_ip for iperf traffic) ---
# Format: prox_host:vmid:datanet_ip:ssh_host:label
GUEST_TARGETS="
pfv-tsys1:102:10.100.100.10:pfv-k8s-cnode1:cnode1
pfv-tsys7:705:10.100.100.11:pfv-k8s-cnode2:cnode2
pfv-tsys6:603:10.100.100.12:pfv-k8s-cnode3:cnode3
pfv-tsys3:313:10.100.100.13:pfv-k8s-wnode-tsys3:wnode-tsys3
pfv-tsys5:500:10.100.100.14:pfv-k8s-wnode-tsys5:wnode-tsys5
pfv-tsys6:601:10.100.100.15:pfv-k8s-wnode-tsys6:wnode-tsys6
pfv-tsys7:701:10.100.100.16:pfv-k8s-wnode-tsys7:wnode-tsys7
pfv-tsys9:905:10.100.100.17:pfv-k8s-wnode-tsys9:wnode-tsys9
pfv-tsys5:5111:10.100.100.18:ultix-streaming:ultix-streaming
pfv-tsys5:5112:10.100.100.19:ultix-offstage:ultix-offstage
"

DURATION="${DURATION:-3}"    # seconds per iperf3 test
STREAMS="${STREAMS:-4}"      # parallel streams

# ============================================================================
# Helpers
# ============================================================================
start_iperf_servers() {
    echo "--- Starting iperf3 servers on all hosts ---"
    for h in $HOSTS; do
        PROX_HOST="$h" bash "$REMOTE_SH" prox \
            'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \
            >/dev/null 2>&1 &
    done
    wait
    echo "    All servers started."
}

stop_iperf_servers() {
    echo "--- Stopping iperf3 servers on all hosts ---"
    for h in $HOSTS; do
        PROX_HOST="$h" bash "$REMOTE_SH" prox \
            'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \
            >/dev/null 2>&1 &
    done
    wait
    echo "    All servers stopped."
}

# ============================================================================
# A. Host-to-host datanet matrix
# ============================================================================
test_datanet() {
    echo ""
    echo "==================================================================="
    echo "  DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix"
    echo "  ${STREAMS} parallel streams, ${DURATION}s per test"
    echo "==================================================================="
    start_iperf_servers

    local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv"
    echo "host_from,host_to,mbps" > "$outfile"

    for client in $HOSTS; do
        for server in $HOSTS; do
            [ "$client" = "$server" ] && continue
            local sip="${DATANET_IP[$server]}"
            local result
            result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \
                "iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
                | awk '/SUM.*receiver/{printf "%.0f", $6}')
            if [ -n "$result" ]; then
                printf "  %-14s → %-14s : %s Mbps\n" "$client" "$server" "$result"
                echo "$client,$server,$result" >> "$outfile"
            else
                printf "  %-14s → %-14s : FAIL\n" "$client" "$server"
                echo "$client,$server,FAIL" >> "$outfile"
            fi
        done
    done

    stop_iperf_servers
    echo ""
    echo "  Results saved: $outfile"
}

# ============================================================================
# B. Guest-to-guest datanet
# ============================================================================
test_guests() {
    echo ""
    echo "==================================================================="
    echo "  DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)"
    echo "  ${STREAMS} parallel streams, ${DURATION}s per test"
    echo "==================================================================="

    # Start iperf3 server on the first guest (cnode1)
    local server_entry
    server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1)
    local s_ip s_ssh s_label
    s_ip=$(echo "$server_entry" | cut -d: -f3)
    s_ssh=$(echo "$server_entry" | cut -d: -f4)
    s_label=$(echo "$server_entry" | cut -d: -f5)

    echo "  Starting iperf3 server on $s_label ($s_ip)..."
    VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
        'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1
    sleep 1

    local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv"
    echo "guest_from,guest_to,mbps" > "$outfile"

    while read -r entry; do
        [ -z "$entry" ] && continue
        local c_ip c_ssh c_label
        c_ip=$(echo "$entry" | cut -d: -f3)
        c_ssh=$(echo "$entry" | cut -d: -f4)
        c_label=$(echo "$entry" | cut -d: -f5)
        [ "$c_ip" = "$s_ip" ] && continue

        local result
        result=$(VM_IP="$c_ssh" bash "$REMOTE_SH" vmroot \
            "iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
            | awk '/SUM.*receiver/{printf "%.0f", $6}')
        if [ -n "$result" ]; then
            printf "  %-18s → %-18s : %s Mbps\n" "$c_label" "$s_label" "$result"
            echo "$c_label,$s_label,$result" >> "$outfile"
        else
            printf "  %-18s → %-18s : FAIL\n" "$c_label" "$s_label"
            echo "$c_label,$s_label,FAIL" >> "$outfile"
        fi
    done <<< "$GUEST_TARGETS"

    # Cleanup
    VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
        'pkill iperf3' >/dev/null 2>&1

    echo ""
    echo "  Results saved: $outfile"
}

# ============================================================================
# C. Storage I/O (NFS read/write)
# ============================================================================
test_storage() {
    echo ""
    echo "==================================================================="
    echo "  STORAGE I/O — NFS Read/Write (100MB dd)"
    echo "==================================================================="

    local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv"
    echo "host,mount,write_mbps,read_mbps" > "$outfile"

    local ddscript="/tmp/perf-dd-$$.sh"
    cat > "$ddscript" <<'DDSCRIPT'
#!/bin/bash
nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc)
for m in $nfs; do
    tf="$m/.perf-$$"
    w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
    r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
    rm -f "$tf" 2>/dev/null
    echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s"
done
DDSCRIPT

    for h in $HOSTS; do
        echo ""
        echo "  --- $h ---"
        PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do
            [ -n "$line" ] && echo "    $line"
        done
    done
    rm -f "$ddscript"

    echo ""
    echo "  Results saved: $outfile"
}

# ============================================================================
# Main
# ============================================================================
MODE="${1:-all}"

case "$MODE" in
    datanet|a)  test_datanet ;;
    guests|b)   test_guests ;;
    storage|c)  test_storage ;;
    all|"")     test_datanet; test_guests; test_storage ;;
    *)
        echo "Usage: $0 [datanet|guests|storage|all]"
        echo ""
        echo "  datanet  — host-to-host bandwidth matrix over VLAN 1000"
        echo "  guests   — guest-to-guest (k8s/ultix VMs over VLAN 1000)"
        echo "  storage  — NFS read/write I/O"
        echo "  all      — run all three (default)"
        exit 1
        ;;
esac

echo ""
echo "==================================================================="
echo "  Perf testing complete. Logs in: $LOG_DIR/"
echo "==================================================================="
