#!/bin/bash ############################################################################### # perf-matrix.sh — Any-to-any performance testing across the PFVCluster # # Tests three network planes: # A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network # B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x) # C. Storage I/O: dd read/write to NFS mounts # # Prerequisites: # - iperf3 installed on all hosts (systemd service: iperf3-server) # - iperf3 installed inside guest VMs # - SSH key + passwordless sudo on all guest VMs (remote.sh; sshd is the # only approved access channel — see AGENTS.md) # # Usage: # bash perf-matrix.sh # run all tests # bash perf-matrix.sh datanet # host-to-host datanet only # bash perf-matrix.sh guests # guest-to-guest datanet only # bash perf-matrix.sh storage # NFS I/O only # # Environment: # REMOTE_SH path to tests/remote.sh (auto-detected) ############################################################################### set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}" LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs" mkdir -p "$LOG_DIR" TIMESTAMP="$(date +%Y%m%d-%H%M%S)" # --- Host datanet IPs (VLAN 1000) --- declare -A DATANET_IP DATANET_IP[pfv-tsys1]="10.100.100.1" DATANET_IP[pfv-tsys3]="10.100.100.3" DATANET_IP[pfv-tsys4]="10.100.100.4" DATANET_IP[pfv-tsys5]="10.100.100.5" DATANET_IP[pfv-tsys6]="10.100.100.6" DATANET_IP[pfv-tsys7]="10.100.100.7" DATANET_IP[pfv-tsys9]="10.100.100.9" HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9" # --- k8s/ultix VM targets (SSH for control; datanet_ip for iperf traffic) --- # Format: prox_host:vmid:datanet_ip:ssh_host:label GUEST_TARGETS=" pfv-tsys1:102:10.100.100.10:pfv-k8s-cnode1:cnode1 pfv-tsys7:705:10.100.100.11:pfv-k8s-cnode2:cnode2 pfv-tsys6:603:10.100.100.12:pfv-k8s-cnode3:cnode3 pfv-tsys3:313:10.100.100.13:pfv-k8s-wnode-tsys3:wnode-tsys3 pfv-tsys5:500:10.100.100.14:pfv-k8s-wnode-tsys5:wnode-tsys5 pfv-tsys6:601:10.100.100.15:pfv-k8s-wnode-tsys6:wnode-tsys6 pfv-tsys7:701:10.100.100.16:pfv-k8s-wnode-tsys7:wnode-tsys7 pfv-tsys9:905:10.100.100.17:pfv-k8s-wnode-tsys9:wnode-tsys9 pfv-tsys5:5111:10.100.100.18:ultix-streaming:ultix-streaming pfv-tsys5:5112:10.100.100.19:ultix-offstage:ultix-offstage " DURATION="${DURATION:-3}" # seconds per iperf3 test STREAMS="${STREAMS:-4}" # parallel streams # ============================================================================ # Helpers # ============================================================================ start_iperf_servers() { echo "--- Starting iperf3 servers on all hosts ---" for h in $HOSTS; do PROX_HOST="$h" bash "$REMOTE_SH" prox \ 'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \ >/dev/null 2>&1 & done wait echo " All servers started." } stop_iperf_servers() { echo "--- Stopping iperf3 servers on all hosts ---" for h in $HOSTS; do PROX_HOST="$h" bash "$REMOTE_SH" prox \ 'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \ >/dev/null 2>&1 & done wait echo " All servers stopped." } # ============================================================================ # A. Host-to-host datanet matrix # ============================================================================ test_datanet() { echo "" echo "===================================================================" echo " DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix" echo " ${STREAMS} parallel streams, ${DURATION}s per test" echo "===================================================================" start_iperf_servers local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv" echo "host_from,host_to,mbps" > "$outfile" for client in $HOSTS; do for server in $HOSTS; do [ "$client" = "$server" ] && continue local sip="${DATANET_IP[$server]}" local result result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \ "iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" > "$outfile" else printf " %-14s → %-14s : FAIL\n" "$client" "$server" echo "$client,$server,FAIL" >> "$outfile" fi done done stop_iperf_servers echo "" echo " Results saved: $outfile" } # ============================================================================ # B. Guest-to-guest datanet # ============================================================================ test_guests() { echo "" echo "===================================================================" echo " DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)" echo " ${STREAMS} parallel streams, ${DURATION}s per test" echo "===================================================================" # Start iperf3 server on the first guest (cnode1) local server_entry server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1) local s_ip s_ssh s_label s_ip=$(echo "$server_entry" | cut -d: -f3) s_ssh=$(echo "$server_entry" | cut -d: -f4) s_label=$(echo "$server_entry" | cut -d: -f5) echo " Starting iperf3 server on $s_label ($s_ip)..." VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \ 'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1 sleep 1 local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv" echo "guest_from,guest_to,mbps" > "$outfile" while read -r entry; do [ -z "$entry" ] && continue local c_ip c_ssh c_label c_ip=$(echo "$entry" | cut -d: -f3) c_ssh=$(echo "$entry" | cut -d: -f4) c_label=$(echo "$entry" | cut -d: -f5) [ "$c_ip" = "$s_ip" ] && continue local result result=$(VM_IP="$c_ssh" bash "$REMOTE_SH" vmroot \ "iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" > "$outfile" else printf " %-18s → %-18s : FAIL\n" "$c_label" "$s_label" echo "$c_label,$s_label,FAIL" >> "$outfile" fi done <<< "$GUEST_TARGETS" # Cleanup VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \ 'pkill iperf3' >/dev/null 2>&1 echo "" echo " Results saved: $outfile" } # ============================================================================ # C. Storage I/O (NFS read/write) # ============================================================================ test_storage() { echo "" echo "===================================================================" echo " STORAGE I/O — NFS Read/Write (100MB dd)" echo "===================================================================" local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv" echo "host,mount,write_mbps,read_mbps" > "$outfile" local ddscript="/tmp/perf-dd-$$.sh" cat > "$ddscript" <<'DDSCRIPT' #!/bin/bash nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc) for m in $nfs; do tf="$m/.perf-$$" w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}') r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}') rm -f "$tf" 2>/dev/null echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s" done DDSCRIPT for h in $HOSTS; do echo "" echo " --- $h ---" PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do [ -n "$line" ] && echo " $line" done done rm -f "$ddscript" echo "" echo " Results saved: $outfile" } # ============================================================================ # Main # ============================================================================ MODE="${1:-all}" case "$MODE" in datanet|a) test_datanet ;; guests|b) test_guests ;; storage|c) test_storage ;; all|"") test_datanet; test_guests; test_storage ;; *) echo "Usage: $0 [datanet|guests|storage|all]" echo "" echo " datanet — host-to-host bandwidth matrix over VLAN 1000" echo " guests — guest-to-guest (k8s/ultix VMs over VLAN 1000)" echo " storage — NFS read/write I/O" echo " all — run all three (default)" exit 1 ;; esac echo "" echo "===================================================================" echo " Perf testing complete. Logs in: $LOG_DIR/" echo "==================================================================="