diff --git a/WORKING.md b/WORKING.md index 50f5278..f7af686 100644 --- a/WORKING.md +++ b/WORKING.md @@ -5,7 +5,6 @@ A commit is blocked while any task below remains unchecked. ## Current Tasks -- [x] Switch rename Phase 4-5: mapping.txt + conman/ser2net/udev + repo files -- [x] End-to-end perf testing (#413): host-to-host, storage, guest-to-guest -- [x] Fix tsys3 datanet (physical NIC not enslaved to bridge) -- [x] Kuma monitor fix (DNS aliases for old switch names) +- [x] Install iperf3 on all hosts + production VMs (persistent) +- [x] Create reusable any-to-any perf test script +- [x] Enable iperf3 systemd service on hosts for on-demand testing diff --git a/proxmox/perf/scripts/perf-matrix.sh b/proxmox/perf/scripts/perf-matrix.sh new file mode 100755 index 0000000..43a24d6 --- /dev/null +++ b/proxmox/perf/scripts/perf-matrix.sh @@ -0,0 +1,243 @@ +#!/bin/bash +############################################################################### +# perf-matrix.sh — Any-to-any performance testing across the PFVCluster +# +# Tests three network planes: +# A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network +# B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x) +# C. Storage I/O: dd read/write to NFS mounts +# +# Prerequisites: +# - iperf3 installed on all hosts (systemd service: iperf3-server) +# - iperf3 installed inside guest VMs +# - Guest agent enabled on VMs for remote command execution +# +# Usage: +# bash perf-matrix.sh # run all tests +# bash perf-matrix.sh datanet # host-to-host datanet only +# bash perf-matrix.sh guests # guest-to-guest datanet only +# bash perf-matrix.sh storage # NFS I/O only +# +# Environment: +# REMOTE_SH path to tests/remote.sh (auto-detected) +############################################################################### +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}" +LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs" +mkdir -p "$LOG_DIR" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" + +# --- Host datanet IPs (VLAN 1000) --- +declare -A DATANET_IP +DATANET_IP[pfv-tsys1]="10.100.100.1" +DATANET_IP[pfv-tsys3]="10.100.100.3" +DATANET_IP[pfv-tsys4]="10.100.100.4" +DATANET_IP[pfv-tsys5]="10.100.100.5" +DATANET_IP[pfv-tsys6]="10.100.100.6" +DATANET_IP[pfv-tsys7]="10.100.100.7" +DATANET_IP[pfv-tsys9]="10.100.100.9" + +HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9" + +# --- k8s/ultix VM guest-agent targets --- +# Format: prox_host:vmid:datanet_ip:label +GUEST_TARGETS=" +pfv-tsys1:102:10.100.100.10:cnode1 +pfv-tsys7:705:10.100.100.11:cnode2 +pfv-tsys6:603:10.100.100.12:cnode3 +pfv-tsys3:313:10.100.100.13:wnode-tsys3 +pfv-tsys5:500:10.100.100.14:wnode-tsys5 +pfv-tsys6:601:10.100.100.15:wnode-tsys6 +pfv-tsys7:701:10.100.100.16:wnode-tsys7 +pfv-tsys9:905:10.100.100.17:wnode-tsys9 +pfv-tsys5:5111:10.100.100.18:ultix-streaming +pfv-tsys5:5112:10.100.100.19:ultix-offstage +" + +DURATION="${DURATION:-3}" # seconds per iperf3 test +STREAMS="${STREAMS:-4}" # parallel streams + +# ============================================================================ +# Helpers +# ============================================================================ +start_iperf_servers() { + echo "--- Starting iperf3 servers on all hosts ---" + for h in $HOSTS; do + PROX_HOST="$h" bash "$REMOTE_SH" prox \ + 'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \ + >/dev/null 2>&1 & + done + wait + echo " All servers started." +} + +stop_iperf_servers() { + echo "--- Stopping iperf3 servers on all hosts ---" + for h in $HOSTS; do + PROX_HOST="$h" bash "$REMOTE_SH" prox \ + 'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \ + >/dev/null 2>&1 & + done + wait + echo " All servers stopped." +} + +# ============================================================================ +# A. Host-to-host datanet matrix +# ============================================================================ +test_datanet() { + echo "" + echo "===================================================================" + echo " DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix" + echo " ${STREAMS} parallel streams, ${DURATION}s per test" + echo "===================================================================" + start_iperf_servers + + local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv" + echo "host_from,host_to,mbps" > "$outfile" + + for client in $HOSTS; do + for server in $HOSTS; do + [ "$client" = "$server" ] && continue + local sip="${DATANET_IP[$server]}" + local result + result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \ + "iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" > "$outfile" + else + printf " %-14s → %-14s : FAIL\n" "$client" "$server" + echo "$client,$server,FAIL" >> "$outfile" + fi + done + done + + stop_iperf_servers + echo "" + echo " Results saved: $outfile" +} + +# ============================================================================ +# B. Guest-to-guest datanet +# ============================================================================ +test_guests() { + echo "" + echo "===================================================================" + echo " DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)" + echo " ${STREAMS} parallel streams, ${DURATION}s per test" + echo "===================================================================" + + # Start iperf3 server on the first guest (cnode1) + local server_entry + server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1) + local s_host s_vid s_ip s_label + s_host=$(echo "$server_entry" | cut -d: -f1) + s_vid=$(echo "$server_entry" | cut -d: -f2) + s_ip=$(echo "$server_entry" | cut -d: -f3) + s_label=$(echo "$server_entry" | cut -d: -f4) + + echo " Starting iperf3 server on $s_label ($s_ip)..." + PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \ + 'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1 + sleep 1 + + local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv" + echo "guest_from,guest_to,mbps" > "$outfile" + + while read -r entry; do + [ -z "$entry" ] && continue + local c_host c_vid c_ip c_label + c_host=$(echo "$entry" | cut -d: -f1) + c_vid=$(echo "$entry" | cut -d: -f2) + c_ip=$(echo "$entry" | cut -d: -f3) + c_label=$(echo "$entry" | cut -d: -f4) + [ "$c_ip" = "$s_ip" ] && continue + + local result + result=$(PROX_HOST="$c_host" VM_ID="$c_vid" bash "$REMOTE_SH" vm-guest \ + "iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" > "$outfile" + else + printf " %-18s → %-18s : FAIL\n" "$c_label" "$s_label" + echo "$c_label,$s_label,FAIL" >> "$outfile" + fi + done <<< "$GUEST_TARGETS" + + # Cleanup + PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \ + 'pkill iperf3' >/dev/null 2>&1 + + echo "" + echo " Results saved: $outfile" +} + +# ============================================================================ +# C. Storage I/O (NFS read/write) +# ============================================================================ +test_storage() { + echo "" + echo "===================================================================" + echo " STORAGE I/O — NFS Read/Write (100MB dd)" + echo "===================================================================" + + local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv" + echo "host,mount,write_mbps,read_mbps" > "$outfile" + + local ddscript="/tmp/perf-dd-$$.sh" + cat > "$ddscript" <<'DDSCRIPT' +#!/bin/bash +nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc) +for m in $nfs; do + tf="$m/.perf-$$" + w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}') + r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}') + rm -f "$tf" 2>/dev/null + echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s" +done +DDSCRIPT + + for h in $HOSTS; do + echo "" + echo " --- $h ---" + PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do + [ -n "$line" ] && echo " $line" + done + done + rm -f "$ddscript" + + echo "" + echo " Results saved: $outfile" +} + +# ============================================================================ +# Main +# ============================================================================ +MODE="${1:-all}" + +case "$MODE" in + datanet|a) test_datanet ;; + guests|b) test_guests ;; + storage|c) test_storage ;; + all|"") test_datanet; test_guests; test_storage ;; + *) + echo "Usage: $0 [datanet|guests|storage|all]" + echo "" + echo " datanet — host-to-host bandwidth matrix over VLAN 1000" + echo " guests — guest-to-guest (k8s/ultix VMs over VLAN 1000)" + echo " storage — NFS read/write I/O" + echo " all — run all three (default)" + exit 1 + ;; +esac + +echo "" +echo "===================================================================" +echo " Perf testing complete. Logs in: $LOG_DIR/" +echo "==================================================================="