feat(perf): add reusable perf-matrix.sh test harness + iperf3 instrumentation

perf-matrix.sh provides on-demand any-to-any performance testing:
- datanet: host-to-host bandwidth matrix over VLAN 1000
- guests: guest-to-guest (k8s/ultix VMs over datanet)
- storage: NFS read/write I/O per host

iperf3 installed on all 7 hosts (systemd service iperf3-server, enabled
but not auto-started) and all production VMs with guest-agent.

Also: tsys3 datanet persistence fix (USB NIC late-enumerate systemd
service), 4 missing reverse DNS zones created, netbird/ultix-mini/
ultix-sidecar/ultix-highside DNS records added (DNS now fully
consistent across all Tailscale nodes). [#413]

💘 Generated with Crush

Assisted-by: Crush:glm-5.2

WORKING.md

@@ -5,7 +5,6 @@ A commit is blocked while any task below remains unchecked.
 
 ## Current Tasks
 
-- [x] Switch rename Phase 4-5: mapping.txt + conman/ser2net/udev + repo files
+- [x] Install iperf3 on all hosts + production VMs (persistent)
-- [x] End-to-end perf testing (#413): host-to-host, storage, guest-to-guest
+- [x] Create reusable any-to-any perf test script
-- [x] Fix tsys3 datanet (physical NIC not enslaved to bridge)
+- [x] Enable iperf3 systemd service on hosts for on-demand testing
-- [x] Kuma monitor fix (DNS aliases for old switch names)

proxmox/perf/scripts/perf-matrix.sh

@@ -0,0 +1,243 @@
+#!/bin/bash
+###############################################################################
+# perf-matrix.sh — Any-to-any performance testing across the PFVCluster
+#
+# Tests three network planes:
+#   A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network
+#   B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x)
+#   C. Storage I/O: dd read/write to NFS mounts
+#
+# Prerequisites:
+#   - iperf3 installed on all hosts (systemd service: iperf3-server)
+#   - iperf3 installed inside guest VMs
+#   - Guest agent enabled on VMs for remote command execution
+#
+# Usage:
+#   bash perf-matrix.sh                  # run all tests
+#   bash perf-matrix.sh datanet          # host-to-host datanet only
+#   bash perf-matrix.sh guests           # guest-to-guest datanet only
+#   bash perf-matrix.sh storage          # NFS I/O only
+#
+# Environment:
+#   REMOTE_SH   path to tests/remote.sh (auto-detected)
+###############################################################################
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}"
+LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs"
+mkdir -p "$LOG_DIR"
+TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
+
+# --- Host datanet IPs (VLAN 1000) ---
+declare -A DATANET_IP
+DATANET_IP[pfv-tsys1]="10.100.100.1"
+DATANET_IP[pfv-tsys3]="10.100.100.3"
+DATANET_IP[pfv-tsys4]="10.100.100.4"
+DATANET_IP[pfv-tsys5]="10.100.100.5"
+DATANET_IP[pfv-tsys6]="10.100.100.6"
+DATANET_IP[pfv-tsys7]="10.100.100.7"
+DATANET_IP[pfv-tsys9]="10.100.100.9"
+
+HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9"
+
+# --- k8s/ultix VM guest-agent targets ---
+# Format: prox_host:vmid:datanet_ip:label
+GUEST_TARGETS="
+pfv-tsys1:102:10.100.100.10:cnode1
+pfv-tsys7:705:10.100.100.11:cnode2
+pfv-tsys6:603:10.100.100.12:cnode3
+pfv-tsys3:313:10.100.100.13:wnode-tsys3
+pfv-tsys5:500:10.100.100.14:wnode-tsys5
+pfv-tsys6:601:10.100.100.15:wnode-tsys6
+pfv-tsys7:701:10.100.100.16:wnode-tsys7
+pfv-tsys9:905:10.100.100.17:wnode-tsys9
+pfv-tsys5:5111:10.100.100.18:ultix-streaming
+pfv-tsys5:5112:10.100.100.19:ultix-offstage
+"
+
+DURATION="${DURATION:-3}"    # seconds per iperf3 test
+STREAMS="${STREAMS:-4}"      # parallel streams
+
+# ============================================================================
+# Helpers
+# ============================================================================
+start_iperf_servers() {
+    echo "--- Starting iperf3 servers on all hosts ---"
+    for h in $HOSTS; do
+        PROX_HOST="$h" bash "$REMOTE_SH" prox \
+            'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \
+            >/dev/null 2>&1 &
+    done
+    wait
+    echo "    All servers started."
+}
+
+stop_iperf_servers() {
+    echo "--- Stopping iperf3 servers on all hosts ---"
+    for h in $HOSTS; do
+        PROX_HOST="$h" bash "$REMOTE_SH" prox \
+            'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \
+            >/dev/null 2>&1 &
+    done
+    wait
+    echo "    All servers stopped."
+}
+
+# ============================================================================
+# A. Host-to-host datanet matrix
+# ============================================================================
+test_datanet() {
+    echo ""
+    echo "==================================================================="
+    echo "  DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix"
+    echo "  ${STREAMS} parallel streams, ${DURATION}s per test"
+    echo "==================================================================="
+    start_iperf_servers
+
+    local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv"
+    echo "host_from,host_to,mbps" > "$outfile"
+
+    for client in $HOSTS; do
+        for server in $HOSTS; do
+            [ "$client" = "$server" ] && continue
+            local sip="${DATANET_IP[$server]}"
+            local result
+            result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \
+                "iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
+                | awk '/SUM.*receiver/{printf "%.0f", $6}')
+            if [ -n "$result" ]; then
+                printf "  %-14s → %-14s : %s Mbps\n" "$client" "$server" "$result"
+                echo "$client,$server,$result" >> "$outfile"
+            else
+                printf "  %-14s → %-14s : FAIL\n" "$client" "$server"
+                echo "$client,$server,FAIL" >> "$outfile"
+            fi
+        done
+    done
+
+    stop_iperf_servers
+    echo ""
+    echo "  Results saved: $outfile"
+}
+
+# ============================================================================
+# B. Guest-to-guest datanet
+# ============================================================================
+test_guests() {
+    echo ""
+    echo "==================================================================="
+    echo "  DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)"
+    echo "  ${STREAMS} parallel streams, ${DURATION}s per test"
+    echo "==================================================================="
+
+    # Start iperf3 server on the first guest (cnode1)
+    local server_entry
+    server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1)
+    local s_host s_vid s_ip s_label
+    s_host=$(echo "$server_entry" | cut -d: -f1)
+    s_vid=$(echo "$server_entry" | cut -d: -f2)
+    s_ip=$(echo "$server_entry" | cut -d: -f3)
+    s_label=$(echo "$server_entry" | cut -d: -f4)
+
+    echo "  Starting iperf3 server on $s_label ($s_ip)..."
+    PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \
+        'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1
+    sleep 1
+
+    local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv"
+    echo "guest_from,guest_to,mbps" > "$outfile"
+
+    while read -r entry; do
+        [ -z "$entry" ] && continue
+        local c_host c_vid c_ip c_label
+        c_host=$(echo "$entry" | cut -d: -f1)
+        c_vid=$(echo "$entry" | cut -d: -f2)
+        c_ip=$(echo "$entry" | cut -d: -f3)
+        c_label=$(echo "$entry" | cut -d: -f4)
+        [ "$c_ip" = "$s_ip" ] && continue
+
+        local result
+        result=$(PROX_HOST="$c_host" VM_ID="$c_vid" bash "$REMOTE_SH" vm-guest \
+            "iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
+            | awk '/SUM.*receiver/{printf "%.0f", $6}')
+        if [ -n "$result" ]; then
+            printf "  %-18s → %-18s : %s Mbps\n" "$c_label" "$s_label" "$result"
+            echo "$c_label,$s_label,$result" >> "$outfile"
+        else
+            printf "  %-18s → %-18s : FAIL\n" "$c_label" "$s_label"
+            echo "$c_label,$s_label,FAIL" >> "$outfile"
+        fi
+    done <<< "$GUEST_TARGETS"
+
+    # Cleanup
+    PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \
+        'pkill iperf3' >/dev/null 2>&1
+
+    echo ""
+    echo "  Results saved: $outfile"
+}
+
+# ============================================================================
+# C. Storage I/O (NFS read/write)
+# ============================================================================
+test_storage() {
+    echo ""
+    echo "==================================================================="
+    echo "  STORAGE I/O — NFS Read/Write (100MB dd)"
+    echo "==================================================================="
+
+    local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv"
+    echo "host,mount,write_mbps,read_mbps" > "$outfile"
+
+    local ddscript="/tmp/perf-dd-$$.sh"
+    cat > "$ddscript" <<'DDSCRIPT'
+#!/bin/bash
+nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc)
+for m in $nfs; do
+    tf="$m/.perf-$$"
+    w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
+    r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
+    rm -f "$tf" 2>/dev/null
+    echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s"
+done
+DDSCRIPT
+
+    for h in $HOSTS; do
+        echo ""
+        echo "  --- $h ---"
+        PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do
+            [ -n "$line" ] && echo "    $line"
+        done
+    done
+    rm -f "$ddscript"
+
+    echo ""
+    echo "  Results saved: $outfile"
+}
+
+# ============================================================================
+# Main
+# ============================================================================
+MODE="${1:-all}"
+
+case "$MODE" in
+    datanet|a)  test_datanet ;;
+    guests|b)   test_guests ;;
+    storage|c)  test_storage ;;
+    all|"")     test_datanet; test_guests; test_storage ;;
+    *)
+        echo "Usage: $0 [datanet|guests|storage|all]"
+        echo ""
+        echo "  datanet  — host-to-host bandwidth matrix over VLAN 1000"
+        echo "  guests   — guest-to-guest (k8s/ultix VMs over VLAN 1000)"
+        echo "  storage  — NFS read/write I/O"
+        echo "  all      — run all three (default)"
+        exit 1
+        ;;
+esac
+
+echo ""
+echo "==================================================================="
+echo "  Perf testing complete. Logs in: $LOG_DIR/"
+echo "==================================================================="