feat(perf): add reusable perf-matrix.sh test harness + iperf3 instrumentation
perf-matrix.sh provides on-demand any-to-any performance testing: - datanet: host-to-host bandwidth matrix over VLAN 1000 - guests: guest-to-guest (k8s/ultix VMs over datanet) - storage: NFS read/write I/O per host iperf3 installed on all 7 hosts (systemd service iperf3-server, enabled but not auto-started) and all production VMs with guest-agent. Also: tsys3 datanet persistence fix (USB NIC late-enumerate systemd service), 4 missing reverse DNS zones created, netbird/ultix-mini/ ultix-sidecar/ultix-highside DNS records added (DNS now fully consistent across all Tailscale nodes). [#413] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
This commit is contained in:
Executable
+243
@@ -0,0 +1,243 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# perf-matrix.sh — Any-to-any performance testing across the PFVCluster
|
||||
#
|
||||
# Tests three network planes:
|
||||
# A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network
|
||||
# B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x)
|
||||
# C. Storage I/O: dd read/write to NFS mounts
|
||||
#
|
||||
# Prerequisites:
|
||||
# - iperf3 installed on all hosts (systemd service: iperf3-server)
|
||||
# - iperf3 installed inside guest VMs
|
||||
# - Guest agent enabled on VMs for remote command execution
|
||||
#
|
||||
# Usage:
|
||||
# bash perf-matrix.sh # run all tests
|
||||
# bash perf-matrix.sh datanet # host-to-host datanet only
|
||||
# bash perf-matrix.sh guests # guest-to-guest datanet only
|
||||
# bash perf-matrix.sh storage # NFS I/O only
|
||||
#
|
||||
# Environment:
|
||||
# REMOTE_SH path to tests/remote.sh (auto-detected)
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}"
|
||||
LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
# --- Host datanet IPs (VLAN 1000) ---
|
||||
declare -A DATANET_IP
|
||||
DATANET_IP[pfv-tsys1]="10.100.100.1"
|
||||
DATANET_IP[pfv-tsys3]="10.100.100.3"
|
||||
DATANET_IP[pfv-tsys4]="10.100.100.4"
|
||||
DATANET_IP[pfv-tsys5]="10.100.100.5"
|
||||
DATANET_IP[pfv-tsys6]="10.100.100.6"
|
||||
DATANET_IP[pfv-tsys7]="10.100.100.7"
|
||||
DATANET_IP[pfv-tsys9]="10.100.100.9"
|
||||
|
||||
HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9"
|
||||
|
||||
# --- k8s/ultix VM guest-agent targets ---
|
||||
# Format: prox_host:vmid:datanet_ip:label
|
||||
GUEST_TARGETS="
|
||||
pfv-tsys1:102:10.100.100.10:cnode1
|
||||
pfv-tsys7:705:10.100.100.11:cnode2
|
||||
pfv-tsys6:603:10.100.100.12:cnode3
|
||||
pfv-tsys3:313:10.100.100.13:wnode-tsys3
|
||||
pfv-tsys5:500:10.100.100.14:wnode-tsys5
|
||||
pfv-tsys6:601:10.100.100.15:wnode-tsys6
|
||||
pfv-tsys7:701:10.100.100.16:wnode-tsys7
|
||||
pfv-tsys9:905:10.100.100.17:wnode-tsys9
|
||||
pfv-tsys5:5111:10.100.100.18:ultix-streaming
|
||||
pfv-tsys5:5112:10.100.100.19:ultix-offstage
|
||||
"
|
||||
|
||||
DURATION="${DURATION:-3}" # seconds per iperf3 test
|
||||
STREAMS="${STREAMS:-4}" # parallel streams
|
||||
|
||||
# ============================================================================
|
||||
# Helpers
|
||||
# ============================================================================
|
||||
start_iperf_servers() {
|
||||
echo "--- Starting iperf3 servers on all hosts ---"
|
||||
for h in $HOSTS; do
|
||||
PROX_HOST="$h" bash "$REMOTE_SH" prox \
|
||||
'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \
|
||||
>/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
echo " All servers started."
|
||||
}
|
||||
|
||||
stop_iperf_servers() {
|
||||
echo "--- Stopping iperf3 servers on all hosts ---"
|
||||
for h in $HOSTS; do
|
||||
PROX_HOST="$h" bash "$REMOTE_SH" prox \
|
||||
'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \
|
||||
>/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
echo " All servers stopped."
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# A. Host-to-host datanet matrix
|
||||
# ============================================================================
|
||||
test_datanet() {
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix"
|
||||
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
|
||||
echo "==================================================================="
|
||||
start_iperf_servers
|
||||
|
||||
local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv"
|
||||
echo "host_from,host_to,mbps" > "$outfile"
|
||||
|
||||
for client in $HOSTS; do
|
||||
for server in $HOSTS; do
|
||||
[ "$client" = "$server" ] && continue
|
||||
local sip="${DATANET_IP[$server]}"
|
||||
local result
|
||||
result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \
|
||||
"iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
|
||||
| awk '/SUM.*receiver/{printf "%.0f", $6}')
|
||||
if [ -n "$result" ]; then
|
||||
printf " %-14s → %-14s : %s Mbps\n" "$client" "$server" "$result"
|
||||
echo "$client,$server,$result" >> "$outfile"
|
||||
else
|
||||
printf " %-14s → %-14s : FAIL\n" "$client" "$server"
|
||||
echo "$client,$server,FAIL" >> "$outfile"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
stop_iperf_servers
|
||||
echo ""
|
||||
echo " Results saved: $outfile"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# B. Guest-to-guest datanet
|
||||
# ============================================================================
|
||||
test_guests() {
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)"
|
||||
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
|
||||
echo "==================================================================="
|
||||
|
||||
# Start iperf3 server on the first guest (cnode1)
|
||||
local server_entry
|
||||
server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1)
|
||||
local s_host s_vid s_ip s_label
|
||||
s_host=$(echo "$server_entry" | cut -d: -f1)
|
||||
s_vid=$(echo "$server_entry" | cut -d: -f2)
|
||||
s_ip=$(echo "$server_entry" | cut -d: -f3)
|
||||
s_label=$(echo "$server_entry" | cut -d: -f4)
|
||||
|
||||
echo " Starting iperf3 server on $s_label ($s_ip)..."
|
||||
PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \
|
||||
'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1
|
||||
sleep 1
|
||||
|
||||
local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv"
|
||||
echo "guest_from,guest_to,mbps" > "$outfile"
|
||||
|
||||
while read -r entry; do
|
||||
[ -z "$entry" ] && continue
|
||||
local c_host c_vid c_ip c_label
|
||||
c_host=$(echo "$entry" | cut -d: -f1)
|
||||
c_vid=$(echo "$entry" | cut -d: -f2)
|
||||
c_ip=$(echo "$entry" | cut -d: -f3)
|
||||
c_label=$(echo "$entry" | cut -d: -f4)
|
||||
[ "$c_ip" = "$s_ip" ] && continue
|
||||
|
||||
local result
|
||||
result=$(PROX_HOST="$c_host" VM_ID="$c_vid" bash "$REMOTE_SH" vm-guest \
|
||||
"iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
|
||||
| awk '/SUM.*receiver/{printf "%.0f", $6}')
|
||||
if [ -n "$result" ]; then
|
||||
printf " %-18s → %-18s : %s Mbps\n" "$c_label" "$s_label" "$result"
|
||||
echo "$c_label,$s_label,$result" >> "$outfile"
|
||||
else
|
||||
printf " %-18s → %-18s : FAIL\n" "$c_label" "$s_label"
|
||||
echo "$c_label,$s_label,FAIL" >> "$outfile"
|
||||
fi
|
||||
done <<< "$GUEST_TARGETS"
|
||||
|
||||
# Cleanup
|
||||
PROX_HOST="$s_host" VM_ID="$s_vid" bash "$REMOTE_SH" vm-guest \
|
||||
'pkill iperf3' >/dev/null 2>&1
|
||||
|
||||
echo ""
|
||||
echo " Results saved: $outfile"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# C. Storage I/O (NFS read/write)
|
||||
# ============================================================================
|
||||
test_storage() {
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STORAGE I/O — NFS Read/Write (100MB dd)"
|
||||
echo "==================================================================="
|
||||
|
||||
local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv"
|
||||
echo "host,mount,write_mbps,read_mbps" > "$outfile"
|
||||
|
||||
local ddscript="/tmp/perf-dd-$$.sh"
|
||||
cat > "$ddscript" <<'DDSCRIPT'
|
||||
#!/bin/bash
|
||||
nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc)
|
||||
for m in $nfs; do
|
||||
tf="$m/.perf-$$"
|
||||
w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
|
||||
r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
|
||||
rm -f "$tf" 2>/dev/null
|
||||
echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s"
|
||||
done
|
||||
DDSCRIPT
|
||||
|
||||
for h in $HOSTS; do
|
||||
echo ""
|
||||
echo " --- $h ---"
|
||||
PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do
|
||||
[ -n "$line" ] && echo " $line"
|
||||
done
|
||||
done
|
||||
rm -f "$ddscript"
|
||||
|
||||
echo ""
|
||||
echo " Results saved: $outfile"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Main
|
||||
# ============================================================================
|
||||
MODE="${1:-all}"
|
||||
|
||||
case "$MODE" in
|
||||
datanet|a) test_datanet ;;
|
||||
guests|b) test_guests ;;
|
||||
storage|c) test_storage ;;
|
||||
all|"") test_datanet; test_guests; test_storage ;;
|
||||
*)
|
||||
echo "Usage: $0 [datanet|guests|storage|all]"
|
||||
echo ""
|
||||
echo " datanet — host-to-host bandwidth matrix over VLAN 1000"
|
||||
echo " guests — guest-to-guest (k8s/ultix VMs over VLAN 1000)"
|
||||
echo " storage — NFS read/write I/O"
|
||||
echo " all — run all three (default)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Perf testing complete. Logs in: $LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Reference in New Issue
Block a user