mrcharles
1d7c74676c
Previous sessions used `qm guest exec` to back-door SSH keys into ~30 VMs, bypassing sshd's audit trail in an ITAR/CMMC environment. Wire the ban deep so it cannot recur: - tests/remote.sh: remove the vm-guest mode + qm-guest-exec path entirely - scripts/check-rules.sh: rule #11 fails on any `qm guest exec` / `vm-guest` pattern in code (scans .sh/.bash/.py; docs may describe the ban freely) - AGENTS.md: codify "Access-channel policy: SSH only" as non-negotiable; add "Questions" rule banning harness question tools (use questions-v1.md) - tests/vm-validation.sh: drop guest-agent key re-injection; SSH-only - proxmox/perf/scripts/perf-matrix.sh + deploy-tuned-guests.sh: convert guest-agent execution to SSH (vmroot) now that VMs have key + sudo - bootstrap-all.sh: re-target the 8 remaining locked-out systems with correct users/methods; print a console one-liner for publickey-only Pis Guest-agent remains installable/checkable for Proxmox state visibility — never as an execution or key-delivery path. 💘 Generated with Crush Assisted-by: Crush:glm-5.2
243 lines
9.0 KiB
Bash
Executable File
243 lines
9.0 KiB
Bash
Executable File
#!/bin/bash
|
|
###############################################################################
|
|
# perf-matrix.sh — Any-to-any performance testing across the PFVCluster
|
|
#
|
|
# Tests three network planes:
|
|
# A. Datanet (VLAN 1000): hypervisor-to-hypervisor over storage network
|
|
# B. Guest-to-guest: k8s/ultix VMs over datanet (10.100.100.x)
|
|
# C. Storage I/O: dd read/write to NFS mounts
|
|
#
|
|
# Prerequisites:
|
|
# - iperf3 installed on all hosts (systemd service: iperf3-server)
|
|
# - iperf3 installed inside guest VMs
|
|
# - SSH key + passwordless sudo on all guest VMs (remote.sh; sshd is the
|
|
# only approved access channel — see AGENTS.md)
|
|
#
|
|
# Usage:
|
|
# bash perf-matrix.sh # run all tests
|
|
# bash perf-matrix.sh datanet # host-to-host datanet only
|
|
# bash perf-matrix.sh guests # guest-to-guest datanet only
|
|
# bash perf-matrix.sh storage # NFS I/O only
|
|
#
|
|
# Environment:
|
|
# REMOTE_SH path to tests/remote.sh (auto-detected)
|
|
###############################################################################
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REMOTE_SH="${REMOTE_SH:-$(cd "$SCRIPT_DIR/../../.." && pwd)/tests/remote.sh}"
|
|
LOG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)/returned-logs"
|
|
mkdir -p "$LOG_DIR"
|
|
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
|
|
|
# --- Host datanet IPs (VLAN 1000) ---
|
|
declare -A DATANET_IP
|
|
DATANET_IP[pfv-tsys1]="10.100.100.1"
|
|
DATANET_IP[pfv-tsys3]="10.100.100.3"
|
|
DATANET_IP[pfv-tsys4]="10.100.100.4"
|
|
DATANET_IP[pfv-tsys5]="10.100.100.5"
|
|
DATANET_IP[pfv-tsys6]="10.100.100.6"
|
|
DATANET_IP[pfv-tsys7]="10.100.100.7"
|
|
DATANET_IP[pfv-tsys9]="10.100.100.9"
|
|
|
|
HOSTS="pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9"
|
|
|
|
# --- k8s/ultix VM targets (SSH for control; datanet_ip for iperf traffic) ---
|
|
# Format: prox_host:vmid:datanet_ip:ssh_host:label
|
|
GUEST_TARGETS="
|
|
pfv-tsys1:102:10.100.100.10:pfv-k8s-cnode1:cnode1
|
|
pfv-tsys7:705:10.100.100.11:pfv-k8s-cnode2:cnode2
|
|
pfv-tsys6:603:10.100.100.12:pfv-k8s-cnode3:cnode3
|
|
pfv-tsys3:313:10.100.100.13:pfv-k8s-wnode-tsys3:wnode-tsys3
|
|
pfv-tsys5:500:10.100.100.14:pfv-k8s-wnode-tsys5:wnode-tsys5
|
|
pfv-tsys6:601:10.100.100.15:pfv-k8s-wnode-tsys6:wnode-tsys6
|
|
pfv-tsys7:701:10.100.100.16:pfv-k8s-wnode-tsys7:wnode-tsys7
|
|
pfv-tsys9:905:10.100.100.17:pfv-k8s-wnode-tsys9:wnode-tsys9
|
|
pfv-tsys5:5111:10.100.100.18:ultix-streaming:ultix-streaming
|
|
pfv-tsys5:5112:10.100.100.19:ultix-offstage:ultix-offstage
|
|
"
|
|
|
|
DURATION="${DURATION:-3}" # seconds per iperf3 test
|
|
STREAMS="${STREAMS:-4}" # parallel streams
|
|
|
|
# ============================================================================
|
|
# Helpers
|
|
# ============================================================================
|
|
start_iperf_servers() {
|
|
echo "--- Starting iperf3 servers on all hosts ---"
|
|
for h in $HOSTS; do
|
|
PROX_HOST="$h" bash "$REMOTE_SH" prox \
|
|
'systemctl start iperf3-server 2>/dev/null || iperf3 -s -D; echo ok' \
|
|
>/dev/null 2>&1 &
|
|
done
|
|
wait
|
|
echo " All servers started."
|
|
}
|
|
|
|
stop_iperf_servers() {
|
|
echo "--- Stopping iperf3 servers on all hosts ---"
|
|
for h in $HOSTS; do
|
|
PROX_HOST="$h" bash "$REMOTE_SH" prox \
|
|
'systemctl stop iperf3-server 2>/dev/null; pkill iperf3 2>/dev/null; true' \
|
|
>/dev/null 2>&1 &
|
|
done
|
|
wait
|
|
echo " All servers stopped."
|
|
}
|
|
|
|
# ============================================================================
|
|
# A. Host-to-host datanet matrix
|
|
# ============================================================================
|
|
test_datanet() {
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " DATANET (VLAN 1000) — Host-to-Host Bandwidth Matrix"
|
|
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
|
|
echo "==================================================================="
|
|
start_iperf_servers
|
|
|
|
local outfile="$LOG_DIR/datanet-host-${TIMESTAMP}.csv"
|
|
echo "host_from,host_to,mbps" > "$outfile"
|
|
|
|
for client in $HOSTS; do
|
|
for server in $HOSTS; do
|
|
[ "$client" = "$server" ] && continue
|
|
local sip="${DATANET_IP[$server]}"
|
|
local result
|
|
result=$(PROX_HOST="$client" bash "$REMOTE_SH" prox \
|
|
"iperf3 -c $sip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
|
|
| awk '/SUM.*receiver/{printf "%.0f", $6}')
|
|
if [ -n "$result" ]; then
|
|
printf " %-14s → %-14s : %s Mbps\n" "$client" "$server" "$result"
|
|
echo "$client,$server,$result" >> "$outfile"
|
|
else
|
|
printf " %-14s → %-14s : FAIL\n" "$client" "$server"
|
|
echo "$client,$server,FAIL" >> "$outfile"
|
|
fi
|
|
done
|
|
done
|
|
|
|
stop_iperf_servers
|
|
echo ""
|
|
echo " Results saved: $outfile"
|
|
}
|
|
|
|
# ============================================================================
|
|
# B. Guest-to-guest datanet
|
|
# ============================================================================
|
|
test_guests() {
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " DATANET (VLAN 1000) — Guest-to-Guest (k8s + ultix VMs)"
|
|
echo " ${STREAMS} parallel streams, ${DURATION}s per test"
|
|
echo "==================================================================="
|
|
|
|
# Start iperf3 server on the first guest (cnode1)
|
|
local server_entry
|
|
server_entry=$(echo "$GUEST_TARGETS" | head -2 | tail -1)
|
|
local s_ip s_ssh s_label
|
|
s_ip=$(echo "$server_entry" | cut -d: -f3)
|
|
s_ssh=$(echo "$server_entry" | cut -d: -f4)
|
|
s_label=$(echo "$server_entry" | cut -d: -f5)
|
|
|
|
echo " Starting iperf3 server on $s_label ($s_ip)..."
|
|
VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
|
|
'pkill iperf3 2>/dev/null; iperf3 -s -D' >/dev/null 2>&1
|
|
sleep 1
|
|
|
|
local outfile="$LOG_DIR/datanet-guest-${TIMESTAMP}.csv"
|
|
echo "guest_from,guest_to,mbps" > "$outfile"
|
|
|
|
while read -r entry; do
|
|
[ -z "$entry" ] && continue
|
|
local c_ip c_ssh c_label
|
|
c_ip=$(echo "$entry" | cut -d: -f3)
|
|
c_ssh=$(echo "$entry" | cut -d: -f4)
|
|
c_label=$(echo "$entry" | cut -d: -f5)
|
|
[ "$c_ip" = "$s_ip" ] && continue
|
|
|
|
local result
|
|
result=$(VM_IP="$c_ssh" bash "$REMOTE_SH" vmroot \
|
|
"iperf3 -c $s_ip -t $DURATION -P $STREAMS -f m 2>&1" </dev/null \
|
|
| awk '/SUM.*receiver/{printf "%.0f", $6}')
|
|
if [ -n "$result" ]; then
|
|
printf " %-18s → %-18s : %s Mbps\n" "$c_label" "$s_label" "$result"
|
|
echo "$c_label,$s_label,$result" >> "$outfile"
|
|
else
|
|
printf " %-18s → %-18s : FAIL\n" "$c_label" "$s_label"
|
|
echo "$c_label,$s_label,FAIL" >> "$outfile"
|
|
fi
|
|
done <<< "$GUEST_TARGETS"
|
|
|
|
# Cleanup
|
|
VM_IP="$s_ssh" bash "$REMOTE_SH" vmroot \
|
|
'pkill iperf3' >/dev/null 2>&1
|
|
|
|
echo ""
|
|
echo " Results saved: $outfile"
|
|
}
|
|
|
|
# ============================================================================
|
|
# C. Storage I/O (NFS read/write)
|
|
# ============================================================================
|
|
test_storage() {
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " STORAGE I/O — NFS Read/Write (100MB dd)"
|
|
echo "==================================================================="
|
|
|
|
local outfile="$LOG_DIR/storage-io-${TIMESTAMP}.csv"
|
|
echo "host,mount,write_mbps,read_mbps" > "$outfile"
|
|
|
|
local ddscript="/tmp/perf-dd-$$.sh"
|
|
cat > "$ddscript" <<'DDSCRIPT'
|
|
#!/bin/bash
|
|
nfs=$(mount | awk '/type nfs/{print $3}' | grep -v proc)
|
|
for m in $nfs; do
|
|
tf="$m/.perf-$$"
|
|
w=$(dd if=/dev/zero of="$tf" bs=1M count=100 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
|
|
r=$(dd if="$tf" of=/dev/null bs=1M 2>&1 | awk '/copied/{printf "%.0f", 100/($8+0.001)}')
|
|
rm -f "$tf" 2>/dev/null
|
|
echo "$m write=${w:-FAIL}MB/s read=${r:-N/A}MB/s"
|
|
done
|
|
DDSCRIPT
|
|
|
|
for h in $HOSTS; do
|
|
echo ""
|
|
echo " --- $h ---"
|
|
PROX_HOST="$h" bash "$REMOTE_SH" prox-file "$ddscript" 2>&1 | while read -r line; do
|
|
[ -n "$line" ] && echo " $line"
|
|
done
|
|
done
|
|
rm -f "$ddscript"
|
|
|
|
echo ""
|
|
echo " Results saved: $outfile"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Main
|
|
# ============================================================================
|
|
MODE="${1:-all}"
|
|
|
|
case "$MODE" in
|
|
datanet|a) test_datanet ;;
|
|
guests|b) test_guests ;;
|
|
storage|c) test_storage ;;
|
|
all|"") test_datanet; test_guests; test_storage ;;
|
|
*)
|
|
echo "Usage: $0 [datanet|guests|storage|all]"
|
|
echo ""
|
|
echo " datanet — host-to-host bandwidth matrix over VLAN 1000"
|
|
echo " guests — guest-to-guest (k8s/ultix VMs over VLAN 1000)"
|
|
echo " storage — NFS read/write I/O"
|
|
echo " all — run all three (default)"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo " Perf testing complete. Logs in: $LOG_DIR/"
|
|
echo "==================================================================="
|