chore: initialize repo with full project state

Performance optimization engagement for a 7-host Proxmox R&D cluster. Captures the accumulated work across host tuning, network analysis, fleet assessment, and kubernetes architecture planning. Contents: - Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS nconnect, tuned profiles -- complete on 5 of 7 hosts - Validation + benchmarking scripts: iperf matrix, bond/NFS fixes - Collected host data (returned-logs/): check.sh output from all 7 hosts + iperf results, including newly-validated pfv-tsys9 - AGENTS.md: operating context for AI agents - PROJECT.md: board-ready fleet assessment with VM placement and storage redundancy analysis (40 VMs across 7 hosts) - K8S.md: kubernetes architecture deep-dive covering cnode/wnode distribution, StorageClass design, and ETL/HPC workload planning 💘 Generated with Crush Assisted-by: Crush:glm-5.2
2026-07-27 11:31:29 -05:00
commit 422999bf3c
106 changed files with 114159 additions and 0 deletions
@@ -0,0 +1,119 @@
+#!/bin/bash
+# iperf-tsys6-tsys7.sh - validate 2Gbps LACP between the two tuned hosts.
+set -uo pipefail
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
+LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
+mkdir -p "$LOG_DIR"
+
+TSYS6="10.100.100.6"
+TSYS7="10.100.100.7"
+
+run_test() {
+    local client="$1" server="$2" server_ip="$3" label="$4" logfile="$5"
+    shift 4
+    local extra_args="$*"
+
+    echo -n "  [$label] ... "
+
+    # Start server in one-shot mode
+    ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
+    sleep 1
+
+    {
+        echo "=== iperf3: $label ==="
+        echo "Client: $client  Server: $server ($server_ip)"
+        echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+        echo "Args: $extra_args"
+        echo ""
+        ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${server_ip} $extra_args" 2>&1
+        echo ""
+        echo "=== END ==="
+    } > "$logfile" 2>&1
+
+    # Extract result
+    sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
+    if [ -n "$sum" ]; then
+        bitrate=$(echo "$sum" | awk '{print $6, $7}')
+        retrans=$(echo "$sum" | awk '{print $8}')
+    else
+        single=$(grep 'sender$' "$logfile" | tail -1)
+        bitrate=$(echo "$single" | awk '{print $7, $8}')
+        retrans=$(echo "$single" | awk '{print $9}')
+    fi
+    echo "${bitrate} (retrans: ${retrans:-0})"
+}
+
+echo "==================================================================="
+echo " iperf3: tsys6 ↔ tsys7 (both have 2×1G LACP + layer3+4 hash)"
+echo " Expectation: ~1.8-2.0 Gbps for 8-stream TCP"
+echo "==================================================================="
+echo ""
+
+# Pre-flight: confirm bond state on both
+echo "--- bond0 state on tsys6 ---"
+ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
+echo ""
+echo "--- bond0 state on tsys7 ---"
+ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
+echo ""
+
+echo "--- Running tests ---"
+echo ""
+
+# Test 1: tsys7 → tsys6, 8-stream TCP forward
+run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
+    "tsys7→tsys6 TCP 8-stream forward" \
+    "$LOG_DIR/tsys6-tsys7-01-tcp-8stream-forward.log" \
+    "-P 8 -t 30 -l 128k -O 2"
+
+# Test 2: tsys6 → tsys7, 8-stream TCP forward (reverse direction)
+run_test pfv-tsys6 pfv-tsys7 "$TSYS7" \
+    "tsys6→tsys7 TCP 8-stream forward" \
+    "$LOG_DIR/tsys6-tsys7-02-tcp-8stream-forward.log" \
+    "-P 8 -t 30 -l 128k -O 2"
+
+# Test 3: tsys7 → tsys6, single stream (should be ~940 Mbps — single flow)
+run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
+    "tsys7→tsys6 TCP single-stream" \
+    "$LOG_DIR/tsys6-tsys7-03-tcp-single.log" \
+    "-t 20 -O 2"
+
+# Test 4: tsys7 → tsys6, 4-stream (nconnect=4 mirrors this)
+run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
+    "tsys7→tsys6 TCP 4-stream" \
+    "$LOG_DIR/tsys6-tsys7-04-tcp-4stream.log" \
+    "-P 4 -t 30 -l 128k -O 2"
+
+# Test 5: UDP saturation
+run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
+    "tsys7→tsys6 UDP saturation" \
+    "$LOG_DIR/tsys6-tsys7-05-udp.log" \
+    "-u -b 3G -t 10 -l 8972"
+
+# Cleanup
+ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
+ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
+
+echo ""
+echo "==================================================================="
+echo " SUMMARY"
+echo "==================================================================="
+echo ""
+printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
+printf "%-45s %15s %10s\n" "----" "----------" "-------"
+for f in "$LOG_DIR"/tsys6-tsys7-*.log; do
+    [ -r "$f" ] || continue
+    label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
+    sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
+    if [ -n "$sum" ]; then
+        bitrate=$(echo "$sum" | awk '{print $6, $7}')
+        retrans=$(echo "$sum" | awk '{print $8}')
+    else
+        single=$(grep 'sender$' "$f" | tail -1)
+        bitrate=$(echo "$single" | awk '{print $7, $8}')
+        retrans=$(echo "$single" | awk '{print $9}')
+    fi
+    printf "%-45s %15s %10s\n" "$label" "$bitrate" "${retrans:--}"
+done
+echo ""
+echo "Expected: 8-stream ~1.8-2.0 Gbps, single-stream ~940 Mbps"