feat: ensure-cpu-performance.sh — fleet CPU governor/EPP audit+enforce [#737][#769]

Founder directive 2026-09-04. Fleet verified at max: tsys1/3/4/6/7/9 governor=performance (+EPP on pstate hosts) under tuned virtual-host (profile-persisted, tuned-adm verify passes); tsys5 = fixed-max by hardware (E5620, no cpufreq driver, cores pinned above base clock). Script audits and can enforce (--fix) per host via the SSH chokepoint. https://projects.knownelement.com/issues/769#note-4152
2026-09-04 07:15:40 -05:00
parent 7a899b8ac6
commit 8780a226a5
1 changed files with 56 additions and 0 deletions
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# ensure-cpu-performance.sh — audit + enforce max CPU performance per host [#737][#769]
+#
+# Founder directive 2026-09-04: "all hosts set for maximum CPU performance
+# at all times". Audit (default) checks tuned profile + per-CPU governor +
+# EPP hint; --fix re-asserts governor (and EPP where the driver exposes it)
+# via sysfs for the running system — persistence stays with the tuned
+# virtual-host profile (verified to carry these settings; `tuned-adm verify`).
+#
+# Usage: PROX_HOST=pfv-tsys1 bash ensure-cpu-performance.sh [--fix]
+# Exit 0 = at max performance, 1 = drift found (run with --fix to correct).
+set -uo pipefail
+FIX=""
+[ "${1:-}" = "--fix" ] && FIX="yes"
+
+REMOTE="${REMOTE:-$HOME/projects/PFVCluster/tests/remote.sh}"
+[ -e "$REMOTE" ] || { echo "FAIL: SSH chokepoint not found at $REMOTE" >&2; exit 1; }
+prox() { timeout "${PROX_TIMEOUT:-60}" env PROX_HOST="$PROX_HOST" bash "$REMOTE" prox "$1"; }
+
+echo "== $PROX_HOST"
+tuned="$(prox 'tuned-adm active 2>/dev/null' | head -1)"
+echo "  $tuned"
+
+govs="$(prox 'grep -h . /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor 2>/dev/null | sort | uniq -c')"
+if [ -z "$govs" ]; then
+    mhz="$(prox 'grep MHz /proc/cpuinfo | head -1' | awk '{print $4}')"
+    echo "  NO cpufreq driver — cores run fixed at firmware max (e.g. ${mhz} MHz). Nothing to set."
+    exit 0
+fi
+echo "  governors: $govs"
+
+nongov="$(printf '%s\n' "$govs" | awk '$2!="performance" {print $2}' | head -1)"
+epp="$(prox 'cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference 2>/dev/null')"
+
+if [ -n "$nongov" ]; then
+    if [ -z "$FIX" ]; then
+        echo "  DRIFT: governor=$nongov on some CPUs (run with --fix)"
+        exit 1
+    fi
+    # shellcheck disable=SC2016  # must expand on the remote host
+    prox 'for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "$g"; done'
+    echo "  FIXED: governors -> performance"
+fi
+if [ -n "$epp" ] && [ "$epp" != "performance" ]; then
+    if [ -z "$FIX" ]; then
+        echo "  DRIFT: EPP=$epp (run with --fix)"
+        exit 1
+    fi
+    # shellcheck disable=SC2016  # must expand on the remote host
+    prox 'for e in /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference; do echo performance > "$e" 2>/dev/null; done'
+    echo "  FIXED: EPP -> performance"
+fi
+if [ -z "$nongov" ] && ! printf '%s' "$epp" | grep -qv performance; then
+    echo "  OK: max performance (governor=performance$( [ -n "$epp" ] && echo ", EPP=performance" ))"
+fi
+exit 0