KNELPerf v2 wave 1: Ultix scheduler breakout + loop skeleton + reports (#826)

- scheduler/: day/night engine (conf-driven redesign), psi/perfsnap collectors,
  proxmox-ctl, systemd timer templates — ported from ~/projects/ultix per the
  breakout map in scheduler/README.md
- loop/: perf-loop driver (baseline/audit/tweak/rebaseline), cpu-bench (new CPU
  leg), baseline-diff (tolerance compare) — smoke-tested locally
- docs/ARCHITECTURE.md: full program design (loop, resource groups, VM/spindle
  balancing, k8s-vs-Slurm, beszel+RAPL/iDRAC telemetry spine, workload classes,
  solar/HA + two-site power economics)
- docs/report-amt-power-telemetry.md + docs/report-moonlight-desktop.md

Redmine: https://projects.knownelement.com/issues/826#note-2

loop/perf-loop.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# KNELPerf loop driver — baseline -> audit/benchmark -> tweak -> re-baseline.
+#
+# The heart of KNELPerf v2 (#826). One host, one cycle phase at a time:
+#
+#   perf-loop.sh baseline <host>       capture baseline bundle (cpu/net/storage/psi)
+#   perf-loop.sh audit <host>          read-only audit vs baseline (alerts on drift)
+#   perf-loop.sh tweak <host> <id>     apply a tweak (from tweaks/) after CR gate
+#   perf-loop.sh rebaseline <host>     capture post-tweak bundle + diff report
+#   perf-loop.sh compare <host> <a> <b>  diff two stored bundles
+#
+# Bundles live in data/baselines/<host>/<ts>/ and are plain TSV/JSON so diff
+# and jq stay the toolchain. Nothing here mutates a host except `tweak`.
+set -euo pipefail
+REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd)
+DATA_DIR=${KNELPERF_DATA_DIR:-$REPO_ROOT/data/baselines}
+SSH_OPTS=(-o ConnectTimeout=8 -o BatchMode=yes)
+
+usage() { sed -n '2,12p' "$0"; exit 2; }
+[ $# -ge 2 ] || usage
+cmd=$1; host=$2
+
+now() { date +%Y%m%d-%H%M%S; }
+host_dir() { mkdir -p "$DATA_DIR/$1"; }
+
+run_on_host() { # host script-path args...
+  local h=$1 script=$2; shift 2
+  case $h in
+    local) bash "$REPO_ROOT/$script" "$@" ;;
+    *) ssh "${SSH_OPTS[@]}" "root@$h" "bash -s" < "$REPO_ROOT/$script" "$@" ;;
+  esac
+}
+
+case "$cmd" in
+baseline|rebaseline)
+  ts=$(now); dir="$DATA_DIR/$host/$ts"; host_dir "$host"; mkdir -p "$dir"
+  echo "== KNELPerf $cmd: $host -> $dir"
+  # shellcheck disable=SC2043  # single-part list by design; grows as legs are added
+  for part in cpu-bench; do
+    run_on_host "$host" "loop/$part.sh" > "$dir/$part.tsv" 2> "$dir/$part.err" \
+      || echo "WARN: $part failed (see $dir/$part.err)"
+  done
+  # storage + network reuse the existing bench harness outputs
+  run_on_host "$host" "bench/bench-run.sh" > "$dir/bench-run.tsv" 2> "$dir/bench.err" \
+    || echo "WARN: bench-run failed (see $dir/bench.err)"
+  echo "$cmd $host $ts" > "$dir/MANIFEST"
+  echo "done: $(ls "$dir")"
+  ;;
+audit)
+  latest=$(find "$DATA_DIR/$host" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' 2>/dev/null | sort | tail -1)
+  [ -n "$latest" ] || { echo "no baseline for $host" >&2; exit 3; }
+  echo "== KNELPerf audit: $host vs baseline $latest"
+  "$REPO_ROOT/loop/baseline-diff.sh" "$DATA_DIR/$host/$latest" "$host"
+  ;;
+tweak)
+  [ $# -ge 3 ] || usage; tweak_id=$3
+  echo "== KNELPerf tweak $tweak_id on $host (CR gate applies - see AGENTS.md)"
+  run_on_host "$host" "tweaks/$tweak_id.sh"
+  ;;
+compare)
+  [ $# -ge 4 ] || usage
+  "$REPO_ROOT/loop/baseline-diff.sh" "$DATA_DIR/$host/$3" "$DATA_DIR/$host/$4"
+  ;;
+*) usage ;;
+esac