KNELPerf v2 wave 1: Ultix scheduler breakout + loop skeleton + reports (#826)

- scheduler/: day/night engine (conf-driven redesign), psi/perfsnap collectors,
  proxmox-ctl, systemd timer templates — ported from ~/projects/ultix per the
  breakout map in scheduler/README.md
- loop/: perf-loop driver (baseline/audit/tweak/rebaseline), cpu-bench (new CPU
  leg), baseline-diff (tolerance compare) — smoke-tested locally
- docs/ARCHITECTURE.md: full program design (loop, resource groups, VM/spindle
  balancing, k8s-vs-Slurm, beszel+RAPL/iDRAC telemetry spine, workload classes,
  solar/HA + two-site power economics)
- docs/report-amt-power-telemetry.md + docs/report-moonlight-desktop.md

Redmine: https://projects.knownelement.com/issues/826#note-2

scheduler/daynight/knelperf-daynight.conf.example

@@ -0,0 +1,28 @@
+# /etc/knelperf/daynight.conf — knelperf-daynight.sh configuration.
+# Copy to /etc/knelperf/daynight.conf and edit per host. Every field is
+# optional; omit slices/sections you do not manage on this host.
+#
+# Slice spec format (space-separated list):
+#   slice[:CPUWeight[:AllowedCPUs[:MemoryHigh]]]
+# Empty field = skip that property. Day AND night lists are required for
+# any slice you list (reboot-safety invariant: day values == unit-file
+# defaults).
+
+# Example (workstation, 8 vCPU — mirrors the ultix values):
+# DAY_SLICES="reachableceo-batch.slice:25:0-5:12G reachableceo-gateway.slice:900::"
+# NIGHT_SLICES="reachableceo-batch.slice:400:0-6:30G reachableceo-gateway.slice:500::"
+
+# Example (PVE host with a batch pool):
+# DAY_SLICES="knelperf-batch.slice:25:0-3:8G"
+# NIGHT_SLICES="knelperf-batch.slice:400:0-7:24G"
+
+# Block devices for night readahead boost (partitions auto-included):
+# BLOCK_DEVS="/dev/sdb"
+
+# Defaults shown in the script; override only what differs:
+# DATA_RA_DAY=256
+# DATA_RA_NIGHT=2048
+# DIRTY_DAY=1073741824
+# DIRTY_NIGHT=2147483648
+# DIRTY_BG_DAY=268435456
+# DIRTY_BG_NIGHT=536870912

scheduler/daynight/knelperf-daynight.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# KNELPerf day/night resource profile engine.
+#
+# Broken out of ultix staged/ukrrs-daynight.sh (2026-09-06, #826) and
+# redesigned: slice names, devices, and all numeric knobs are CONF-driven
+# (no host-specific hardcoding); slice targets are a list so any host can
+# run it. Runtime-only changes (set-property --runtime + sysctl + readahead):
+# a reboot always lands safely in day mode, because slice unit files carry
+# the day defaults (unchanged ultix invariant).
+#
+# Usage: knelperf-daynight.sh [--conf FILE] day|night
+set -euo pipefail
+
+CONF=${KNELPERF_DAYNIGHT_CONF:-/etc/knelperf/daynight.conf}
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --conf) CONF=$2; shift 2 ;;
+    day|night) MODE=$1; shift ;;
+    *) echo "usage: $0 [--conf FILE] day|night" >&2; exit 2 ;;
+  esac
+done
+[ "${MODE:-}" ] || { echo "usage: $0 [--conf FILE] day|night" >&2; exit 2; }
+[ -r "$CONF" ] && . "$CONF"
+
+# Targets: "slice:CPUWeight:AllowedCPUs:MemoryHigh" per mode, space-separated.
+# Empty fields mean "skip that property". See knelperf-daynight.conf.example.
+: "${DAY_SLICES:=}";  : "${NIGHT_SLICES:=}"
+# Block devices for readahead tuning (space-separated; partitions auto-appended).
+: "${BLOCK_DEVS:=}"
+: "${DATA_RA_DAY:=256}";          : "${DATA_RA_NIGHT:=2048}"
+: "${DIRTY_DAY:=1073741824}";     : "${DIRTY_NIGHT:=2147483648}"
+: "${DIRTY_BG_DAY:=268435456}";   : "${DIRTY_BG_NIGHT:=536870912}"
+
+log() { echo "[knelperf-daynight] $*"; }
+ra=$DATA_RA_DAY; dirty=$DIRTY_DAY; dirtybg=$DIRTY_BG_DAY; slices=$DAY_SLICES
+[ "$MODE" = night ] && { ra=$DATA_RA_NIGHT; dirty=$DIRTY_NIGHT; dirtybg=$DIRTY_BG_NIGHT; slices=$NIGHT_SLICES; }
+
+apply_slice() { # "slice:weight:cpus:memhigh"
+  local spec=$1 slice weight cpus memhigh
+  IFS=: read -r slice weight cpus memhigh <<< "$spec"
+  [ -n "$slice" ] || return 0
+  local -a props=()
+  [ -n "$weight" ]  && props+=(CPUWeight="$weight")
+  [ -n "$cpus" ]    && props+=(AllowedCPUs="$cpus")
+  [ -n "$memhigh" ] && props+=(MemoryHigh="$memhigh")
+  if [ ${#props[@]} -gt 0 ]; then
+    systemctl set-property --runtime "$slice" "${props[@]}" \
+      || log "WARN: set-property failed for $slice (slice not loaded?)"
+  fi
+}
+
+for spec in $slices; do apply_slice "$spec"; done
+
+if [ -n "$BLOCK_DEVS" ]; then
+  for dev in $BLOCK_DEVS; do
+    blockdev --setra "$ra" "$dev" 2>/dev/null || log "readahead skip: $dev"
+    for part in "${dev}"[0-9]*; do
+      [ -e "$part" ] && blockdev --setra "$ra" "$part" 2>/dev/null || true
+    done
+  done
+fi
+sysctl -q -w vm.dirty_bytes="$dirty" vm.dirty_background_bytes="$dirtybg"
+log "profile $MODE applied $(date -Is)"

scheduler/daynight/systemd/README.md

@@ -0,0 +1,24 @@
+# systemd units — day/night profile timers
+
+Install per host (adjust names to your unit namespace if not knelperf-):
+
+    install -m 0644 knelperf-dayprofile.service  /etc/systemd/system/
+    install -m 0644 knelperf-dayprofile.timer    /etc/systemd/system/
+    install -m 0644 knelperf-nightprofile.service /etc/systemd/system/
+    install -m 0644 knelperf-nightprofile.timer   /etc/systemd/system/
+    systemctl daemon-reload
+    systemctl enable --now knelperf-dayprofile.timer knelperf-nightprofile.timer
+
+Design notes (carried from ultix, keep true here):
+
+- `Persistent=true` — a missed flip (host powered off) replays at next boot,
+  so the host can never get stuck in the wrong profile across a reboot cycle.
+- Services are `Type=oneshot` calling the profile engine with the SAME conf
+  file; the conf — not the timer — is the only place values live (fixes the
+  ultix wart where the day-flip time was sed-edited on the live timer).
+- Day schedule 07:00, night 22:00 local time (batch window 22:00–07:00).
+  Change the OnCalendar here when a host needs a different window; the timers
+  stay trivial and identical in shape.
+
+Reboot-safety invariant: slice unit files must carry the DAY values as their
+static defaults, so an unattended reboot always lands in day mode.

scheduler/daynight/systemd/knelperf-dayprofile.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=KNELPerf day resource profile (interactive-first)
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/knelperf-daynight.sh day
+# Never restart a profile flip; the timer owns scheduling
+Restart=no

scheduler/daynight/systemd/knelperf-dayprofile.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=KNELPerf day profile flip (07:00 local)
+
+[Timer]
+OnCalendar=*-*-* 07:00:00
+Persistent=true
+Unit=knelperf-dayprofile.service
+
+[Install]
+WantedBy=timers.target

scheduler/daynight/systemd/knelperf-nightprofile.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=KNELPerf night resource profile (batch window 22:00-07:00)
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/knelperf-daynight.sh night
+Restart=no

scheduler/daynight/systemd/knelperf-nightprofile.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=KNELPerf night profile flip (22:00 local)
+
+[Timer]
+OnCalendar=*-*-* 22:00:00
+Persistent=true
+Unit=knelperf-nightprofile.service
+
+[Install]
+WantedBy=timers.target