#!/usr/bin/env bash
# KNELPerf node monitoring installer — sysstat tuning + collectors + timers.
# Idempotent; run as ROOT on a PVE node from a repo checkout:
#   bash monitoring/node-install.sh
# Surface (all details in monitoring/README.md):
#   - /etc/sysstat/sysstat        HISTORY 7 -> 28, COMPRESSAFTER 10 -> 31
#   - /usr/local/sbin/knelperf-{perfsnap,psi-textfile,iotop-snap}.sh
#   - /etc/systemd/system/knelperf-*.{service,timer}  (enabled timers)
#   - /etc/knelperf/perfsnap.conf  per-node ifaces/cgroup globs
#   - logs under /var/log/knelperf-{perf,iotop}, textfiles /var/lib/knelperf/textfile
# No reboots, no package installs (PVE ships sysstat + iotop already).
# Docs: https://community.turnsys.com/t/328
set -euo pipefail
[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
here=$(cd "$(dirname "$0")/.." && pwd)
say() { echo "[knelperf-install] $*"; }

# 1) sysstat: keep 28 days of sar history (two-week program + re-baseline window)
if [ -f /etc/sysstat/sysstat ]; then
  sed -i -E 's/^HISTORY=.*/HISTORY=28/; s/^COMPRESSAFTER=.*/COMPRESSAFTER=31/' /etc/sysstat/sysstat
  sed -i -E 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat
  systemctl restart sysstat 2>/dev/null || true
  /usr/libexec/sysstat/debian-sa1 1 1 2>/dev/null || true
  today=/var/log/sysstat/sa$(date +%d)
  if [ -f "$today" ]; then say "sar ok: $today present"; else say "sar WARN: $today missing (check /etc/cron.d/sysstat)"; fi
else
  say "WARN: no /etc/sysstat/sysstat on this host — sar tuning skipped"
fi

# 2) collectors + units
install -m 0755 "$here/scheduler/collectors/knelperf-perfsnap.sh"     /usr/local/sbin/knelperf-perfsnap.sh
install -m 0755 "$here/scheduler/collectors/knelperf-psi-textfile.sh" /usr/local/sbin/knelperf-psi-textfile.sh
install -m 0755 "$here/monitoring/collectors/knelperf-iotop-snap.sh"  /usr/local/sbin/knelperf-iotop-snap.sh
for u in perfsnap psi-textfile iotop-snap; do
  install -m 0644 "$here/monitoring/systemd/knelperf-$u.service" /etc/systemd/system/knelperf-$u.service
  install -m 0644 "$here/monitoring/systemd/knelperf-$u.timer"   /etc/systemd/system/knelperf-$u.timer
done

# 3) per-node conf: watch PVE guest cgroups + real bridges/tailscale
ifaces=""
for i in vmbr0 datanet tailscale0; do
  ip link show "$i" >/dev/null 2>&1 && ifaces="$ifaces $i"
done
mkdir -p /etc/knelperf
cat > /etc/knelperf/perfsnap.conf <<EOF
# /etc/knelperf/perfsnap.conf — generated by KNELPerf node-install.sh $(date -I)
KNELPERF_SNAP_CGROUPS="qemu:/sys/fs/cgroup/qemu.slice lxc:/sys/fs/cgroup/lxc"
KNELPERF_SNAP_IFACES="${ifaces# }"
KNELPERF_CG_GLOB="/sys/fs/cgroup/qemu.slice/*.scope /sys/fs/cgroup/lxc/*"
KNELPERF_TEXTFILE_DIR=/var/lib/knelperf/textfile
KNELPERF_PERF_LOGDIR=/var/log/knelperf-perf
KNELPERF_IOTOP_LOGDIR=/var/log/knelperf-iotop
EOF
say "conf written (ifaces: ${ifaces# })"

# 4) timers on + first tick of each collector
systemctl daemon-reload
for u in knelperf-perfsnap knelperf-psi-textfile knelperf-iotop-snap; do
  systemctl enable --now "$u.timer" >/dev/null 2>&1
done
/usr/local/sbin/knelperf-perfsnap.sh
/usr/local/sbin/knelperf-psi-textfile.sh
/usr/local/sbin/knelperf-iotop-snap.sh
say "timers enabled; first ticks logged"

# 5) evidence
say "--- timers ---"
systemctl list-timers 'knelperf-*' --no-pager | sed -n '1,5p'
say "--- snap.log tail ---"
tail -1 /var/log/knelperf-perf/snap.log
say "--- textfile ---"
head -3 /var/lib/knelperf/textfile/knelperf_pressure.prom 2>/dev/null || true
say "done."