Fleet monitoring: sysstat 28d + perfsnap/PSI/iotop timers + on-demand toolbox container (#826)

- monitoring/node-install.sh: idempotent per-node installer (sar HISTORY=28, three oneshot timers, per-node conf with qemu.slice/lxc cgroup globs) - monitoring/collectors + systemd units: iotop-snap 15min, perfsnap 60s, psi-textfile 15s (now sources /etc/knelperf/perfsnap.conf; multi-glob) - monitoring/toolbox: pct LXC (vmid 950, privileged, apparmor unconfined, stopped at rest) + knelperf-toolbox wrapper + Dockerfile variant - monitoring/deploy-fleet.sh: workstation-side rollout driver - piloted on pfv-tsys1: per-VM PSI gauges verified live Redmine: https://projects.knownelement.com/issues/826
2026-09-06 18:50:07 -05:00
parent f1729868fa
commit 3680f1dba1
16 changed files with 407 additions and 1 deletions
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# knelperf-toolbox LXC — the on-demand perf analysis container for a PVE node.
+# STOPS to zero footprint; started only when a human/agent runs a workload.
+#   lxc-toolbox.sh [--vmid N] {--create|--upgrade|--remove|--facts}
+# Why privileged + apparmor unconfined: bpftrace/perf need host kernel BPF,
+# tracefs and host /proc. Bind mounts: /host/proc, /host/cgroup, host debugfs
+# + tracing, /host/var/log (ro) for offline sar analysis, block devs for
+# smart/nvme. Package list lives in toolbox-pkgs.txt next to this script.
+# Docs: https://community.turnsys.com/t/328
+set -euo pipefail
+[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
+here=$(cd "$(dirname "$0")" && pwd)
+VMID=950
+TPL=debian-12-standard_12.12-1_amd64.tar.zst
+case "${1:-}" in
+  --vmid) VMID=$2; shift 2 ;;
+esac
+case "${1:-}" in
+  --facts)
+    pct config "$VMID" 2>/dev/null || echo "vmid $VMID not present"
+    pct status "$VMID" 2>/dev/null || true
+    exit 0 ;;
+  --remove)
+    pct shutdown "$VMID" --timeout 60 2>/dev/null || pct stop "$VMID" 2>/dev/null || true
+    pct destroy "$VMID" --purge 1
+    rm -f /usr/local/sbin/knelperf-toolbox
+    echo "toolbox $VMID removed"
+    exit 0 ;;
+  --create|--upgrade) action=$1 ;;
+  *) sed -n 's/^# \?//p' "$0" | sed -n '2,5p'; exit 1 ;;
+esac
+
+if [ "$action" = --create ]; then
+  if pct status "$VMID" >/dev/null 2>&1; then
+    echo "vmid $VMID already present — use --upgrade" >&2; exit 1
+  fi
+  pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL"
+  # block-device passthrough for smartctl/nvme (first 8 devices)
+  dev_args=()
+  for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do
+    [ -b "$d" ] || continue
+    [ "${#dev_args[@]}" -ge 16 ] && break
+    dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d")
+  done
+  pct create "$VMID" "local:vztmpl/$TPL" \
+    --hostname knelperf-toolbox \
+    --ostype debian \
+    --cores 2 --memory 2048 --swap 512 \
+    --rootfs local-lvm:8 \
+    --net0 name=eth0,bridge=vmbr0,ip=dhcp \
+    --unprivileged 0 --onboot 0 \
+    --mp0 /proc,mp=/host/proc \
+    --mp1 /sys/kernel/debug,mp=/sys/kernel/debug \
+    --mp2 /sys/kernel/tracing,mp=/sys/kernel/tracing \
+    --mp3 /sys/fs/cgroup,mp=/host/cgroup \
+    --mp4 /var/log,mp=/host/var/log,ro=1 \
+    "${dev_args[@]}"
+  pct set "$VMID" --raw.lxc 'lxc.apparmor.profile=unconfined'
+fi
+
+# --create continues here; --upgrade rejoins an existing container
+was_running=1
+pct status "$VMID" 2>/dev/null | grep -q running || { was_running=0; pct start "$VMID"; }
+i=0
+until pct exec "$VMID" -- true 2>/dev/null; do
+  i=$((i + 1)); [ $i -gt 60 ] && { echo "boot timeout" >&2; exit 1; }
+  sleep 2
+done
+pkgs=$(sed 's/#.*//' "$here/toolbox-pkgs.txt" | tr '\n' ' ')
+pct exec "$VMID" -- bash -c \
+  "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install $pkgs"
+install -m 0755 "$here/knelperf-toolbox" /usr/local/sbin/knelperf-toolbox
+[ "$was_running" = 0 ] && pct shutdown "$VMID" --timeout 60
+echo "toolbox $VMID $action done: knelperf-toolbox wrapper installed"
+pct status "$VMID"