Fleet monitoring: sysstat 28d + perfsnap/PSI/iotop timers + on-demand toolbox container (#826)

- monitoring/node-install.sh: idempotent per-node installer (sar HISTORY=28, three oneshot timers, per-node conf with qemu.slice/lxc cgroup globs) - monitoring/collectors + systemd units: iotop-snap 15min, perfsnap 60s, psi-textfile 15s (now sources /etc/knelperf/perfsnap.conf; multi-glob) - monitoring/toolbox: pct LXC (vmid 950, privileged, apparmor unconfined, stopped at rest) + knelperf-toolbox wrapper + Dockerfile variant - monitoring/deploy-fleet.sh: workstation-side rollout driver - piloted on pfv-tsys1: per-VM PSI gauges verified live Redmine: https://projects.knownelement.com/issues/826
2026-09-06 18:50:07 -05:00
parent f1729868fa
commit 3680f1dba1
16 changed files with 407 additions and 1 deletions
@@ -0,0 +1,16 @@
+# knelperf-toolbox — Docker image variant for docker-capable hosts
+# (workstation, VMs). On PVE nodes the LXC variant (lxc-toolbox.sh) is
+# canonical. Package list is shared: toolbox-pkgs.txt.
+# Build:  docker build -t git.knownelement.com/knel/knelperf-toolbox:v1 \
+#           -f monitoring/toolbox/Dockerfile monitoring/toolbox/
+# Docs: https://community.turnsys.com/t/328
+FROM debian:12.9-slim
+
+COPY toolbox-pkgs.txt /tmp/toolbox-pkgs.txt
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      $(sed 's/#.*//' /tmp/toolbox-pkgs.txt | tr '\n' ' ') && \
+    rm -rf /var/lib/apt/lists/* /tmp/toolbox-pkgs.txt
+
+WORKDIR /work
+ENTRYPOINT ["/bin/bash"]
@@ -0,0 +1,27 @@
+#!/bin/sh
+# knelerf-toolbox — front door to the on-demand perf toolbox LXC on this node.
+# usage: knelperf-toolbox run '<cmd>'   (starts the LXC if stopped, runs, leaves it up)
+#        knelperf-toolbox shell         (interactive)
+#        knelperf-toolbox start|stop|status
+# The LXC (vmid 950) is STOPPED at rest — zero footprint; stop it when done.
+# Docs: https://community.turnsys.com/t/328
+VMID=${KNELPERF_TOOLBOX_VMID:-950}
+
+ensure_running() {
+  pct status "$VMID" 2>/dev/null | grep -q running && return 0
+  pct start "$VMID"
+  i=0
+  until pct exec "$VMID" -- true 2>/dev/null; do
+    i=$((i + 1)); [ $i -gt 45 ] && { echo "boot timeout" >&2; return 1; }
+    sleep 2
+  done
+}
+
+case "${1:-help}" in
+  run)    shift; ensure_running && pct exec "$VMID" -- sh -c "$*" ;;
+  shell)  ensure_running && pct enter "$VMID" ;;
+  start)  ensure_running ;;
+  stop)   pct shutdown "$VMID" --timeout 60 2>/dev/null || pct stop "$VMID" ;;
+  status) pct status "$VMID" 2>/dev/null || echo "vmid $VMID not present" ;;
+  *)      sed -n '2,6p' "$0" | sed 's/^# \?//' ;;
+esac
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# knelperf-toolbox LXC — the on-demand perf analysis container for a PVE node.
+# STOPS to zero footprint; started only when a human/agent runs a workload.
+#   lxc-toolbox.sh [--vmid N] {--create|--upgrade|--remove|--facts}
+# Why privileged + apparmor unconfined: bpftrace/perf need host kernel BPF,
+# tracefs and host /proc. Bind mounts: /host/proc, /host/cgroup, host debugfs
+# + tracing, /host/var/log (ro) for offline sar analysis, block devs for
+# smart/nvme. Package list lives in toolbox-pkgs.txt next to this script.
+# Docs: https://community.turnsys.com/t/328
+set -euo pipefail
+[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
+here=$(cd "$(dirname "$0")" && pwd)
+VMID=950
+TPL=debian-12-standard_12.12-1_amd64.tar.zst
+case "${1:-}" in
+  --vmid) VMID=$2; shift 2 ;;
+esac
+case "${1:-}" in
+  --facts)
+    pct config "$VMID" 2>/dev/null || echo "vmid $VMID not present"
+    pct status "$VMID" 2>/dev/null || true
+    exit 0 ;;
+  --remove)
+    pct shutdown "$VMID" --timeout 60 2>/dev/null || pct stop "$VMID" 2>/dev/null || true
+    pct destroy "$VMID" --purge 1
+    rm -f /usr/local/sbin/knelperf-toolbox
+    echo "toolbox $VMID removed"
+    exit 0 ;;
+  --create|--upgrade) action=$1 ;;
+  *) sed -n 's/^# \?//p' "$0" | sed -n '2,5p'; exit 1 ;;
+esac
+
+if [ "$action" = --create ]; then
+  if pct status "$VMID" >/dev/null 2>&1; then
+    echo "vmid $VMID already present — use --upgrade" >&2; exit 1
+  fi
+  pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL"
+  # block-device passthrough for smartctl/nvme (first 8 devices)
+  dev_args=()
+  for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do
+    [ -b "$d" ] || continue
+    [ "${#dev_args[@]}" -ge 16 ] && break
+    dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d")
+  done
+  pct create "$VMID" "local:vztmpl/$TPL" \
+    --hostname knelperf-toolbox \
+    --ostype debian \
+    --cores 2 --memory 2048 --swap 512 \
+    --rootfs local-lvm:8 \
+    --net0 name=eth0,bridge=vmbr0,ip=dhcp \
+    --unprivileged 0 --onboot 0 \
+    --mp0 /proc,mp=/host/proc \
+    --mp1 /sys/kernel/debug,mp=/sys/kernel/debug \
+    --mp2 /sys/kernel/tracing,mp=/sys/kernel/tracing \
+    --mp3 /sys/fs/cgroup,mp=/host/cgroup \
+    --mp4 /var/log,mp=/host/var/log,ro=1 \
+    "${dev_args[@]}"
+  pct set "$VMID" --raw.lxc 'lxc.apparmor.profile=unconfined'
+fi
+
+# --create continues here; --upgrade rejoins an existing container
+was_running=1
+pct status "$VMID" 2>/dev/null | grep -q running || { was_running=0; pct start "$VMID"; }
+i=0
+until pct exec "$VMID" -- true 2>/dev/null; do
+  i=$((i + 1)); [ $i -gt 60 ] && { echo "boot timeout" >&2; exit 1; }
+  sleep 2
+done
+pkgs=$(sed 's/#.*//' "$here/toolbox-pkgs.txt" | tr '\n' ' ')
+pct exec "$VMID" -- bash -c \
+  "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install $pkgs"
+install -m 0755 "$here/knelperf-toolbox" /usr/local/sbin/knelperf-toolbox
+[ "$was_running" = 0 ] && pct shutdown "$VMID" --timeout 60
+echo "toolbox $VMID $action done: knelperf-toolbox wrapper installed"
+pct status "$VMID"
@@ -0,0 +1,37 @@
+# knelperf-toolbox package manifest — shared by the LXC bootstrap and the
+# Docker image variant. One package (or apt argument) per line; # = comment.
+# Docs: https://community.turnsys.com/t/328
+bpftrace
+bcc-tools
+sysstat
+iotop
+sysbench
+fio
+iperf3
+stress-ng
+linux-perf
+jq
+bc
+curl
+procps
+psmisc
+util-linux
+pciutils
+usbutils
+ethtool
+net-tools
+bind9-dnsutils
+smartmontools
+nvme-cli
+hdparm
+lm-sensors
+numactl
+hwloc-nox
+strace
+ltrace
+file
+rsync
+openssh-client
+vim-tiny
+less
+tmux