From 2d2aa3ef388b663c9b9e9aaefd0e03528dac23f5 Mon Sep 17 00:00:00 2001 From: ic-builder Date: Sun, 6 Sep 2026 19:07:22 -0500 Subject: [PATCH] Toolbox: PVE9 apparmor via lxc.conf, runtime tracefs/debugfs mounts (#826) - PVE 9 dropped 'pct set --raw.lxc': lxc.apparmor.profile goes straight into /etc/pve/lxc/.conf and PVE::LXC honors it over the generated profile (verified: /proc/self/attr/current = unconfined) - pct skips virtual-fs bind sources (mp /proc,/sys/kernel/* landed empty): keep cgroupfs + /var/log binds, wrapper mounts tracefs/debugfs at start - bpftrace PROVEN against host kernel (2689 tracepoints, live syscall counts showing host pvestatd/vgs); bcc-tools -> bpfcc-tools (bookworm) - smartctl on SAT devices needs -d sat (recipes updated) Redmine: https://projects.knownelement.com/issues/826 --- monitoring/README.md | 20 +++++----- monitoring/toolbox/knelperf-toolbox | 23 ++++++++---- monitoring/toolbox/lxc-toolbox.sh | 57 +++++++++++++++-------------- monitoring/toolbox/toolbox-pkgs.txt | 2 +- 4 files changed, 57 insertions(+), 45 deletions(-) mode change 100644 => 100755 monitoring/toolbox/knelperf-toolbox diff --git a/monitoring/README.md b/monitoring/README.md index 5dd6842..ec46ad0 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -39,17 +39,19 @@ installer (`monitoring/node-install.sh`) is idempotent — safe to re-run. ## The toolbox container (vmid 950, `knelperf-toolbox`) PVE-native LXC on each node — Debian 12, privileged, apparmor -unconfined, bind mounts for host `/proc`, `/sys/fs/cgroup`, debugfs, -tracingfs, `/var/log` (ro) and up to 8 block devices. **Stopped at -rest; `onboot=0`.** Tools: bpftrace, BCC, sysstat, iotop, sysbench, -fio, iperf3, stress-ng, perf, smartmontools, nvme-cli, hwloc… +unconfined, bind mounts for host cgroupfs and `/var/log` (ro), up to 8 +block devices, and runtime-mounted tracefs/debugfs (pct skips +virtual-fs bind sources — the wrapper mounts them on start). +**Stopped at rest; `onboot=0`.** Tools: bpftrace, BCC, sysstat, iotop, +sysbench, fio, iperf3, stress-ng, perf, smartmontools, nvme-cli, hwloc… ```bash -knelperf-toolbox run 'iostat -x 1 5' # starts LXC if stopped -knelperf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"' -knelperf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06' # host sar, offline -knelperf-toolbox shell # interactive -knelperf-toolbox stop # back to zero footprint +knelerf-toolbox run 'iostat -x 1 5' # starts LXC if stopped +knelerf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"' +knelerf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06' # host sar, offline +knelerf-toolbox run 'smartctl -d sat -H /dev/sda' # SAT devices need -d sat; NVMe doesn't +knelerf-toolbox shell # interactive +knelerf-toolbox stop # back to zero footprint ``` Docker-capable hosts use the same manifest as an image: diff --git a/monitoring/toolbox/knelperf-toolbox b/monitoring/toolbox/knelperf-toolbox old mode 100644 new mode 100755 index a85fd4b..68779c4 --- a/monitoring/toolbox/knelperf-toolbox +++ b/monitoring/toolbox/knelperf-toolbox @@ -1,5 +1,5 @@ #!/bin/sh -# knelerf-toolbox — front door to the on-demand perf toolbox LXC on this node. +# knelperf-toolbox — front door to the on-demand perf toolbox LXC on this node. # usage: knelperf-toolbox run '' (starts the LXC if stopped, runs, leaves it up) # knelperf-toolbox shell (interactive) # knelperf-toolbox start|stop|status @@ -8,13 +8,20 @@ VMID=${KNELPERF_TOOLBOX_VMID:-950} ensure_running() { - pct status "$VMID" 2>/dev/null | grep -q running && return 0 - pct start "$VMID" - i=0 - until pct exec "$VMID" -- true 2>/dev/null; do - i=$((i + 1)); [ $i -gt 45 ] && { echo "boot timeout" >&2; return 1; } - sleep 2 - done + if ! pct status "$VMID" 2>/dev/null | grep -q running; then + pct start "$VMID" + i=0 + until pct exec "$VMID" -- true 2>/dev/null; do + i=$((i + 1)); [ $i -gt 45 ] && { echo "boot timeout" >&2; return 1; } + sleep 2 + done + fi + # host-kernel analysis mounts — pct skips virtual-fs bind sources, so + # tracefs/debugfs are mounted at runtime (container is privileged + + # apparmor unconfined precisely for this). + pct exec "$VMID" -- sh -c ' + mountpoint -q /sys/kernel/tracing || mount -t tracefs none /sys/kernel/tracing 2>/dev/null + mountpoint -q /sys/kernel/debug || mount -t debugfs none /sys/kernel/debug 2>/dev/null' } case "${1:-help}" in diff --git a/monitoring/toolbox/lxc-toolbox.sh b/monitoring/toolbox/lxc-toolbox.sh index e2abf39..07e2341 100644 --- a/monitoring/toolbox/lxc-toolbox.sh +++ b/monitoring/toolbox/lxc-toolbox.sh @@ -3,9 +3,10 @@ # STOPS to zero footprint; started only when a human/agent runs a workload. # lxc-toolbox.sh [--vmid N] {--create|--upgrade|--remove|--facts} # Why privileged + apparmor unconfined: bpftrace/perf need host kernel BPF, -# tracefs and host /proc. Bind mounts: /host/proc, /host/cgroup, host debugfs -# + tracing, /host/var/log (ro) for offline sar analysis, block devs for -# smart/nvme. Package list lives in toolbox-pkgs.txt next to this script. +# tracefs and debugfs (pct skips virtual-fs bind sources, so the knelperf-toolbox +# wrapper mounts tracefs/debugfs at runtime). Binds: /host/cgroup, /host/var/log +# (ro, offline sar analysis), block devs for smart/nvme. Package list lives in +# toolbox-pkgs.txt next to this script. # Docs: https://community.turnsys.com/t/328 set -euo pipefail [ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; } @@ -31,31 +32,33 @@ case "${1:-}" in esac if [ "$action" = --create ]; then - if pct status "$VMID" >/dev/null 2>&1; then - echo "vmid $VMID already present — use --upgrade" >&2; exit 1 + if ! pct status "$VMID" >/dev/null 2>&1; then + pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL" + # block-device passthrough for smartctl/nvme (first 8 devices) + dev_args=() + for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do + [ -b "$d" ] || continue + [ "${#dev_args[@]}" -ge 16 ] && break + dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d") + done + pct create "$VMID" "local:vztmpl/$TPL" \ + --hostname knelperf-toolbox \ + --ostype debian \ + --cores 2 --memory 2048 --swap 512 \ + --rootfs local-lvm:8 \ + --net0 name=eth0,bridge=vmbr0,ip=dhcp \ + --unprivileged 0 --onboot 0 \ + --mp3 /sys/fs/cgroup,mp=/host/cgroup \ + --mp4 /var/log,mp=/host/var/log,ro=1 \ + "${dev_args[@]}" + fi + # apparmor unconfined for host-kernel BPF/tracing access. PVE 9 dropped + # `pct set --raw.lxc`; lxc.* keys go straight into the config file and + # PVE::LXC honors an explicit profile over its generated one. + if ! grep -q '^lxc.apparmor.profile:' /etc/pve/lxc/"$VMID".conf; then + echo 'lxc.apparmor.profile: unconfined' >> /etc/pve/lxc/"$VMID".conf + pct status "$VMID" 2>/dev/null | grep -q running && pct reboot "$VMID" fi - pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL" - # block-device passthrough for smartctl/nvme (first 8 devices) - dev_args=() - for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do - [ -b "$d" ] || continue - [ "${#dev_args[@]}" -ge 16 ] && break - dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d") - done - pct create "$VMID" "local:vztmpl/$TPL" \ - --hostname knelperf-toolbox \ - --ostype debian \ - --cores 2 --memory 2048 --swap 512 \ - --rootfs local-lvm:8 \ - --net0 name=eth0,bridge=vmbr0,ip=dhcp \ - --unprivileged 0 --onboot 0 \ - --mp0 /proc,mp=/host/proc \ - --mp1 /sys/kernel/debug,mp=/sys/kernel/debug \ - --mp2 /sys/kernel/tracing,mp=/sys/kernel/tracing \ - --mp3 /sys/fs/cgroup,mp=/host/cgroup \ - --mp4 /var/log,mp=/host/var/log,ro=1 \ - "${dev_args[@]}" - pct set "$VMID" --raw.lxc 'lxc.apparmor.profile=unconfined' fi # --create continues here; --upgrade rejoins an existing container diff --git a/monitoring/toolbox/toolbox-pkgs.txt b/monitoring/toolbox/toolbox-pkgs.txt index 68f0cd6..014dded 100644 --- a/monitoring/toolbox/toolbox-pkgs.txt +++ b/monitoring/toolbox/toolbox-pkgs.txt @@ -2,7 +2,7 @@ # Docker image variant. One package (or apt argument) per line; # = comment. # Docs: https://community.turnsys.com/t/328 bpftrace -bcc-tools +bpfcc-tools sysstat iotop sysbench