Toolbox: PVE9 apparmor via lxc.conf, runtime tracefs/debugfs mounts (#826)

- PVE 9 dropped 'pct set --raw.lxc': lxc.apparmor.profile goes straight
  into /etc/pve/lxc/<vmid>.conf and PVE::LXC honors it over the generated
  profile (verified: /proc/self/attr/current = unconfined)
- pct skips virtual-fs bind sources (mp /proc,/sys/kernel/* landed empty):
  keep cgroupfs + /var/log binds, wrapper mounts tracefs/debugfs at start
- bpftrace PROVEN against host kernel (2689 tracepoints, live syscall
  counts showing host pvestatd/vgs); bcc-tools -> bpfcc-tools (bookworm)
- smartctl on SAT devices needs -d sat (recipes updated)

Redmine: https://projects.knownelement.com/issues/826

monitoring/README.md

@@ -39,17 +39,19 @@ installer (`monitoring/node-install.sh`) is idempotent — safe to re-run.
 ## The toolbox container (vmid 950, `knelperf-toolbox`)
 
 PVE-native LXC on each node — Debian 12, privileged, apparmor
-unconfined, bind mounts for host `/proc`, `/sys/fs/cgroup`, debugfs,
+unconfined, bind mounts for host cgroupfs and `/var/log` (ro), up to 8
-tracingfs, `/var/log` (ro) and up to 8 block devices. **Stopped at
+block devices, and runtime-mounted tracefs/debugfs (pct skips
-rest; `onboot=0`.** Tools: bpftrace, BCC, sysstat, iotop, sysbench,
+virtual-fs bind sources — the wrapper mounts them on start).
-fio, iperf3, stress-ng, perf, smartmontools, nvme-cli, hwloc…
+**Stopped at rest; `onboot=0`.** Tools: bpftrace, BCC, sysstat, iotop,
+sysbench, fio, iperf3, stress-ng, perf, smartmontools, nvme-cli, hwloc…
 
 ```bash
-knelperf-toolbox run 'iostat -x 1 5'          # starts LXC if stopped
+knelerf-toolbox run 'iostat -x 1 5'          # starts LXC if stopped
-knelperf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"'
+knelerf-toolbox run 'bpftrace -e "tracepoint:block:block_rq_issue { @[comm] = count(); }" -c "sleep 5"'
-knelperf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06'   # host sar, offline
+knelerf-toolbox run 'sar -A -f /host/var/log/sysstat/sa06'   # host sar, offline
-knelperf-toolbox shell                          # interactive
+knelerf-toolbox run 'smartctl -d sat -H /dev/sda'            # SAT devices need -d sat; NVMe doesn't
-knelperf-toolbox stop                           # back to zero footprint
+knelerf-toolbox shell                          # interactive
+knelerf-toolbox stop                           # back to zero footprint
 ```
 
 Docker-capable hosts use the same manifest as an image:

monitoring/toolbox/knelperf-toolbox

@@ -1,5 +1,5 @@
 #!/bin/sh
-# knelerf-toolbox — front door to the on-demand perf toolbox LXC on this node.
+# knelperf-toolbox — front door to the on-demand perf toolbox LXC on this node.
 # usage: knelperf-toolbox run '<cmd>'   (starts the LXC if stopped, runs, leaves it up)
 #        knelperf-toolbox shell         (interactive)
 #        knelperf-toolbox start|stop|status
@@ -8,13 +8,20 @@
 VMID=${KNELPERF_TOOLBOX_VMID:-950}
 
 ensure_running() {
-  pct status "$VMID" 2>/dev/null | grep -q running && return 0
+  if ! pct status "$VMID" 2>/dev/null | grep -q running; then
-  pct start "$VMID"
+    pct start "$VMID"
-  i=0
+    i=0
-  until pct exec "$VMID" -- true 2>/dev/null; do
+    until pct exec "$VMID" -- true 2>/dev/null; do
-    i=$((i + 1)); [ $i -gt 45 ] && { echo "boot timeout" >&2; return 1; }
+      i=$((i + 1)); [ $i -gt 45 ] && { echo "boot timeout" >&2; return 1; }
-    sleep 2
+      sleep 2
-  done
+    done
+  fi
+  # host-kernel analysis mounts — pct skips virtual-fs bind sources, so
+  # tracefs/debugfs are mounted at runtime (container is privileged +
+  # apparmor unconfined precisely for this).
+  pct exec "$VMID" -- sh -c '
+    mountpoint -q /sys/kernel/tracing || mount -t tracefs none /sys/kernel/tracing 2>/dev/null
+    mountpoint -q /sys/kernel/debug || mount -t debugfs none /sys/kernel/debug 2>/dev/null'
 }
 
 case "${1:-help}" in

monitoring/toolbox/lxc-toolbox.sh

@@ -3,9 +3,10 @@
 # STOPS to zero footprint; started only when a human/agent runs a workload.
 #   lxc-toolbox.sh [--vmid N] {--create|--upgrade|--remove|--facts}
 # Why privileged + apparmor unconfined: bpftrace/perf need host kernel BPF,
-# tracefs and host /proc. Bind mounts: /host/proc, /host/cgroup, host debugfs
+# tracefs and debugfs (pct skips virtual-fs bind sources, so the knelperf-toolbox
-# + tracing, /host/var/log (ro) for offline sar analysis, block devs for
+# wrapper mounts tracefs/debugfs at runtime). Binds: /host/cgroup, /host/var/log
-# smart/nvme. Package list lives in toolbox-pkgs.txt next to this script.
+# (ro, offline sar analysis), block devs for smart/nvme. Package list lives in
+# toolbox-pkgs.txt next to this script.
 # Docs: https://community.turnsys.com/t/328
 set -euo pipefail
 [ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
@@ -31,31 +32,33 @@ case "${1:-}" in
 esac
 
 if [ "$action" = --create ]; then
-  if pct status "$VMID" >/dev/null 2>&1; then
+  if ! pct status "$VMID" >/dev/null 2>&1; then
-    echo "vmid $VMID already present — use --upgrade" >&2; exit 1
+    pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL"
+    # block-device passthrough for smartctl/nvme (first 8 devices)
+    dev_args=()
+    for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do
+      [ -b "$d" ] || continue
+      [ "${#dev_args[@]}" -ge 16 ] && break
+      dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d")
+    done
+    pct create "$VMID" "local:vztmpl/$TPL" \
+      --hostname knelperf-toolbox \
+      --ostype debian \
+      --cores 2 --memory 2048 --swap 512 \
+      --rootfs local-lvm:8 \
+      --net0 name=eth0,bridge=vmbr0,ip=dhcp \
+      --unprivileged 0 --onboot 0 \
+      --mp3 /sys/fs/cgroup,mp=/host/cgroup \
+      --mp4 /var/log,mp=/host/var/log,ro=1 \
+      "${dev_args[@]}"
+  fi
+  # apparmor unconfined for host-kernel BPF/tracing access. PVE 9 dropped
+  # `pct set --raw.lxc`; lxc.* keys go straight into the config file and
+  # PVE::LXC honors an explicit profile over its generated one.
+  if ! grep -q '^lxc.apparmor.profile:' /etc/pve/lxc/"$VMID".conf; then
+    echo 'lxc.apparmor.profile: unconfined' >> /etc/pve/lxc/"$VMID".conf
+    pct status "$VMID" 2>/dev/null | grep -q running && pct reboot "$VMID"
   fi
-  pveam list local 2>/dev/null | grep -q "$TPL" || pveam download local "$TPL"
-  # block-device passthrough for smartctl/nvme (first 8 devices)
-  dev_args=()
-  for d in /dev/nvme[0-9]n1 /dev/sd[a-z]; do
-    [ -b "$d" ] || continue
-    [ "${#dev_args[@]}" -ge 16 ] && break
-    dev_args+=("--dev$(( ${#dev_args[@]} / 2 ))" "$d")
-  done
-  pct create "$VMID" "local:vztmpl/$TPL" \
-    --hostname knelperf-toolbox \
-    --ostype debian \
-    --cores 2 --memory 2048 --swap 512 \
-    --rootfs local-lvm:8 \
-    --net0 name=eth0,bridge=vmbr0,ip=dhcp \
-    --unprivileged 0 --onboot 0 \
-    --mp0 /proc,mp=/host/proc \
-    --mp1 /sys/kernel/debug,mp=/sys/kernel/debug \
-    --mp2 /sys/kernel/tracing,mp=/sys/kernel/tracing \
-    --mp3 /sys/fs/cgroup,mp=/host/cgroup \
-    --mp4 /var/log,mp=/host/var/log,ro=1 \
-    "${dev_args[@]}"
-  pct set "$VMID" --raw.lxc 'lxc.apparmor.profile=unconfined'
 fi
 
 # --create continues here; --upgrade rejoins an existing container

monitoring/toolbox/toolbox-pkgs.txt

@@ -2,7 +2,7 @@
 # Docker image variant. One package (or apt argument) per line; # = comment.
 # Docs: https://community.turnsys.com/t/328
 bpftrace
-bcc-tools
+bpfcc-tools
 sysstat
 iotop
 sysbench