ic-builder
3680f1dba1
ci / audit (push) Successful in 26s
- monitoring/node-install.sh: idempotent per-node installer (sar HISTORY=28, three oneshot timers, per-node conf with qemu.slice/lxc cgroup globs) - monitoring/collectors + systemd units: iotop-snap 15min, perfsnap 60s, psi-textfile 15s (now sources /etc/knelperf/perfsnap.conf; multi-glob) - monitoring/toolbox: pct LXC (vmid 950, privileged, apparmor unconfined, stopped at rest) + knelperf-toolbox wrapper + Dockerfile variant - monitoring/deploy-fleet.sh: workstation-side rollout driver - piloted on pfv-tsys1: per-VM PSI gauges verified live Redmine: https://projects.knownelement.com/issues/826
74 lines
3.4 KiB
Bash
74 lines
3.4 KiB
Bash
#!/usr/bin/env bash
|
|
# KNELPerf node monitoring installer — sysstat tuning + collectors + timers.
|
|
# Idempotent; run as ROOT on a PVE node from a repo checkout:
|
|
# bash monitoring/node-install.sh
|
|
# Surface (all details in monitoring/README.md):
|
|
# - /etc/sysstat/sysstat HISTORY 7 -> 28, COMPRESSAFTER 10 -> 31
|
|
# - /usr/local/sbin/knelperf-{perfsnap,psi-textfile,iotop-snap}.sh
|
|
# - /etc/systemd/system/knelperf-*.{service,timer} (enabled timers)
|
|
# - /etc/knelperf/perfsnap.conf per-node ifaces/cgroup globs
|
|
# - logs under /var/log/knelperf-{perf,iotop}, textfiles /var/lib/knelperf/textfile
|
|
# No reboots, no package installs (PVE ships sysstat + iotop already).
|
|
# Docs: https://community.turnsys.com/t/328
|
|
set -euo pipefail
|
|
[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
|
|
here=$(cd "$(dirname "$0")/.." && pwd)
|
|
say() { echo "[knelperf-install] $*"; }
|
|
|
|
# 1) sysstat: keep 28 days of sar history (two-week program + re-baseline window)
|
|
if [ -f /etc/sysstat/sysstat ]; then
|
|
sed -i -E 's/^HISTORY=.*/HISTORY=28/; s/^COMPRESSAFTER=.*/COMPRESSAFTER=31/' /etc/sysstat/sysstat
|
|
sed -i -E 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat
|
|
systemctl restart sysstat 2>/dev/null || true
|
|
/usr/libexec/sysstat/debian-sa1 1 1 2>/dev/null || true
|
|
today=/var/log/sysstat/sa$(date +%d)
|
|
if [ -f "$today" ]; then say "sar ok: $today present"; else say "sar WARN: $today missing (check /etc/cron.d/sysstat)"; fi
|
|
else
|
|
say "WARN: no /etc/sysstat/sysstat on this host — sar tuning skipped"
|
|
fi
|
|
|
|
# 2) collectors + units
|
|
install -m 0755 "$here/scheduler/collectors/knelperf-perfsnap.sh" /usr/local/sbin/knelperf-perfsnap.sh
|
|
install -m 0755 "$here/scheduler/collectors/knelperf-psi-textfile.sh" /usr/local/sbin/knelperf-psi-textfile.sh
|
|
install -m 0755 "$here/monitoring/collectors/knelperf-iotop-snap.sh" /usr/local/sbin/knelperf-iotop-snap.sh
|
|
for u in perfsnap psi-textfile iotop-snap; do
|
|
install -m 0644 "$here/monitoring/systemd/knelperf-$u.service" /etc/systemd/system/knelperf-$u.service
|
|
install -m 0644 "$here/monitoring/systemd/knelperf-$u.timer" /etc/systemd/system/knelperf-$u.timer
|
|
done
|
|
|
|
# 3) per-node conf: watch PVE guest cgroups + real bridges/tailscale
|
|
ifaces=""
|
|
for i in vmbr0 datanet tailscale0; do
|
|
ip link show "$i" >/dev/null 2>&1 && ifaces="$ifaces $i"
|
|
done
|
|
mkdir -p /etc/knelperf
|
|
cat > /etc/knelperf/perfsnap.conf <<EOF
|
|
# /etc/knelperf/perfsnap.conf — generated by KNELPerf node-install.sh $(date -I)
|
|
KNELPERF_SNAP_CGROUPS="qemu:/sys/fs/cgroup/qemu.slice lxc:/sys/fs/cgroup/lxc"
|
|
KNELPERF_SNAP_IFACES="${ifaces# }"
|
|
KNELPERF_CG_GLOB="/sys/fs/cgroup/qemu.slice/*.scope /sys/fs/cgroup/lxc/*"
|
|
KNELPERF_TEXTFILE_DIR=/var/lib/knelperf/textfile
|
|
KNELPERF_PERF_LOGDIR=/var/log/knelperf-perf
|
|
KNELPERF_IOTOP_LOGDIR=/var/log/knelperf-iotop
|
|
EOF
|
|
say "conf written (ifaces: ${ifaces# })"
|
|
|
|
# 4) timers on + first tick of each collector
|
|
systemctl daemon-reload
|
|
for u in knelperf-perfsnap knelperf-psi-textfile knelperf-iotop-snap; do
|
|
systemctl enable --now "$u.timer" >/dev/null 2>&1
|
|
done
|
|
/usr/local/sbin/knelperf-perfsnap.sh
|
|
/usr/local/sbin/knelperf-psi-textfile.sh
|
|
/usr/local/sbin/knelperf-iotop-snap.sh
|
|
say "timers enabled; first ticks logged"
|
|
|
|
# 5) evidence
|
|
say "--- timers ---"
|
|
systemctl list-timers 'knelperf-*' --no-pager | sed -n '1,5p'
|
|
say "--- snap.log tail ---"
|
|
tail -1 /var/log/knelperf-perf/snap.log
|
|
say "--- textfile ---"
|
|
head -3 /var/lib/knelperf/textfile/knelperf_pressure.prom 2>/dev/null || true
|
|
say "done."
|