ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111

Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
2026-08-31 16:46:13 -05:00
commit cba2a64d67
43 changed files with 3137 additions and 0 deletions
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# UKRRS host audit for pfv-tsys5 (Proxmox VE). READ-ONLY, safe to run anytime.
+# Usage (from a machine that can ssh):
+#   ssh root@pfv-tsys5.knel.net 'bash -s' < host-audit.sh > host-audit.out
+set -uo pipefail
+hdr() { echo; echo "########## $* ##########"; }
+
+hdr "node"; hostname; uptime; pveversion 2>/dev/null || echo not-pve
+
+hdr "dmi system+baseboard"
+dmidecode -t system 2>/dev/null | grep -E 'Manufacturer|Product Name|Version|Wake-up'
+dmidecode -t baseboard 2>/dev/null | grep -E 'Manufacturer|Product Name|Version'
+
+hdr "processors"
+dmidecode -t processor 2>/dev/null | grep -E 'Socket Designation|Version|Max Speed|Current Speed|^Status|Core Count|Thread Count|Upgrade:'
+lscpu 2>/dev/null | grep -E 'Model name|^CPU\(s\)|Thread|Core|Socket|NUMA|MHz'
+
+hdr "cpu frequency policy"
+for p in scaling_driver scaling_governor scaling_max_freq scaling_min_freq; do
+  v=$(cat /sys/devices/system/cpu/cpu0/cpufreq/$p 2>/dev/null) && echo "$p = $v"
+done
+
+hdr "memory"
+free -h; grep MemTotal /proc/meminfo
+dmidecode -t memory 2>/dev/null | grep -E '^\s*(Size|Speed|Type:|Locator|Form Factor)' | grep -v 'No Module'
+
+hdr "storage: block devices"
+lsblk -o NAME,SIZE,TYPE,ROTA,FSTYPE,MOUNTPOINTS,MODEL 2>/dev/null
+
+hdr "storage: pve"
+pvesm status 2>/dev/null
+
+hdr "storage: zfs"
+if command -v zpool >/dev/null 2>&1; then
+  zpool status 2>/dev/null
+  zfs list -o name,used,avail,recordsize,compression 2>/dev/null | head -40
+  grep -E '^(size|c_max|c_min) ' /proc/spl/kstat/zfs/arcstats 2>/dev/null
+else
+  echo no-zfs
+fi
+
+hdr "smart health"
+if command -v smartctl >/dev/null 2>&1; then
+  for d in $(ls /dev | grep -E '^(sd[a-z]|nvme[0-9]+)$'); do
+    echo "--- /dev/$d"
+    smartctl -H /dev/$d 2>/dev/null | grep -iE 'result|overall|health'
+    smartctl -i /dev/$d 2>/dev/null | grep -iE 'model|firmware|serial' | head -3
+  done
+else
+  echo no-smartctl
+fi
+
+hdr "kernel + cmdline"
+uname -a
+cat /proc/cmdline
+[ -f /etc/kernel/cmdline ] && cat /etc/kernel/cmdline
+proxmox-boot-tool status 2>/dev/null | head -20
+
+hdr "iommu/dmar"
+dmesg 2>/dev/null | grep -iE 'iommu|dmar|vtd' | head -25
+
+hdr "lspci (all)"
+lspci -nn 2>/dev/null
+
+hdr "iommu groups"
+if [ -d /sys/kernel/iommu_groups ]; then
+  for g in /sys/kernel/iommu_groups/*; do
+    [ -d "$g" ] || continue
+    echo "group ${g##*/}:"
+    for d in "$g"/devices/*; do
+      printf '   '; lspci -s "${d##*/}" 2>/dev/null
+    done
+  done
+else
+  echo "no iommu groups (intel_iommu not enabled?)"
+fi
+
+hdr "pci link speeds (gpu/usb/nvme candidates)"
+for d in /sys/bus/pci/devices/*; do
+  s=$(cat "$d/current_link_speed" 2>/dev/null); w=$(cat "$d/current_link_width" 2>/dev/null)
+  [ -n "$s" ] && echo "${d##*/} $s x$w"
+done
+
+hdr "network"
+ip -br a 2>/dev/null || cat /proc/net/dev
+bridge link 2>/dev/null
+
+hdr "qemu vms"
+qm list 2>/dev/null
+
+hdr "vm configs"
+for vm in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
+  echo "--- VM $vm"; qm config "$vm" 2>/dev/null
+done
+
+hdr "lxc containers"
+pct list 2>/dev/null
+for ct in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
+  echo "--- CT $ct"; pct config "$ct" 2>/dev/null
+done
+
+hdr "sensors"
+sensors 2>/dev/null || echo no-lm-sensors
+
+hdr "guest agent check"
+qm agent 100 ping 2>/dev/null && echo "agent ok on vm 100" || true
+
+echo
+echo "AUDIT DONE"