commit cba2a64d67aff46be6f3f9204e825666e94c122c Author: reachableceo Date: Mon Aug 31 16:46:13 2026 -0500 ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111 Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0e494a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.crush/ diff --git a/1-guest-prep.sh b/1-guest-prep.sh new file mode 100755 index 0000000..735aaaa --- /dev/null +++ b/1-guest-prep.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# ONE-SHOT guest prep (idempotent; safe to re-run). Run ON ultix-streaming: +# sudo ~/optimize/1-guest-prep.sh +# 1. All staged configs (docker restarts once) + sddm autologin. +# 2. Human-tier slices for the two human accounts (KDE/CAD/video priority). +# NO storage operations (root growth is manual: GROW-ROOT-RUNBOOK.md). +set -euo pipefail +[ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; } +DIR=$(cd "$(dirname "$0")" && pwd) + +echo "== [1/3] all staged configs ==" +RUN=1 CONFIRM=1 bash "$DIR/staged/apply-guest.sh" all + +echo "== [2/3] human-tier slices (CPUWeight 600, no cpu pinning, 12G/16G) ==" +RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo 1001 12G 16G all human +RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo-offstage 1010 12G 16G all human + +echo "== [3/3] done ==" +echo "GUEST READY. Next, the host one-shot (safe to run from THIS VM; the" +echo "VM bounce is detached on the host and survives your ssh dying):" +echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/optimize/2-host-one-shot.sh" +echo +echo "Root growth to ~505G stays MANUAL: ~/optimize/GROW-ROOT-RUNBOOK.md" diff --git a/2-host-one-shot.sh b/2-host-one-shot.sh new file mode 100755 index 0000000..3f1f10b --- /dev/null +++ b/2-host-one-shot.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111. +# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh +# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh +# ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ... # optional +# +# 1. VM 5111's disk/net/onboot flags are written as PENDING config: they +# activate automatically at the VM's NEXT reboot (tonight, your last step +# today, Friday — whenever the VM naturally restarts). Nothing forces it. +# 2. CPU priority (live): VM 5111 = 9000, sectestbed/preprod fleet = 50. +# 3. Reboot wave: every OTHER running VM. +# Optional --with-vm-restart: also gracefully bounce VM 5111 at the end. +# THE HOST ITSELF IS NEVER REBOOTED (NFS server). NO DISK IS RESIZED +# (storage ops are manual-only: GROW-ROOT-RUNBOOK.md). +set -euo pipefail +GO=0; VMR=0; SKIPWAVE=0 +for a in "$@"; do + case "$a" in + --go) GO=1 ;; + --with-vm-restart) VMR=1 ;; + --skip-wave) SKIPWAVE=1 ;; + esac +done +VM=5111 +FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108" +run() { if [ "$GO" = 1 ]; then "$@"; else echo "DRY: $*"; fi; } +staget() { # pending-stage a qm set; tolerate refusal on a running VM + if [ "$GO" = 1 ]; then + qm set "$@" || echo " NOTE: not staged while running; use --with-vm-restart (or set while stopped) later" + else + echo "DRY(pending): qm set $*" + fi +} + +echo "== 1. VM $VM flags as PENDING config (live at its next reboot) ==" +staget "$VM" --onboot 1 --startup order=10,up=180 +staget "$VM" -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G +staget "$VM" -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G +staget "$VM" -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G +staget "$VM" --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4 +staget "$VM" --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2 +staget "$VM" --delete ide2 + +echo "== 2. CPU priority (live, reversible) ==" +run qm set "$VM" --cpuunits 9000 +for id in $FLEET; do run qm set "$id" --cpuunits 50; done + +echo "== 3. reboot wave: every other running VM ==" +if [ "$SKIPWAVE" = 1 ]; then + echo "skipped (--skip-wave)" +else +OTHERS=$(qm list | awk 'NR>1 && $3=="running" && $1!="'"$VM"'" {print $1}') +echo "targets: ${OTHERS//$'\n'/ }" +for id in $OTHERS; do + if [ "$GO" = 1 ]; then + qm reboot "$id" 2>/dev/null || { qm shutdown "$id" --timeout 120 || true; sleep 2; qm start "$id" 2>/dev/null || true; } + else + echo "DRY: reboot $id" + fi +done +fi + +if [ "$GO" = 1 ]; then + echo; echo "== pending queue for VM $VM (activates at its next restart) ==" + qm pending "$VM" 2>/dev/null || true +fi + +LOG=/var/log/ukrrs-vm5111-bounce.log +if [ "$VMR" = 1 ]; then + echo "== graceful bounce of VM $VM (detached; flags applied while STOPPED) ==" + qm status "$VM" + if [ "$GO" = 1 ]; then + setsid bash -c " + echo bounce-start \$(date -Is) + qm shutdown $VM --timeout 120 + qm wait $VM --timeout 180 || true + qm set $VM --onboot 1 --startup order=10,up=180 + qm set $VM -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G + qm set $VM -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G + qm set $VM -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G + qm set $VM --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4 + qm set $VM --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2 + qm set $VM --delete ide2 || true + qm start $VM + echo bounce-done \$(date -Is) + " "$LOG" 2>&1 & + echo "dispatched. Downtime ~3-4 min; it comes back on its own." + echo "Watch from anywhere: ssh root@pfv-tsys5.knel.net tail -f $LOG" + else + echo "DRY: would detach-bounce $VM, applying flags while stopped, log $LOG" + fi +else + echo + echo "VM $VM was NOT rebooted. Flags sit pending and go live at its next restart." +fi diff --git a/3-post-reboot-fixes.sh b/3-post-reboot-fixes.sh new file mode 100755 index 0000000..7eefdc5 --- /dev/null +++ b/3-post-reboot-fixes.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Post-reboot validation fallout (2026-08-31). Run with sudo, once: +# sudo ~/optimize/3-post-reboot-fixes.sh +# 1. Installs ethtool + enables virtio multiqueue (host side now offers +# 4 queues on ens18, 2 on ens19; the guest defaults to 1 until told). +# 2. Installs the gateway ensure-up unit (prod boot race on the tailscale-IP +# port bind + live-restore endpoint loss, both lanes; idempotent). +set -euo pipefail +[ "$(id -u)" = 0 ] || { echo "run with sudo"; exit 1; } +cd "$(dirname "$0")" +export DEBIAN_FRONTEND=noninteractive + +apt-get update -qq +apt-get install -y -qq ethtool + +install -m 0755 staged/ukrrs-gateway-ensure.sh /usr/local/sbin/ +install -m 0644 staged/systemd/ukrrs-gateway-ensure.service /etc/systemd/system/ +install -m 0644 staged/systemd/ukrrs-net-multiqueue.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now ukrrs-net-multiqueue.service +systemctl enable --now ukrrs-gateway-ensure.service + +echo; echo "== verify ==" +ethtool -l ens18 +ethtool -l ens19 +timeout 5 bash -c '/dev/null && echo "gateway prod: serving on :4000" diff --git a/4-host-netcheck.sh b/4-host-netcheck.sh new file mode 100755 index 0000000..482505b --- /dev/null +++ b/4-host-netcheck.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Read-only host check: why did the net multiqueue flags not land on VM 5111? +# Run from the guest: ~/optimize/4-host-netcheck.sh +# Everything is saved to ~/optimize/host-netcheck.out for crush to read. +set -euo pipefail +OUT="$(cd "$(dirname "$0")" && pwd)/host-netcheck.out" +timeout 120 ssh root@pfv-tsys5.knel.net ' + echo "== qm config 5111 (relevant lines) ==" + qm config 5111 | grep -E "^(net|scsi|onboot|startup|cpuunits|ide|machine|cores|sockets|threads|memory)" || true + echo + echo "== qm pending 5111 (unapplied staged changes) ==" + qm pending 5111 2>&1 || true + echo + echo "== last bounce log (/var/log/ukrrs-vm5111-bounce.log) ==" + tail -n 40 /var/log/ukrrs-vm5111-bounce.log 2>&1 || true + echo + echo "== pve version ==" + pveversion 2>&1 | head -3 +' | tee "$OUT" +echo +echo "saved: $OUT" diff --git a/GROW-ROOT-RUNBOOK.md b/GROW-ROOT-RUNBOOK.md new file mode 100644 index 0000000..467cbc8 --- /dev/null +++ b/GROW-ROOT-RUNBOOK.md @@ -0,0 +1,95 @@ +# Manual runbook: grow / from 279G to ~505G (VM 5111, ultix-streaming) + +YOU run every command, one at a time, checking the output before the next. +Nothing here is automated, scheduled, or boot-triggered. Storage ops are the +only steps in this whole effort that no script touches. + +## Layout change +``` +before: sda1 root 279.2G | sda2 extended [ sda5 swap 8.8G ] | ~150G unallocated +after: sda1 root ~505G | sda2 swap ~13G (new, at the tail) +``` +MBR stays MBR, ext4 grows online, VM stays up the whole time. You can safely +stop after any step; the system runs fine at any intermediate size. Takes +about 10 minutes end to end. + +## Step 0 — preconditions +``` +command -v growpart || sudo apt-get install -y cloud-guest-utils +df -h / /data1 /data2 # know your starting point +lsblk /dev/sda +``` + +## Step 1 — backups (30 seconds, do not skip) +``` +sudo sfdisk --dump /dev/sda | sudo tee /root/sda.sfdisk.bak.$(date +%F) +sudo cp /etc/fstab /root/fstab.bak.$(date +%F) +``` +Check: `head /root/sda.sfdisk.bak.*` lists sda1, sda2, sda5. + +## Step 2 — grow the virtual disk at the host (hot, VM stays up) +``` +ssh root@pfv-tsys5.knel.net 'qm config 5111 | grep scsi0' # eyeball the target +ssh root@pfv-tsys5.knel.net 'qm resize 5111 scsi0 +80G' +``` +Check in guest: `lsblk /dev/sda` shows sda = 518G, sda1 still 279.2G. + +## Step 3 — retire the old swap +``` +free -h # swap "used" must be ~0 +sudo swapoff /dev/sda5 +OLDUUID=$(sudo blkid -s UUID -o value /dev/sda5); echo "OLDUUID=$OLDUUID" +cat /proc/swaps # check: empty +``` + +## Step 4 — remove the extended partition (the strip between root and free space) +``` +sudo sfdisk --delete /dev/sda 2 +sudo partprobe /dev/sda +lsblk /dev/sda +``` +Check: only sda1 remains. If sda1 is missing: STOP, do not reboot, do not +write anything; restore the table: +`sudo sfdisk /dev/sda < /root/sda.sfdisk.bak.` + +## Step 5 — grow partition 1 into the free space +``` +sudo growpart /dev/sda 1 +# parted alternative: sudo parted /dev/sda resizepart 1 100% +lsblk /dev/sda # sda1 now ~505G +``` + +## Step 6 — grow the filesystem (online) +``` +sudo resize2fs /dev/sda1 +df -h / # ~500G available +``` + +## Step 7 — new swap at the tail +``` +sudo parted -s /dev/sda mkpart primary linux-swap 505GB 100% +sudo partprobe /dev/sda +lsblk /dev/sda +NEWSWAP=$(lsblk -no NAME,TYPE /dev/sda | awk '$2=="part"{print $1}' | tail -1); echo "NEWSWAP=$NEWSWAP" +sudo mkswap /dev/$NEWSWAP +NEWUUID=$(sudo blkid -s UUID -o value /dev/$NEWSWAP); echo "NEWUUID=$NEWUUID" +``` + +## Step 8 — swap fstab line, activate +``` +sudo sed -i "s/^UUID=$OLDUUID/#UUID=$OLDUUID retired-sda5 $(date +%F)/" /etc/fstab +echo "UUID=$NEWUUID none swap sw 0 0" | sudo tee -a /etc/fstab +sudo swapon -a +cat /proc/swaps # new swap active +sudo fstrim -v / # optional: reclaim (discard is on now) +``` + +## Rollback map +- Anytime before step 4: nothing changed except two backup files. +- After partition edits, before resize2fs: restore table from the sfdisk dump + (step 4 note); no data has moved, only the table. +- After resize2fs: growth is one-way by design; reverting size means restore + from backups, so this is the one step to do when calm. (It is also the + safest operation in the list: online ext4 grow is journaled.) +- fstab: backups in /root; only the swap line changes, and a bad swap line is + non-fatal at boot (root entry is untouched). diff --git a/NEXT.md b/NEXT.md new file mode 100644 index 0000000..bb3ac8e --- /dev/null +++ b/NEXT.md @@ -0,0 +1,38 @@ +# NEXT — perf-opt closeout COMPLETE (2026-08-31 16:4x) + +## Final state — nothing owed +- All guest tuning live and verified (see 16:04-16:15 pass below). +- Host flags live: ssd=1/discard (ROTA=0), iothread, onboot=1, startup + order=10, cpuunits 9000/50, ide2 gone. +- Net multiqueue: queues=4/2 now IN live qm config (staged 16:40 by crush + via ssh; activates at next VM start = Friday). Guest oneshot unit + ukrrs-net-multiqueue auto-runs ethtool -L on that boot. +- Gateway boot-race + live-restore endpoint loss: permanently fixed by + ukrrs-gateway-ensure unit (enabled, both lanes). +- Root cause of the 15:59 no-op: the script copy executed then predates the + queues= fix (silent same-value rewrite); current config verified correct. + +## Verified live after the 16:01 bounce (all read-only from the guest) +- Disks: ROTA=0, discard 4K/1G on sda+sdb+sdc → ssd=1/discard=on ACTIVE ✅ +- Kernel: bbr+fq, 16M socket buffers, dirty_bytes 1G/256M, min_free 384M, + aio 1M, inotify 512/1M, port range 10240-65535, slow_start_after_idle=0, + tcp_tw_reuse=1, THP=madvise, DefaultLimitNOFILE 65536 ✅ +- Mounts /,/data1,/data2 = noatime; swap sda5 prio -2 unused ✅ +- Units: all ukrrs slices + day/night + psi + builder-prune + thp units + installed and enabled; PSI textfile writing (16:06) ✅ +- Docker: daemon.json fully live (live-restore, log caps 20m×3, 172.16/12 + pools, metrics :9323); 21 containers up ✅ +- Desktop: sddm autologin worked (Relogin=true), session alive on seat0/tty2, + greeter burn GONE ✅ +- Gateway prod was DOWN at boot (bind race + live-restore endpoint loss) → + recovered via compose --force-recreate; /status = mode normal, serving ✅ + +## Nothing owed. Optional sanity check any time (crush can ssh now): + ~/optimize/4-host-netcheck.sh + +## Noted, no action taken +- mopac-demo/mcli fake containers: restart=no, exited at the first bounce, + left down (test stubs; your call). +- Prometheus/harness daemons not deployed here yet (that is OPT-9's wiring). +- Friday unchanged: CPU/RAM swap (plan NFS outage for -02), 2nd USB3 card, + q35 + 20 vCPU/128G reshape, GPUs. Enable the ensure unit BEFORE Friday. diff --git a/README.md b/README.md new file mode 100644 index 0000000..977ae03 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# ultix + +Perf/ops workbench for **ultix-streaming** (KVM VM 5111 on Proxmox host +pfv-tsys5). Everything here is human-run, gated, and reversible; storage ops +are manual-only by ruling. + +- Redmine: project 55 "Known Element Enterprises - Technology & Facility + Services" — open work: #601 (Fri 2026-09-04 hardware window, urgent), + #603 root growth, #604 metrics/PSI wiring, #605 k8s + proxmox token, + #606 GPU passthrough, #607 account map/mkacct; pass record: #602. + https://projects.knownelement.com/projects/55 +- Discourse doc: pending (house cross-link rule: create at next doc pass). +- Docs live in-repo (runbook exception per house rules): + - REPORT.md — full optimization report + host audit results (section 8) + - NEXT.md — current state / what is owed (usually: nothing) + - TRACKING.md — rolling HUD table + decisions log + - GROW-ROOT-RUNBOOK.md — manual root-growth checklist (#603) + - RUNBOOK-TODAY.md, questions-v1.md +- Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh, + 3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh); + staged/ holds the gated configs those scripts install. +- crush.md = session preferences for agents working in this repo. diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 0000000..3457662 --- /dev/null +++ b/REPORT.md @@ -0,0 +1,574 @@ +# ultix-streaming optimization report v1 (2026-08-31) + +Scope: the ultix-streaming KVM guest on pfv-tsys5, tuned for ~9 autonomous agent +accounts + gateway fleet + builds + ETL + SDR + GPU, all concurrent. Inputs: +direct measurement of the guest, the ukrrs repo tree, and your answers in chat. +Companion files: `questions-v1.md` (answer inline), `host-audit.sh` (run on PVE), +`staged/` (ready-to-apply configs, gated by `RUN=1`). + +## Exec summary + +One 2010-era 4-core Xeon currently serves this entire host: your dev +workstation plus 24 other running VMs, with zero resource limits anywhere in +the fleet, and the workstation does not even auto-start after a host reboot. +The fix is roughly $300 of used parts, one maintenance window, and the +configurations already staged in this directory. + +| spend item | rough cost (used, 2026) | effect | +|---|---|---| +| 2× Xeon X5675 | $60-80 | host 8 → 24 threads | +| 6× 16GB DDR3 ECC RDIMM 1333 | $120-180 | 96G → 192G | +| Dell T7500 CPU2 heatsink + paste | $25-40 | required for socket 2 | +| **platform total** | **~$210-300** | **3× threads, dev VM 128G** | +| optional: 1T DRAM-buffered TLC NVMe | $60-100 | only if the Crucial P1 QLC throttles builds | +| 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) | + +Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes, +gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn +window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by +host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven +backpressure wired end to end. All software pieces are staged and $0. + +## 0. TL;DR, ordered by leverage + +1. Host CPUs: CONFIRMED by audit: one E5620 (4C/8T) serves the ENTIRE host, + including 24 running VMs (load 7.5/8 at idle). CPU2 socket is empty. + Drop-in fix: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) = 24 + threads, plus CPU2 DIMM slots unlock 192G. See section 8. +2. One maintenance window fixes VM shape: machine q35 (i440fx today, blocks clean + GPU passthrough), 20 vCPU / 128G / ballooning off, virtio-scsi-single + + iothread + ssd=1 + discard=on per disk (guest currently sees ALL disks as + ROTATIONAL: the SSD hint is not passed through), net0 multiqueue. +3. No resource limits exist anywhere in the fleet: 18 live containers, zero + cpus/mem/cgroup_parent in any compose, and `dev.sh check` builders default to + GOMAXPROCS=8 on an 8-vCPU box. Staged: cgroup v2 slice architecture + (ukrrs-{gateway,pmo,lsp,batch,rt} + per-account slices via mkacct.sh). +4. Docker daemon is 100% stock: unbounded json-file logs, live-restore off, + default address pool (~15 networks; you already have 12), no metrics socket. + Staged daemon.json + weekly builder-prune timer (build cache is 9.3G/187 entries). +5. Guest kernel: THP=always (bad for the two Postgres), 66M min_free_kbytes, + tiny socket buffers (212K) for long LLM streams, dirty ratios in % of 48G + (multi-second writeback stalls under ETL). Staged sysctl.d + THP=madvise. +6. Signals: harness already reads loadavg/MemAvailable/PSI-io + (harness/internal/quota/resources.go:24). Missing: PSI cpu+memory, per-slice + pressure, docker awareness. Cheapest full stack: node_exporter as a compose + service into your existing mopac prometheus + PSI textfile script. Beszel + agent already runs (PID 940). k8s and Proxmox both have clean shedding APIs. +7. sddm-greeter burns ~22% of one core 24/7 (stuck on login screen). Fix: + autologin + lock. Sunshine is already installed; it becomes excellent once + the GPU lands (NVENC), replacing any xrdp idea. +8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3). +9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool, + memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep + window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST). + Guest has no CPU governor to remove (KVM: host owns P-states; set host + governor to performance once, done). + +## 1. Measured current state + +### Guest (ultix-streaming) +| aspect | measured | +|---|---| +| CPU | 8 vCPU presented as 2 sockets × 4 cores × 1 thread; Xeon E5620 2.40GHz (Westmere-EP, LGA1366, 2010; no AVX); 1 NUMA node | +| RAM | 47G usable + 8.8G swap on sda5 (priority -2, unused) | +| Machine type | i440fx + SeaBIOS (blocks clean GPU passthrough; q35 preferred) | +| Kernel/OS | Debian 13 trixie, 6.12.105, cgroup v2 unified, systemd | +| Disks | sda 438G: sda1 root 279G ext4 (29% used), sda5 swap; sdb 400G ext4 /data2 (empty); sdc 200G ext4 /data1 (empty). MBR table. ALL show ROTA=1 | +| Sched/mount | scheduler `none` on all disks (good), readahead 128K, relatime everywhere, fstrim.timer present | +| Desktop | KDE + sddm, greeter idling at ~22% of a core since boot (118 min CPU); no autologin; Sunshine installed, unused; no xrdp; no nvidia userspace | +| PSI now | cpu some avg60 ~1.2%, memory/io ~0 (idle baseline) | +| Desktop default | readlink default.target empty (graphical) | + +Key tunables (measured): swappiness 60, dirty_ratio 20/10 (percent-based), +min_free_kbytes 66M, max_map_count 1048576 (fine), aio-max-nr 65536, inotify +watches 386K / instances 128, somaxconn 4096 (fine), conntrack 262144 (fine), +ip_local_port_range 32768-60999, cubic only (bbr module not loaded), +rmem_max/wmem_max 212992, slow_start_after_idle=1, THP enabled=always, +autogroup=1 (good, keep for screen sessions), pid_max fine. tuned not installed. + +### Docker +| aspect | measured | +|---|---| +| Version/driver | 29.7.2, overlayfs, systemd cgroup driver, cgroup v2 | +| Config | NO daemon.json: unbounded json-file logs, live-restore false, data-root /var/lib/docker on root fs | +| Footprint | 24 images 14.3G, build cache 9.3G (187 entries), 18 containers up, 6 volumes | +| Networks | 12 total (9 user bridges). Default pool = 172.17-31/16, caps at ~15 networks | +| Running | 2 gateway stacks + 2 postgres (prod+beta), 9 LSP, openwebui, cli-mred, 2 fakes | + +### Fleet facts (from ukrrs tree) +- 9 harness daemons planned on THIS host (harness/deploy/accounts.tsv, idx 0-8: + reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO, reachableceo-offstage, + COSRCEO-Personal, COSRCEO-Biz, COSWFO). NOTE: /home currently shows different + names (COS-RCEO, COS-TSG, ...) so the account map is not final (Q7). +- Gateway: fleet semaphore 15 parallel provider calls, policy ladder + normal/peak/cooldown/hard recomputed every 30s, pacing cruise 90%. +- Harness gate order: resources → quota → peak → soft-defer. Resource reads: + loadavg, MemAvailable, PSI io only. Known gaps: harness still polls retired + cop :8110 (#585), credits are estimates, no PSI cpu/mem. +- Scale target: ~100 work streams, 24×7, 750 credits/hour. +- `./dev.sh check` = pinned golang builder, go build+vet+test, NO -p/cpus flags + → each builder defaults to all 8 vCPUs. 42 MCP + 10 LSP always-on. +- Beszel agent running (PID 940); prometheus with 30d TSDB already in harness + compose (mopac-harness-prometheus). + +## 2. Bottleneck model and the policy + +Who fights whom on this box: + +| workload | cpu | ram | disk | net | latency sensitivity | +|---|---|---|---|---|---| +| LLM harness loops ×9 | low | low | low | 9 × long streams | medium (streaming) | +| gateway + 2× postgres | low-med | med (shared_buffers+cache) | WAL writes | all LLM traffic, via tailscale | HIGH | +| docker pulls (burst) | med (gzip) | low | layer writes (NVMe) | burst 100s MB | low | +| compiles (dev.sh check) | saturating, GOMAXPROCS=8 each | 2-6G each | cache reads (NVMe) | low | low | +| ETL | med | page cache heavy | sequential r/w (SSD) | low | low | +| LSP ×10 + MCP 42 | idle mostly | 0.3-1G total-ish | low | none | low (interactive bursts) | +| SDR DSP (future) | 1-2 cores steady | low | low | low | HARD real-time | +| Sunshine encode (future) | 2-4 cores CPU / ~0 with NVENC | low | low | LAN | medium | + +Design in one sentence: schedule by intent, not by luck. Three CPU pools +(interactive/gateway, batch/build, rt/sdr), per-account fair shares with cgroup +v2 weights + memory fences, spindle separation per I/O class, and PSI as the +universal pressure signal your harness already half-reads. + +``` + +----------------- system.slice -----------------+ + | ukrrs-gateway.slice CPUWeight 900 (gateway+pg stacks) + | ukrrs-pmo.slice CPUWeight 200 (9 PMO dispatch loops) + | ukrrs-lsp.slice CPUWeight 150 (LSP + MCP tier) + | ukrrs-batch.slice CPUWeight 25 AllowedCPUs pool A + | ukrrs-acct-<9>.slice CPUWeight 75 AllowedCPUs pool A + | ukrrs-rt.slice AllowedCPUs pool B (SDR, encode) + | docker default (everything not tagged) = last + +---------------------------------------------------+ + interim 8 vCPU: pool A = 0-5, pool B = 6-7 + post-upgrade 20: pool A = 0-15, pool B = 18-19 (gateway may use 0-19) +``` + +Fairness math: with 9 accounts each compiling at weight 75 and gateway at 900, +gateway keeps ~13% of CPU even under total compile storm; with weights alone an +idle machine still lets ONE account use every core (burstable, no quotas). I/O +weights are inert under the `none` scheduler, so fairness comes from separation +(NVMe = images/builds, /data1 = databases, /data2 = ETL/backup/scratch) plus +optional device throttles if a specific ETL goes rogue. + +## 3. Host plan (pfv-tsys5) + +### 3.1 Audit (do this first) +My tooling cannot ssh (hard-banned), so run: +``` +ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out +``` +I parse the output and finalize §3.2-3.7 numbers. The script is read-only. + +### 3.2 CPU upgrade (drop-in, LGA1366) +| option | result | notes | +|---|---|---| +| add 2nd E5620 (~$10) | 8C/16T | cheapest, matches existing | +| 2× X5675 (recommended) | 12C/24T @3.06/3.46 | 95W each, DDR3-1333, ~$25 each | +| 2× X5690 | 12C/24T @3.46/3.73 | 130W each, hot, marginal gain, PSU risk with GPUs | +| 2× L5640 | 12C/24T @2.27 | low power, slower | + +Caveats the audit resolves: chassis must be dual-socket (T5500/T7500; T3500 is +single), BIOS revision (A17 supports 5600-series), current DIMM layout, PSU +rating. Westmere lacks AVX/AVX2: Go/Rust/CUDA fine, but some prebuilt binaries +(newer Node native modules, llama.cpp CPU builds, some distro packages) assume +AVX; plan on building from source or using GPU for inference. + +### 3.3 RAM +96G now is CPU0 DIMMs only (6 slots). Second CPU unlocks 6 more slots → 192G. +Give the VM 128G, leave 64G for host + preprod VMs. Use DDR3 ECC RDIMM 1333 +(12800R); mixing 10600R works but clocks down. Balanced population per branch. + +### 3.4 VM re-shape (one maintenance window, VM off) +``` +qm set --machine q35 # from i440fx; expect NIC rename in guest +qm set --cpu host --sockets 2 --cores 5 --threads 2 --numa 1 # 20 vCPU +qm set --memory 131072 --balloon 0 +qm set --scsihw virtio-scsi-single +# per disk (real disk ids from audit): +qm set -scsi0 :vm--disk-0,ssd=1,discard=1,iothread=1 # repeat sdb sdc +qm set --net0 virtio=,bridge=
,multiqueue=4 +``` +q35 + NIC rename: prepare a systemd .link file or just let udev rename and +restart networking; tailscale state survives. Guest then sets 4 RSS queues +(ethtool -L, handled by a tiny oneshot unit, staged). If ballooning was on, off. +i440fx→q35 is required for the GPU plan; do it once, with console access. + +### 3.5 Storage backend +Fixes: ssd=1 (kills the ROTA=1 lie the guest sees today), discard=on (with the +fstrim.timer already present), virtio-scsi-single + iothread=1 (one iothread per +disk spreads interrupts), cache=none on ZFS backend / writeback only on +LVM-thin with UPS. Audit reports backend + link speeds. Known ceiling: if the +two SSDs sit on the onboard ICH10R SATA2 ports they cap ~280MB/s each (3Gbps +link); the NVMe on a PCIe2.0 x4 card caps ~1.7GB/s. Fine for this fleet, just +set expectations for ETL throughput. + +### 3.6 GPU passthrough +Now: old NVIDIA on host. Soon: two newer GPUs. Plan: +1. IOMMU: already active with clean per-device groups (audit section 8). No + ACS override needed. Optional at next host reboot: add `iommu=pt` for + cheaper DMA on passthrough devices. +2. The Quadro 4000: skip entirely. Fermi has no support in any CUDA toolkit + this decade (compute capability 2.x was dropped at CUDA 9) and no driver + that builds on kernel 6.x. It stays as the host console card. +3. The two newer GPUs are COMPUTE cards (per ruling 2026-08-31), which is + easier than display passthrough: no dummy plug, usually no romfile, pass + without x-vga (`qm set 5111 -hostpci0 0000:xx:00.0`), modern NVIDIA + drivers (465+) officially support VMs. Remaining risk: this-era BIOS has + no Above-4G-Decoding, so big-VRAM cards with large BARs may fail to map. + Sweet spot for this box: 2× used RTX A2000 12GB (~$150-200 each): 75W + slot power, no aux cables, no PSU change, 24G VRAM total, BARs fit. Larger + cards (3060 12G / 4060Ti 16G class) need the PSU check + may hit the BAR + wall. q35 recommended when they land; OVMF only if a UEFI-only vBIOS + forces it. +4. In-guest: nvidia driver + nvidia-container-toolkit; builders get `--gpus + all` in the batch slice (house-style digest-pinned CUDA image). +5. PSU budget if going bigger: 2× 95W CPU + 2× 200-250W GPUs + platform ≈ + 750-850W sustained: needs the 875W PSU option (Q5 sticker check). +6. Sunshine (already installed) still benefits even though the cards are + compute-only: capture the KDE session via X11/KMS on the emulated VGA, + encode on the GPU with NVENC headless. CPU encode (current state, + 2-4 Westmere cores at 1080p60) remains tolerable-but-temporary. No xrdp: + it gives a second synthetic session with no accel; Sunshine mirrors the + real session with audio. SSH stays break-glass. + +### 3.7 SDR passthrough +You have two PCI USB3 cards, SDRs load-balanced across them: pass the CARDS, +never devices. `qm set -hostpci1 -hostpci2 `. Guest gets native +xHCI controllers with real MSI interrupts (emulated USB adds latency and jitter +that ruins tuner sample streams). SDR processes run in ukrrs-rt.slice +(AllowedCPUs pool B) so compile storms cannot preempt DSP. Bandwidth: tens of +MB/s per SDR, trivial for USB3. libusb latency in a VM is fine for SDR work. + +### 3.8 Proxmox control from the harness +PVE API token (appendix B has exact pveum commands), then +`staged/proxmox-ctl.sh vms|shutdown|start|snapshot` with a hard PROTECTED list +containing this VM and anything untouchable. Natural uses: stand preprod VMs +down during compile storms, snapshot before risky agent work, bring up scratch +VMs. Scoping question Q12. + +### 3.9 Host-side fleet arbitration (dev VM over sectestbed/preprod) +Yes, the audit pulled every VM config (26 VMs: 24 running, 1 stopped, 1 +template). Two live levers, no reboots: +``` +# priority: PVE maps cpuunits to cgroup v2 CPUWeight (clamped 1-10000) +qm set 5111 --cpuunits 9000 +FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108" +for id in $FLEET; do qm set $id --cpuunits 50; done +# optional hard caps where the test fleet never needs burst: +# qm set --cpulimit 1 +# (VM 500 k8s-wnode was removed 2026-08-31: ultix-streaming becomes the k8s worker) +``` +At equal demand the dev VM holds ~87% of CPU; the fleet still bursts to full +idle capacity when the dev VM is quiet. Post-upgrade, physical partitioning +on 24 threads: `qm set 5111 --affinity 0-19` and the fleet `--affinity 20-23` +(so test VMs can never preempt dev cores at all). Add proxmox-ctl standing +down idle sectestbed VMs entirely (frees host RAM too) and their disks are +already separate (local-lvm spinner vs your dedicated SSDs), so no I/O +arbitration needed. Optional host cmdline at next reboot: `iommu=pt`. + +## 4. Guest plan (staged in ~/optimize/staged, apply via apply-guest.sh) + +### 4.1 Kernel (60-ukrrs-vm.conf) +| knob | now → set | why | +|---|---|---| +| dirty_background_bytes / dirty_bytes | % ratios → 256M / 1G | bytes-based caps writeback stalls (ETL) regardless of RAM size | +| min_free_kbytes | 66M → 384M (768M post-upgrade) | survive bursty reclaim under compile+ETL | +| aio-max-nr | 65536 → 1M | postgres AIO / io_uring era defaults | +| inotify instances | 128 → 512; watches 386K → 1M | 9 harness daemons + LSPs + crush sessions watching repos | +| ip_local_port_range | 32768-60999 → 10240-65535 | 9 accounts × long-lived streams + tailscale + docker NAT | +| tcp_tw_reuse | 2 → 1 | outbound provider connection churn | +| tcp_slow_start_after_idle | 1 → 0 | LLM turns idle minutes between bursts on live sockets | +| rmem_max/wmem_max + tcp_r/wmem | 212K → 16M | large SSE/JSON streams through gateway | +| congestion control | cubic → bbr + fq | module load staged; falls back to cubic if absent | +| DefaultLimitNOFILE | 1024 soft → 65536 | harness daemons, many sockets/files | + +Left alone deliberately: swappiness 60, overcommit 0, vfs_cache_pressure, +pid_max, somaxconn 4096, conntrack 262144, autogroup 1, page-cluster. +THP: always → madvise (oneshot unit, staged). Postgres dislikes always-THP; +Go/builds get THP via madvise anyway where it matters. + +### 4.2 Mounts and root growth to 500G (MBR shuffle, online except swapoff) +Ruling 2026-08-31: storage operations are MANUAL ONLY, executed by the human, +one command at a time. Nothing automatic, nothing boot-triggered. The +step-by-step checklist with per-step verification is GROW-ROOT-RUNBOOK.md; +the commands below are reference for what it does. +Current MBR: sda1 root ends 279G, then extended sda2 holding sda5 swap 8.8G, +then ~150G unallocated (disk 438G). Target: root 500G → grow disk at PVE first: +`qm resize scsi0 +80G` (→ 518G), then in-guest: +``` +sfdisk --dump /dev/sda > /root/sda.sfdisk.bak; cp /etc/fstab /root/fstab.bak +swapoff /dev/sda5 +sfdisk --delete /dev/sda 2 # removes extended + logical swap +growpart /dev/sda 1 # cloud-guest-utils; grows root partition +resize2fs /dev/sda1 # online ext4 grow → 505G +parted /dev/sda mkpart primary linux-swap 505GB 100%; mkswap /dev/sdaX +# fix fstab swap UUID (blkid), swapon -a; verify with lsblk + df -h / +``` +Rollback: fstab + sfdisk dumps kept; original layout restorable offline. Also: +add `noatime` to /, /data1, /data2 (remount online; relatime today). + +### 4.3 Swap +Keep sda5-style swap at disk tail (fresh 10G from the shuffle). Skip zram: old +CPU (compression burns cores) and RAM is tripling soon. Optional: 32G low-pri +swapfile on /data2 as OOM insurance post-upgrade (staged flag, default off). + +### 4.4 Docker (daemon.json staged) +live-restore true (daemon restarts stop killing agent containers; aligns with +your "never bounce prod for convenience" rule), json-file logs capped 20m × 3 +(9 accounts × 42 MCP without caps = quiet-log rule violation waiting to happen), +max-concurrent-downloads 6 (parallel pulls across accounts), address pool +172.16/12 as /24s (you are at 12 of ~15 default networks; the fleet adds one +per compose project), metrics on 127.0.0.1:9323 (experimental:true is required +by docker for the engine metrics endpoint; drop both lines if unwanted). +Weekly `docker builder prune --keep-storage 25GB` timer staged (9.3G today, +unbounded growth otherwise). Applying daemon.json restarts docker once (do it in +a quiet moment; live-restore protects future restarts only). + +### 4.5 cgroup v2 architecture (the core deliverable) +Static slices staged: gateway (CPUWeight 900, MemoryMin 2G, MemoryHigh 12G), +pmo (200, High 1G/Max 1.5G; the dispatch control plane: cheap but weighted +above workers so a compile storm can never starve dispatch), lsp (150, High +6G/Max 8G), batch (25, High 12G/Max 16G, pool A cpus), rt (pool B cpus, +weight 10000). Night profile overrides batch/gateway at runtime (§5.5). Per-account slices via `mkacct.sh `: +CPUWeight 75, TasksMax 4096, Memory fences (interim 48G: High 3G/Max 4.5G ×9; +post-upgrade 128G: High 10G/Max 12G ×9), plus a matching user-.slice +drop-in so the harness daemon process itself is fenced too. +Containers join their account slice via compose `cgroup_parent:` (snippet +printed by mkacct). Builders and ETL join ukrrs-batch via dev.sh/compose. +Everything untagged lands in plain docker scope = effectively lowest priority. +NOTE: with the `none` I/O scheduler, IOWeight is inert; fairness = separation +across the three physical devices + (rare) per-container device rate limits. + +### 4.6 Desktop +Fix the greeter burn: sddm autologin into an UNLOCKED session (ruling +2026-08-31: no autolock, ever; instant Jump/iPad re-attach outranks lock +security on this box). `~/optimize/fix-kde.sh` does it (sudo, idempotent, +optional --restart). Result: greeter stops rendering, session idles near 0%, +an always-alive unlocked KDE session is attachable instantly. Delete sddm-greeter +CPU cost entirely the day you stop wanting console KDE (multi-user.target) but +that kills Sunshine's session, so default plan keeps KDE. + +KDE vs XFCE verdict: do not switch for resource reasons. Locked/idle Plasma is +~0% CPU and roughly 600-900M RSS vs XFCE ~350-450M; on a box going to 128G +that delta is noise. The 22% greeter burn was a greeter-stuck artifact, not +Plasma weight, and lightdm+XFCE can do the same trick. The one real argument +for XFCE later: it is X11-only, and X11 is still the most battle-tested +Sunshine capture path with the NVIDIA proprietary driver; Plasma Wayland +capture on NVIDIA is good in 2026 but younger. Sequence: keep KDE now, fix +autologin+lock, land the GPU, test Sunshine on Wayland+NVENC, switch to XFCE +only if capture disappoints. Revisit when Agent Zero lands (Q15): if it ever +runs headed-browser tasks, the desktop story matters more than RAM. + +## 5. Signals and integrations + +### 5.1 What the harness reads today and the minimal patch +resources.go reads loadavg, MemAvailable, PSI-io avg60, disk free, with clean +seams. Extend (same pattern, ~40 lines + tests, ticket per repo): +- generalize readIODelay → readPressure(res) for cpu and memory +- optional: read /sys/fs/cgroup/ukrrs-batch.slice/cpu.pressure to gate on "is + the batch pool saturated" rather than host-wide load (loadavg counts 24 + post-upgrade vCPUs; PSI is the honest signal) +- suggested defer thresholds: cpu some avg60 > 20, io avg60 > 25, mem some + avg60 > 10 (tune with real data; the metric names already exist + harness_resource_load_avg_1m / _io_delay_pct) + +### 5.2 Metrics stack (your "super lightweight" ask) +Nothing new to run on the host OS: node_exporter as a compose service +(read-only binds of / and /var/lib/node_exporter/textfile, house-compliant, +digest-pinned at adoption), scraped by the ALREADY RUNNING +mopac-harness-prometheus (30d TSDB). PSI textfile script (staged) adds +per-slice pressure gauges every 15s. Docker engine metrics land on +127.0.0.1:9323 via daemon.json. Beszel (already installed) keeps the human +dashboard role. Gateway choice (Q10): poll node_exporter directly in its 30s +tick (no new dependency, ~30 lines) vs PromQL against existing prom. Either +beats the retired cop :8110 (#585) it still polls. + +### 5.3 k8s (yes, the APIs shed work cleanly) +- `kubectl cordon ultix-streaming` stops new scheduling; `kubectl drain + --ignore-daemonsets --delete-emptydir-data --grace-period=120` evicts. +- Set all k8s workloads to a low PriorityClass + preemption policy, so burst + pods die first under node pressure automatically. +- kubelet eviction thresholds (memory.available<2Gi, imagefs.available<10Gi) + auto-shed when builds eat disk/RAM. Plus --system-reserved/--kube-reserved. +- Recommendation: join AFTER the host upgrade, as worker only, never control + plane, kubelet+containerd units get CPUWeight 20 + MemoryHigh fences so the + compose fleet always wins arbitration. Control plane location Q11. + +### 5.4 Proxmox +See §3.8; skeleton staged, token + PROTECTED list via Q12. + +### 5.5 Day/night dynamic profile + PMO dispatch backpressure +Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the +z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset +(gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect: +22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO +burn maxes, 05:00 everything returns to day bias. + +What flips at night (systemctl set-property --runtime + sysctl, all +non-persistent; a reboot lands safely in day mode; values in +/etc/ukrrs/daynight.conf, staged): +| object | day | night (48G interim / 128G post) | +|---|---|---| +| batch slice CPUWeight | 25 | 400 | +| batch AllowedCPUs | 0-5 (post 0-15) | 0-6 (post 0-17) | +| batch MemoryHigh | 12G (post 24G) | 30G (post 80G) | +| gateway CPUWeight | 900 | 500 (still far above idle accounts) | +| /data2 readahead | 256K | 1M (ETL sequential) | +| dirty_bytes / background | 1G / 256M | 2G / 512M | +Honest limit: the guest has no CPU governor to "take off"; KVM host owns +P-states. Set the host governor to performance once (audit confirms current +state); an always-busy 15-year-old Xeon saves nothing from dynamic P-states. +Night is also the maintenance window: builder prune Sun 22:30, fstrim, +postgres vacuum/reindex, image pre-pull for the morning fleet. + +PMO backpressure: yes, gate at dispatch, it is the earliest and cheapest +point. Each account's PMO loop claims from Redmine only when, in order (all +zero-dependency reads, same pattern as resources.go): +1. gateway /status policy mode is normal (else trickle: ladder already + handles 01:00-05:00), +2. /proc/pressure/{cpu,memory,io} avg60 under thresholds, +3. ukrrs-batch.slice cpu/io pressure under threshold (worker pool headroom), +4. wall clock: night = max parallel builders + bigger tickets (compiles, + ETL, doc builds); day = fewer, interactive-sized tickets. +The 9 PMO loops live in ukrrs-pmo.slice so worker storms cannot starve the +dispatcher. This gate composes with the existing harness gate order +(resources → quota → peak → soft-defer, harness/README.md:298-312) instead of +replacing it. + +Agent Zero: containerized python agent runtime. Placement: interactive +agents → their account slice (or gateway-adjacent weights), background +agents → ukrrs-batch.slice. Decide via Q15; no OS change either way, the +slice architecture absorbs it. + +## 6. Sequencing and rollback +- Phase 0 (now, reversible, ~1h, no reboot): apply-guest.sh sysctl+thp+slices+ + noatime+timers+daynight+desktop; daemon.json in a quiet window (one docker + restart). +- Phase 1: host audit (one ssh line), order CPUs/RAM (Q2/Q3). +- Phase 2 (window, VM off): q35 + resize + disk flags + multiqueue; then root + growth to 500G; then GPU + USB3 passthrough; then VM vCPU/RAM at new values. +- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in compose, + resources.go PSI extension, gateway pressure input, #585 cop removal. +- Phase 4: k8s join, proxmox-ctl wiring. +Every phase independently revertible: sysctl.d and units are files, qm changes +are per-flag, partition work has dumps. Never a bare `down`, per house rules. + +## 7. Open items +See questions-v1.md, answer inline; version the file if you want another round. + +## 8. Host audit results (2026-08-31, parsed from host-audit.out) + +### Confirmed +- Dell Precision T7500 (baseboard 06FW8P A02), dual socket, **CPU2 EMPTY**. + One E5620 4C/8T (HT on) serves 24 RUNNING VMs (~57G/96G RAM allocated); + host load was 7.5 on 8 threads while mostly idle. Your VM gets all 8 + threads only because the rest are quiet. The 2× X5675 upgrade is now + urgent, not optional: 3× threads. DMI quirk: "Upgrade: Socket LGA771" is a + Dell string error (E5620 exists only in LGA1366); still verify visually + before ordering. +- RAM: 6× 16GB DDR3-1600 filling ALL six CPU1 slots. Second CPU unlocks six + more: order 6× 16GB more → 192G. Runs at CPU max (1066 on E5620, 1333 on + X5675). +- IOMMU active with CLEAN per-device groups: Quadro alone (20), Renesas + USB3 alone (5), NVMe alone (21), NIC ports separate (3/4). **No ACS + override needed** (better than section 3.6 assumed). +- No CPU frequency driver exists on this host (no /sys cpufreq at all, this + kernel + 5520). There is no governor to tune anywhere, host or guest. + Day/night dynamics are cgroup-only, exactly as staged in section 5.5. +- Host storage: PVE root on 2TB Hitachi spinner via LVM-thin (local-lvm, + where the 24 sectestbed VMs live); SMART PASSED on everything; temps fine. + +### Corrected +- Your "NVMe" = **Crucial P1 500G QLC (DRAM-less)** as qcow2 on ext4 dir + storage (NVME:), dedicated to this VM, 59% used at storage level. /data1 = + Samsung 860 PRO 256G, /data2 = SK hynix SC300B 512G, both dedicated, both + on ICH10R SATA2 3Gbps links (~280MB/s ceiling each). None carry ssd=1 or + discard (that is why the guest sees ROTA=1). Expect P1 sustained-write + collapse under heavy builder churn (QLC, no DRAM): monitor; if it bites, + the fix is a better NVMe, not config. +- Old GPU = **Quadro 4000 (Fermi, 2GB)**. Verdict: skip passthrough. Fermi + has no NVENC (starts with Kepler) and no driver that builds on kernel 6.x + (390xx is dead): it buys display outputs only. Keep it as host console. + Sunshine stays software-encode until the new GPUs arrive; their clean + IOMMU groups are already waiting. +- Only **ONE** Renesas uPD720201 USB3 card visible (24:00.0). You said two. + Check the second: unseated, dead, or behind the (empty) Pericom PCI-X + bridge at 01:00.0. +- VM 5111 has **no onboot flag**: it will not auto-start after a host + reboot. debian13.iso still attached to ide2 (nit). +- k8s on this host consolidated (2026-08-31): VM 500 pfv-k8s-wnode-tsys5 was + removed; ultix-streaming itself becomes the k8s worker on pfv-tsys5 + (12G RAM + 2 vCPU of host pressure freed; timing/control plane per Q11). +- The 18-VM sectestbed fleet runs 24/7 on the same 8 threads and the 2TB + spinner. proxmox-ctl.sh from this VM is the natural on/off switch (Q12). + +### QM checklist for the next touch (any time, hot where noted) +``` +qm set 5111 --onboot 1 --startup order=10,up=180 +qm set 5111 -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G +qm set 5111 -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G +qm set 5111 -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G +qm set 5111 --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,multiqueue=4 +qm set 5111 --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,multiqueue=2 +qm set 5111 --delete ide2 +# q35 + 20 vCPU + 128G + sockets/threads reshape: only at the CPU/RAM window (3.4) +# root growth to 500G: MANUAL only, GROW-ROOT-RUNBOOK.md (discard=on live first; fstrim reclaims qcow2) +# net multiqueue needs a guest-side ethtool -L oneshot after reboot +``` + +## Appendix A: staged file map +``` +staged/60-ukrrs-vm.conf → /etc/sysctl.d/ (kernel) +staged/modules-load.d/tcp_bbr.conf → /etc/modules-load.d/ +staged/systemd/*.slice|*.service|*.timer → /etc/systemd/system/ +staged/ukrrs-psi-textfile.sh → /usr/local/sbin/ (with .service/.timer) +staged/ukrrs-daynight.sh → /usr/local/sbin/ + day/night profile timers + + /etc/ukrrs/daynight.conf (all staged values) +staged/mkacct.sh → run per account: name uid [high] [max] [cpus] +staged/apply-guest.sh → installer, RUN=1 to mutate, subcommands +staged/docker/daemon.json → /etc/docker/ (restarts docker once) +staged/proxmox-ctl.sh → PVE API wrapper, PROTECTED guard +``` + +## Appendix B: repo patches (ticketed, not applied from here) +1. compose builder caps (each repo with a builder/check svc): +``` +x-ukrrs-batch: &ukrrs-batch + cgroup_parent: ukrrs-batch.slice + cpus: 4 + mem_limit: 4g +services: + check: { <<: *ukrrs-batch, ...existing... } +``` +2. dev.sh one-shot builders: `docker run --rm --cgroup-parent ukrrs-batch.slice + --cpus 4 --memory 4g ...` (GOMAXPROCS then follows cpuset/cpus). +3. resources.go: generalize PSI read to cpu/mem + optional slice pressure + (fields + gates + tests, mirrors readIODelay at resources.go:90). +4. gateway pressure poller: in the 30s tick (gateway.go:802), GET + 127.0.0.1:9100/metrics, parse node_pressure_cpu_waiting_seconds... or the + textfile gauges; feed the ladder as a new host-pressure input class. +5. PVE token: +``` +pveum user add ukrrs-infra@pam +pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0 +pveum acl modify /pool/ -user ukrrs-infra@pam -role PVEVMUser +``` +6. GPU vBIOS dump (on host, card must be unused): +``` +cd /sys/bus/pci/devices/0000:0X:00.0; echo 1 > remove +cat rom > /root/gpu-.rom; echo 0 > remove +``` +7. node_exporter service (into harness deploy compose, digest pin at adoption): +``` +node-exporter: + image: prom/node-exporter@sha256: + container_name: ukrrs-mopac-nodeexporter + command: [--path.rootfs=/host, --collector.pressure, + --collector.textfile.directory=/textfile] + pid: host + volumes: [/:/host:ro,rslave, /var/lib/node_exporter/textfile:/textfile:ro] + network_mode: host + restart: unless-stopped +``` diff --git a/RUNBOOK-TODAY.md b/RUNBOOK-TODAY.md new file mode 100644 index 0000000..71770db --- /dev/null +++ b/RUNBOOK-TODAY.md @@ -0,0 +1,30 @@ +# Today: two commands, one reboot wave + +**1. On the VM** (ultix-streaming, any time before step 2): +``` +sudo ~/optimize/1-guest-prep.sh +``` +Installs everything (kernel tunables, BBR, THP, cgroup slices, PSI collector, +day/night timers, docker daemon.json, sddm autologin, no autolock). Docker +restarts once. Touches NO storage. + +**2. From your desk** (dry-run first if you like, then execute): +``` +ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh +ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh +``` +Graceful stop of 5111, ssd/discard/multiqueue/onboot flags, CPU priority over +the sectestbed fleet, start 5111, then reboots every other running VM. +No disk resizing, no host reboot (NFS role). + +**3.** Say "done" here; I verify everything read-only and report. + +**Separate, whenever you choose** (fully manual, step-by-step with checks): +root growth 279G -> ~505G via `~/optimize/GROW-ROOT-RUNBOOK.md`. No script +runs it; no boot triggers it; you type every command yourself. + +Rollback: apply-guest backs up every file it touches; qm flags are +individually reversible. + +Friday still owns: CPU/RAM swap (host power-off -> plan the NFS outage for +-02 first), second USB3 card reseat, q35 + 20 vCPU / 128G reshape, GPUs. diff --git a/TRACKING.md b/TRACKING.md new file mode 100644 index 0000000..96787a7 --- /dev/null +++ b/TRACKING.md @@ -0,0 +1,35 @@ +# ultix-streaming optimization — tracking HUD + +One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / blocked ⛔ / next ⏳ + +| id | lane | state | item | +|---|---|---|---| +| OPT-1 | guest | ✅ | recon: system + ukrrs fleet (REPORT §1) | +| OPT-2 | guest | ✅ | REPORT.md + staged configs + questions-v1.md | +| OPT-3 | host | ✅ | audit parsed → REPORT §8 (T7500, CPU2 empty, clean IOMMU, Q4000 skip) | +| OPT-4 | host | ⏳ | Fri 2026-09-04 #601: 2× X5675 + 6× 16GB → 24T/192G (NFS outage for -02 first; USB3 reseat; q35/20c/128G) | +| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue now in qm config (live at Friday's start) | +| OPT-6 | guest | ✅ | guest prep applied + verified (bbr, THP, slices, timers, noatime, daemon.json, autologin) | +| OPT-7 | guest | ⏳ | #603: root growth MANUAL ONLY, GROW-ROOT-RUNBOOK.md, user-executed | +| OPT-8 | repos | ✅ | PMO backpressure design note in ukrrs/docs/harness; code = ticketed | +| OPT-9 | integ | ⏳ | #604: node_exporter svc + gateway pressure scrape (Q10) | +| OPT-10 | integ | ⏳ | #605: k8s join (Q11, blocked by #601) + proxmox-ctl token (Q12); GPUs #606 | + +Inbox (mid-task interrupts): none. + +Decisions log (latest wins): +- 2026-08-31 16:5x: Redmine = record: perf pass + RCA filed as #602 (project 55, tracker Support, left New for human review); open items #601 (Fri window, Urgent, due 09-04) #603 root growth #604 metrics #605 k8s+proxmox (blocked by #601) #606 GPUs (relates #601). mred = docker exec wrapper (~/.local/bin/mred → mopac-cli-mred container). +- 2026-08-31 16:40: net multiqueue CLOSED: queues=4/2 written to live qm config via crush ssh (user authorized non-disruptive host mods); activates at next VM start; guest oneshot enables channels on that boot. Earlier silent no-op explained: the 15:59 run used a pre-fix script copy. +- 2026-08-31 16:3x: crush CAN ssh to the host from this workspace (old "hard-banned" note obsolete); still human-gated for disruptive ops. +- 2026-08-31 (post-reboot validation): prod gateway down at boot ROOT-CAUSED twice over: (a) port bind to tailscale IP lost the dockerd-vs-tailscaled race (docker never retries failed starts), (b) live-restore dropped the container's network endpoint (host resolver → gateway-db unresolvable → crash loop). Recovered via compose --force-recreate (healthy, mode=normal). Permanent fix staged: ukrrs-gateway-ensure unit, BOTH lanes, MUST be enabled before Friday's bounces. +- 2026-08-31: multiqueue host flags verified live (ROTA=0 + discard 1G on all 3 disks), but guest still runs 1 queue/NIC: ethtool not installed; 3-post-reboot-fixes.sh staged (installs ethtool + oneshot unit, sets 4/2 queues). +- 2026-08-31: mopac-demo/mcli fake containers left DOWN (restart=no, exited at first bounce); prometheus/harness daemons not deployed on this box (pre-existing; belongs to OPT-9). +- 2026-08-31: STORAGE OPS MANUAL ONLY: boot-time auto-grow service removed, qm resize removed from the one-shot, grow-root.sh deleted. Runbook = GROW-ROOT-RUNBOOK.md, every command typed by the human. +- 2026-08-31: KDE autologin into UNLOCKED always-alive session (Relogin=true); autolock OFF (instant Jump/iPad attach outranks lock). XFCE only if Sunshine capture disappoints post-GPU. +- 2026-08-31: streamlined to 1-guest-prep.sh + 2-host-one-shot.sh (one reboot wave; no post-reboot steps; no storage). +- 2026-08-31: constraint: host pfv-tsys5 NEVER reboots casually (NFS server for -02); Friday CPU swap needs a planned NFS outage first. +- 2026-08-31: GPUs are COMPUTE cards; Quadro 4000 skipped (Fermi: no NVENC, no kernel-6 driver, no modern CUDA); clean IOMMU groups verified. +- 2026-08-31: day/night = runtime-only cgroup flips 22:00/05:00; NO cpufreq driver exists host or guest. +- 2026-08-31: zram rejected; no docker data-root move; skip tuned (sysctl.d + units). +- 2026-08-31: second USB3 SDR card missing on host (only one Renesas visible); Friday reseat/check. +---- \ No newline at end of file diff --git a/crush.md b/crush.md new file mode 100644 index 0000000..6497772 --- /dev/null +++ b/crush.md @@ -0,0 +1,28 @@ +# crush.md — project preferences for crush sessions in ~/optimize + +## Locale (ruling 2026-08-31) +- Human + this host are in TEXAS: America/Chicago (Central Time; UTC-5 CDT in + summer). "1638 CST" in chat means Central local time. +- 2026-08-31 is a MONDAY. "Friday" in these docs = 2026-09-04. +- Docker/ISO timestamps print UTC (trailing Z) = local +5h; never mislabel + weekday or TZ when narrating logs. + +## Host + no-Python rules (ruling 2026-08-31, also in global AGENTS.md) +- No Python from agents: text/JSON/CSV = bash/sed/awk/perl/cut/jq. NEVER + python one-liners. python in dev containers is fine. +- Host stays clean: common shell commands fine, curl + jq + ripgrep etc + ALLOWED (old curl ban retired). NO installs on the host (apt-get/pip/etc). +- Missing CLI tool? docker pull it, standing permission — run via + `docker run --rm `. Work in containers. + +## Output format (user preference, 2026-08-31) +- The crush TUI sidebar truncates long chat output. NEVER hand the user a long + command in chat. ALWAYS write runnable commands as a numbered script in this + directory (next free number, e.g. 5-foo.sh) and tell them only the one-liner + to execute it. Scripts that produce diagnostics must tee output to a .out + file in this directory so the next turn can read it. +- Scripts that need root must SELF-ELEVATE (re-exec via `exec sudo bash "$0"` + when EUID != 0) so the user never types sudo. Run-from-guest ssh-root scripts + need no elevation; say so in the script header. +- Keep chat replies short-line-width; details live in files (NEXT.md, REPORT.md, + questions-v*.md, script headers). diff --git a/fix-kde.sh b/fix-kde.sh new file mode 100755 index 0000000..12854ad --- /dev/null +++ b/fix-kde.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# UKRRS KDE fix: sddm autologin into an UNLOCKED, always-alive session. +# Deliberately NO autolock (instant Jump/iPad re-attach requirement). +# Relogin=true: if the session ever exits/crashes, sddm logs straight back in, +# so there is always a session to attach to. +# +# Usage: +# sudo ./fix-kde.sh config only; takes effect next boot/restart +# sudo ./fix-kde.sh --restart also restart sddm NOW. WARNING: restart kills +# any ACTIVE local GUI session. Right now only +# the greeter runs, so it is safe; verify with +# `loginctl list-sessions` if unsure. +# Override user with: sudo DESKTOP_USER=someone ./fix-kde.sh +set -euo pipefail +[ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; } +USER_NAME=${DESKTOP_USER:-reachableceo} +CONF_DIR=/etc/sddm.conf.d +CONF=$CONF_DIR/50-ukrrs-autologin.conf + +mkdir -p "$CONF_DIR" +[ -f "$CONF" ] && cp "$CONF" "$CONF.bak.$(date +%s)" + +cat > "$CONF" < mtu 1500 master vmbr0 state forwarding priority 32 cost 5 +7: bond0: mtu 1500 master datanet state forwarding priority 32 cost 5 +9: tap515i0: mtu 1500 master fwbr515i0 state forwarding priority 32 cost 2 +11: fwpr515p0@fwln515i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +12: fwln515i0@fwpr515p0: mtu 1500 master fwbr515i0 state forwarding priority 32 cost 2 +100: tap51015i0: mtu 1500 master fwbr51015i0 state forwarding priority 32 cost 2 +102: fwpr51015p0@fwln51015i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +103: fwln51015i0@fwpr51015p0: mtu 1500 master fwbr51015i0 state forwarding priority 32 cost 2 +118: tap501i0: mtu 1500 master fwbr501i0 state forwarding priority 32 cost 2 +120: fwpr501p0@fwln501i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +121: fwln501i0@fwpr501p0: mtu 1500 master fwbr501i0 state forwarding priority 32 cost 2 +122: tap5000i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +123: tap5101i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +124: tap5103i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +125: tap5102i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +126: tap5104i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +127: tap5105i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +128: tap5106i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +129: tap5107i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +130: tap5108i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +131: tap5109i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +132: tap51012i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +133: tap51011i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +134: tap51013i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +136: tap51016i0: mtu 1500 master fwbr51016i0 state forwarding priority 32 cost 2 +138: fwpr51016p0@fwln51016i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +139: fwln51016i0@fwpr51016p0: mtu 1500 master fwbr51016i0 state forwarding priority 32 cost 2 +140: tap53100i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +141: tap53101i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +142: tap51014i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +143: tap53102i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +144: tap53103i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +145: tap53107i0: mtu 1500 master fwbr53107i0 state forwarding priority 32 cost 2 +147: fwpr53107p0@fwln53107i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +148: fwln53107i0@fwpr53107p0: mtu 1500 master fwbr53107i0 state forwarding priority 32 cost 2 +149: tap53106i0: mtu 1500 master fwbr53106i0 state forwarding priority 32 cost 2 +151: fwpr53106p0@fwln53106i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +152: fwln53106i0@fwpr53106p0: mtu 1500 master fwbr53106i0 state forwarding priority 32 cost 2 +153: tap53105i0: mtu 1500 master fwbr53105i0 state forwarding priority 32 cost 2 +155: fwpr53105p0@fwln53105i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +156: fwln53105i0@fwpr53105p0: mtu 1500 master fwbr53105i0 state forwarding priority 32 cost 2 +157: tap53104i0: mtu 1500 master fwbr53104i0 state forwarding priority 32 cost 2 +159: fwpr53104p0@fwln53104i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +160: fwln53104i0@fwpr53104p0: mtu 1500 master fwbr53104i0 state forwarding priority 32 cost 2 +162: tap53108i0: mtu 1500 master fwbr53108i0 state forwarding priority 32 cost 2 +164: fwpr53108p0@fwln53108i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +165: fwln53108i0@fwpr53108p0: mtu 1500 master fwbr53108i0 state forwarding priority 32 cost 2 +169: tap500i0: mtu 1500 master fwbr500i0 state forwarding priority 32 cost 2 +171: fwpr500p0@fwln500i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +172: fwln500i0@fwpr500p0: mtu 1500 master fwbr500i0 state forwarding priority 32 cost 2 +173: tap500i1: mtu 1500 master datanet state forwarding priority 32 cost 2 +177: tap5111i0: mtu 1500 master vmbr0 state forwarding priority 32 cost 2 +178: tap5111i1: mtu 1500 master datanet state forwarding priority 32 cost 2 + +########## qemu vms ########## + VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID + 500 pfv-k8s-wnode-tsys5 running 12000 32.00 2793335 + 501 devbox-cloudron running 4096 32.00 2159654 + 515 preprod-hfnoc-uisp running 2048 100.00 1966 + 5000 sectestbed-sandbox running 2000 32.00 2159873 + 5100 sectestbed-template stopped 4096 32.00 0 + 5101 sectestbed-siem running 10000 132.00 2160027 + 5102 sectestbed-proxmox-pve running 3500 32.00 2161874 + 5103 sectestbed-proxmox-datacenter running 2048 32.00 2161792 + 5104 sectestbed-proxmox-pbs running 2048 32.00 2162075 + 5105 sectestbed-awx running 12000 288.00 2162239 + 5106 sectestbed-k8s-cnode running 4096 32.00 2162409 + 5107 sectestbed-k8s-wnode running 2048 32.00 2162514 + 5108 sectestbed-librenms running 2048 32.00 2162625 + 5109 sectestbed-netinfra running 2048 32.00 2162807 + 5111 ultix-streaming running 50000 438.00 3537748 + 5500 RestoreTemplate stopped 2048 32.00 0 + 51011 sectestbed-cloudron running 4096 32.00 2163252 + 51012 sectestbed-hfnoc-uisp running 2048 32.00 2163232 + 51013 sectestbed-rancherplatform running 4096 32.00 2163428 + 51014 sectestbed-proxmox-mailgw running 2048 32.00 2164206 + 51015 sectestbed-ca running 2048 32.00 2068605 + 51016 sectestbed-voip running 2048 32.00 2163789 + 53100 preprod-awx running 9000 160.00 2163973 + 53101 preprod-siem running 12000 32.00 2164164 + 53102 preprod-rancherplatform running 8000 32.00 2164659 + 53103 preprod-proxmox-mailgw running 4096 32.00 2164799 + 53104 preprod-ca running 2048 32.00 2165630 + 53105 preprod-proxmox-datacenter running 4096 32.00 2165432 + 53106 preprod-librenms running 2048 32.00 2165231 + 53107 preprod-voip running 2048 32.00 2164983 + 53108 preprod-cloudron running 2048 32.00 2170908 + +########## vm configs ########## +--- VM 500 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 12000 +meta: creation-qemu=11.0.2,ctime=1785942363 +name: pfv-k8s-wnode-tsys5 +net0: virtio=BC:24:11:C7:A8:6C,bridge=vmbr0,firewall=1 +net1: virtio=BC:24:11:91:5D:31,bridge=datanet +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-500-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=27f5e916-ed42-41a4-a0ab-cc1c811e2062 +sockets: 1 +vmgenid: b2acc351-ce33-45e9-a6f2-c2876e57e7fb +--- VM 501 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 4096 +meta: creation-qemu=11.0.2,ctime=1785952444 +name: devbox-cloudron +net0: virtio=BC:24:11:F7:B1:07,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-501-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=97c04fe7-a1ca-4675-b1e5-1a6d6d19761e +sockets: 1 +vmgenid: a9bb1154-7e2c-40a8-89f9-4a37f7743983 +--- VM 515 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1776451466 +name: preprod-hfnoc-uisp +net0: virtio=BC:24:11:74:D6:8A,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-515-disk-0,iothread=1,size=100G +scsihw: virtio-scsi-single +smbios1: uuid=ec5d795e-6814-46e4-920b-bd473959bd5f +sockets: 2 +vmgenid: 18e1f03c-7660-428c-894f-c92c8c04f13e +--- VM 5000 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-sandbox +net0: virtio=BC:24:11:EB:F0:0F,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-6000-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=569a6be3-7977-40f5-a3f0-e05719d68f52 +sockets: 2 +vmgenid: 95899daa-9d73-467c-bfd6-cefadf9340ab +--- VM 5100 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: x86-64-v2-AES +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 4096 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-template +net0: virtio=BC:24:11:BB:73:31,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:base-5100-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=c444ec6b-e601-46d9-a19f-204a91331f63 +sockets: 2 +template: 1 +vmgenid: eeecff9b-c423-4d67-aa31-2d0c5f87f962 +--- VM 5101 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 10000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-siem +net0: virtio=BC:24:11:C7:11:71,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5101-disk-0,iothread=1,size=132G +scsihw: virtio-scsi-single +smbios1: uuid=79489ab0-c040-4479-b102-0f4772059356 +sockets: 2 +vmgenid: bc34e73d-14a5-4c4d-a91d-afc079b369d9 +--- VM 5102 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-ve_9.2-1.iso,media=cdrom,size=1666190K +memory: 3500 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-proxmox-pve +net0: virtio=BC:24:11:AC:1A:51,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5102-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=4fd9e0f1-3139-4c10-b5b8-460865aa4ff5 +sockets: 2 +vmgenid: 3182fc1a-0109-4125-a326-9856ad7c7503 +--- VM 5103 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-datacenter-manager_1.1-1.iso,media=cdrom,size=1424692K +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-proxmox-datacenter +net0: virtio=BC:24:11:DC:45:D0,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5103-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=58bdbcb6-19c9-488d-bcc1-c7270a683ce2 +sockets: 2 +vmgenid: 24a287c9-a823-4a87-a0d7-59b4aabc39b1 +--- VM 5104 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-backup-server_4.2-1.iso,media=cdrom,size=1445344K +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-proxmox-pbs +net0: virtio=BC:24:11:8E:1B:23,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5104-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=e2ec6afd-0ff0-4228-8735-602e9ee5709a +sockets: 2 +vmgenid: 5205599e-7b2b-4763-b796-6056338e7eca +--- VM 5105 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 12000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-awx +net0: virtio=BC:24:11:1E:FE:57,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5105-disk-0,iothread=1,size=288G +scsihw: virtio-scsi-single +smbios1: uuid=02704679-75b0-4180-8906-071ca00a98a6 +sockets: 2 +vmgenid: 5106467f-6b7c-441a-8393-977045e0a36a +--- VM 5106 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 4096 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-k8s-cnode +net0: virtio=BC:24:11:CA:B0:CD,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5106-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=0eff8d5d-b8f3-4fdc-92b5-f93a9eeecc5e +sockets: 2 +vmgenid: 5846c8f5-518f-4491-835c-a5df8ebae413 +--- VM 5107 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-k8s-wnode +net0: virtio=BC:24:11:36:16:3C,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5107-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=65d9c31b-b4db-4655-8ed9-5b58eb801b42 +sockets: 2 +vmgenid: 25a5ffef-5f47-4ef0-ac62-e158f09f7ca5 +--- VM 5108 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-librenms +net0: virtio=BC:24:11:E2:1A:09,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5108-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=0664920e-125a-4ef3-8505-56b8c4e0f31c +sockets: 2 +vmgenid: 8c8641b5-50d0-4d87-8817-9c79f19c1c9a +--- VM 5109 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-netinfra +net0: virtio=BC:24:11:80:A1:3C,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-5109-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=f1664d3d-4dd1-451a-87b1-07724c12ca70 +sockets: 2 +vmgenid: 47f317dc-baaf-4aa7-832f-1d447d3f3392 +--- VM 5111 +agent: 1 +balloon: 0 +boot: order=scsi0;net0 +cores: 4 +cpu: host,flags=+nested-virt +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 50000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: ultix-streaming +net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0 +net1: virtio=BC:24:11:E3:32:D9,bridge=datanet +numa: 0 +ostype: l26 +scsi0: NVME:5111/vm-5111-disk-0.qcow2,iothread=1,size=438G +scsi1: ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,size=400G +scsi2: SSD:5111/vm-5111-disk-0.qcow2,iothread=1,size=200G +scsihw: virtio-scsi-single +smbios1: uuid=bc74eb91-b156-46e2-b946-808da9b4f037 +sockets: 2 +vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f +--- VM 5500 +agent: 1 +allow-ksm: 0 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: none,media=cdrom +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1773957951 +name: RestoreTemplate +net0: virtio=BC:24:11:E8:78:F3,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: D1:5500/base-5500-disk-0.qcow2,size=32G +scsihw: virtio-scsi-pci +smbios1: uuid=0d66e977-633a-4111-83a7-cc5d03c5c3b2 +sockets: 2 +template: 1 +vmgenid: c46b7a9e-6275-4ce9-811c-57380dbedbe9 +--- VM 51011 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 4096 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-cloudron +net0: virtio=BC:24:11:C3:DC:75,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-51011-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=d509a43d-a3d8-45e3-9e93-f6ddf148054e +sockets: 2 +vmgenid: 01694276-61da-4219-a665-5b114a6d7155 +--- VM 51012 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-hfnoc-uisp +net0: virtio=BC:24:11:7A:44:5F,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-51012-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=f3f05416-829c-4cfa-9c15-09034cdc18b6 +sockets: 2 +vmgenid: 5ca81070-ed5e-4d1f-a23a-39f5bd613819 +--- VM 51013 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 4096 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-rancherplatform +net0: virtio=BC:24:11:78:11:AC,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-51013-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=94efa581-b963-4fdd-b36a-ddb1b2000860 +sockets: 2 +vmgenid: f4ef5349-ef23-4f86-b22b-9053bd172dc3 +--- VM 51014 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-mail-gateway_9.1-1.iso,media=cdrom,size=1654614K +memory: 2048 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: sectestbed-proxmox-mailgw +net0: virtio=BC:24:11:8D:C0:F6,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-51014-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=1d9e15da-16c0-4c9c-8a6a-6a6868e88485 +sockets: 2 +vmgenid: 928b7e49-2a7c-4482-ad29-7ccbf2cc8411 +--- VM 51015 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785891906 +name: sectestbed-ca +net0: virtio=BC:24:11:0C:AD:B0,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-51015-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=b9a1bc29-d7e9-4d56-a281-c548ec79eca6 +sockets: 1 +vmgenid: 6c0044b3-c3b3-4cbe-be7e-d3e598cc9b0d +--- VM 51016 +agent: 1 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785909068 +name: sectestbed-voip +net0: virtio=BC:24:11:A4:5D:E1,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-51016-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=30cffa22-f92c-4069-9079-f408db14f9ab +sockets: 1 +vmgenid: e7842e31-1667-413c-8e3a-7456d3a96fbe +--- VM 53100 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 9000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: preprod-awx +net0: virtio=BC:24:11:FB:7D:38,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-53100-disk-0,iothread=1,size=160G +scsihw: virtio-scsi-single +smbios1: uuid=36763d6b-c72e-4207-975c-fc8c46aebc56 +sockets: 2 +vmgenid: 176bfe48-2e3c-488b-a9b6-bcc070d4b7b0 +--- VM 53101 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 12000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: preprod-siem +net0: virtio=BC:24:11:CB:F7:AA,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-53101-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=e29825a2-eebb-444b-b590-46377075023b +sockets: 2 +vmgenid: 35ccbc15-72e2-465b-9890-784792df8412 +--- VM 53102 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 8000 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: preprod-rancherplatform +net0: virtio=BC:24:11:6F:B0:06,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-53102-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=f3de9511-0212-4907-8d30-ef58891b5568 +sockets: 2 +vmgenid: 3654bc7e-8744-4561-ab07-a6c7aa357387 +--- VM 53103 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-mail-gateway_9.1-1.iso,media=cdrom,size=1654614K +memory: 4096 +meta: creation-qemu=10.1.2,ctime=1784819584 +name: preprod-proxmox-mailgw +net0: virtio=BC:24:11:EA:01:CD,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local-lvm:vm-53103-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=6cbc9c35-beed-4090-ad0f-8e80ac7bdc2c +sockets: 2 +vmgenid: b774c009-2573-4a34-87a1-caecb8e8fddd +--- VM 53104 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 1 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785891839 +name: preprod-ca +net0: virtio=BC:24:11:A7:53:C6,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-53104-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=73c2622e-5c88-4d53-a04b-1738de695107 +sockets: 2 +vmgenid: 48ea9fc8-3ada-4565-8190-8c8126106ef9 +--- VM 53105 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/proxmox-datacenter-manager_1.1-1.iso,media=cdrom,size=1424692K +memory: 4096 +meta: creation-qemu=11.0.2,ctime=1785901419 +name: preprod-proxmox-datacenter +net0: virtio=BC:24:11:4E:AC:7E,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-53105-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=3fb146fb-f25f-484a-987e-23ba6bb9508c +sockets: 1 +vmgenid: 4857c79e-30d3-4f60-b3b7-376679f05697 +--- VM 53106 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785901918 +name: preprod-librenms +net0: virtio=BC:24:11:6C:65:A0,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-53106-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=dec3ad07-3a55-47a4-8aea-59d89bd6435d +sockets: 1 +vmgenid: ba211ed8-020d-4b50-9fd0-87f0ae817036 +--- VM 53107 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/debian13.iso,media=cdrom,size=754M +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785908849 +name: preprod-voip +net0: virtio=BC:24:11:45:0A:1D,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-53107-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=bb910dc6-ce0b-4b08-9f3c-cfafd76f37d0 +sockets: 1 +vmgenid: ee1a0227-cd09-4741-aa52-660c37701200 +--- VM 53108 +agent: 1 +balloon: 0 +boot: order=scsi0;ide2;net0 +cores: 2 +cpu: host +ide2: local:iso/ubuntur-24.04.4.iso,media=cdrom,size=3226020K +memory: 2048 +meta: creation-qemu=11.0.2,ctime=1785972218 +name: preprod-cloudron +net0: virtio=BC:24:11:EE:F0:FF,bridge=vmbr0,firewall=1 +numa: 0 +onboot: 1 +ostype: l26 +scsi0: local-lvm:vm-53108-disk-0,iothread=1,size=32G +scsihw: virtio-scsi-single +smbios1: uuid=edca399c-72ac-4d44-939c-4388cb1aa88b +sockets: 1 +vmgenid: 5057dce7-e776-4c7e-a51b-c6d14de13733 + +########## lxc containers ########## + +########## sensors ########## +nouveau-pci-0300 +Adapter: PCI adapter +GPU core: 887.00 mV (min = +0.82 V, max = +1.21 V) +fan1: 2340 RPM +temp1: +66.0°C (high = +95.0°C, hyst = +3.0°C) + (crit = +105.0°C, hyst = +5.0°C) + (emerg = +135.0°C, hyst = +5.0°C) + +coretemp-isa-0000 +Adapter: ISA adapter +Core 0: +41.0°C (high = +85.0°C, crit = +95.0°C) +Core 1: +42.0°C (high = +85.0°C, crit = +95.0°C) +Core 9: +42.0°C (high = +85.0°C, crit = +95.0°C) +Core 10: +35.0°C (high = +85.0°C, crit = +95.0°C) + +nvme-pci-0400 +Adapter: PCI adapter +Composite: +42.9°C (low = -273.1°C, high = +69.8°C) + (crit = +79.8°C) +Sensor 1: +44.9°C (low = -273.1°C, high = +65261.8°C) +Sensor 2: +40.9°C (low = -273.1°C, high = +65261.8°C) +Sensor 5: +56.9°C (low = -273.1°C, high = +65261.8°C) + +adt7473-i2c-3-2e +Adapter: nvkm-0000:03:00.0-bus-0002 +in1: 3.00 V (min = +0.00 V, max = +2.99 V) ++3.3V: 3.28 V (min = +0.00 V, max = +4.39 V) +fan1: 2329 RPM (min = 0 RPM) +fan2: 0 RPM (min = 0 RPM) +fan3: 0 RPM (min = 164 RPM) ALARM +temp1: +52.2°C (low = +76.0°C, high = +96.0°C) ALARM + (crit = +93.0°C, hyst = +92.0°C) +Board Temp: +47.0°C (low = +20.0°C, high = +60.0°C) + (crit = +100.0°C, hyst = +100.0°C) +temp3: +52.0°C (low = +71.0°C, high = +86.0°C) ALARM + (crit = +98.0°C, hyst = +94.0°C) +pwm1: 46% (freq = 22500 Hz) +pwm3: 128% (freq = 22500 Hz) + +dell_smm-isa-00de +Adapter: ISA adapter +Processor Fan: 721 RPM (min = 0 RPM, max = 2000 RPM) +Motherboard Fan: 1136 RPM (min = 0 RPM, max = 1500 RPM) +Motherboard Fan: 1189 RPM (min = 0 RPM, max = 2500 RPM) +Chipset Fan: 2465 RPM (min = 0 RPM, max = 3700 RPM) +Ambient: +22.0°C +pwm1: N/A +pwm2: N/A +pwm3: N/A +pwm4: N/A + + +########## guest agent check ########## + +AUDIT DONE diff --git a/host-audit.sh b/host-audit.sh new file mode 100755 index 0000000..cc36fa9 --- /dev/null +++ b/host-audit.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# UKRRS host audit for pfv-tsys5 (Proxmox VE). READ-ONLY, safe to run anytime. +# Usage (from a machine that can ssh): +# ssh root@pfv-tsys5.knel.net 'bash -s' < host-audit.sh > host-audit.out +set -uo pipefail +hdr() { echo; echo "########## $* ##########"; } + +hdr "node"; hostname; uptime; pveversion 2>/dev/null || echo not-pve + +hdr "dmi system+baseboard" +dmidecode -t system 2>/dev/null | grep -E 'Manufacturer|Product Name|Version|Wake-up' +dmidecode -t baseboard 2>/dev/null | grep -E 'Manufacturer|Product Name|Version' + +hdr "processors" +dmidecode -t processor 2>/dev/null | grep -E 'Socket Designation|Version|Max Speed|Current Speed|^Status|Core Count|Thread Count|Upgrade:' +lscpu 2>/dev/null | grep -E 'Model name|^CPU\(s\)|Thread|Core|Socket|NUMA|MHz' + +hdr "cpu frequency policy" +for p in scaling_driver scaling_governor scaling_max_freq scaling_min_freq; do + v=$(cat /sys/devices/system/cpu/cpu0/cpufreq/$p 2>/dev/null) && echo "$p = $v" +done + +hdr "memory" +free -h; grep MemTotal /proc/meminfo +dmidecode -t memory 2>/dev/null | grep -E '^\s*(Size|Speed|Type:|Locator|Form Factor)' | grep -v 'No Module' + +hdr "storage: block devices" +lsblk -o NAME,SIZE,TYPE,ROTA,FSTYPE,MOUNTPOINTS,MODEL 2>/dev/null + +hdr "storage: pve" +pvesm status 2>/dev/null + +hdr "storage: zfs" +if command -v zpool >/dev/null 2>&1; then + zpool status 2>/dev/null + zfs list -o name,used,avail,recordsize,compression 2>/dev/null | head -40 + grep -E '^(size|c_max|c_min) ' /proc/spl/kstat/zfs/arcstats 2>/dev/null +else + echo no-zfs +fi + +hdr "smart health" +if command -v smartctl >/dev/null 2>&1; then + for d in $(ls /dev | grep -E '^(sd[a-z]|nvme[0-9]+)$'); do + echo "--- /dev/$d" + smartctl -H /dev/$d 2>/dev/null | grep -iE 'result|overall|health' + smartctl -i /dev/$d 2>/dev/null | grep -iE 'model|firmware|serial' | head -3 + done +else + echo no-smartctl +fi + +hdr "kernel + cmdline" +uname -a +cat /proc/cmdline +[ -f /etc/kernel/cmdline ] && cat /etc/kernel/cmdline +proxmox-boot-tool status 2>/dev/null | head -20 + +hdr "iommu/dmar" +dmesg 2>/dev/null | grep -iE 'iommu|dmar|vtd' | head -25 + +hdr "lspci (all)" +lspci -nn 2>/dev/null + +hdr "iommu groups" +if [ -d /sys/kernel/iommu_groups ]; then + for g in /sys/kernel/iommu_groups/*; do + [ -d "$g" ] || continue + echo "group ${g##*/}:" + for d in "$g"/devices/*; do + printf ' '; lspci -s "${d##*/}" 2>/dev/null + done + done +else + echo "no iommu groups (intel_iommu not enabled?)" +fi + +hdr "pci link speeds (gpu/usb/nvme candidates)" +for d in /sys/bus/pci/devices/*; do + s=$(cat "$d/current_link_speed" 2>/dev/null); w=$(cat "$d/current_link_width" 2>/dev/null) + [ -n "$s" ] && echo "${d##*/} $s x$w" +done + +hdr "network" +ip -br a 2>/dev/null || cat /proc/net/dev +bridge link 2>/dev/null + +hdr "qemu vms" +qm list 2>/dev/null + +hdr "vm configs" +for vm in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do + echo "--- VM $vm"; qm config "$vm" 2>/dev/null +done + +hdr "lxc containers" +pct list 2>/dev/null +for ct in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do + echo "--- CT $ct"; pct config "$ct" 2>/dev/null +done + +hdr "sensors" +sensors 2>/dev/null || echo no-lm-sensors + +hdr "guest agent check" +qm agent 100 ping 2>/dev/null && echo "agent ok on vm 100" || true + +echo +echo "AUDIT DONE" diff --git a/host-netcheck.out b/host-netcheck.out new file mode 100644 index 0000000..0547e79 --- /dev/null +++ b/host-netcheck.out @@ -0,0 +1,45 @@ +== qm config 5111 (relevant lines) == +cores: 4 +cpuunits: 9000 +memory: 50000 +net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0 +net1: virtio=BC:24:11:E3:32:D9,bridge=datanet +onboot: 1 +scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1 +scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1 +scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1 +scsihw: virtio-scsi-single +sockets: 2 +startup: order=10,up=180 + +== qm pending 5111 (unapplied staged changes) == +cur agent: 1 +cur balloon: 0 +cur boot: order=scsi0;net0 +cur cores: 4 +cur cpu: host,flags=+nested-virt +cur cpuunits: 9000 +cur memory: 50000 +cur meta: creation-qemu=10.1.2,ctime=1784819584 +cur name: ultix-streaming +cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0 +cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet +cur numa: 0 +cur onboot: 1 +cur ostype: l26 +cur scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1 +cur scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1 +cur scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1 +cur scsihw: virtio-scsi-single +cur smbios1: uuid=bc74eb91-b156-46e2-b946-808da9b4f037 +cur sockets: 2 +cur startup: order=10,up=180 +cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f + +== last bounce log (/var/log/ukrrs-vm5111-bounce.log) == +bounce-start 2026-08-31T15:59:56-05:00 +CPU flag 'nested-virt' resolved to 'vmx' +bounce-done 2026-08-31T16:01:41-05:00 + +== pve version == +pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve) diff --git a/questions-v1.md b/questions-v1.md new file mode 100644 index 0000000..6ddbf28 --- /dev/null +++ b/questions-v1.md @@ -0,0 +1,140 @@ +# questions-v1 — ultix-streaming optimization + +Answer inline under each A:. Where I have a recommendation it is marked REC; +"ok" is a sufficient answer. Version the file (v2) for another round. + +## Q1. Host audit (required first, blocks host-side numbers) +I cannot ssh from crush (tool policy), so run: +``` +ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out +``` +then just say "audit done". I read the file and finalize §3 of REPORT.md. +A: + +## Q2. CPU upgrade +REC: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) → 24 threads. +Cheaper alt: second E5620 (~$10) → 16 threads. Audit confirms the chassis is +dual-socket before you buy. +A: + +## Q3. RAM +REC: populate CPU2 DIMM slots to 192G total, give this VM 128G, ballooning off. +Which DIMMs to order comes from the audit (current population + speed). +A: + +## Q4. Root growth to 500G +REC: you run `qm resize scsi0 +80G`, I do the in-guest MBR shuffle +(online except a brief swapoff; backups of fstab + partition table first). +When? +A: + +## Q5. GPUs +Models + VRAM of the two incoming cards, and when? PSU wattage on the host +(sticker)? Old card: passthrough to this VM now, or leave for preprod? +Why it matters: no Above-4G-Decoding on this-era BIOS; big-VRAM cards may not +map. UEFI-only vBIOS cards would force the OVMF migration. +A: + +## Q6. q35 switch +REC: yes, one offline window, NIC rename expected in guest (tailscale survives). +Needed for clean GPU/USB3 passthrough. +A: + +## Q7. Account → slice names +accounts.tsv says: reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO, +reachableceo-offstage, COSRCEO-Personal, COSRCEO-Biz, COSWFO. +/home currently shows: COS-RCEO, COS-TSG, COS-WFO, ... Which 9 login names are +final on THIS host? (I generate one slice + user drop-in per account.) +A: + +## Q8. Per-account memory fences +REC interim (48G): each account MemoryHigh 3G / Max 4.5G. +REC post-upgrade (128G): High 10G / Max 12G. CPU: weight 75, no hard quota +(burstable). OK? +A: + +## Q9. Desktop +REC: keep KDE, fix the greeter (autologin into a locked session), Sunshine + +Moonlight after the GPU lands (NVENC). XFCE only if Sunshine capture on KDE +Wayland disappoints. OK to apply the autologin+lock config? +A: + +## Q10. Metrics/pressure wiring +REC: node_exporter as a compose service (house-style, digest-pinned) scraped +by your existing mopac-harness prometheus; PSI textfile script for per-slice +pressure; gateway polls node_exporter directly in its 30s tick (option A, no +new dependency). Keep beszel for the human dashboard. Where does the beszel +hub live? +A: + +## Q11. k8s +Control plane: where/what (k3s? existing cluster?)? REC: join AFTER host +upgrade, worker-only, low PriorityClass for burst workloads, kubelet eviction +thresholds, kubelet+containerd fenced to CPUWeight 20 + MemoryHigh. OK? +A: + +## Q12. Proxmox control scope +REC: dedicated API token (ukrrs-infra@pam harness) scoped to a preprod pool +only. Which VMIDs may the harness stand down/snapshot, and which are +PROTECTED (this VM certainly)? I hard-code the PROTECTED list into +proxmox-ctl.sh. +A: + +## Q13. SDRs +REC: passthrough both PCI USB3 controller cards whole (never per-device), +SDR processes pinned to the rt CPU pool. Confirm: both cards → THIS VM, +always-on? Any latency budget I should know about? +A: + +## Q14. Day/night windows +REC: OS night profile 22:00-05:00 local; gateway z.ai peak ladder stays +01:00-05:00 CST as configured. Confirm 22:00-05:00 (vs your words "2200 to +0500") and the timezone the guest should key off (it currently sees system TZ). +A: + +## Q15. Agent Zero +Which accounts host it, expected concurrent agents, docker-based? REC: +background agents → ukrrs-batch.slice, interactive ones → their account slice. +A: + +## Q16. TCP BBR +REC: load tcp_bbr module + bbr/fq for the long provider streams. If you +prefer stock cubic, say so and I drop two lines. +A: + +## Audit update 2026-08-31 (host-audit.out parsed; details in REPORT.md section 8) +- Q1 DONE. Q2 upgraded to URGENT: CPU2 socket confirmed empty; one E5620 + serves 24 running VMs. 2× X5675 triples threads. (DMI claims LGA771, a Dell + string quirk; E5620 is LGA1366; verify visually before ordering.) +- Q3 refined: 6× 16GB DDR3 already fill all CPU1 slots; order 6 more 16GB. +- Q5 verdict: skip Quadro 4000 passthrough (Fermi: no NVENC, no modern CUDA + driver). Still need: new GPU models/VRAM + PSU wattage sticker. +- Q13 wrinkle: only ONE USB3 card visible on the host. Where is the second? +- Q11 wrinkle: VM 500 pfv-k8s-wnode-tsys5 already exists here: join or + consolidate? +- IOMMU groups are clean (no ACS override needed). No cpufreq driver exists + anywhere, so day/night stays pure cgroup (already the design). + +## Q17. VM 5111 auto-start +Audit shows onboot unset: after any host reboot your workstation stays DOWN +until manually started. OK to set onboot=1 with startup order 10? +A: + +## Q18. Remote-desktop matrix (audit correction: xrdp IS running, sesman too) +Confirmed live: xrdp + xrdp-sesman = your Jump RDP path (my earlier "no xrdp" +was a bad probe, my error). Sunshine installed but not running. Two human +accounts, both with always-alive KDE (reachableceo 1001, -offstage 1010). +REC: keep xrdp for iPad quick-attach (verify /etc/xrdp/sesman.ini +KillDisconnected=false so sessions persist); start Sunshine only after the +compute GPUs land and use Moonlight for the video/audio editing sessions +(NVENC + proper audio); sddm autologin keeps a warm console session for that. +Confirm? And which account hosts the media-editing workload? +A: + +## Q11 partial (2026-08-31): consolidation decided +VM 500 pfv-k8s-wnode-tsys5 removed; ultix-streaming will be THE k8s worker on +this host. Still open: control plane location/type (k3s?), and timing (rec: +join after Friday's CPU/RAM resize, worker-only, kubelet+containerd fenced at +CPUWeight 20 + MemoryHigh, low PriorityClass + eviction thresholds per REPORT +5.3). +A (control plane + timing): diff --git a/staged/60-ukrrs-vm.conf b/staged/60-ukrrs-vm.conf new file mode 100644 index 0000000..4ee13c9 --- /dev/null +++ b/staged/60-ukrrs-vm.conf @@ -0,0 +1,34 @@ +# /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile +# Measured-before values and rationale: ~/optimize/REPORT.md §4.1. +# Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure, +# pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3). + +# Predictable writeback under ETL bursts (bytes-based, RAM-size independent). +# Night profile (ukrrs-daynight.sh) raises these to 2G/512M. +vm.dirty_background_bytes = 268435456 +vm.dirty_bytes = 1073741824 + +# Survive bursty reclaim when compiles+ETL hit at once. 768M post-upgrade. +vm.min_free_kbytes = 393216 + +# Postgres AIO / io_uring era; 9 harness daemons + LSPs watching many repos. +fs.aio-max-nr = 1048576 +fs.inotify.max_user_watches = 1048576 +fs.inotify.max_user_instances = 512 + +# 9 accounts x long-lived provider streams + tailscale + docker NAT. +net.ipv4.ip_local_port_range = 10240 65535 +net.ipv4.tcp_tw_reuse = 1 + +# LLM turns idle minutes between bursts on live sockets; large SSE/JSON. +net.ipv4.tcp_slow_start_after_idle = 0 +net.core.rmem_max = 16777216 +net.core.wmem_max = 16777216 +net.ipv4.tcp_rmem = 4096 131072 16777216 +net.ipv4.tcp_wmem = 4096 65536 16777216 +net.core.netdev_max_backlog = 8192 + +# BBR requires the module: staged modules-load.d/tcp_bbr.conf loads it. +# If bbr is unavailable, comment the last two lines out (cubic is fine). +net.ipv4.tcp_congestion_control = bbr +net.core.default_qdisc = fq diff --git a/staged/apply-guest.sh b/staged/apply-guest.sh new file mode 100755 index 0000000..9db12c6 --- /dev/null +++ b/staged/apply-guest.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first. +# sudo RUN=1 ./apply-guest.sh +# RUN=1 required to mutate anything (default: plan only). +# daemonjson and desktop additionally require CONFIRM=1 (one-time disruption). +# Steps: sysctl modules thp slices psi prune nofile noatime daemonjson desktop +# daynight all +set -euo pipefail +RUN=${RUN:-0} +CONFIRM=${CONFIRM:-0} +DESKTOP_USER=${DESKTOP_USER:-reachableceo} +here=$(cd "$(dirname "$0")" && pwd) + +say() { echo "[apply-guest] $*"; } +die() { echo "[apply-guest] $*" >&2; exit 2; } +need_root() { [ "$(id -u)" = 0 ] || die "needs root"; } +gate() { + if [ "$RUN" != 1 ]; then say "DRY: would $* (rerun with RUN=1)"; return 1; fi + need_root; return 0 +} +inst() { # src dst mode + install -m "$3" "$1" "$2" && say "installed $2" +} + +step_sysctl() { + gate "install sysctl.d profile" || return 0 + inst "$here/60-ukrrs-vm.conf" /etc/sysctl.d/60-ukrrs-vm.conf 0644 + sysctl --system >/dev/null && say "sysctl applied (bbr lines need the module: step modules)" +} +step_modules() { + gate "install tcp_bbr module load" || return 0 + inst "$here/modules-load.d/tcp_bbr.conf" /etc/modules-load.d/tcp_bbr.conf 0644 + modprobe tcp_bbr 2>/dev/null || say "tcp_bbr not loaded now (will load at boot)" +} +step_thp() { + gate "install THP madvise unit" || return 0 + inst "$here/systemd/ukrrs-thp-madvise.service" /etc/systemd/system/ukrrs-thp-madvise.service 0644 + systemctl daemon-reload + systemctl enable --now ukrrs-thp-madvise.service + say "THP=$(cat /sys/kernel/mm/transparent_hugepage/enabled)" +} +step_slices() { + gate "install ukrrs slices" || return 0 + for s in gateway pmo lsp batch rt; do + inst "$here/systemd/ukrrs-$s.slice" "/etc/systemd/system/ukrrs-$s.slice" 0644 + done + systemctl daemon-reload + say "slices ready; containers opt in via cgroup_parent=ukrrs-.slice" +} +step_psi() { + gate "install PSI textfile collector" || return 0 + inst "$here/ukrrs-psi-textfile.sh" /usr/local/sbin/ukrrs-psi-textfile.sh 0755 + mkdir -p /var/lib/node_exporter/textfile && chmod 755 /var/lib/node_exporter /var/lib/node_exporter/textfile + inst "$here/systemd/ukrrs-psi-textfile.service" /etc/systemd/system/ukrrs-psi-textfile.service 0644 + inst "$here/systemd/ukrrs-psi-textfile.timer" /etc/systemd/system/ukrrs-psi-textfile.timer 0644 + systemctl daemon-reload + systemctl enable --now ukrrs-psi-textfile.timer + say "pressure gauges: /var/lib/node_exporter/textfile/ukrrs_pressure.prom" +} +step_prune() { + gate "install builder-prune timer" || return 0 + inst "$here/systemd/ukrrs-builder-prune.service" /etc/systemd/system/ukrrs-builder-prune.service 0644 + inst "$here/systemd/ukrrs-builder-prune.timer" /etc/systemd/system/ukrrs-builder-prune.timer 0644 + systemctl daemon-reload + systemctl enable --now ukrrs-builder-prune.timer +} +step_nofile() { + gate "raise DefaultLimitNOFILE" || return 0 + mkdir -p /etc/systemd/system.conf.d + cat > /etc/systemd/system.conf.d/50-ukrrs.conf <<'EOF' +[Manager] +DefaultLimitNOFILE=65536:1048576 +EOF + systemctl daemon-reload + say "DefaultLimitNOFILE raised (new sessions)" +} +step_noatime() { + gate "add noatime to /, /data1, /data2" || return 0 + cp /etc/fstab "/etc/fstab.bak.ukrrs.$(date +%s)" + for mnt in / /data1 /data2; do + opts=$(findmnt -n -o OPTIONS "$mnt") || continue + case ",$opts," in *,noatime,*) say "$mnt already noatime"; continue ;; esac + mount -o remount,noatime "$mnt" + awk -v m="$mnt" 'BEGIN{FS=OFS=" "} $2==m && $3=="ext4" { if ($4 !~ /(^|,)noatime(,|$)/) $4=$4",noatime" } 1' \ + /etc/fstab > /etc/fstab.ukrrs.new && mv /etc/fstab.ukrrs.new /etc/fstab + say "$mnt: remounted noatime + fstab updated" + done +} +step_daemonjson() { + [ "$CONFIRM" = 1 ] || die "daemonjson needs CONFIRM=1: restarts docker ONCE (live-restore protects future restarts)" + gate "install daemon.json" || return 0 + [ -f /etc/docker/daemon.json ] && cp /etc/docker/daemon.json "/etc/docker/daemon.json.bak.ukrrs.$(date +%s)" + inst "$here/docker/daemon.json" /etc/docker/daemon.json 0644 + systemctl restart docker + say "docker restarted with new config; check: docker info | grep -E 'Live|Logging'" +} +step_desktop() { + [ "$CONFIRM" = 1 ] || die "desktop needs CONFIRM=1: enables sddm autologin for $DESKTOP_USER" + gate "configure sddm autologin+lock" || return 0 + mkdir -p /etc/sddm.conf.d + cat > /etc/sddm.conf.d/50-ukrrs-autologin.conf < [mem_high] [mem_max] [allowed_cpus] [tier] +# +# tier=agent (default): CPUWeight=75, pinned to a CPU pool — the 9 PMO/worker +# lanes. interim rec: 3G 4.5G 0-5 post-upgrade: 10G 12G 0-15 +# tier=human: CPUWeight=600, NO cpu restriction, generous memory — the two +# human interactive accounts (KDE/CAD/EDA/video via xrdp live in these). +# interim rec: 12G 16G all post-upgrade: 32G 40G all +# +# Examples (uids verified 2026-08-31): +# sudo RUN=1 ./mkacct.sh reachableceo 1001 12G 16G all human +# sudo RUN=1 ./mkacct.sh reachableceo-offstage 1010 12G 16G all human +# sudo RUN=1 ./mkacct.sh TSGBOD 3G 4.5G 0-5 agent +# +# RUN=1 installs (needs root); default prints the plan + snippets only. +set -euo pipefail +acct=${1:?account}; uid=${2:?uid} +high=${3:-3G}; max=${4:-4.5G}; cpus=${5:-0-5}; tier=${6:-agent} +RUN=${RUN:-0} + +case "$tier" in + human) weight=600; cpuline="" ;; + agent) weight=75; cpuline="AllowedCPUs=$cpus" ;; + *) echo "tier must be human or agent" >&2; exit 2 ;; +esac + +slice="/etc/systemd/system/ukrrs-acct-$acct.slice" +userdrop="/etc/systemd/system/user-$uid.slice.d/50-ukrrs.conf" + +cat <&2; exit 1; } + cat > "$slice" < "$userdrop" < +# proxmox-ctl.sh start +# proxmox-ctl.sh snapshot +# Token setup (on the pve host, see REPORT.md appendix B): +# pveum user add ukrrs-infra@pam +# pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0 +# pveum acl modify /pool/ -user ukrrs-infra@pam -role PVEVMUser +set -euo pipefail +: "${PROTECTED:?set PROTECTED=vmid1,vmid2,... (this VM must be in the list)}" + +pvesh_() { + if [ "${PVE_LOCAL:-0}" = 1 ]; then + command pvesh "$@" + else + : "${PVE_HOST:?}" "${PVE_TOKEN:?}" + command pvesh --host "$PVE_HOST" --api-token "$PVE_TOKEN" "$@" + fi +} + +guard() { + case ",$PROTECTED," in + *",$1,"*) echo "REFUSED: vmid $1 is PROTECTED" >&2; exit 3 ;; + esac +} + +cmd=${1:-}; shift || true +case "$cmd" in + vms) + pvesh_ get /cluster/resources --type vm ;; + shutdown) + [ $# = 1 ] || { echo "usage: $0 shutdown " >&2; exit 2; } + guard "$1"; : "${PVE_NODE:?}" + pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/shutdown" --timeout 120 ;; + start) + [ $# = 1 ] || { echo "usage: $0 start " >&2; exit 2; } + guard "$1"; : "${PVE_NODE:?}" + pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/start" ;; + snapshot) + [ $# = 2 ] || { echo "usage: $0 snapshot " >&2; exit 2; } + guard "$1"; : "${PVE_NODE:?}" + pvesh_ create "/nodes/$PVE_NODE/qemu/$1/snapshot" snapname="$2" ;; + *) + echo "usage: $0 vms|shutdown |start |snapshot " >&2 + exit 2 ;; +esac diff --git a/staged/systemd/ukrrs-batch.slice b/staged/systemd/ukrrs-batch.slice new file mode 100644 index 0000000..c1e09f7 --- /dev/null +++ b/staged/systemd/ukrrs-batch.slice @@ -0,0 +1,14 @@ +[Unit] +Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents +Documentation=file:///home/reachableceo/optimize/REPORT.md + +[Slice] +# Day bias: lowest weight, restricted to pool A. The night profile timer +# (ukrrs-nightprofile.service) raises weight/MemoryHigh and widens AllowedCPUs +# via `systemctl set-property --runtime`; reboot lands back in day mode. +# IOWeight deliberately absent: inert under the `none` I/O scheduler. +CPUWeight=25 +AllowedCPUs=0-5 +MemoryHigh=12884901888 +MemoryMax=17179869184 +TasksMax=16384 diff --git a/staged/systemd/ukrrs-builder-prune.service b/staged/systemd/ukrrs-builder-prune.service new file mode 100644 index 0000000..3928c6e --- /dev/null +++ b/staged/systemd/ukrrs-builder-prune.service @@ -0,0 +1,7 @@ +[Unit] +Description=UKRRS: weekly docker build-cache prune (25G ceiling) +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/docker builder prune --force --keep-storage 25GB diff --git a/staged/systemd/ukrrs-builder-prune.timer b/staged/systemd/ukrrs-builder-prune.timer new file mode 100644 index 0000000..da1d734 --- /dev/null +++ b/staged/systemd/ukrrs-builder-prune.timer @@ -0,0 +1,9 @@ +[Unit] +Description=UKRRS: build-cache prune, Sunday night window + +[Timer] +OnCalendar=Sun *-*-* 22:30:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/staged/systemd/ukrrs-dayprofile.service b/staged/systemd/ukrrs-dayprofile.service new file mode 100644 index 0000000..1b471b4 --- /dev/null +++ b/staged/systemd/ukrrs-dayprofile.service @@ -0,0 +1,6 @@ +[Unit] +Description=UKRRS: switch back to day profile (interactive bias) + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ukrrs-daynight.sh day diff --git a/staged/systemd/ukrrs-dayprofile.timer b/staged/systemd/ukrrs-dayprofile.timer new file mode 100644 index 0000000..fbca759 --- /dev/null +++ b/staged/systemd/ukrrs-dayprofile.timer @@ -0,0 +1,9 @@ +[Unit] +Description=UKRRS: day profile at 05:00 + +[Timer] +OnCalendar=*-*-* 05:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/staged/systemd/ukrrs-gateway-ensure.service b/staged/systemd/ukrrs-gateway-ensure.service new file mode 100644 index 0000000..f17b522 --- /dev/null +++ b/staged/systemd/ukrrs-gateway-ensure.service @@ -0,0 +1,12 @@ +[Unit] +Description=ukrrs gateway ensure-up (tailscale port race + live-restore endpoint loss) +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/ukrrs-gateway-ensure.sh + +[Install] +WantedBy=multi-user.target diff --git a/staged/systemd/ukrrs-gateway.slice b/staged/systemd/ukrrs-gateway.slice new file mode 100644 index 0000000..84c7a90 --- /dev/null +++ b/staged/systemd/ukrrs-gateway.slice @@ -0,0 +1,12 @@ +[Unit] +Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta) +Documentation=file:///home/reachableceo/optimize/REPORT.md + +[Slice] +# Latency-sensitive: all provider traffic + spend accounting. Weight beats any +# single account even under total compile storm (see REPORT section 2 math). +CPUWeight=900 +# Hard floor so accounts can never squeeze the accounting DB into reclaim. +MemoryMin=2147483648 +MemoryHigh=12884901888 +TasksMax=infinity diff --git a/staged/systemd/ukrrs-lsp.slice b/staged/systemd/ukrrs-lsp.slice new file mode 100644 index 0000000..5c7c5b0 --- /dev/null +++ b/staged/systemd/ukrrs-lsp.slice @@ -0,0 +1,9 @@ +[Unit] +Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads) +Documentation=file:///home/reachableceo/optimize/REPORT.md + +[Slice] +CPUWeight=150 +MemoryHigh=6442450944 +MemoryMax=8589934592 +TasksMax=8192 diff --git a/staged/systemd/ukrrs-net-multiqueue.service b/staged/systemd/ukrrs-net-multiqueue.service new file mode 100644 index 0000000..5bd1030 --- /dev/null +++ b/staged/systemd/ukrrs-net-multiqueue.service @@ -0,0 +1,12 @@ +[Unit] +Description=Enable virtio NIC multiqueue (host offers 4 queues on ens18, 2 on ens19) +After=network-pre.target +Before=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/sbin/ethtool -L ens18 combined 4 || true; /usr/sbin/ethtool -L ens19 combined 2 || true; true' + +[Install] +WantedBy=multi-user.target diff --git a/staged/systemd/ukrrs-nightprofile.service b/staged/systemd/ukrrs-nightprofile.service new file mode 100644 index 0000000..95b58c0 --- /dev/null +++ b/staged/systemd/ukrrs-nightprofile.service @@ -0,0 +1,6 @@ +[Unit] +Description=UKRRS: switch to night profile (batch burn window 22:00-05:00) + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ukrrs-daynight.sh night diff --git a/staged/systemd/ukrrs-nightprofile.timer b/staged/systemd/ukrrs-nightprofile.timer new file mode 100644 index 0000000..b3ffc51 --- /dev/null +++ b/staged/systemd/ukrrs-nightprofile.timer @@ -0,0 +1,9 @@ +[Unit] +Description=UKRRS: night profile at 22:00 + +[Timer] +OnCalendar=*-*-* 22:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/staged/systemd/ukrrs-pmo.slice b/staged/systemd/ukrrs-pmo.slice new file mode 100644 index 0000000..ed1604d --- /dev/null +++ b/staged/systemd/ukrrs-pmo.slice @@ -0,0 +1,11 @@ +[Unit] +Description=UKRRS PMO dispatch loops (one per account): control plane +Documentation=file:///home/reachableceo/optimize/REPORT.md + +[Slice] +# Dispatcher must never starve behind worker storms: cheap but weighted above +# batch and idle accounts. Night profile leaves this untouched. +CPUWeight=200 +MemoryHigh=1073741824 +MemoryMax=1610612736 +TasksMax=512 diff --git a/staged/systemd/ukrrs-psi-textfile.service b/staged/systemd/ukrrs-psi-textfile.service new file mode 100644 index 0000000..213ae75 --- /dev/null +++ b/staged/systemd/ukrrs-psi-textfile.service @@ -0,0 +1,7 @@ +[Unit] +Description=UKRRS: PSI textfile collector tick +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ukrrs-psi-textfile.sh diff --git a/staged/systemd/ukrrs-psi-textfile.timer b/staged/systemd/ukrrs-psi-textfile.timer new file mode 100644 index 0000000..ccb4774 --- /dev/null +++ b/staged/systemd/ukrrs-psi-textfile.timer @@ -0,0 +1,10 @@ +[Unit] +Description=UKRRS: PSI textfile collector (15s) + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15 +AccuracySec=5 + +[Install] +WantedBy=timers.target diff --git a/staged/systemd/ukrrs-rt.slice b/staged/systemd/ukrrs-rt.slice new file mode 100644 index 0000000..450bb45 --- /dev/null +++ b/staged/systemd/ukrrs-rt.slice @@ -0,0 +1,12 @@ +[Unit] +Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work +Documentation=file:///home/reachableceo/optimize/REPORT.md + +[Slice] +# Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores, +# so anything placed here keeps headroom even under full compile storm. +# Interim 8 vCPU: 6-7. Post-upgrade 20 vCPU: 18-19 (edit after resize). +CPUWeight=10000 +AllowedCPUs=6-7 +MemoryHigh=2147483648 +MemoryMax=4294967296 diff --git a/staged/systemd/ukrrs-thp-madvise.service b/staged/systemd/ukrrs-thp-madvise.service new file mode 100644 index 0000000..b3b01dc --- /dev/null +++ b/staged/systemd/ukrrs-thp-madvise.service @@ -0,0 +1,11 @@ +[Unit] +Description=UKRRS: transparent hugepages to madvise (postgres-friendly) +After=multi-user.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled' + +[Install] +WantedBy=multi-user.target diff --git a/staged/ukrrs-daynight.sh b/staged/ukrrs-daynight.sh new file mode 100755 index 0000000..47d6f2a --- /dev/null +++ b/staged/ukrrs-daynight.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# UKRRS day/night resource profile (REPORT.md section 5.5). +# Runtime-only changes (set-property --runtime + sysctl): a reboot always +# lands safely in day mode (slice unit files carry the day defaults). +set -euo pipefail +mode=${1:?usage: ukrrs-daynight.sh day|night} +CONF=/etc/ukrrs/daynight.conf +[ -r "$CONF" ] && . "$CONF" + +: "${DATA2_DEV:=/dev/sdb}" +: "${BATCH_WEIGHT_DAY:=25}"; : "${BATCH_WEIGHT_NIGHT:=400}" +: "${BATCH_CPUS_DAY:=0-5}"; : "${BATCH_CPUS_NIGHT:=0-6}" +: "${BATCH_HIGH_DAY:=12G}"; : "${BATCH_HIGH_NIGHT:=30G}" +: "${GW_WEIGHT_DAY:=900}"; : "${GW_WEIGHT_NIGHT:=500}" +: "${DATA2_RA_DAY:=256}"; : "${DATA2_RA_NIGHT:=2048}" +: "${DIRTY_DAY:=1073741824}"; : "${DIRTY_NIGHT:=2147483648}" +: "${DIRTY_BG_DAY:=268435456}"; : "${DIRTY_BG_NIGHT:=536870912}" + +log() { echo "[ukrrs-daynight] $*"; } +setprop() { systemctl set-property --runtime "$@"; } +setra() { blockdev --setra "$1" "$2" 2>/dev/null || log "readahead skip: $2"; } + +case "$mode" in +night) + setprop ukrrs-batch.slice \ + CPUWeight="$BATCH_WEIGHT_NIGHT" \ + AllowedCPUs="$BATCH_CPUS_NIGHT" \ + MemoryHigh="$BATCH_HIGH_NIGHT" + setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_NIGHT" + setra "$DATA2_RA_NIGHT" "$DATA2_DEV"; setra "$DATA2_RA_NIGHT" "${DATA2_DEV}1" + sysctl -q -w vm.dirty_bytes="$DIRTY_NIGHT" vm.dirty_background_bytes="$DIRTY_BG_NIGHT" + ;; +day) + setprop ukrrs-batch.slice \ + CPUWeight="$BATCH_WEIGHT_DAY" \ + AllowedCPUs="$BATCH_CPUS_DAY" \ + MemoryHigh="$BATCH_HIGH_DAY" + setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_DAY" + setra "$DATA2_RA_DAY" "$DATA2_DEV"; setra "$DATA2_RA_DAY" "${DATA2_DEV}1" + sysctl -q -w vm.dirty_bytes="$DIRTY_DAY" vm.dirty_background_bytes="$DIRTY_BG_DAY" + ;; +*) + echo "unknown mode: $mode" >&2; exit 2 ;; +esac +log "profile $mode applied $(date -Is)" diff --git a/staged/ukrrs-gateway-ensure.sh b/staged/ukrrs-gateway-ensure.sh new file mode 100755 index 0000000..0970acb --- /dev/null +++ b/staged/ukrrs-gateway-ensure.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Boots are racy for the gateway containers (both root-caused 2026-08-31): +# - prod binds :4000/:9090 to the tailscale IP; if dockerd wins the race over +# tailscaled, the bind fails and docker NEVER retries failed starts; +# - live-restore can drop a container's network endpoint across a boot, +# leaving it started with the host resolver (gateway-db unresolvable, +# crash loop). A plain `docker start` does NOT reattach the endpoint. +# Waits for the tailscale IP, then recreates any lane not actually serving. +# Idempotent; recreating is safe at boot time (nothing in flight). +set -u +GW_DIR=/home/reachableceo/projects/ukrrs/gateway +TS_IP=100.101.187.119 + +for _ in $(seq 1 60); do + /usr/sbin/ip -o addr | grep -q "$TS_IP/" && break + sleep 2 +done + +ensure_lane() { # name host:port project compose_file + local name=$1 hostport=$2 project=$3 file=$4 + if timeout 5 bash -c "/dev/null; then + echo "$name: serving on $hostport" + return 0 + fi + echo "$name: NOT serving on $hostport; recreating" + docker compose -p "$project" -f "$GW_DIR/$file" up -d --no-deps --force-recreate gateway +} + +ensure_lane prod "$TS_IP:4000" mopac-gateway docker-compose.yml +ensure_lane beta "127.0.0.1:4002" mopac-gateway-beta compose.beta.yaml +exit 0 diff --git a/staged/ukrrs-psi-textfile.sh b/staged/ukrrs-psi-textfile.sh new file mode 100755 index 0000000..464de2d --- /dev/null +++ b/staged/ukrrs-psi-textfile.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Per-slice + host PSI (pressure stall info) avg60 -> node_exporter textfile. +# Consumed by the existing mopac-harness prometheus; gateway may also read it. +set -u +OUT_DIR=${UKRRS_TEXTFILE_DIR:-/var/lib/node_exporter/textfile} +CGROOT=${UKRRS_CGROOT:-/sys/fs/cgroup} +mkdir -p "$OUT_DIR" +tmp=$(mktemp "$OUT_DIR/.ukrrs.XXXXXX") + +some60() { # file -> prints avg60 value or nothing + awk '$1=="some"{for(i=2;i<=NF;i++) if($i ~ /^avg60=/){sub("avg60=","",$i); print $i; exit}}' "$1" 2>/dev/null +} + +emit() { # name cgroup_path + local res v + for res in cpu memory io; do + [ -r "$2/$res.pressure" ] || continue + v=$(some60 "$2/$res.pressure") + [ -n "${v:-}" ] && echo "ukrrs_slice_pressure_some60{slice=\"$1\",res=\"$res\"} $v" + done +} + +echo "# HELP ukrrs_slice_pressure_some60 PSI some avg60 (percent) for ukrrs slices" >>"$tmp" +echo "# TYPE ukrrs_slice_pressure_some60 gauge" >>"$tmp" +for cg in "$CGROOT"/system.slice/ukrrs-*.slice; do + [ -d "$cg" ] && emit "${cg##*/}" "$cg" +done +[ -d "$CGROOT/user.slice" ] && emit "user.slice" "$CGROOT/user.slice" + +echo "# HELP ukrrs_host_pressure_some60 PSI some avg60 (percent) host-wide" >>"$tmp" +echo "# TYPE ukrrs_host_pressure_some60 gauge" >>"$tmp" +for res in cpu memory io; do + [ -r "/proc/pressure/$res" ] || continue + v=$(some60 "/proc/pressure/$res") + [ -n "${v:-}" ] && echo "ukrrs_host_pressure_some60{res=\"$res\"} $v" +done + +mv "$tmp" "$OUT_DIR/ukrrs_pressure.prom" +chmod 0644 "$OUT_DIR/ukrrs_pressure.prom"