ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111

Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
2026-08-31 16:46:13 -05:00
commit cba2a64d67
43 changed files with 3137 additions and 0 deletions
@@ -0,0 +1 @@
 .crush/
@@ -0,0 +1,23 @@
 #!/usr/bin/env bash
 # ONE-SHOT guest prep (idempotent; safe to re-run). Run ON ultix-streaming:
 #   sudo ~/optimize/1-guest-prep.sh
 # 1. All staged configs (docker restarts once) + sddm autologin.
 # 2. Human-tier slices for the two human accounts (KDE/CAD/video priority).
 # NO storage operations (root growth is manual: GROW-ROOT-RUNBOOK.md).
 set -euo pipefail
 [ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; }
 DIR=$(cd "$(dirname "$0")" && pwd)
 echo "== [1/3] all staged configs =="
 RUN=1 CONFIRM=1 bash "$DIR/staged/apply-guest.sh" all
 echo "== [2/3] human-tier slices (CPUWeight 600, no cpu pinning, 12G/16G) =="
 RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo 1001 12G 16G all human
 RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo-offstage 1010 12G 16G all human
 echo "== [3/3] done =="
 echo "GUEST READY. Next, the host one-shot (safe to run from THIS VM; the"
 echo "VM bounce is detached on the host and survives your ssh dying):"
 echo "  ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/optimize/2-host-one-shot.sh"
 echo
 echo "Root growth to ~505G stays MANUAL: ~/optimize/GROW-ROOT-RUNBOOK.md"
@@ -0,0 +1,95 @@
 #!/usr/bin/env bash
 # ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111.
 #   ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
 #   ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
 #   ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ...   # optional
 #
 # 1. VM 5111's disk/net/onboot flags are written as PENDING config: they
 #    activate automatically at the VM's NEXT reboot (tonight, your last step
 #    today, Friday — whenever the VM naturally restarts). Nothing forces it.
 # 2. CPU priority (live): VM 5111 = 9000, sectestbed/preprod fleet = 50.
 # 3. Reboot wave: every OTHER running VM.
 # Optional --with-vm-restart: also gracefully bounce VM 5111 at the end.
 # THE HOST ITSELF IS NEVER REBOOTED (NFS server). NO DISK IS RESIZED
 # (storage ops are manual-only: GROW-ROOT-RUNBOOK.md).
 set -euo pipefail
 GO=0; VMR=0; SKIPWAVE=0
 for a in "$@"; do
  case "$a" in
    --go) GO=1 ;;
    --with-vm-restart) VMR=1 ;;
    --skip-wave) SKIPWAVE=1 ;;
  esac
 done
 VM=5111
 FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108"
 run() { if [ "$GO" = 1 ]; then "$@"; else echo "DRY: $*"; fi; }
 staget() { # pending-stage a qm set; tolerate refusal on a running VM
  if [ "$GO" = 1 ]; then
    qm set "$@" || echo "  NOTE: not staged while running; use --with-vm-restart (or set while stopped) later"
  else
    echo "DRY(pending): qm set $*"
  fi
 }
 echo "== 1. VM $VM flags as PENDING config (live at its next reboot) =="
 staget "$VM" --onboot 1 --startup order=10,up=180
 staget "$VM" -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
 staget "$VM" -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
 staget "$VM" -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
 staget "$VM" --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
 staget "$VM" --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
 staget "$VM" --delete ide2
 echo "== 2. CPU priority (live, reversible) =="
 run qm set "$VM" --cpuunits 9000
 for id in $FLEET; do run qm set "$id" --cpuunits 50; done
 echo "== 3. reboot wave: every other running VM =="
 if [ "$SKIPWAVE" = 1 ]; then
  echo "skipped (--skip-wave)"
 else
 OTHERS=$(qm list | awk 'NR>1 && $3=="running" && $1!="'"$VM"'" {print $1}')
 echo "targets: ${OTHERS//$'\n'/ }"
 for id in $OTHERS; do
  if [ "$GO" = 1 ]; then
    qm reboot "$id" 2>/dev/null || { qm shutdown "$id" --timeout 120 || true; sleep 2; qm start "$id" 2>/dev/null || true; }
  else
    echo "DRY: reboot $id"
  fi
 done
 fi
 if [ "$GO" = 1 ]; then
  echo; echo "== pending queue for VM $VM (activates at its next restart) =="
  qm pending "$VM" 2>/dev/null || true
 fi
 LOG=/var/log/ukrrs-vm5111-bounce.log
 if [ "$VMR" = 1 ]; then
  echo "== graceful bounce of VM $VM (detached; flags applied while STOPPED) =="
  qm status "$VM"
  if [ "$GO" = 1 ]; then
    setsid bash -c "
      echo bounce-start \$(date -Is)
      qm shutdown $VM --timeout 120
      qm wait $VM --timeout 180 || true
      qm set $VM --onboot 1 --startup order=10,up=180
      qm set $VM -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
      qm set $VM -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
      qm set $VM -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
      qm set $VM --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
      qm set $VM --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
      qm set $VM --delete ide2 || true
      qm start $VM
      echo bounce-done \$(date -Is)
    " </dev/null >"$LOG" 2>&1 &
    echo "dispatched. Downtime ~3-4 min; it comes back on its own."
    echo "Watch from anywhere: ssh root@pfv-tsys5.knel.net tail -f $LOG"
  else
    echo "DRY: would detach-bounce $VM, applying flags while stopped, log $LOG"
  fi
 else
  echo
  echo "VM $VM was NOT rebooted. Flags sit pending and go live at its next restart."
 fi
@@ -0,0 +1,26 @@
 #!/usr/bin/env bash
 # Post-reboot validation fallout (2026-08-31). Run with sudo, once:
 #   sudo ~/optimize/3-post-reboot-fixes.sh
 # 1. Installs ethtool + enables virtio multiqueue (host side now offers
 #    4 queues on ens18, 2 on ens19; the guest defaults to 1 until told).
 # 2. Installs the gateway ensure-up unit (prod boot race on the tailscale-IP
 #    port bind + live-restore endpoint loss, both lanes; idempotent).
 set -euo pipefail
 [ "$(id -u)" = 0 ] || { echo "run with sudo"; exit 1; }
 cd "$(dirname "$0")"
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
 apt-get install -y -qq ethtool
 install -m 0755 staged/ukrrs-gateway-ensure.sh  /usr/local/sbin/
 install -m 0644 staged/systemd/ukrrs-gateway-ensure.service  /etc/systemd/system/
 install -m 0644 staged/systemd/ukrrs-net-multiqueue.service  /etc/systemd/system/
 systemctl daemon-reload
 systemctl enable --now ukrrs-net-multiqueue.service
 systemctl enable --now ukrrs-gateway-ensure.service
 echo; echo "== verify =="
 ethtool -l ens18
 ethtool -l ens19
 timeout 5 bash -c '</dev/tcp/100.101.187.119/4000' 2>/dev/null && echo "gateway prod: serving on :4000"
@@ -0,0 +1,21 @@
 #!/usr/bin/env bash
 # Read-only host check: why did the net multiqueue flags not land on VM 5111?
 # Run from the guest:  ~/optimize/4-host-netcheck.sh
 # Everything is saved to ~/optimize/host-netcheck.out for crush to read.
 set -euo pipefail
 OUT="$(cd "$(dirname "$0")" && pwd)/host-netcheck.out"
 timeout 120 ssh root@pfv-tsys5.knel.net '
  echo "== qm config 5111 (relevant lines) =="
  qm config 5111 | grep -E "^(net|scsi|onboot|startup|cpuunits|ide|machine|cores|sockets|threads|memory)" || true
  echo
  echo "== qm pending 5111 (unapplied staged changes) =="
  qm pending 5111 2>&1 || true
  echo
  echo "== last bounce log (/var/log/ukrrs-vm5111-bounce.log) =="
  tail -n 40 /var/log/ukrrs-vm5111-bounce.log 2>&1 || true
  echo
  echo "== pve version =="
  pveversion 2>&1 | head -3
 ' | tee "$OUT"
 echo
 echo "saved: $OUT"
@@ -0,0 +1,95 @@
 # Manual runbook: grow / from 279G to ~505G (VM 5111, ultix-streaming)
 YOU run every command, one at a time, checking the output before the next.
 Nothing here is automated, scheduled, or boot-triggered. Storage ops are the
 only steps in this whole effort that no script touches.
 ## Layout change
 ```
 before: sda1 root 279.2G | sda2 extended [ sda5 swap 8.8G ] | ~150G unallocated
 after:  sda1 root ~505G   | sda2 swap ~13G (new, at the tail)
 ```
 MBR stays MBR, ext4 grows online, VM stays up the whole time. You can safely
 stop after any step; the system runs fine at any intermediate size. Takes
 about 10 minutes end to end.
 ## Step 0 — preconditions
 ```
 command -v growpart || sudo apt-get install -y cloud-guest-utils
 df -h / /data1 /data2          # know your starting point
 lsblk /dev/sda
 ```
 ## Step 1 — backups (30 seconds, do not skip)
 ```
 sudo sfdisk --dump /dev/sda | sudo tee /root/sda.sfdisk.bak.$(date +%F)
 sudo cp /etc/fstab /root/fstab.bak.$(date +%F)
 ```
 Check: `head /root/sda.sfdisk.bak.*` lists sda1, sda2, sda5.
 ## Step 2 — grow the virtual disk at the host (hot, VM stays up)
 ```
 ssh root@pfv-tsys5.knel.net 'qm config 5111 | grep scsi0'    # eyeball the target
 ssh root@pfv-tsys5.knel.net 'qm resize 5111 scsi0 +80G'
 ```
 Check in guest: `lsblk /dev/sda` shows sda = 518G, sda1 still 279.2G.
 ## Step 3 — retire the old swap
 ```
 free -h                                  # swap "used" must be ~0
 sudo swapoff /dev/sda5
 OLDUUID=$(sudo blkid -s UUID -o value /dev/sda5); echo "OLDUUID=$OLDUUID"
 cat /proc/swaps                          # check: empty
 ```
 ## Step 4 — remove the extended partition (the strip between root and free space)
 ```
 sudo sfdisk --delete /dev/sda 2
 sudo partprobe /dev/sda
 lsblk /dev/sda
 ```
 Check: only sda1 remains. If sda1 is missing: STOP, do not reboot, do not
 write anything; restore the table:
 `sudo sfdisk /dev/sda < /root/sda.sfdisk.bak.<date>`
 ## Step 5 — grow partition 1 into the free space
 ```
 sudo growpart /dev/sda 1
 # parted alternative: sudo parted /dev/sda resizepart 1 100%
 lsblk /dev/sda                           # sda1 now ~505G
 ```
 ## Step 6 — grow the filesystem (online)
 ```
 sudo resize2fs /dev/sda1
 df -h /                                  # ~500G available
 ```
 ## Step 7 — new swap at the tail
 ```
 sudo parted -s /dev/sda mkpart primary linux-swap 505GB 100%
 sudo partprobe /dev/sda
 lsblk /dev/sda
 NEWSWAP=$(lsblk -no NAME,TYPE /dev/sda | awk '$2=="part"{print $1}' | tail -1); echo "NEWSWAP=$NEWSWAP"
 sudo mkswap /dev/$NEWSWAP
 NEWUUID=$(sudo blkid -s UUID -o value /dev/$NEWSWAP); echo "NEWUUID=$NEWUUID"
 ```
 ## Step 8 — swap fstab line, activate
 ```
 sudo sed -i "s/^UUID=$OLDUUID/#UUID=$OLDUUID retired-sda5 $(date +%F)/" /etc/fstab
 echo "UUID=$NEWUUID none swap sw 0 0" | sudo tee -a /etc/fstab
 sudo swapon -a
 cat /proc/swaps                           # new swap active
 sudo fstrim -v /                          # optional: reclaim (discard is on now)
 ```
 ## Rollback map
 - Anytime before step 4: nothing changed except two backup files.
 - After partition edits, before resize2fs: restore table from the sfdisk dump
  (step 4 note); no data has moved, only the table.
 - After resize2fs: growth is one-way by design; reverting size means restore
  from backups, so this is the one step to do when calm. (It is also the
  safest operation in the list: online ext4 grow is journaled.)
 - fstab: backups in /root; only the swap line changes, and a bad swap line is
  non-fatal at boot (root entry is untouched).
@@ -0,0 +1,38 @@
 # NEXT — perf-opt closeout COMPLETE (2026-08-31 16:4x)
 ## Final state — nothing owed
 - All guest tuning live and verified (see 16:04-16:15 pass below).
 - Host flags live: ssd=1/discard (ROTA=0), iothread, onboot=1, startup
  order=10, cpuunits 9000/50, ide2 gone.
 - Net multiqueue: queues=4/2 now IN live qm config (staged 16:40 by crush
  via ssh; activates at next VM start = Friday). Guest oneshot unit
  ukrrs-net-multiqueue auto-runs ethtool -L on that boot.
 - Gateway boot-race + live-restore endpoint loss: permanently fixed by
  ukrrs-gateway-ensure unit (enabled, both lanes).
 - Root cause of the 15:59 no-op: the script copy executed then predates the
  queues= fix (silent same-value rewrite); current config verified correct.
 ## Verified live after the 16:01 bounce (all read-only from the guest)
 - Disks: ROTA=0, discard 4K/1G on sda+sdb+sdc → ssd=1/discard=on ACTIVE ✅
 - Kernel: bbr+fq, 16M socket buffers, dirty_bytes 1G/256M, min_free 384M,
  aio 1M, inotify 512/1M, port range 10240-65535, slow_start_after_idle=0,
  tcp_tw_reuse=1, THP=madvise, DefaultLimitNOFILE 65536 ✅
 - Mounts /,/data1,/data2 = noatime; swap sda5 prio -2 unused ✅
 - Units: all ukrrs slices + day/night + psi + builder-prune + thp units
  installed and enabled; PSI textfile writing (16:06) ✅
 - Docker: daemon.json fully live (live-restore, log caps 20m×3, 172.16/12
  pools, metrics :9323); 21 containers up ✅
 - Desktop: sddm autologin worked (Relogin=true), session alive on seat0/tty2,
  greeter burn GONE ✅
 - Gateway prod was DOWN at boot (bind race + live-restore endpoint loss) →
  recovered via compose --force-recreate; /status = mode normal, serving ✅
 ## Nothing owed. Optional sanity check any time (crush can ssh now):
  ~/optimize/4-host-netcheck.sh
 ## Noted, no action taken
 - mopac-demo/mcli fake containers: restart=no, exited at the first bounce,
  left down (test stubs; your call).
 - Prometheus/harness daemons not deployed here yet (that is OPT-9's wiring).
 - Friday unchanged: CPU/RAM swap (plan NFS outage for -02), 2nd USB3 card,
  q35 + 20 vCPU/128G reshape, GPUs. Enable the ensure unit BEFORE Friday.
@@ -0,0 +1,22 @@
 # ultix
 Perf/ops workbench for **ultix-streaming** (KVM VM 5111 on Proxmox host
 pfv-tsys5). Everything here is human-run, gated, and reversible; storage ops
 are manual-only by ruling.
 - Redmine: project 55 "Known Element Enterprises - Technology & Facility
  Services" — open work: #601 (Fri 2026-09-04 hardware window, urgent),
  #603 root growth, #604 metrics/PSI wiring, #605 k8s + proxmox token,
  #606 GPU passthrough, #607 account map/mkacct; pass record: #602.
  https://projects.knownelement.com/projects/55
 - Discourse doc: pending (house cross-link rule: create at next doc pass).
 - Docs live in-repo (runbook exception per house rules):
  - REPORT.md — full optimization report + host audit results (section 8)
  - NEXT.md — current state / what is owed (usually: nothing)
  - TRACKING.md — rolling HUD table + decisions log
  - GROW-ROOT-RUNBOOK.md — manual root-growth checklist (#603)
  - RUNBOOK-TODAY.md, questions-v1.md
 - Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh,
  3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh);
  staged/ holds the gated configs those scripts install.
 - crush.md = session preferences for agents working in this repo.
@@ -0,0 +1,574 @@
 # ultix-streaming optimization report v1 (2026-08-31)
 Scope: the ultix-streaming KVM guest on pfv-tsys5, tuned for ~9 autonomous agent
 accounts + gateway fleet + builds + ETL + SDR + GPU, all concurrent. Inputs:
 direct measurement of the guest, the ukrrs repo tree, and your answers in chat.
 Companion files: `questions-v1.md` (answer inline), `host-audit.sh` (run on PVE),
 `staged/` (ready-to-apply configs, gated by `RUN=1`).
 ## Exec summary
 One 2010-era 4-core Xeon currently serves this entire host: your dev
 workstation plus 24 other running VMs, with zero resource limits anywhere in
 the fleet, and the workstation does not even auto-start after a host reboot.
 The fix is roughly $300 of used parts, one maintenance window, and the
 configurations already staged in this directory.
 | spend item | rough cost (used, 2026) | effect |
 |---|---|---|
 | 2× Xeon X5675 | $60-80 | host 8 → 24 threads |
 | 6× 16GB DDR3 ECC RDIMM 1333 | $120-180 | 96G → 192G |
 | Dell T7500 CPU2 heatsink + paste | $25-40 | required for socket 2 |
 | **platform total** | **~$210-300** | **3× threads, dev VM 128G** |
 | optional: 1T DRAM-buffered TLC NVMe | $60-100 | only if the Crucial P1 QLC throttles builds |
 | 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) |
 Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes,
 gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn
 window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by
 host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven
 backpressure wired end to end. All software pieces are staged and $0.
 ## 0. TL;DR, ordered by leverage
 1. Host CPUs: CONFIRMED by audit: one E5620 (4C/8T) serves the ENTIRE host,
   including 24 running VMs (load 7.5/8 at idle). CPU2 socket is empty.
   Drop-in fix: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) = 24
   threads, plus CPU2 DIMM slots unlock 192G. See section 8.
 2. One maintenance window fixes VM shape: machine q35 (i440fx today, blocks clean
   GPU passthrough), 20 vCPU / 128G / ballooning off, virtio-scsi-single +
   iothread + ssd=1 + discard=on per disk (guest currently sees ALL disks as
   ROTATIONAL: the SSD hint is not passed through), net0 multiqueue.
 3. No resource limits exist anywhere in the fleet: 18 live containers, zero
   cpus/mem/cgroup_parent in any compose, and `dev.sh check` builders default to
   GOMAXPROCS=8 on an 8-vCPU box. Staged: cgroup v2 slice architecture
   (ukrrs-{gateway,pmo,lsp,batch,rt} + per-account slices via mkacct.sh).
 4. Docker daemon is 100% stock: unbounded json-file logs, live-restore off,
   default address pool (~15 networks; you already have 12), no metrics socket.
   Staged daemon.json + weekly builder-prune timer (build cache is 9.3G/187 entries).
 5. Guest kernel: THP=always (bad for the two Postgres), 66M min_free_kbytes,
   tiny socket buffers (212K) for long LLM streams, dirty ratios in % of 48G
   (multi-second writeback stalls under ETL). Staged sysctl.d + THP=madvise.
 6. Signals: harness already reads loadavg/MemAvailable/PSI-io
   (harness/internal/quota/resources.go:24). Missing: PSI cpu+memory, per-slice
   pressure, docker awareness. Cheapest full stack: node_exporter as a compose
   service into your existing mopac prometheus + PSI textfile script. Beszel
   agent already runs (PID 940). k8s and Proxmox both have clean shedding APIs.
 7. sddm-greeter burns ~22% of one core 24/7 (stuck on login screen). Fix:
   autologin + lock. Sunshine is already installed; it becomes excellent once
   the GPU lands (NVENC), replacing any xrdp idea.
 8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3).
 9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool,
   memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep
   window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
   Guest has no CPU governor to remove (KVM: host owns P-states; set host
   governor to performance once, done).
 ## 1. Measured current state
 ### Guest (ultix-streaming)
 | aspect | measured |
 |---|---|
 | CPU | 8 vCPU presented as 2 sockets × 4 cores × 1 thread; Xeon E5620 2.40GHz (Westmere-EP, LGA1366, 2010; no AVX); 1 NUMA node |
 | RAM | 47G usable + 8.8G swap on sda5 (priority -2, unused) |
 | Machine type | i440fx + SeaBIOS (blocks clean GPU passthrough; q35 preferred) |
 | Kernel/OS | Debian 13 trixie, 6.12.105, cgroup v2 unified, systemd |
 | Disks | sda 438G: sda1 root 279G ext4 (29% used), sda5 swap; sdb 400G ext4 /data2 (empty); sdc 200G ext4 /data1 (empty). MBR table. ALL show ROTA=1 |
 | Sched/mount | scheduler `none` on all disks (good), readahead 128K, relatime everywhere, fstrim.timer present |
 | Desktop | KDE + sddm, greeter idling at ~22% of a core since boot (118 min CPU); no autologin; Sunshine installed, unused; no xrdp; no nvidia userspace |
 | PSI now | cpu some avg60 ~1.2%, memory/io ~0 (idle baseline) |
 | Desktop default | readlink default.target empty (graphical) |
 Key tunables (measured): swappiness 60, dirty_ratio 20/10 (percent-based),
 min_free_kbytes 66M, max_map_count 1048576 (fine), aio-max-nr 65536, inotify
 watches 386K / instances 128, somaxconn 4096 (fine), conntrack 262144 (fine),
 ip_local_port_range 32768-60999, cubic only (bbr module not loaded),
 rmem_max/wmem_max 212992, slow_start_after_idle=1, THP enabled=always,
 autogroup=1 (good, keep for screen sessions), pid_max fine. tuned not installed.
 ### Docker
 | aspect | measured |
 |---|---|
 | Version/driver | 29.7.2, overlayfs, systemd cgroup driver, cgroup v2 |
 | Config | NO daemon.json: unbounded json-file logs, live-restore false, data-root /var/lib/docker on root fs |
 | Footprint | 24 images 14.3G, build cache 9.3G (187 entries), 18 containers up, 6 volumes |
 | Networks | 12 total (9 user bridges). Default pool = 172.17-31/16, caps at ~15 networks |
 | Running | 2 gateway stacks + 2 postgres (prod+beta), 9 LSP, openwebui, cli-mred, 2 fakes |
 ### Fleet facts (from ukrrs tree)
 - 9 harness daemons planned on THIS host (harness/deploy/accounts.tsv, idx 0-8:
  reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO, reachableceo-offstage,
  COSRCEO-Personal, COSRCEO-Biz, COSWFO). NOTE: /home currently shows different
  names (COS-RCEO, COS-TSG, ...) so the account map is not final (Q7).
 - Gateway: fleet semaphore 15 parallel provider calls, policy ladder
  normal/peak/cooldown/hard recomputed every 30s, pacing cruise 90%.
 - Harness gate order: resources → quota → peak → soft-defer. Resource reads:
  loadavg, MemAvailable, PSI io only. Known gaps: harness still polls retired
  cop :8110 (#585), credits are estimates, no PSI cpu/mem.
 - Scale target: ~100 work streams, 24×7, 750 credits/hour.
 - `./dev.sh check` = pinned golang builder, go build+vet+test, NO -p/cpus flags
  → each builder defaults to all 8 vCPUs. 42 MCP + 10 LSP always-on.
 - Beszel agent running (PID 940); prometheus with 30d TSDB already in harness
  compose (mopac-harness-prometheus).
 ## 2. Bottleneck model and the policy
 Who fights whom on this box:
 | workload | cpu | ram | disk | net | latency sensitivity |
 |---|---|---|---|---|---|
 | LLM harness loops ×9 | low | low | low | 9 × long streams | medium (streaming) |
 | gateway + 2× postgres | low-med | med (shared_buffers+cache) | WAL writes | all LLM traffic, via tailscale | HIGH |
 | docker pulls (burst) | med (gzip) | low | layer writes (NVMe) | burst 100s MB | low |
 | compiles (dev.sh check) | saturating, GOMAXPROCS=8 each | 2-6G each | cache reads (NVMe) | low | low |
 | ETL | med | page cache heavy | sequential r/w (SSD) | low | low |
 | LSP ×10 + MCP 42 | idle mostly | 0.3-1G total-ish | low | none | low (interactive bursts) |
 | SDR DSP (future) | 1-2 cores steady | low | low | low | HARD real-time |
 | Sunshine encode (future) | 2-4 cores CPU / ~0 with NVENC | low | low | LAN | medium |
 Design in one sentence: schedule by intent, not by luck. Three CPU pools
 (interactive/gateway, batch/build, rt/sdr), per-account fair shares with cgroup
 v2 weights + memory fences, spindle separation per I/O class, and PSI as the
 universal pressure signal your harness already half-reads.
 ```
                    +----------------- system.slice -----------------+
                    |  ukrrs-gateway.slice   CPUWeight 900  (gateway+pg stacks)
                    |  ukrrs-pmo.slice       CPUWeight 200  (9 PMO dispatch loops)
                    |  ukrrs-lsp.slice       CPUWeight 150  (LSP + MCP tier)
                    |  ukrrs-batch.slice     CPUWeight 25   AllowedCPUs pool A
                    |  ukrrs-acct-<9>.slice  CPUWeight 75   AllowedCPUs pool A
                    |  ukrrs-rt.slice        AllowedCPUs pool B (SDR, encode)
                    |  docker default (everything not tagged) = last
                    +---------------------------------------------------+
  interim 8 vCPU:  pool A = 0-5, pool B = 6-7
  post-upgrade 20: pool A = 0-15, pool B = 18-19 (gateway may use 0-19)
 ```
 Fairness math: with 9 accounts each compiling at weight 75 and gateway at 900,
 gateway keeps ~13% of CPU even under total compile storm; with weights alone an
 idle machine still lets ONE account use every core (burstable, no quotas). I/O
 weights are inert under the `none` scheduler, so fairness comes from separation
 (NVMe = images/builds, /data1 = databases, /data2 = ETL/backup/scratch) plus
 optional device throttles if a specific ETL goes rogue.
 ## 3. Host plan (pfv-tsys5)
 ### 3.1 Audit (do this first)
 My tooling cannot ssh (hard-banned), so run:
 ```
 ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
 ```
 I parse the output and finalize §3.2-3.7 numbers. The script is read-only.
 ### 3.2 CPU upgrade (drop-in, LGA1366)
 | option | result | notes |
 |---|---|---|
 | add 2nd E5620 (~$10) | 8C/16T | cheapest, matches existing |
 | 2× X5675 (recommended) | 12C/24T @3.06/3.46 | 95W each, DDR3-1333, ~$25 each |
 | 2× X5690 | 12C/24T @3.46/3.73 | 130W each, hot, marginal gain, PSU risk with GPUs |
 | 2× L5640 | 12C/24T @2.27 | low power, slower |
 Caveats the audit resolves: chassis must be dual-socket (T5500/T7500; T3500 is
 single), BIOS revision (A17 supports 5600-series), current DIMM layout, PSU
 rating. Westmere lacks AVX/AVX2: Go/Rust/CUDA fine, but some prebuilt binaries
 (newer Node native modules, llama.cpp CPU builds, some distro packages) assume
 AVX; plan on building from source or using GPU for inference.
 ### 3.3 RAM
 96G now is CPU0 DIMMs only (6 slots). Second CPU unlocks 6 more slots → 192G.
 Give the VM 128G, leave 64G for host + preprod VMs. Use DDR3 ECC RDIMM 1333
 (12800R); mixing 10600R works but clocks down. Balanced population per branch.
 ### 3.4 VM re-shape (one maintenance window, VM off)
 ```
 qm set <id> --machine q35                 # from i440fx; expect NIC rename in guest
 qm set <id> --cpu host --sockets 2 --cores 5 --threads 2 --numa 1   # 20 vCPU
 qm set <id> --memory 131072 --balloon 0
 qm set <id> --scsihw virtio-scsi-single
 # per disk (real disk ids from audit):
 qm set <id> -scsi0 <store>:vm-<id>-disk-0,ssd=1,discard=1,iothread=1   # repeat sdb sdc
 qm set <id> --net0 virtio=<mac>,bridge=<br>,multiqueue=4
 ```
 q35 + NIC rename: prepare a systemd .link file or just let udev rename and
 restart networking; tailscale state survives. Guest then sets 4 RSS queues
 (ethtool -L, handled by a tiny oneshot unit, staged). If ballooning was on, off.
 i440fx→q35 is required for the GPU plan; do it once, with console access.
 ### 3.5 Storage backend
 Fixes: ssd=1 (kills the ROTA=1 lie the guest sees today), discard=on (with the
 fstrim.timer already present), virtio-scsi-single + iothread=1 (one iothread per
 disk spreads interrupts), cache=none on ZFS backend / writeback only on
 LVM-thin with UPS. Audit reports backend + link speeds. Known ceiling: if the
 two SSDs sit on the onboard ICH10R SATA2 ports they cap ~280MB/s each (3Gbps
 link); the NVMe on a PCIe2.0 x4 card caps ~1.7GB/s. Fine for this fleet, just
 set expectations for ETL throughput.
 ### 3.6 GPU passthrough
 Now: old NVIDIA on host. Soon: two newer GPUs. Plan:
 1. IOMMU: already active with clean per-device groups (audit section 8). No
   ACS override needed. Optional at next host reboot: add `iommu=pt` for
   cheaper DMA on passthrough devices.
 2. The Quadro 4000: skip entirely. Fermi has no support in any CUDA toolkit
   this decade (compute capability 2.x was dropped at CUDA 9) and no driver
   that builds on kernel 6.x. It stays as the host console card.
 3. The two newer GPUs are COMPUTE cards (per ruling 2026-08-31), which is
   easier than display passthrough: no dummy plug, usually no romfile, pass
   without x-vga (`qm set 5111 -hostpci0 0000:xx:00.0`), modern NVIDIA
   drivers (465+) officially support VMs. Remaining risk: this-era BIOS has
   no Above-4G-Decoding, so big-VRAM cards with large BARs may fail to map.
   Sweet spot for this box: 2× used RTX A2000 12GB (~$150-200 each): 75W
   slot power, no aux cables, no PSU change, 24G VRAM total, BARs fit. Larger
   cards (3060 12G / 4060Ti 16G class) need the PSU check + may hit the BAR
   wall. q35 recommended when they land; OVMF only if a UEFI-only vBIOS
   forces it.
 4. In-guest: nvidia driver + nvidia-container-toolkit; builders get `--gpus
   all` in the batch slice (house-style digest-pinned CUDA image).
 5. PSU budget if going bigger: 2× 95W CPU + 2× 200-250W GPUs + platform ≈
   750-850W sustained: needs the 875W PSU option (Q5 sticker check).
 6. Sunshine (already installed) still benefits even though the cards are
   compute-only: capture the KDE session via X11/KMS on the emulated VGA,
   encode on the GPU with NVENC headless. CPU encode (current state,
   2-4 Westmere cores at 1080p60) remains tolerable-but-temporary. No xrdp:
   it gives a second synthetic session with no accel; Sunshine mirrors the
   real session with audio. SSH stays break-glass.
 ### 3.7 SDR passthrough
 You have two PCI USB3 cards, SDRs load-balanced across them: pass the CARDS,
 never devices. `qm set <id> -hostpci1 <id1> -hostpci2 <id2>`. Guest gets native
 xHCI controllers with real MSI interrupts (emulated USB adds latency and jitter
 that ruins tuner sample streams). SDR processes run in ukrrs-rt.slice
 (AllowedCPUs pool B) so compile storms cannot preempt DSP. Bandwidth: tens of
 MB/s per SDR, trivial for USB3. libusb latency in a VM is fine for SDR work.
 ### 3.8 Proxmox control from the harness
 PVE API token (appendix B has exact pveum commands), then
 `staged/proxmox-ctl.sh vms|shutdown|start|snapshot` with a hard PROTECTED list
 containing this VM and anything untouchable. Natural uses: stand preprod VMs
 down during compile storms, snapshot before risky agent work, bring up scratch
 VMs. Scoping question Q12.
 ### 3.9 Host-side fleet arbitration (dev VM over sectestbed/preprod)
 Yes, the audit pulled every VM config (26 VMs: 24 running, 1 stopped, 1
 template). Two live levers, no reboots:
 ```
 # priority: PVE maps cpuunits to cgroup v2 CPUWeight (clamped 1-10000)
 qm set 5111 --cpuunits 9000
 FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108"
 for id in $FLEET; do qm set $id --cpuunits 50; done
 # optional hard caps where the test fleet never needs burst:
 #   qm set <id> --cpulimit 1
 # (VM 500 k8s-wnode was removed 2026-08-31: ultix-streaming becomes the k8s worker)
 ```
 At equal demand the dev VM holds ~87% of CPU; the fleet still bursts to full
 idle capacity when the dev VM is quiet. Post-upgrade, physical partitioning
 on 24 threads: `qm set 5111 --affinity 0-19` and the fleet `--affinity 20-23`
 (so test VMs can never preempt dev cores at all). Add proxmox-ctl standing
 down idle sectestbed VMs entirely (frees host RAM too) and their disks are
 already separate (local-lvm spinner vs your dedicated SSDs), so no I/O
 arbitration needed. Optional host cmdline at next reboot: `iommu=pt`.
 ## 4. Guest plan (staged in ~/optimize/staged, apply via apply-guest.sh)
 ### 4.1 Kernel (60-ukrrs-vm.conf)
 | knob | now → set | why |
 |---|---|---|
 | dirty_background_bytes / dirty_bytes | % ratios → 256M / 1G | bytes-based caps writeback stalls (ETL) regardless of RAM size |
 | min_free_kbytes | 66M → 384M (768M post-upgrade) | survive bursty reclaim under compile+ETL |
 | aio-max-nr | 65536 → 1M | postgres AIO / io_uring era defaults |
 | inotify instances | 128 → 512; watches 386K → 1M | 9 harness daemons + LSPs + crush sessions watching repos |
 | ip_local_port_range | 32768-60999 → 10240-65535 | 9 accounts × long-lived streams + tailscale + docker NAT |
 | tcp_tw_reuse | 2 → 1 | outbound provider connection churn |
 | tcp_slow_start_after_idle | 1 → 0 | LLM turns idle minutes between bursts on live sockets |
 | rmem_max/wmem_max + tcp_r/wmem | 212K → 16M | large SSE/JSON streams through gateway |
 | congestion control | cubic → bbr + fq | module load staged; falls back to cubic if absent |
 | DefaultLimitNOFILE | 1024 soft → 65536 | harness daemons, many sockets/files |
 Left alone deliberately: swappiness 60, overcommit 0, vfs_cache_pressure,
 pid_max, somaxconn 4096, conntrack 262144, autogroup 1, page-cluster.
 THP: always → madvise (oneshot unit, staged). Postgres dislikes always-THP;
 Go/builds get THP via madvise anyway where it matters.
 ### 4.2 Mounts and root growth to 500G (MBR shuffle, online except swapoff)
 Ruling 2026-08-31: storage operations are MANUAL ONLY, executed by the human,
 one command at a time. Nothing automatic, nothing boot-triggered. The
 step-by-step checklist with per-step verification is GROW-ROOT-RUNBOOK.md;
 the commands below are reference for what it does.
 Current MBR: sda1 root ends 279G, then extended sda2 holding sda5 swap 8.8G,
 then ~150G unallocated (disk 438G). Target: root 500G → grow disk at PVE first:
 `qm resize <id> scsi0 +80G` (→ 518G), then in-guest:
 ```
 sfdisk --dump /dev/sda > /root/sda.sfdisk.bak; cp /etc/fstab /root/fstab.bak
 swapoff /dev/sda5
 sfdisk --delete /dev/sda 2            # removes extended + logical swap
 growpart /dev/sda 1                   # cloud-guest-utils; grows root partition
 resize2fs /dev/sda1                   # online ext4 grow → 505G
 parted /dev/sda mkpart primary linux-swap 505GB 100%; mkswap /dev/sdaX
 # fix fstab swap UUID (blkid), swapon -a; verify with lsblk + df -h /
 ```
 Rollback: fstab + sfdisk dumps kept; original layout restorable offline. Also:
 add `noatime` to /, /data1, /data2 (remount online; relatime today).
 ### 4.3 Swap
 Keep sda5-style swap at disk tail (fresh 10G from the shuffle). Skip zram: old
 CPU (compression burns cores) and RAM is tripling soon. Optional: 32G low-pri
 swapfile on /data2 as OOM insurance post-upgrade (staged flag, default off).
 ### 4.4 Docker (daemon.json staged)
 live-restore true (daemon restarts stop killing agent containers; aligns with
 your "never bounce prod for convenience" rule), json-file logs capped 20m × 3
 (9 accounts × 42 MCP without caps = quiet-log rule violation waiting to happen),
 max-concurrent-downloads 6 (parallel pulls across accounts), address pool
 172.16/12 as /24s (you are at 12 of ~15 default networks; the fleet adds one
 per compose project), metrics on 127.0.0.1:9323 (experimental:true is required
 by docker for the engine metrics endpoint; drop both lines if unwanted).
 Weekly `docker builder prune --keep-storage 25GB` timer staged (9.3G today,
 unbounded growth otherwise). Applying daemon.json restarts docker once (do it in
 a quiet moment; live-restore protects future restarts only).
 ### 4.5 cgroup v2 architecture (the core deliverable)
 Static slices staged: gateway (CPUWeight 900, MemoryMin 2G, MemoryHigh 12G),
 pmo (200, High 1G/Max 1.5G; the dispatch control plane: cheap but weighted
 above workers so a compile storm can never starve dispatch), lsp (150, High
 6G/Max 8G), batch (25, High 12G/Max 16G, pool A cpus), rt (pool B cpus,
 weight 10000). Night profile overrides batch/gateway at runtime (§5.5). Per-account slices via `mkacct.sh <name> <uid>`:
 CPUWeight 75, TasksMax 4096, Memory fences (interim 48G: High 3G/Max 4.5G ×9;
 post-upgrade 128G: High 10G/Max 12G ×9), plus a matching user-<uid>.slice
 drop-in so the harness daemon process itself is fenced too.
 Containers join their account slice via compose `cgroup_parent:` (snippet
 printed by mkacct). Builders and ETL join ukrrs-batch via dev.sh/compose.
 Everything untagged lands in plain docker scope = effectively lowest priority.
 NOTE: with the `none` I/O scheduler, IOWeight is inert; fairness = separation
 across the three physical devices + (rare) per-container device rate limits.
 ### 4.6 Desktop
 Fix the greeter burn: sddm autologin into an UNLOCKED session (ruling
 2026-08-31: no autolock, ever; instant Jump/iPad re-attach outranks lock
 security on this box). `~/optimize/fix-kde.sh` does it (sudo, idempotent,
 optional --restart). Result: greeter stops rendering, session idles near 0%,
 an always-alive unlocked KDE session is attachable instantly. Delete sddm-greeter
 CPU cost entirely the day you stop wanting console KDE (multi-user.target) but
 that kills Sunshine's session, so default plan keeps KDE.
 KDE vs XFCE verdict: do not switch for resource reasons. Locked/idle Plasma is
 ~0% CPU and roughly 600-900M RSS vs XFCE ~350-450M; on a box going to 128G
 that delta is noise. The 22% greeter burn was a greeter-stuck artifact, not
 Plasma weight, and lightdm+XFCE can do the same trick. The one real argument
 for XFCE later: it is X11-only, and X11 is still the most battle-tested
 Sunshine capture path with the NVIDIA proprietary driver; Plasma Wayland
 capture on NVIDIA is good in 2026 but younger. Sequence: keep KDE now, fix
 autologin+lock, land the GPU, test Sunshine on Wayland+NVENC, switch to XFCE
 only if capture disappoints. Revisit when Agent Zero lands (Q15): if it ever
 runs headed-browser tasks, the desktop story matters more than RAM.
 ## 5. Signals and integrations
 ### 5.1 What the harness reads today and the minimal patch
 resources.go reads loadavg, MemAvailable, PSI-io avg60, disk free, with clean
 seams. Extend (same pattern, ~40 lines + tests, ticket per repo):
 - generalize readIODelay → readPressure(res) for cpu and memory
 - optional: read /sys/fs/cgroup/ukrrs-batch.slice/cpu.pressure to gate on "is
  the batch pool saturated" rather than host-wide load (loadavg counts 24
  post-upgrade vCPUs; PSI is the honest signal)
 - suggested defer thresholds: cpu some avg60 > 20, io avg60 > 25, mem some
  avg60 > 10 (tune with real data; the metric names already exist
  harness_resource_load_avg_1m / _io_delay_pct)
 ### 5.2 Metrics stack (your "super lightweight" ask)
 Nothing new to run on the host OS: node_exporter as a compose service
 (read-only binds of / and /var/lib/node_exporter/textfile, house-compliant,
 digest-pinned at adoption), scraped by the ALREADY RUNNING
 mopac-harness-prometheus (30d TSDB). PSI textfile script (staged) adds
 per-slice pressure gauges every 15s. Docker engine metrics land on
 127.0.0.1:9323 via daemon.json. Beszel (already installed) keeps the human
 dashboard role. Gateway choice (Q10): poll node_exporter directly in its 30s
 tick (no new dependency, ~30 lines) vs PromQL against existing prom. Either
 beats the retired cop :8110 (#585) it still polls.
 ### 5.3 k8s (yes, the APIs shed work cleanly)
 - `kubectl cordon ultix-streaming` stops new scheduling; `kubectl drain
  --ignore-daemonsets --delete-emptydir-data --grace-period=120` evicts.
 - Set all k8s workloads to a low PriorityClass + preemption policy, so burst
  pods die first under node pressure automatically.
 - kubelet eviction thresholds (memory.available<2Gi, imagefs.available<10Gi)
  auto-shed when builds eat disk/RAM. Plus --system-reserved/--kube-reserved.
 - Recommendation: join AFTER the host upgrade, as worker only, never control
  plane, kubelet+containerd units get CPUWeight 20 + MemoryHigh fences so the
  compose fleet always wins arbitration. Control plane location Q11.
 ### 5.4 Proxmox
 See §3.8; skeleton staged, token + PROTECTED list via Q12.
 ### 5.5 Day/night dynamic profile + PMO dispatch backpressure
 Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the
 z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset
 (gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect:
 22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO
 burn maxes, 05:00 everything returns to day bias.
 What flips at night (systemctl set-property --runtime + sysctl, all
 non-persistent; a reboot lands safely in day mode; values in
 /etc/ukrrs/daynight.conf, staged):
 | object | day | night (48G interim / 128G post) |
 |---|---|---|
 | batch slice CPUWeight | 25 | 400 |
 | batch AllowedCPUs | 0-5 (post 0-15) | 0-6 (post 0-17) |
 | batch MemoryHigh | 12G (post 24G) | 30G (post 80G) |
 | gateway CPUWeight | 900 | 500 (still far above idle accounts) |
 | /data2 readahead | 256K | 1M (ETL sequential) |
 | dirty_bytes / background | 1G / 256M | 2G / 512M |
 Honest limit: the guest has no CPU governor to "take off"; KVM host owns
 P-states. Set the host governor to performance once (audit confirms current
 state); an always-busy 15-year-old Xeon saves nothing from dynamic P-states.
 Night is also the maintenance window: builder prune Sun 22:30, fstrim,
 postgres vacuum/reindex, image pre-pull for the morning fleet.
 PMO backpressure: yes, gate at dispatch, it is the earliest and cheapest
 point. Each account's PMO loop claims from Redmine only when, in order (all
 zero-dependency reads, same pattern as resources.go):
 1. gateway /status policy mode is normal (else trickle: ladder already
   handles 01:00-05:00),
 2. /proc/pressure/{cpu,memory,io} avg60 under thresholds,
 3. ukrrs-batch.slice cpu/io pressure under threshold (worker pool headroom),
 4. wall clock: night = max parallel builders + bigger tickets (compiles,
   ETL, doc builds); day = fewer, interactive-sized tickets.
 The 9 PMO loops live in ukrrs-pmo.slice so worker storms cannot starve the
 dispatcher. This gate composes with the existing harness gate order
 (resources → quota → peak → soft-defer, harness/README.md:298-312) instead of
 replacing it.
 Agent Zero: containerized python agent runtime. Placement: interactive
 agents → their account slice (or gateway-adjacent weights), background
 agents → ukrrs-batch.slice. Decide via Q15; no OS change either way, the
 slice architecture absorbs it.
 ## 6. Sequencing and rollback
 - Phase 0 (now, reversible, ~1h, no reboot): apply-guest.sh sysctl+thp+slices+
  noatime+timers+daynight+desktop; daemon.json in a quiet window (one docker
  restart).
 - Phase 1: host audit (one ssh line), order CPUs/RAM (Q2/Q3).
 - Phase 2 (window, VM off): q35 + resize + disk flags + multiqueue; then root
  growth to 500G; then GPU + USB3 passthrough; then VM vCPU/RAM at new values.
 - Phase 3 (repo tickets, independent): builder caps + cgroup_parent in compose,
  resources.go PSI extension, gateway pressure input, #585 cop removal.
 - Phase 4: k8s join, proxmox-ctl wiring.
 Every phase independently revertible: sysctl.d and units are files, qm changes
 are per-flag, partition work has dumps. Never a bare `down`, per house rules.
 ## 7. Open items
 See questions-v1.md, answer inline; version the file if you want another round.
 ## 8. Host audit results (2026-08-31, parsed from host-audit.out)
 ### Confirmed
 - Dell Precision T7500 (baseboard 06FW8P A02), dual socket, **CPU2 EMPTY**.
  One E5620 4C/8T (HT on) serves 24 RUNNING VMs (~57G/96G RAM allocated);
  host load was 7.5 on 8 threads while mostly idle. Your VM gets all 8
  threads only because the rest are quiet. The 2× X5675 upgrade is now
  urgent, not optional: 3× threads. DMI quirk: "Upgrade: Socket LGA771" is a
  Dell string error (E5620 exists only in LGA1366); still verify visually
  before ordering.
 - RAM: 6× 16GB DDR3-1600 filling ALL six CPU1 slots. Second CPU unlocks six
  more: order 6× 16GB more → 192G. Runs at CPU max (1066 on E5620, 1333 on
  X5675).
 - IOMMU active with CLEAN per-device groups: Quadro alone (20), Renesas
  USB3 alone (5), NVMe alone (21), NIC ports separate (3/4). **No ACS
  override needed** (better than section 3.6 assumed).
 - No CPU frequency driver exists on this host (no /sys cpufreq at all, this
  kernel + 5520). There is no governor to tune anywhere, host or guest.
  Day/night dynamics are cgroup-only, exactly as staged in section 5.5.
 - Host storage: PVE root on 2TB Hitachi spinner via LVM-thin (local-lvm,
  where the 24 sectestbed VMs live); SMART PASSED on everything; temps fine.
 ### Corrected
 - Your "NVMe" = **Crucial P1 500G QLC (DRAM-less)** as qcow2 on ext4 dir
  storage (NVME:), dedicated to this VM, 59% used at storage level. /data1 =
  Samsung 860 PRO 256G, /data2 = SK hynix SC300B 512G, both dedicated, both
  on ICH10R SATA2 3Gbps links (~280MB/s ceiling each). None carry ssd=1 or
  discard (that is why the guest sees ROTA=1). Expect P1 sustained-write
  collapse under heavy builder churn (QLC, no DRAM): monitor; if it bites,
  the fix is a better NVMe, not config.
 - Old GPU = **Quadro 4000 (Fermi, 2GB)**. Verdict: skip passthrough. Fermi
  has no NVENC (starts with Kepler) and no driver that builds on kernel 6.x
  (390xx is dead): it buys display outputs only. Keep it as host console.
  Sunshine stays software-encode until the new GPUs arrive; their clean
  IOMMU groups are already waiting.
 - Only **ONE** Renesas uPD720201 USB3 card visible (24:00.0). You said two.
  Check the second: unseated, dead, or behind the (empty) Pericom PCI-X
  bridge at 01:00.0.
 - VM 5111 has **no onboot flag**: it will not auto-start after a host
  reboot. debian13.iso still attached to ide2 (nit).
 - k8s on this host consolidated (2026-08-31): VM 500 pfv-k8s-wnode-tsys5 was
  removed; ultix-streaming itself becomes the k8s worker on pfv-tsys5
  (12G RAM + 2 vCPU of host pressure freed; timing/control plane per Q11).
 - The 18-VM sectestbed fleet runs 24/7 on the same 8 threads and the 2TB
  spinner. proxmox-ctl.sh from this VM is the natural on/off switch (Q12).
 ### QM checklist for the next touch (any time, hot where noted)
 ```
 qm set 5111 --onboot 1 --startup order=10,up=180
 qm set 5111 -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
 qm set 5111 -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
 qm set 5111 -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
 qm set 5111 --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,multiqueue=4
 qm set 5111 --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,multiqueue=2
 qm set 5111 --delete ide2
 # q35 + 20 vCPU + 128G + sockets/threads reshape: only at the CPU/RAM window (3.4)
 # root growth to 500G: MANUAL only, GROW-ROOT-RUNBOOK.md (discard=on live first; fstrim reclaims qcow2)
 # net multiqueue needs a guest-side ethtool -L oneshot after reboot
 ```
 ## Appendix A: staged file map
 ```
 staged/60-ukrrs-vm.conf          → /etc/sysctl.d/ (kernel)
 staged/modules-load.d/tcp_bbr.conf → /etc/modules-load.d/
 staged/systemd/*.slice|*.service|*.timer → /etc/systemd/system/
 staged/ukrrs-psi-textfile.sh     → /usr/local/sbin/ (with .service/.timer)
 staged/ukrrs-daynight.sh         → /usr/local/sbin/ + day/night profile timers
                                  + /etc/ukrrs/daynight.conf (all staged values)
 staged/mkacct.sh                 → run per account: name uid [high] [max] [cpus]
 staged/apply-guest.sh            → installer, RUN=1 to mutate, subcommands
 staged/docker/daemon.json        → /etc/docker/ (restarts docker once)
 staged/proxmox-ctl.sh            → PVE API wrapper, PROTECTED guard
 ```
 ## Appendix B: repo patches (ticketed, not applied from here)
 1. compose builder caps (each repo with a builder/check svc):
 ```
 x-ukrrs-batch: &ukrrs-batch
  cgroup_parent: ukrrs-batch.slice
  cpus: 4
  mem_limit: 4g
 services:
  check: { <<: *ukrrs-batch, ...existing... }
 ```
 2. dev.sh one-shot builders: `docker run --rm --cgroup-parent ukrrs-batch.slice
   --cpus 4 --memory 4g ...` (GOMAXPROCS then follows cpuset/cpus).
 3. resources.go: generalize PSI read to cpu/mem + optional slice pressure
   (fields + gates + tests, mirrors readIODelay at resources.go:90).
 4. gateway pressure poller: in the 30s tick (gateway.go:802), GET
   127.0.0.1:9100/metrics, parse node_pressure_cpu_waiting_seconds... or the
   textfile gauges; feed the ladder as a new host-pressure input class.
 5. PVE token:
 ```
 pveum user add ukrrs-infra@pam
 pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0
 pveum acl modify /pool/<preprod-pool> -user ukrrs-infra@pam -role PVEVMUser
 ```
 6. GPU vBIOS dump (on host, card must be unused):
 ```
 cd /sys/bus/pci/devices/0000:0X:00.0; echo 1 > remove
 cat rom > /root/gpu-<model>.rom; echo 0 > remove
 ```
 7. node_exporter service (into harness deploy compose, digest pin at adoption):
 ```
 node-exporter:
  image: prom/node-exporter@sha256:<pin>
  container_name: ukrrs-mopac-nodeexporter
  command: [--path.rootfs=/host, --collector.pressure,
            --collector.textfile.directory=/textfile]
  pid: host
  volumes: [/:/host:ro,rslave, /var/lib/node_exporter/textfile:/textfile:ro]
  network_mode: host
  restart: unless-stopped
 ```
@@ -0,0 +1,30 @@
 # Today: two commands, one reboot wave
 **1. On the VM** (ultix-streaming, any time before step 2):
 ```
 sudo ~/optimize/1-guest-prep.sh
 ```
 Installs everything (kernel tunables, BBR, THP, cgroup slices, PSI collector,
 day/night timers, docker daemon.json, sddm autologin, no autolock). Docker
 restarts once. Touches NO storage.
 **2. From your desk** (dry-run first if you like, then execute):
 ```
 ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
 ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
 ```
 Graceful stop of 5111, ssd/discard/multiqueue/onboot flags, CPU priority over
 the sectestbed fleet, start 5111, then reboots every other running VM.
 No disk resizing, no host reboot (NFS role).
 **3.** Say "done" here; I verify everything read-only and report.
 **Separate, whenever you choose** (fully manual, step-by-step with checks):
 root growth 279G -> ~505G via `~/optimize/GROW-ROOT-RUNBOOK.md`. No script
 runs it; no boot triggers it; you type every command yourself.
 Rollback: apply-guest backs up every file it touches; qm flags are
 individually reversible.
 Friday still owns: CPU/RAM swap (host power-off -> plan the NFS outage for
 -02 first), second USB3 card reseat, q35 + 20 vCPU / 128G reshape, GPUs.
@@ -0,0 +1,35 @@
 # ultix-streaming optimization — tracking HUD
 One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / blocked ⛔ / next ⏳
 | id | lane | state | item |
 |---|---|---|---|
 | OPT-1 | guest | ✅ | recon: system + ukrrs fleet (REPORT §1) |
 | OPT-2 | guest | ✅ | REPORT.md + staged configs + questions-v1.md |
 | OPT-3 | host | ✅ | audit parsed → REPORT §8 (T7500, CPU2 empty, clean IOMMU, Q4000 skip) |
 | OPT-4 | host | ⏳ | Fri 2026-09-04 #601: 2× X5675 + 6× 16GB → 24T/192G (NFS outage for -02 first; USB3 reseat; q35/20c/128G) |
 | OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue now in qm config (live at Friday's start) |
 | OPT-6 | guest | ✅ | guest prep applied + verified (bbr, THP, slices, timers, noatime, daemon.json, autologin) |
 | OPT-7 | guest | ⏳ | #603: root growth MANUAL ONLY, GROW-ROOT-RUNBOOK.md, user-executed |
 | OPT-8 | repos | ✅ | PMO backpressure design note in ukrrs/docs/harness; code = ticketed |
 | OPT-9 | integ | ⏳ | #604: node_exporter svc + gateway pressure scrape (Q10) |
 | OPT-10 | integ | ⏳ | #605: k8s join (Q11, blocked by #601) + proxmox-ctl token (Q12); GPUs #606 |
 Inbox (mid-task interrupts): none.
 Decisions log (latest wins):
 - 2026-08-31 16:5x: Redmine = record: perf pass + RCA filed as #602 (project 55, tracker Support, left New for human review); open items #601 (Fri window, Urgent, due 09-04) #603 root growth #604 metrics #605 k8s+proxmox (blocked by #601) #606 GPUs (relates #601). mred = docker exec wrapper (~/.local/bin/mred → mopac-cli-mred container).
 - 2026-08-31 16:40: net multiqueue CLOSED: queues=4/2 written to live qm config via crush ssh (user authorized non-disruptive host mods); activates at next VM start; guest oneshot enables channels on that boot. Earlier silent no-op explained: the 15:59 run used a pre-fix script copy.
 - 2026-08-31 16:3x: crush CAN ssh to the host from this workspace (old "hard-banned" note obsolete); still human-gated for disruptive ops.
 - 2026-08-31 (post-reboot validation): prod gateway down at boot ROOT-CAUSED twice over: (a) port bind to tailscale IP lost the dockerd-vs-tailscaled race (docker never retries failed starts), (b) live-restore dropped the container's network endpoint (host resolver → gateway-db unresolvable → crash loop). Recovered via compose --force-recreate (healthy, mode=normal). Permanent fix staged: ukrrs-gateway-ensure unit, BOTH lanes, MUST be enabled before Friday's bounces.
 - 2026-08-31: multiqueue host flags verified live (ROTA=0 + discard 1G on all 3 disks), but guest still runs 1 queue/NIC: ethtool not installed; 3-post-reboot-fixes.sh staged (installs ethtool + oneshot unit, sets 4/2 queues).
 - 2026-08-31: mopac-demo/mcli fake containers left DOWN (restart=no, exited at first bounce); prometheus/harness daemons not deployed on this box (pre-existing; belongs to OPT-9).
 - 2026-08-31: STORAGE OPS MANUAL ONLY: boot-time auto-grow service removed, qm resize removed from the one-shot, grow-root.sh deleted. Runbook = GROW-ROOT-RUNBOOK.md, every command typed by the human.
 - 2026-08-31: KDE autologin into UNLOCKED always-alive session (Relogin=true); autolock OFF (instant Jump/iPad attach outranks lock). XFCE only if Sunshine capture disappoints post-GPU.
 - 2026-08-31: streamlined to 1-guest-prep.sh + 2-host-one-shot.sh (one reboot wave; no post-reboot steps; no storage).
 - 2026-08-31: constraint: host pfv-tsys5 NEVER reboots casually (NFS server for -02); Friday CPU swap needs a planned NFS outage first.
 - 2026-08-31: GPUs are COMPUTE cards; Quadro 4000 skipped (Fermi: no NVENC, no kernel-6 driver, no modern CUDA); clean IOMMU groups verified.
 - 2026-08-31: day/night = runtime-only cgroup flips 22:00/05:00; NO cpufreq driver exists host or guest.
 - 2026-08-31: zram rejected; no docker data-root move; skip tuned (sysctl.d + units).
 - 2026-08-31: second USB3 SDR card missing on host (only one Renesas visible); Friday reseat/check.
 ----
@@ -0,0 +1,28 @@
 # crush.md — project preferences for crush sessions in ~/optimize
 ## Locale (ruling 2026-08-31)
 - Human + this host are in TEXAS: America/Chicago (Central Time; UTC-5 CDT in
  summer). "1638 CST" in chat means Central local time.
 - 2026-08-31 is a MONDAY. "Friday" in these docs = 2026-09-04.
 - Docker/ISO timestamps print UTC (trailing Z) = local +5h; never mislabel
  weekday or TZ when narrating logs.
 ## Host + no-Python rules (ruling 2026-08-31, also in global AGENTS.md)
 - No Python from agents: text/JSON/CSV = bash/sed/awk/perl/cut/jq. NEVER
  python one-liners. python in dev containers is fine.
 - Host stays clean: common shell commands fine, curl + jq + ripgrep etc
  ALLOWED (old curl ban retired). NO installs on the host (apt-get/pip/etc).
 - Missing CLI tool? docker pull it, standing permission — run via
  `docker run --rm <pinned image>`. Work in containers.
 ## Output format (user preference, 2026-08-31)
 - The crush TUI sidebar truncates long chat output. NEVER hand the user a long
  command in chat. ALWAYS write runnable commands as a numbered script in this
  directory (next free number, e.g. 5-foo.sh) and tell them only the one-liner
  to execute it. Scripts that produce diagnostics must tee output to a .out
  file in this directory so the next turn can read it.
 - Scripts that need root must SELF-ELEVATE (re-exec via `exec sudo bash "$0"`
  when EUID != 0) so the user never types sudo. Run-from-guest ssh-root scripts
  need no elevation; say so in the script header.
 - Keep chat replies short-line-width; details live in files (NEXT.md, REPORT.md,
  questions-v*.md, script headers).
@@ -0,0 +1,36 @@
 #!/usr/bin/env bash
 # UKRRS KDE fix: sddm autologin into an UNLOCKED, always-alive session.
 # Deliberately NO autolock (instant Jump/iPad re-attach requirement).
 # Relogin=true: if the session ever exits/crashes, sddm logs straight back in,
 # so there is always a session to attach to.
 #
 # Usage:
 #   sudo ./fix-kde.sh             config only; takes effect next boot/restart
 #   sudo ./fix-kde.sh --restart   also restart sddm NOW. WARNING: restart kills
 #                                 any ACTIVE local GUI session. Right now only
 #                                 the greeter runs, so it is safe; verify with
 #                                 `loginctl list-sessions` if unsure.
 # Override user with: sudo DESKTOP_USER=someone ./fix-kde.sh
 set -euo pipefail
 [ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; }
 USER_NAME=${DESKTOP_USER:-reachableceo}
 CONF_DIR=/etc/sddm.conf.d
 CONF=$CONF_DIR/50-ukrrs-autologin.conf
 mkdir -p "$CONF_DIR"
 [ -f "$CONF" ] && cp "$CONF" "$CONF.bak.$(date +%s)"
 cat > "$CONF" <<EOF
 [Autologin]
 User=$USER_NAME
 Session=plasma.desktop
 Relogin=true
 EOF
 echo "wrote $CONF (user=$USER_NAME, session=plasma.desktop, unlocked, always-alive)"
 if [ "${1:-}" = "--restart" ]; then
  systemctl restart sddm
  echo "sddm restarted: autologin session should be up now (check: loginctl)"
 else
  echo "takes effect at next sddm start/reboot; or rerun with --restart"
 fi
@@ -0,0 +1,109 @@
 #!/usr/bin/env bash
 # UKRRS host audit for pfv-tsys5 (Proxmox VE). READ-ONLY, safe to run anytime.
 # Usage (from a machine that can ssh):
 #   ssh root@pfv-tsys5.knel.net 'bash -s' < host-audit.sh > host-audit.out
 set -uo pipefail
 hdr() { echo; echo "########## $* ##########"; }
 hdr "node"; hostname; uptime; pveversion 2>/dev/null || echo not-pve
 hdr "dmi system+baseboard"
 dmidecode -t system 2>/dev/null | grep -E 'Manufacturer|Product Name|Version|Wake-up'
 dmidecode -t baseboard 2>/dev/null | grep -E 'Manufacturer|Product Name|Version'
 hdr "processors"
 dmidecode -t processor 2>/dev/null | grep -E 'Socket Designation|Version|Max Speed|Current Speed|^Status|Core Count|Thread Count|Upgrade:'
 lscpu 2>/dev/null | grep -E 'Model name|^CPU\(s\)|Thread|Core|Socket|NUMA|MHz'
 hdr "cpu frequency policy"
 for p in scaling_driver scaling_governor scaling_max_freq scaling_min_freq; do
  v=$(cat /sys/devices/system/cpu/cpu0/cpufreq/$p 2>/dev/null) && echo "$p = $v"
 done
 hdr "memory"
 free -h; grep MemTotal /proc/meminfo
 dmidecode -t memory 2>/dev/null | grep -E '^\s*(Size|Speed|Type:|Locator|Form Factor)' | grep -v 'No Module'
 hdr "storage: block devices"
 lsblk -o NAME,SIZE,TYPE,ROTA,FSTYPE,MOUNTPOINTS,MODEL 2>/dev/null
 hdr "storage: pve"
 pvesm status 2>/dev/null
 hdr "storage: zfs"
 if command -v zpool >/dev/null 2>&1; then
  zpool status 2>/dev/null
  zfs list -o name,used,avail,recordsize,compression 2>/dev/null | head -40
  grep -E '^(size|c_max|c_min) ' /proc/spl/kstat/zfs/arcstats 2>/dev/null
 else
  echo no-zfs
 fi
 hdr "smart health"
 if command -v smartctl >/dev/null 2>&1; then
  for d in $(ls /dev | grep -E '^(sd[a-z]|nvme[0-9]+)$'); do
    echo "--- /dev/$d"
    smartctl -H /dev/$d 2>/dev/null | grep -iE 'result|overall|health'
    smartctl -i /dev/$d 2>/dev/null | grep -iE 'model|firmware|serial' | head -3
  done
 else
  echo no-smartctl
 fi
 hdr "kernel + cmdline"
 uname -a
 cat /proc/cmdline
 [ -f /etc/kernel/cmdline ] && cat /etc/kernel/cmdline
 proxmox-boot-tool status 2>/dev/null | head -20
 hdr "iommu/dmar"
 dmesg 2>/dev/null | grep -iE 'iommu|dmar|vtd' | head -25
 hdr "lspci (all)"
 lspci -nn 2>/dev/null
 hdr "iommu groups"
 if [ -d /sys/kernel/iommu_groups ]; then
  for g in /sys/kernel/iommu_groups/*; do
    [ -d "$g" ] || continue
    echo "group ${g##*/}:"
    for d in "$g"/devices/*; do
      printf '   '; lspci -s "${d##*/}" 2>/dev/null
    done
  done
 else
  echo "no iommu groups (intel_iommu not enabled?)"
 fi
 hdr "pci link speeds (gpu/usb/nvme candidates)"
 for d in /sys/bus/pci/devices/*; do
  s=$(cat "$d/current_link_speed" 2>/dev/null); w=$(cat "$d/current_link_width" 2>/dev/null)
  [ -n "$s" ] && echo "${d##*/} $s x$w"
 done
 hdr "network"
 ip -br a 2>/dev/null || cat /proc/net/dev
 bridge link 2>/dev/null
 hdr "qemu vms"
 qm list 2>/dev/null
 hdr "vm configs"
 for vm in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
  echo "--- VM $vm"; qm config "$vm" 2>/dev/null
 done
 hdr "lxc containers"
 pct list 2>/dev/null
 for ct in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
  echo "--- CT $ct"; pct config "$ct" 2>/dev/null
 done
 hdr "sensors"
 sensors 2>/dev/null || echo no-lm-sensors
 hdr "guest agent check"
 qm agent 100 ping 2>/dev/null && echo "agent ok on vm 100" || true
 echo
 echo "AUDIT DONE"
@@ -0,0 +1,45 @@
 == qm config 5111 (relevant lines) ==
 cores: 4
 cpuunits: 9000
 memory: 50000
 net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
 net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
 onboot: 1
 scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
 scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
 scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1
 scsihw: virtio-scsi-single
 sockets: 2
 startup: order=10,up=180
 == qm pending 5111 (unapplied staged changes) ==
 cur agent: 1
 cur balloon: 0
 cur boot: order=scsi0;net0
 cur cores: 4
 cur cpu: host,flags=+nested-virt
 cur cpuunits: 9000
 cur memory: 50000
 cur meta: creation-qemu=10.1.2,ctime=1784819584
 cur name: ultix-streaming
 cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
 cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
 cur numa: 0
 cur onboot: 1
 cur ostype: l26
 cur scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
 cur scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
 cur scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1
 cur scsihw: virtio-scsi-single
 cur smbios1: uuid=bc74eb91-b156-46e2-b946-808da9b4f037
 cur sockets: 2
 cur startup: order=10,up=180
 cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f
 == last bounce log (/var/log/ukrrs-vm5111-bounce.log) ==
 bounce-start 2026-08-31T15:59:56-05:00
 CPU flag 'nested-virt' resolved to 'vmx'
 bounce-done 2026-08-31T16:01:41-05:00
 == pve version ==
 pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve)
@@ -0,0 +1,140 @@
 # questions-v1 — ultix-streaming optimization
 Answer inline under each A:. Where I have a recommendation it is marked REC;
 "ok" is a sufficient answer. Version the file (v2) for another round.
 ## Q1. Host audit (required first, blocks host-side numbers)
 I cannot ssh from crush (tool policy), so run:
 ```
 ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
 ```
 then just say "audit done". I read the file and finalize §3 of REPORT.md.
 A:
 ## Q2. CPU upgrade
 REC: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) → 24 threads.
 Cheaper alt: second E5620 (~$10) → 16 threads. Audit confirms the chassis is
 dual-socket before you buy.
 A:
 ## Q3. RAM
 REC: populate CPU2 DIMM slots to 192G total, give this VM 128G, ballooning off.
 Which DIMMs to order comes from the audit (current population + speed).
 A:
 ## Q4. Root growth to 500G
 REC: you run `qm resize <id> scsi0 +80G`, I do the in-guest MBR shuffle
 (online except a brief swapoff; backups of fstab + partition table first).
 When?
 A:
 ## Q5. GPUs
 Models + VRAM of the two incoming cards, and when? PSU wattage on the host
 (sticker)? Old card: passthrough to this VM now, or leave for preprod?
 Why it matters: no Above-4G-Decoding on this-era BIOS; big-VRAM cards may not
 map. UEFI-only vBIOS cards would force the OVMF migration.
 A:
 ## Q6. q35 switch
 REC: yes, one offline window, NIC rename expected in guest (tailscale survives).
 Needed for clean GPU/USB3 passthrough.
 A:
 ## Q7. Account → slice names
 accounts.tsv says: reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO,
 reachableceo-offstage, COSRCEO-Personal, COSRCEO-Biz, COSWFO.
 /home currently shows: COS-RCEO, COS-TSG, COS-WFO, ... Which 9 login names are
 final on THIS host? (I generate one slice + user drop-in per account.)
 A:
 ## Q8. Per-account memory fences
 REC interim (48G): each account MemoryHigh 3G / Max 4.5G.
 REC post-upgrade (128G): High 10G / Max 12G. CPU: weight 75, no hard quota
 (burstable). OK?
 A:
 ## Q9. Desktop
 REC: keep KDE, fix the greeter (autologin into a locked session), Sunshine +
 Moonlight after the GPU lands (NVENC). XFCE only if Sunshine capture on KDE
 Wayland disappoints. OK to apply the autologin+lock config?
 A:
 ## Q10. Metrics/pressure wiring
 REC: node_exporter as a compose service (house-style, digest-pinned) scraped
 by your existing mopac-harness prometheus; PSI textfile script for per-slice
 pressure; gateway polls node_exporter directly in its 30s tick (option A, no
 new dependency). Keep beszel for the human dashboard. Where does the beszel
 hub live?
 A:
 ## Q11. k8s
 Control plane: where/what (k3s? existing cluster?)? REC: join AFTER host
 upgrade, worker-only, low PriorityClass for burst workloads, kubelet eviction
 thresholds, kubelet+containerd fenced to CPUWeight 20 + MemoryHigh. OK?
 A:
 ## Q12. Proxmox control scope
 REC: dedicated API token (ukrrs-infra@pam harness) scoped to a preprod pool
 only. Which VMIDs may the harness stand down/snapshot, and which are
 PROTECTED (this VM certainly)? I hard-code the PROTECTED list into
 proxmox-ctl.sh.
 A:
 ## Q13. SDRs
 REC: passthrough both PCI USB3 controller cards whole (never per-device),
 SDR processes pinned to the rt CPU pool. Confirm: both cards → THIS VM,
 always-on? Any latency budget I should know about?
 A:
 ## Q14. Day/night windows
 REC: OS night profile 22:00-05:00 local; gateway z.ai peak ladder stays
 01:00-05:00 CST as configured. Confirm 22:00-05:00 (vs your words "2200 to
 0500") and the timezone the guest should key off (it currently sees system TZ).
 A:
 ## Q15. Agent Zero
 Which accounts host it, expected concurrent agents, docker-based? REC:
 background agents → ukrrs-batch.slice, interactive ones → their account slice.
 A:
 ## Q16. TCP BBR
 REC: load tcp_bbr module + bbr/fq for the long provider streams. If you
 prefer stock cubic, say so and I drop two lines.
 A:
 ## Audit update 2026-08-31 (host-audit.out parsed; details in REPORT.md section 8)
 - Q1 DONE. Q2 upgraded to URGENT: CPU2 socket confirmed empty; one E5620
  serves 24 running VMs. 2× X5675 triples threads. (DMI claims LGA771, a Dell
  string quirk; E5620 is LGA1366; verify visually before ordering.)
 - Q3 refined: 6× 16GB DDR3 already fill all CPU1 slots; order 6 more 16GB.
 - Q5 verdict: skip Quadro 4000 passthrough (Fermi: no NVENC, no modern CUDA
  driver). Still need: new GPU models/VRAM + PSU wattage sticker.
 - Q13 wrinkle: only ONE USB3 card visible on the host. Where is the second?
 - Q11 wrinkle: VM 500 pfv-k8s-wnode-tsys5 already exists here: join or
  consolidate?
 - IOMMU groups are clean (no ACS override needed). No cpufreq driver exists
  anywhere, so day/night stays pure cgroup (already the design).
 ## Q17. VM 5111 auto-start
 Audit shows onboot unset: after any host reboot your workstation stays DOWN
 until manually started. OK to set onboot=1 with startup order 10?
 A:
 ## Q18. Remote-desktop matrix (audit correction: xrdp IS running, sesman too)
 Confirmed live: xrdp + xrdp-sesman = your Jump RDP path (my earlier "no xrdp"
 was a bad probe, my error). Sunshine installed but not running. Two human
 accounts, both with always-alive KDE (reachableceo 1001, -offstage 1010).
 REC: keep xrdp for iPad quick-attach (verify /etc/xrdp/sesman.ini
 KillDisconnected=false so sessions persist); start Sunshine only after the
 compute GPUs land and use Moonlight for the video/audio editing sessions
 (NVENC + proper audio); sddm autologin keeps a warm console session for that.
 Confirm? And which account hosts the media-editing workload?
 A:
 ## Q11 partial (2026-08-31): consolidation decided
 VM 500 pfv-k8s-wnode-tsys5 removed; ultix-streaming will be THE k8s worker on
 this host. Still open: control plane location/type (k3s?), and timing (rec:
 join after Friday's CPU/RAM resize, worker-only, kubelet+containerd fenced at
 CPUWeight 20 + MemoryHigh, low PriorityClass + eviction thresholds per REPORT
 5.3).
 A (control plane + timing):
@@ -0,0 +1,34 @@
 # /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile
 # Measured-before values and rationale: ~/optimize/REPORT.md §4.1.
 # Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure,
 # pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3).
 # Predictable writeback under ETL bursts (bytes-based, RAM-size independent).
 # Night profile (ukrrs-daynight.sh) raises these to 2G/512M.
 vm.dirty_background_bytes = 268435456
 vm.dirty_bytes = 1073741824
 # Survive bursty reclaim when compiles+ETL hit at once. 768M post-upgrade.
 vm.min_free_kbytes = 393216
 # Postgres AIO / io_uring era; 9 harness daemons + LSPs watching many repos.
 fs.aio-max-nr = 1048576
 fs.inotify.max_user_watches = 1048576
 fs.inotify.max_user_instances = 512
 # 9 accounts x long-lived provider streams + tailscale + docker NAT.
 net.ipv4.ip_local_port_range = 10240 65535
 net.ipv4.tcp_tw_reuse = 1
 # LLM turns idle minutes between bursts on live sockets; large SSE/JSON.
 net.ipv4.tcp_slow_start_after_idle = 0
 net.core.rmem_max = 16777216
 net.core.wmem_max = 16777216
 net.ipv4.tcp_rmem = 4096 131072 16777216
 net.ipv4.tcp_wmem = 4096 65536 16777216
 net.core.netdev_max_backlog = 8192
 # BBR requires the module: staged modules-load.d/tcp_bbr.conf loads it.
 # If bbr is unavailable, comment the last two lines out (cubic is fine).
 net.ipv4.tcp_congestion_control = bbr
 net.core.default_qdisc = fq
@@ -0,0 +1,134 @@
 #!/usr/bin/env bash
 # UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first.
 #   sudo RUN=1 ./apply-guest.sh <step>
 # RUN=1 required to mutate anything (default: plan only).
 # daemonjson and desktop additionally require CONFIRM=1 (one-time disruption).
 # Steps: sysctl modules thp slices psi prune nofile noatime daemonjson desktop
 #        daynight all
 set -euo pipefail
 RUN=${RUN:-0}
 CONFIRM=${CONFIRM:-0}
 DESKTOP_USER=${DESKTOP_USER:-reachableceo}
 here=$(cd "$(dirname "$0")" && pwd)
 say() { echo "[apply-guest] $*"; }
 die() { echo "[apply-guest] $*" >&2; exit 2; }
 need_root() { [ "$(id -u)" = 0 ] || die "needs root"; }
 gate() {
  if [ "$RUN" != 1 ]; then say "DRY: would $* (rerun with RUN=1)"; return 1; fi
  need_root; return 0
 }
 inst() { # src dst mode
  install -m "$3" "$1" "$2" && say "installed $2"
 }
 step_sysctl() {
  gate "install sysctl.d profile" || return 0
  inst "$here/60-ukrrs-vm.conf" /etc/sysctl.d/60-ukrrs-vm.conf 0644
  sysctl --system >/dev/null && say "sysctl applied (bbr lines need the module: step modules)"
 }
 step_modules() {
  gate "install tcp_bbr module load" || return 0
  inst "$here/modules-load.d/tcp_bbr.conf" /etc/modules-load.d/tcp_bbr.conf 0644
  modprobe tcp_bbr 2>/dev/null || say "tcp_bbr not loaded now (will load at boot)"
 }
 step_thp() {
  gate "install THP madvise unit" || return 0
  inst "$here/systemd/ukrrs-thp-madvise.service" /etc/systemd/system/ukrrs-thp-madvise.service 0644
  systemctl daemon-reload
  systemctl enable --now ukrrs-thp-madvise.service
  say "THP=$(cat /sys/kernel/mm/transparent_hugepage/enabled)"
 }
 step_slices() {
  gate "install ukrrs slices" || return 0
  for s in gateway pmo lsp batch rt; do
    inst "$here/systemd/ukrrs-$s.slice" "/etc/systemd/system/ukrrs-$s.slice" 0644
  done
  systemctl daemon-reload
  say "slices ready; containers opt in via cgroup_parent=ukrrs-<x>.slice"
 }
 step_psi() {
  gate "install PSI textfile collector" || return 0
  inst "$here/ukrrs-psi-textfile.sh" /usr/local/sbin/ukrrs-psi-textfile.sh 0755
  mkdir -p /var/lib/node_exporter/textfile && chmod 755 /var/lib/node_exporter /var/lib/node_exporter/textfile
  inst "$here/systemd/ukrrs-psi-textfile.service" /etc/systemd/system/ukrrs-psi-textfile.service 0644
  inst "$here/systemd/ukrrs-psi-textfile.timer" /etc/systemd/system/ukrrs-psi-textfile.timer 0644
  systemctl daemon-reload
  systemctl enable --now ukrrs-psi-textfile.timer
  say "pressure gauges: /var/lib/node_exporter/textfile/ukrrs_pressure.prom"
 }
 step_prune() {
  gate "install builder-prune timer" || return 0
  inst "$here/systemd/ukrrs-builder-prune.service" /etc/systemd/system/ukrrs-builder-prune.service 0644
  inst "$here/systemd/ukrrs-builder-prune.timer" /etc/systemd/system/ukrrs-builder-prune.timer 0644
  systemctl daemon-reload
  systemctl enable --now ukrrs-builder-prune.timer
 }
 step_nofile() {
  gate "raise DefaultLimitNOFILE" || return 0
  mkdir -p /etc/systemd/system.conf.d
  cat > /etc/systemd/system.conf.d/50-ukrrs.conf <<'EOF'
 [Manager]
 DefaultLimitNOFILE=65536:1048576
 EOF
  systemctl daemon-reload
  say "DefaultLimitNOFILE raised (new sessions)"
 }
 step_noatime() {
  gate "add noatime to /, /data1, /data2" || return 0
  cp /etc/fstab "/etc/fstab.bak.ukrrs.$(date +%s)"
  for mnt in / /data1 /data2; do
    opts=$(findmnt -n -o OPTIONS "$mnt") || continue
    case ",$opts," in *,noatime,*) say "$mnt already noatime"; continue ;; esac
    mount -o remount,noatime "$mnt"
    awk -v m="$mnt" 'BEGIN{FS=OFS=" "} $2==m && $3=="ext4" { if ($4 !~ /(^|,)noatime(,|$)/) $4=$4",noatime" } 1' \
      /etc/fstab > /etc/fstab.ukrrs.new && mv /etc/fstab.ukrrs.new /etc/fstab
    say "$mnt: remounted noatime + fstab updated"
  done
 }
 step_daemonjson() {
  [ "$CONFIRM" = 1 ] || die "daemonjson needs CONFIRM=1: restarts docker ONCE (live-restore protects future restarts)"
  gate "install daemon.json" || return 0
  [ -f /etc/docker/daemon.json ] && cp /etc/docker/daemon.json "/etc/docker/daemon.json.bak.ukrrs.$(date +%s)"
  inst "$here/docker/daemon.json" /etc/docker/daemon.json 0644
  systemctl restart docker
  say "docker restarted with new config; check: docker info | grep -E 'Live|Logging'"
 }
 step_desktop() {
  [ "$CONFIRM" = 1 ] || die "desktop needs CONFIRM=1: enables sddm autologin for $DESKTOP_USER"
  gate "configure sddm autologin+lock" || return 0
  mkdir -p /etc/sddm.conf.d
  cat > /etc/sddm.conf.d/50-ukrrs-autologin.conf <<EOF
 [Autologin]
 User=$DESKTOP_USER
 Session=plasma.desktop
 Relogin=true
 EOF
  say "sddm autologin configured (UNLOCKED session; autolock deliberately off"
  say "per 2026-08-31 ruling: instant Jump/iPad re-attach outranks lock)"
 }
 step_daynight() {
  gate "install day/night profile timers" || return 0
  inst "$here/ukrrs-daynight.sh" /usr/local/sbin/ukrrs-daynight.sh 0755
  mkdir -p /etc/ukrrs
  inst "$here/etc-ukrrs-daynight.conf" /etc/ukrrs/daynight.conf 0644
  for u in nightprofile.service nightprofile.timer dayprofile.service dayprofile.timer; do
    inst "$here/systemd/ukrrs-$u" "/etc/systemd/system/ukrrs-$u" 0644
  done
  systemctl daemon-reload
  systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer
  say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night"
 }
 step_all() {
  step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune
  step_nofile; step_noatime; step_daynight
  if [ "$CONFIRM" = 1 ]; then step_daemonjson; step_desktop
  else say "skipped daemonjson + desktop (need CONFIRM=1)"; fi
 }
 step=${1:-}
 case "$step" in
  sysctl|modules|thp|slices|psi|prune|nofile|noatime|daemonjson|desktop|daynight|all)
    "step_$step" ;;
  *) die "unknown step: $step" ;;
 esac
@@ -0,0 +1,18 @@
 {
  "live-restore": true,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "20m",
    "max-file": "3"
  },
  "max-concurrent-downloads": 6,
  "max-concurrent-uploads": 4,
  "metrics-addr": "127.0.0.1:9323",
  "experimental": true,
  "default-address-pools": [
    {
      "base": "172.16.0.0/12",
      "size": 24
    }
  ]
 }
@@ -0,0 +1,11 @@
 # /etc/ukrrs/daynight.conf — overrides for ukrrs-daynight.sh
 # Interim values (8 vCPU / 48G) are the script defaults; only set what differs.
 DATA2_DEV=/dev/sdb
 # Post-upgrade values (20 vCPU / 128G), uncomment after the VM resize:
 # BATCH_CPUS_DAY=0-15
 # BATCH_CPUS_NIGHT=0-17
 # BATCH_HIGH_DAY=24G
 # BATCH_HIGH_NIGHT=80G
 # Also edit ukrrs-rt.slice AllowedCPUs to 18-19 and account slices (mkacct.sh).
@@ -0,0 +1,72 @@
 #!/usr/bin/env bash
 # UKRRS: create one account lane = cgroup slice + matching user-slice drop-in.
 # usage: mkacct.sh <account> <uid> [mem_high] [mem_max] [allowed_cpus] [tier]
 #
 #   tier=agent (default): CPUWeight=75, pinned to a CPU pool — the 9 PMO/worker
 #     lanes. interim rec: 3G 4.5G 0-5   post-upgrade: 10G 12G 0-15
 #   tier=human: CPUWeight=600, NO cpu restriction, generous memory — the two
 #     human interactive accounts (KDE/CAD/EDA/video via xrdp live in these).
 #     interim rec: 12G 16G all   post-upgrade: 32G 40G all
 #
 # Examples (uids verified 2026-08-31):
 #   sudo RUN=1 ./mkacct.sh reachableceo 1001 12G 16G all human
 #   sudo RUN=1 ./mkacct.sh reachableceo-offstage 1010 12G 16G all human
 #   sudo RUN=1 ./mkacct.sh TSGBOD <uid> 3G 4.5G 0-5 agent
 #
 # RUN=1 installs (needs root); default prints the plan + snippets only.
 set -euo pipefail
 acct=${1:?account}; uid=${2:?uid}
 high=${3:-3G}; max=${4:-4.5G}; cpus=${5:-0-5}; tier=${6:-agent}
 RUN=${RUN:-0}
 case "$tier" in
  human) weight=600; cpuline="" ;;
  agent) weight=75;  cpuline="AllowedCPUs=$cpus" ;;
  *) echo "tier must be human or agent" >&2; exit 2 ;;
 esac
 slice="/etc/systemd/system/ukrrs-acct-$acct.slice"
 userdrop="/etc/systemd/system/user-$uid.slice.d/50-ukrrs.conf"
 cat <<EOF
 plan ($tier tier):
  $slice
      CPUWeight=$weight ${cpuline:+$cpuline }MemoryHigh=$high MemoryMax=$max TasksMax=4096
  $userdrop
      CPUWeight=$weight MemoryHigh=$high
 compose snippet (this account's agent projects):
  x-ukrrs-acct: &ukrrs_acct
    cgroup_parent: ukrrs-acct-$acct.slice
  services:
    anything: { <<: *ukrrs_acct }
 dev.sh one-shot builders belong in the batch pool, not the account slice:
  docker run --rm --cgroup-parent ukrrs-batch.slice --cpus 4 --memory 4g ...
 EOF
 if [ "$RUN" = 1 ]; then
  [ "$(id -u)" = 0 ] || { echo "RUN=1 needs root" >&2; exit 1; }
  cat > "$slice" <<EOF
 [Unit]
 Description=UKRRS account lane ($tier): $acct
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 CPUWeight=$weight
 ${cpuline}
 MemoryHigh=$high
 MemoryMax=$max
 TasksMax=4096
 EOF
  mkdir -p "$(dirname "$userdrop")"
  cat > "$userdrop" <<EOF
 [Slice]
 CPUWeight=$weight
 MemoryHigh=$high
 EOF
  systemctl daemon-reload
  # push live: drop-ins alone don't re-apply to already-existing user slices
  systemctl set-property "user-$uid.slice" CPUWeight="$weight" MemoryHigh="$high"
  echo "installed + daemon-reload + live-applied ok: $acct ($tier)"
 fi
@@ -0,0 +1,2 @@
 # Load tcp_bbr at boot so sysctl.d/60-ukrrs-vm.conf can set bbr.
 tcp_bbr
@@ -0,0 +1,56 @@
 #!/usr/bin/env bash
 # UKRRS Proxmox control wrapper for harness/human use. HARD RULES baked in:
 #   - vmids listed in PROTECTED are never touched (list THIS VM first).
 #   - graceful shutdown only; no reset, no hard stop, no bare down (house rule).
 # Env:
 #   PVE_LOCAL=1                      run ON the pve host itself (pvesh direct)
 #   or PVE_HOST + PVE_TOKEN          remote, token format: user@realm!tokenid=secret
 #   PVE_NODE                         node name (required for vm ops)
 #   PROTECTED                        required, comma-separated vmids
 # Usage:
 #   proxmox-ctl.sh vms
 #   proxmox-ctl.sh shutdown <vmid>
 #   proxmox-ctl.sh start <vmid>
 #   proxmox-ctl.sh snapshot <vmid> <name>
 # Token setup (on the pve host, see REPORT.md appendix B):
 #   pveum user add ukrrs-infra@pam
 #   pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0
 #   pveum acl modify /pool/<preprod-pool> -user ukrrs-infra@pam -role PVEVMUser
 set -euo pipefail
 : "${PROTECTED:?set PROTECTED=vmid1,vmid2,... (this VM must be in the list)}"
 pvesh_() {
  if [ "${PVE_LOCAL:-0}" = 1 ]; then
    command pvesh "$@"
  else
    : "${PVE_HOST:?}" "${PVE_TOKEN:?}"
    command pvesh --host "$PVE_HOST" --api-token "$PVE_TOKEN" "$@"
  fi
 }
 guard() {
  case ",$PROTECTED," in
    *",$1,"*) echo "REFUSED: vmid $1 is PROTECTED" >&2; exit 3 ;;
  esac
 }
 cmd=${1:-}; shift || true
 case "$cmd" in
  vms)
    pvesh_ get /cluster/resources --type vm ;;
  shutdown)
    [ $# = 1 ] || { echo "usage: $0 shutdown <vmid>" >&2; exit 2; }
    guard "$1"; : "${PVE_NODE:?}"
    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/shutdown" --timeout 120 ;;
  start)
    [ $# = 1 ] || { echo "usage: $0 start <vmid>" >&2; exit 2; }
    guard "$1"; : "${PVE_NODE:?}"
    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/start" ;;
  snapshot)
    [ $# = 2 ] || { echo "usage: $0 snapshot <vmid> <name>" >&2; exit 2; }
    guard "$1"; : "${PVE_NODE:?}"
    pvesh_ create "/nodes/$PVE_NODE/qemu/$1/snapshot" snapname="$2" ;;
  *)
    echo "usage: $0 vms|shutdown <vmid>|start <vmid>|snapshot <vmid> <name>" >&2
    exit 2 ;;
 esac
@@ -0,0 +1,14 @@
 [Unit]
 Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 # Day bias: lowest weight, restricted to pool A. The night profile timer
 # (ukrrs-nightprofile.service) raises weight/MemoryHigh and widens AllowedCPUs
 # via `systemctl set-property --runtime`; reboot lands back in day mode.
 # IOWeight deliberately absent: inert under the `none` I/O scheduler.
 CPUWeight=25
 AllowedCPUs=0-5
 MemoryHigh=12884901888
 MemoryMax=17179869184
 TasksMax=16384
@@ -0,0 +1,7 @@
 [Unit]
 Description=UKRRS: weekly docker build-cache prune (25G ceiling)
 After=docker.service
 [Service]
 Type=oneshot
 ExecStart=/usr/bin/docker builder prune --force --keep-storage 25GB
@@ -0,0 +1,9 @@
 [Unit]
 Description=UKRRS: build-cache prune, Sunday night window
 [Timer]
 OnCalendar=Sun *-*-* 22:30:00
 Persistent=true
 [Install]
 WantedBy=timers.target
@@ -0,0 +1,6 @@
 [Unit]
 Description=UKRRS: switch back to day profile (interactive bias)
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/ukrrs-daynight.sh day
@@ -0,0 +1,9 @@
 [Unit]
 Description=UKRRS: day profile at 05:00
 [Timer]
 OnCalendar=*-*-* 05:00:00
 Persistent=true
 [Install]
 WantedBy=timers.target
@@ -0,0 +1,12 @@
 [Unit]
 Description=ukrrs gateway ensure-up (tailscale port race + live-restore endpoint loss)
 After=docker.service
 Requires=docker.service
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/usr/local/sbin/ukrrs-gateway-ensure.sh
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,12 @@
 [Unit]
 Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta)
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 # Latency-sensitive: all provider traffic + spend accounting. Weight beats any
 # single account even under total compile storm (see REPORT section 2 math).
 CPUWeight=900
 # Hard floor so accounts can never squeeze the accounting DB into reclaim.
 MemoryMin=2147483648
 MemoryHigh=12884901888
 TasksMax=infinity
@@ -0,0 +1,9 @@
 [Unit]
 Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads)
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 CPUWeight=150
 MemoryHigh=6442450944
 MemoryMax=8589934592
 TasksMax=8192
@@ -0,0 +1,12 @@
 [Unit]
 Description=Enable virtio NIC multiqueue (host offers 4 queues on ens18, 2 on ens19)
 After=network-pre.target
 Before=network-online.target
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/sh -c '/usr/sbin/ethtool -L ens18 combined 4 || true; /usr/sbin/ethtool -L ens19 combined 2 || true; true'
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,6 @@
 [Unit]
 Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/ukrrs-daynight.sh night
@@ -0,0 +1,9 @@
 [Unit]
 Description=UKRRS: night profile at 22:00
 [Timer]
 OnCalendar=*-*-* 22:00:00
 Persistent=true
 [Install]
 WantedBy=timers.target
@@ -0,0 +1,11 @@
 [Unit]
 Description=UKRRS PMO dispatch loops (one per account): control plane
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 # Dispatcher must never starve behind worker storms: cheap but weighted above
 # batch and idle accounts. Night profile leaves this untouched.
 CPUWeight=200
 MemoryHigh=1073741824
 MemoryMax=1610612736
 TasksMax=512
@@ -0,0 +1,7 @@
 [Unit]
 Description=UKRRS: PSI textfile collector tick
 After=multi-user.target
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/ukrrs-psi-textfile.sh
@@ -0,0 +1,10 @@
 [Unit]
 Description=UKRRS: PSI textfile collector (15s)
 [Timer]
 OnBootSec=2min
 OnUnitActiveSec=15
 AccuracySec=5
 [Install]
 WantedBy=timers.target
@@ -0,0 +1,12 @@
 [Unit]
 Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work
 Documentation=file:///home/reachableceo/optimize/REPORT.md
 [Slice]
 # Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores,
 # so anything placed here keeps headroom even under full compile storm.
 # Interim 8 vCPU: 6-7. Post-upgrade 20 vCPU: 18-19 (edit after resize).
 CPUWeight=10000
 AllowedCPUs=6-7
 MemoryHigh=2147483648
 MemoryMax=4294967296
@@ -0,0 +1,11 @@
 [Unit]
 Description=UKRRS: transparent hugepages to madvise (postgres-friendly)
 After=multi-user.target
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,45 @@
 #!/usr/bin/env bash
 # UKRRS day/night resource profile (REPORT.md section 5.5).
 # Runtime-only changes (set-property --runtime + sysctl): a reboot always
 # lands safely in day mode (slice unit files carry the day defaults).
 set -euo pipefail
 mode=${1:?usage: ukrrs-daynight.sh day|night}
 CONF=/etc/ukrrs/daynight.conf
 [ -r "$CONF" ] && . "$CONF"
 : "${DATA2_DEV:=/dev/sdb}"
 : "${BATCH_WEIGHT_DAY:=25}";        : "${BATCH_WEIGHT_NIGHT:=400}"
 : "${BATCH_CPUS_DAY:=0-5}";         : "${BATCH_CPUS_NIGHT:=0-6}"
 : "${BATCH_HIGH_DAY:=12G}";         : "${BATCH_HIGH_NIGHT:=30G}"
 : "${GW_WEIGHT_DAY:=900}";          : "${GW_WEIGHT_NIGHT:=500}"
 : "${DATA2_RA_DAY:=256}";           : "${DATA2_RA_NIGHT:=2048}"
 : "${DIRTY_DAY:=1073741824}";       : "${DIRTY_NIGHT:=2147483648}"
 : "${DIRTY_BG_DAY:=268435456}";     : "${DIRTY_BG_NIGHT:=536870912}"
 log() { echo "[ukrrs-daynight] $*"; }
 setprop() { systemctl set-property --runtime "$@"; }
 setra() { blockdev --setra "$1" "$2" 2>/dev/null || log "readahead skip: $2"; }
 case "$mode" in
 night)
  setprop ukrrs-batch.slice \
    CPUWeight="$BATCH_WEIGHT_NIGHT" \
    AllowedCPUs="$BATCH_CPUS_NIGHT" \
    MemoryHigh="$BATCH_HIGH_NIGHT"
  setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_NIGHT"
  setra "$DATA2_RA_NIGHT" "$DATA2_DEV"; setra "$DATA2_RA_NIGHT" "${DATA2_DEV}1"
  sysctl -q -w vm.dirty_bytes="$DIRTY_NIGHT" vm.dirty_background_bytes="$DIRTY_BG_NIGHT"
  ;;
 day)
  setprop ukrrs-batch.slice \
    CPUWeight="$BATCH_WEIGHT_DAY" \
    AllowedCPUs="$BATCH_CPUS_DAY" \
    MemoryHigh="$BATCH_HIGH_DAY"
  setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_DAY"
  setra "$DATA2_RA_DAY" "$DATA2_DEV"; setra "$DATA2_RA_DAY" "${DATA2_DEV}1"
  sysctl -q -w vm.dirty_bytes="$DIRTY_DAY" vm.dirty_background_bytes="$DIRTY_BG_DAY"
  ;;
 *)
  echo "unknown mode: $mode" >&2; exit 2 ;;
 esac
 log "profile $mode applied $(date -Is)"
@@ -0,0 +1,31 @@
 #!/usr/bin/env bash
 # Boots are racy for the gateway containers (both root-caused 2026-08-31):
 #  - prod binds :4000/:9090 to the tailscale IP; if dockerd wins the race over
 #    tailscaled, the bind fails and docker NEVER retries failed starts;
 #  - live-restore can drop a container's network endpoint across a boot,
 #    leaving it started with the host resolver (gateway-db unresolvable,
 #    crash loop). A plain `docker start` does NOT reattach the endpoint.
 # Waits for the tailscale IP, then recreates any lane not actually serving.
 # Idempotent; recreating is safe at boot time (nothing in flight).
 set -u
 GW_DIR=/home/reachableceo/projects/ukrrs/gateway
 TS_IP=100.101.187.119
 for _ in $(seq 1 60); do
  /usr/sbin/ip -o addr | grep -q "$TS_IP/" && break
  sleep 2
 done
 ensure_lane() { # name host:port project compose_file
  local name=$1 hostport=$2 project=$3 file=$4
  if timeout 5 bash -c "</dev/tcp/$hostport" 2>/dev/null; then
    echo "$name: serving on $hostport"
    return 0
  fi
  echo "$name: NOT serving on $hostport; recreating"
  docker compose -p "$project" -f "$GW_DIR/$file" up -d --no-deps --force-recreate gateway
 }
 ensure_lane prod  "$TS_IP:4000"    mopac-gateway       docker-compose.yml
 ensure_lane beta "127.0.0.1:4002" mopac-gateway-beta  compose.beta.yaml
 exit 0
@@ -0,0 +1,39 @@
 #!/usr/bin/env bash
 # Per-slice + host PSI (pressure stall info) avg60 -> node_exporter textfile.
 # Consumed by the existing mopac-harness prometheus; gateway may also read it.
 set -u
 OUT_DIR=${UKRRS_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}
 CGROOT=${UKRRS_CGROOT:-/sys/fs/cgroup}
 mkdir -p "$OUT_DIR"
 tmp=$(mktemp "$OUT_DIR/.ukrrs.XXXXXX")
 some60() { # file -> prints avg60 value or nothing
  awk '$1=="some"{for(i=2;i<=NF;i++) if($i ~ /^avg60=/){sub("avg60=","",$i); print $i; exit}}' "$1" 2>/dev/null
 }
 emit() { # name cgroup_path
  local res v
  for res in cpu memory io; do
    [ -r "$2/$res.pressure" ] || continue
    v=$(some60 "$2/$res.pressure")
    [ -n "${v:-}" ] && echo "ukrrs_slice_pressure_some60{slice=\"$1\",res=\"$res\"} $v"
  done
 }
 echo "# HELP ukrrs_slice_pressure_some60 PSI some avg60 (percent) for ukrrs slices" >>"$tmp"
 echo "# TYPE ukrrs_slice_pressure_some60 gauge" >>"$tmp"
 for cg in "$CGROOT"/system.slice/ukrrs-*.slice; do
  [ -d "$cg" ] && emit "${cg##*/}" "$cg"
 done
 [ -d "$CGROOT/user.slice" ] && emit "user.slice" "$CGROOT/user.slice"
 echo "# HELP ukrrs_host_pressure_some60 PSI some avg60 (percent) host-wide" >>"$tmp"
 echo "# TYPE ukrrs_host_pressure_some60 gauge" >>"$tmp"
 for res in cpu memory io; do
  [ -r "/proc/pressure/$res" ] || continue
  v=$(some60 "/proc/pressure/$res")
  [ -n "${v:-}" ] && echo "ukrrs_host_pressure_some60{res=\"$res\"} $v"
 done
 mv "$tmp" "$OUT_DIR/ukrrs_pressure.prom"
 chmod 0644 "$OUT_DIR/ukrrs_pressure.prom"
		`@@ -0,0 +1,2 @@`
							`# Load tcp_bbr at boot so sysctl.d/60-ukrrs-vm.conf can set bbr.`
							`tcp_bbr`