ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111

Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
2026-08-31 16:46:13 -05:00
commit cba2a64d67
43 changed files with 3137 additions and 0 deletions
@@ -0,0 +1,14 @@
+[Unit]
+Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Day bias: lowest weight, restricted to pool A. The night profile timer
+# (ukrrs-nightprofile.service) raises weight/MemoryHigh and widens AllowedCPUs
+# via `systemctl set-property --runtime`; reboot lands back in day mode.
+# IOWeight deliberately absent: inert under the `none` I/O scheduler.
+CPUWeight=25
+AllowedCPUs=0-5
+MemoryHigh=12884901888
+MemoryMax=17179869184
+TasksMax=16384
@@ -0,0 +1,7 @@
+[Unit]
+Description=UKRRS: weekly docker build-cache prune (25G ceiling)
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/docker builder prune --force --keep-storage 25GB
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: build-cache prune, Sunday night window
+
+[Timer]
+OnCalendar=Sun *-*-* 22:30:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,6 @@
+[Unit]
+Description=UKRRS: switch back to day profile (interactive bias)
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-daynight.sh day
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: day profile at 05:00
+
+[Timer]
+OnCalendar=*-*-* 05:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=ukrrs gateway ensure-up (tailscale port race + live-restore endpoint loss)
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/ukrrs-gateway-ensure.sh
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta)
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Latency-sensitive: all provider traffic + spend accounting. Weight beats any
+# single account even under total compile storm (see REPORT section 2 math).
+CPUWeight=900
+# Hard floor so accounts can never squeeze the accounting DB into reclaim.
+MemoryMin=2147483648
+MemoryHigh=12884901888
+TasksMax=infinity
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads)
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+CPUWeight=150
+MemoryHigh=6442450944
+MemoryMax=8589934592
+TasksMax=8192
@@ -0,0 +1,12 @@
+[Unit]
+Description=Enable virtio NIC multiqueue (host offers 4 queues on ens18, 2 on ens19)
+After=network-pre.target
+Before=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c '/usr/sbin/ethtool -L ens18 combined 4 || true; /usr/sbin/ethtool -L ens19 combined 2 || true; true'
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,6 @@
+[Unit]
+Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-daynight.sh night
@@ -0,0 +1,9 @@
+[Unit]
+Description=UKRRS: night profile at 22:00
+
+[Timer]
+OnCalendar=*-*-* 22:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,11 @@
+[Unit]
+Description=UKRRS PMO dispatch loops (one per account): control plane
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Dispatcher must never starve behind worker storms: cheap but weighted above
+# batch and idle accounts. Night profile leaves this untouched.
+CPUWeight=200
+MemoryHigh=1073741824
+MemoryMax=1610612736
+TasksMax=512
@@ -0,0 +1,7 @@
+[Unit]
+Description=UKRRS: PSI textfile collector tick
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ukrrs-psi-textfile.sh
@@ -0,0 +1,10 @@
+[Unit]
+Description=UKRRS: PSI textfile collector (15s)
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=15
+AccuracySec=5
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,12 @@
+[Unit]
+Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work
+Documentation=file:///home/reachableceo/optimize/REPORT.md
+
+[Slice]
+# Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores,
+# so anything placed here keeps headroom even under full compile storm.
+# Interim 8 vCPU: 6-7. Post-upgrade 20 vCPU: 18-19 (edit after resize).
+CPUWeight=10000
+AllowedCPUs=6-7
+MemoryHigh=2147483648
+MemoryMax=4294967296
@@ -0,0 +1,11 @@
+[Unit]
+Description=UKRRS: transparent hugepages to madvise (postgres-friendly)
+After=multi-user.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'
+
+[Install]
+WantedBy=multi-user.target