ultix perf/ops workbench: complete 2026-08-31 optimization pass for VM 5111
Everything for the ultix-streaming (VM 5111, pfv-tsys5) performance pass: full report + host audit results, staged/gated configs, guest prep + host one-shot + post-reboot-fix + netcheck lifecycle scripts, grow-root manual runbook, rolling tracking HUD, questions v1, and the gateway boot-race hardening units. Applied and verified live 2026-08-31; open work is tracked in Redmine project 55 as #601-#607. [#602] 💘 Generated with Crush Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -0,0 +1 @@
|
||||
.crush/
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE-SHOT guest prep (idempotent; safe to re-run). Run ON ultix-streaming:
|
||||
# sudo ~/optimize/1-guest-prep.sh
|
||||
# 1. All staged configs (docker restarts once) + sddm autologin.
|
||||
# 2. Human-tier slices for the two human accounts (KDE/CAD/video priority).
|
||||
# NO storage operations (root growth is manual: GROW-ROOT-RUNBOOK.md).
|
||||
set -euo pipefail
|
||||
[ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; }
|
||||
DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
echo "== [1/3] all staged configs =="
|
||||
RUN=1 CONFIRM=1 bash "$DIR/staged/apply-guest.sh" all
|
||||
|
||||
echo "== [2/3] human-tier slices (CPUWeight 600, no cpu pinning, 12G/16G) =="
|
||||
RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo 1001 12G 16G all human
|
||||
RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo-offstage 1010 12G 16G all human
|
||||
|
||||
echo "== [3/3] done =="
|
||||
echo "GUEST READY. Next, the host one-shot (safe to run from THIS VM; the"
|
||||
echo "VM bounce is detached on the host and survives your ssh dying):"
|
||||
echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/optimize/2-host-one-shot.sh"
|
||||
echo
|
||||
echo "Root growth to ~505G stays MANUAL: ~/optimize/GROW-ROOT-RUNBOOK.md"
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111.
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ... # optional
|
||||
#
|
||||
# 1. VM 5111's disk/net/onboot flags are written as PENDING config: they
|
||||
# activate automatically at the VM's NEXT reboot (tonight, your last step
|
||||
# today, Friday — whenever the VM naturally restarts). Nothing forces it.
|
||||
# 2. CPU priority (live): VM 5111 = 9000, sectestbed/preprod fleet = 50.
|
||||
# 3. Reboot wave: every OTHER running VM.
|
||||
# Optional --with-vm-restart: also gracefully bounce VM 5111 at the end.
|
||||
# THE HOST ITSELF IS NEVER REBOOTED (NFS server). NO DISK IS RESIZED
|
||||
# (storage ops are manual-only: GROW-ROOT-RUNBOOK.md).
|
||||
set -euo pipefail
|
||||
GO=0; VMR=0; SKIPWAVE=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--go) GO=1 ;;
|
||||
--with-vm-restart) VMR=1 ;;
|
||||
--skip-wave) SKIPWAVE=1 ;;
|
||||
esac
|
||||
done
|
||||
VM=5111
|
||||
FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108"
|
||||
run() { if [ "$GO" = 1 ]; then "$@"; else echo "DRY: $*"; fi; }
|
||||
staget() { # pending-stage a qm set; tolerate refusal on a running VM
|
||||
if [ "$GO" = 1 ]; then
|
||||
qm set "$@" || echo " NOTE: not staged while running; use --with-vm-restart (or set while stopped) later"
|
||||
else
|
||||
echo "DRY(pending): qm set $*"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "== 1. VM $VM flags as PENDING config (live at its next reboot) =="
|
||||
staget "$VM" --onboot 1 --startup order=10,up=180
|
||||
staget "$VM" -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
||||
staget "$VM" -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
|
||||
staget "$VM" -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
|
||||
staget "$VM" --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
||||
staget "$VM" --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
||||
staget "$VM" --delete ide2
|
||||
|
||||
echo "== 2. CPU priority (live, reversible) =="
|
||||
run qm set "$VM" --cpuunits 9000
|
||||
for id in $FLEET; do run qm set "$id" --cpuunits 50; done
|
||||
|
||||
echo "== 3. reboot wave: every other running VM =="
|
||||
if [ "$SKIPWAVE" = 1 ]; then
|
||||
echo "skipped (--skip-wave)"
|
||||
else
|
||||
OTHERS=$(qm list | awk 'NR>1 && $3=="running" && $1!="'"$VM"'" {print $1}')
|
||||
echo "targets: ${OTHERS//$'\n'/ }"
|
||||
for id in $OTHERS; do
|
||||
if [ "$GO" = 1 ]; then
|
||||
qm reboot "$id" 2>/dev/null || { qm shutdown "$id" --timeout 120 || true; sleep 2; qm start "$id" 2>/dev/null || true; }
|
||||
else
|
||||
echo "DRY: reboot $id"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$GO" = 1 ]; then
|
||||
echo; echo "== pending queue for VM $VM (activates at its next restart) =="
|
||||
qm pending "$VM" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
LOG=/var/log/ukrrs-vm5111-bounce.log
|
||||
if [ "$VMR" = 1 ]; then
|
||||
echo "== graceful bounce of VM $VM (detached; flags applied while STOPPED) =="
|
||||
qm status "$VM"
|
||||
if [ "$GO" = 1 ]; then
|
||||
setsid bash -c "
|
||||
echo bounce-start \$(date -Is)
|
||||
qm shutdown $VM --timeout 120
|
||||
qm wait $VM --timeout 180 || true
|
||||
qm set $VM --onboot 1 --startup order=10,up=180
|
||||
qm set $VM -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
||||
qm set $VM -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
|
||||
qm set $VM -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
|
||||
qm set $VM --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
||||
qm set $VM --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
||||
qm set $VM --delete ide2 || true
|
||||
qm start $VM
|
||||
echo bounce-done \$(date -Is)
|
||||
" </dev/null >"$LOG" 2>&1 &
|
||||
echo "dispatched. Downtime ~3-4 min; it comes back on its own."
|
||||
echo "Watch from anywhere: ssh root@pfv-tsys5.knel.net tail -f $LOG"
|
||||
else
|
||||
echo "DRY: would detach-bounce $VM, applying flags while stopped, log $LOG"
|
||||
fi
|
||||
else
|
||||
echo
|
||||
echo "VM $VM was NOT rebooted. Flags sit pending and go live at its next restart."
|
||||
fi
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Post-reboot validation fallout (2026-08-31). Run with sudo, once:
|
||||
# sudo ~/optimize/3-post-reboot-fixes.sh
|
||||
# 1. Installs ethtool + enables virtio multiqueue (host side now offers
|
||||
# 4 queues on ens18, 2 on ens19; the guest defaults to 1 until told).
|
||||
# 2. Installs the gateway ensure-up unit (prod boot race on the tailscale-IP
|
||||
# port bind + live-restore endpoint loss, both lanes; idempotent).
|
||||
set -euo pipefail
|
||||
[ "$(id -u)" = 0 ] || { echo "run with sudo"; exit 1; }
|
||||
cd "$(dirname "$0")"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq ethtool
|
||||
|
||||
install -m 0755 staged/ukrrs-gateway-ensure.sh /usr/local/sbin/
|
||||
install -m 0644 staged/systemd/ukrrs-gateway-ensure.service /etc/systemd/system/
|
||||
install -m 0644 staged/systemd/ukrrs-net-multiqueue.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-net-multiqueue.service
|
||||
systemctl enable --now ukrrs-gateway-ensure.service
|
||||
|
||||
echo; echo "== verify =="
|
||||
ethtool -l ens18
|
||||
ethtool -l ens19
|
||||
timeout 5 bash -c '</dev/tcp/100.101.187.119/4000' 2>/dev/null && echo "gateway prod: serving on :4000"
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Read-only host check: why did the net multiqueue flags not land on VM 5111?
|
||||
# Run from the guest: ~/optimize/4-host-netcheck.sh
|
||||
# Everything is saved to ~/optimize/host-netcheck.out for crush to read.
|
||||
set -euo pipefail
|
||||
OUT="$(cd "$(dirname "$0")" && pwd)/host-netcheck.out"
|
||||
timeout 120 ssh root@pfv-tsys5.knel.net '
|
||||
echo "== qm config 5111 (relevant lines) =="
|
||||
qm config 5111 | grep -E "^(net|scsi|onboot|startup|cpuunits|ide|machine|cores|sockets|threads|memory)" || true
|
||||
echo
|
||||
echo "== qm pending 5111 (unapplied staged changes) =="
|
||||
qm pending 5111 2>&1 || true
|
||||
echo
|
||||
echo "== last bounce log (/var/log/ukrrs-vm5111-bounce.log) =="
|
||||
tail -n 40 /var/log/ukrrs-vm5111-bounce.log 2>&1 || true
|
||||
echo
|
||||
echo "== pve version =="
|
||||
pveversion 2>&1 | head -3
|
||||
' | tee "$OUT"
|
||||
echo
|
||||
echo "saved: $OUT"
|
||||
@@ -0,0 +1,95 @@
|
||||
# Manual runbook: grow / from 279G to ~505G (VM 5111, ultix-streaming)
|
||||
|
||||
YOU run every command, one at a time, checking the output before the next.
|
||||
Nothing here is automated, scheduled, or boot-triggered. Storage ops are the
|
||||
only steps in this whole effort that no script touches.
|
||||
|
||||
## Layout change
|
||||
```
|
||||
before: sda1 root 279.2G | sda2 extended [ sda5 swap 8.8G ] | ~150G unallocated
|
||||
after: sda1 root ~505G | sda2 swap ~13G (new, at the tail)
|
||||
```
|
||||
MBR stays MBR, ext4 grows online, VM stays up the whole time. You can safely
|
||||
stop after any step; the system runs fine at any intermediate size. Takes
|
||||
about 10 minutes end to end.
|
||||
|
||||
## Step 0 — preconditions
|
||||
```
|
||||
command -v growpart || sudo apt-get install -y cloud-guest-utils
|
||||
df -h / /data1 /data2 # know your starting point
|
||||
lsblk /dev/sda
|
||||
```
|
||||
|
||||
## Step 1 — backups (30 seconds, do not skip)
|
||||
```
|
||||
sudo sfdisk --dump /dev/sda | sudo tee /root/sda.sfdisk.bak.$(date +%F)
|
||||
sudo cp /etc/fstab /root/fstab.bak.$(date +%F)
|
||||
```
|
||||
Check: `head /root/sda.sfdisk.bak.*` lists sda1, sda2, sda5.
|
||||
|
||||
## Step 2 — grow the virtual disk at the host (hot, VM stays up)
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'qm config 5111 | grep scsi0' # eyeball the target
|
||||
ssh root@pfv-tsys5.knel.net 'qm resize 5111 scsi0 +80G'
|
||||
```
|
||||
Check in guest: `lsblk /dev/sda` shows sda = 518G, sda1 still 279.2G.
|
||||
|
||||
## Step 3 — retire the old swap
|
||||
```
|
||||
free -h # swap "used" must be ~0
|
||||
sudo swapoff /dev/sda5
|
||||
OLDUUID=$(sudo blkid -s UUID -o value /dev/sda5); echo "OLDUUID=$OLDUUID"
|
||||
cat /proc/swaps # check: empty
|
||||
```
|
||||
|
||||
## Step 4 — remove the extended partition (the strip between root and free space)
|
||||
```
|
||||
sudo sfdisk --delete /dev/sda 2
|
||||
sudo partprobe /dev/sda
|
||||
lsblk /dev/sda
|
||||
```
|
||||
Check: only sda1 remains. If sda1 is missing: STOP, do not reboot, do not
|
||||
write anything; restore the table:
|
||||
`sudo sfdisk /dev/sda < /root/sda.sfdisk.bak.<date>`
|
||||
|
||||
## Step 5 — grow partition 1 into the free space
|
||||
```
|
||||
sudo growpart /dev/sda 1
|
||||
# parted alternative: sudo parted /dev/sda resizepart 1 100%
|
||||
lsblk /dev/sda # sda1 now ~505G
|
||||
```
|
||||
|
||||
## Step 6 — grow the filesystem (online)
|
||||
```
|
||||
sudo resize2fs /dev/sda1
|
||||
df -h / # ~500G available
|
||||
```
|
||||
|
||||
## Step 7 — new swap at the tail
|
||||
```
|
||||
sudo parted -s /dev/sda mkpart primary linux-swap 505GB 100%
|
||||
sudo partprobe /dev/sda
|
||||
lsblk /dev/sda
|
||||
NEWSWAP=$(lsblk -no NAME,TYPE /dev/sda | awk '$2=="part"{print $1}' | tail -1); echo "NEWSWAP=$NEWSWAP"
|
||||
sudo mkswap /dev/$NEWSWAP
|
||||
NEWUUID=$(sudo blkid -s UUID -o value /dev/$NEWSWAP); echo "NEWUUID=$NEWUUID"
|
||||
```
|
||||
|
||||
## Step 8 — swap fstab line, activate
|
||||
```
|
||||
sudo sed -i "s/^UUID=$OLDUUID/#UUID=$OLDUUID retired-sda5 $(date +%F)/" /etc/fstab
|
||||
echo "UUID=$NEWUUID none swap sw 0 0" | sudo tee -a /etc/fstab
|
||||
sudo swapon -a
|
||||
cat /proc/swaps # new swap active
|
||||
sudo fstrim -v / # optional: reclaim (discard is on now)
|
||||
```
|
||||
|
||||
## Rollback map
|
||||
- Anytime before step 4: nothing changed except two backup files.
|
||||
- After partition edits, before resize2fs: restore table from the sfdisk dump
|
||||
(step 4 note); no data has moved, only the table.
|
||||
- After resize2fs: growth is one-way by design; reverting size means restore
|
||||
from backups, so this is the one step to do when calm. (It is also the
|
||||
safest operation in the list: online ext4 grow is journaled.)
|
||||
- fstab: backups in /root; only the swap line changes, and a bad swap line is
|
||||
non-fatal at boot (root entry is untouched).
|
||||
@@ -0,0 +1,38 @@
|
||||
# NEXT — perf-opt closeout COMPLETE (2026-08-31 16:4x)
|
||||
|
||||
## Final state — nothing owed
|
||||
- All guest tuning live and verified (see 16:04-16:15 pass below).
|
||||
- Host flags live: ssd=1/discard (ROTA=0), iothread, onboot=1, startup
|
||||
order=10, cpuunits 9000/50, ide2 gone.
|
||||
- Net multiqueue: queues=4/2 now IN live qm config (staged 16:40 by crush
|
||||
via ssh; activates at next VM start = Friday). Guest oneshot unit
|
||||
ukrrs-net-multiqueue auto-runs ethtool -L on that boot.
|
||||
- Gateway boot-race + live-restore endpoint loss: permanently fixed by
|
||||
ukrrs-gateway-ensure unit (enabled, both lanes).
|
||||
- Root cause of the 15:59 no-op: the script copy executed then predates the
|
||||
queues= fix (silent same-value rewrite); current config verified correct.
|
||||
|
||||
## Verified live after the 16:01 bounce (all read-only from the guest)
|
||||
- Disks: ROTA=0, discard 4K/1G on sda+sdb+sdc → ssd=1/discard=on ACTIVE ✅
|
||||
- Kernel: bbr+fq, 16M socket buffers, dirty_bytes 1G/256M, min_free 384M,
|
||||
aio 1M, inotify 512/1M, port range 10240-65535, slow_start_after_idle=0,
|
||||
tcp_tw_reuse=1, THP=madvise, DefaultLimitNOFILE 65536 ✅
|
||||
- Mounts /,/data1,/data2 = noatime; swap sda5 prio -2 unused ✅
|
||||
- Units: all ukrrs slices + day/night + psi + builder-prune + thp units
|
||||
installed and enabled; PSI textfile writing (16:06) ✅
|
||||
- Docker: daemon.json fully live (live-restore, log caps 20m×3, 172.16/12
|
||||
pools, metrics :9323); 21 containers up ✅
|
||||
- Desktop: sddm autologin worked (Relogin=true), session alive on seat0/tty2,
|
||||
greeter burn GONE ✅
|
||||
- Gateway prod was DOWN at boot (bind race + live-restore endpoint loss) →
|
||||
recovered via compose --force-recreate; /status = mode normal, serving ✅
|
||||
|
||||
## Nothing owed. Optional sanity check any time (crush can ssh now):
|
||||
~/optimize/4-host-netcheck.sh
|
||||
|
||||
## Noted, no action taken
|
||||
- mopac-demo/mcli fake containers: restart=no, exited at the first bounce,
|
||||
left down (test stubs; your call).
|
||||
- Prometheus/harness daemons not deployed here yet (that is OPT-9's wiring).
|
||||
- Friday unchanged: CPU/RAM swap (plan NFS outage for -02), 2nd USB3 card,
|
||||
q35 + 20 vCPU/128G reshape, GPUs. Enable the ensure unit BEFORE Friday.
|
||||
@@ -0,0 +1,22 @@
|
||||
# ultix
|
||||
|
||||
Perf/ops workbench for **ultix-streaming** (KVM VM 5111 on Proxmox host
|
||||
pfv-tsys5). Everything here is human-run, gated, and reversible; storage ops
|
||||
are manual-only by ruling.
|
||||
|
||||
- Redmine: project 55 "Known Element Enterprises - Technology & Facility
|
||||
Services" — open work: #601 (Fri 2026-09-04 hardware window, urgent),
|
||||
#603 root growth, #604 metrics/PSI wiring, #605 k8s + proxmox token,
|
||||
#606 GPU passthrough, #607 account map/mkacct; pass record: #602.
|
||||
https://projects.knownelement.com/projects/55
|
||||
- Discourse doc: pending (house cross-link rule: create at next doc pass).
|
||||
- Docs live in-repo (runbook exception per house rules):
|
||||
- REPORT.md — full optimization report + host audit results (section 8)
|
||||
- NEXT.md — current state / what is owed (usually: nothing)
|
||||
- TRACKING.md — rolling HUD table + decisions log
|
||||
- GROW-ROOT-RUNBOOK.md — manual root-growth checklist (#603)
|
||||
- RUNBOOK-TODAY.md, questions-v1.md
|
||||
- Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh,
|
||||
3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh);
|
||||
staged/ holds the gated configs those scripts install.
|
||||
- crush.md = session preferences for agents working in this repo.
|
||||
@@ -0,0 +1,574 @@
|
||||
# ultix-streaming optimization report v1 (2026-08-31)
|
||||
|
||||
Scope: the ultix-streaming KVM guest on pfv-tsys5, tuned for ~9 autonomous agent
|
||||
accounts + gateway fleet + builds + ETL + SDR + GPU, all concurrent. Inputs:
|
||||
direct measurement of the guest, the ukrrs repo tree, and your answers in chat.
|
||||
Companion files: `questions-v1.md` (answer inline), `host-audit.sh` (run on PVE),
|
||||
`staged/` (ready-to-apply configs, gated by `RUN=1`).
|
||||
|
||||
## Exec summary
|
||||
|
||||
One 2010-era 4-core Xeon currently serves this entire host: your dev
|
||||
workstation plus 24 other running VMs, with zero resource limits anywhere in
|
||||
the fleet, and the workstation does not even auto-start after a host reboot.
|
||||
The fix is roughly $300 of used parts, one maintenance window, and the
|
||||
configurations already staged in this directory.
|
||||
|
||||
| spend item | rough cost (used, 2026) | effect |
|
||||
|---|---|---|
|
||||
| 2× Xeon X5675 | $60-80 | host 8 → 24 threads |
|
||||
| 6× 16GB DDR3 ECC RDIMM 1333 | $120-180 | 96G → 192G |
|
||||
| Dell T7500 CPU2 heatsink + paste | $25-40 | required for socket 2 |
|
||||
| **platform total** | **~$210-300** | **3× threads, dev VM 128G** |
|
||||
| optional: 1T DRAM-buffered TLC NVMe | $60-100 | only if the Crucial P1 QLC throttles builds |
|
||||
| 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) |
|
||||
|
||||
Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes,
|
||||
gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn
|
||||
window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by
|
||||
host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven
|
||||
backpressure wired end to end. All software pieces are staged and $0.
|
||||
|
||||
## 0. TL;DR, ordered by leverage
|
||||
|
||||
1. Host CPUs: CONFIRMED by audit: one E5620 (4C/8T) serves the ENTIRE host,
|
||||
including 24 running VMs (load 7.5/8 at idle). CPU2 socket is empty.
|
||||
Drop-in fix: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) = 24
|
||||
threads, plus CPU2 DIMM slots unlock 192G. See section 8.
|
||||
2. One maintenance window fixes VM shape: machine q35 (i440fx today, blocks clean
|
||||
GPU passthrough), 20 vCPU / 128G / ballooning off, virtio-scsi-single +
|
||||
iothread + ssd=1 + discard=on per disk (guest currently sees ALL disks as
|
||||
ROTATIONAL: the SSD hint is not passed through), net0 multiqueue.
|
||||
3. No resource limits exist anywhere in the fleet: 18 live containers, zero
|
||||
cpus/mem/cgroup_parent in any compose, and `dev.sh check` builders default to
|
||||
GOMAXPROCS=8 on an 8-vCPU box. Staged: cgroup v2 slice architecture
|
||||
(ukrrs-{gateway,pmo,lsp,batch,rt} + per-account slices via mkacct.sh).
|
||||
4. Docker daemon is 100% stock: unbounded json-file logs, live-restore off,
|
||||
default address pool (~15 networks; you already have 12), no metrics socket.
|
||||
Staged daemon.json + weekly builder-prune timer (build cache is 9.3G/187 entries).
|
||||
5. Guest kernel: THP=always (bad for the two Postgres), 66M min_free_kbytes,
|
||||
tiny socket buffers (212K) for long LLM streams, dirty ratios in % of 48G
|
||||
(multi-second writeback stalls under ETL). Staged sysctl.d + THP=madvise.
|
||||
6. Signals: harness already reads loadavg/MemAvailable/PSI-io
|
||||
(harness/internal/quota/resources.go:24). Missing: PSI cpu+memory, per-slice
|
||||
pressure, docker awareness. Cheapest full stack: node_exporter as a compose
|
||||
service into your existing mopac prometheus + PSI textfile script. Beszel
|
||||
agent already runs (PID 940). k8s and Proxmox both have clean shedding APIs.
|
||||
7. sddm-greeter burns ~22% of one core 24/7 (stuck on login screen). Fix:
|
||||
autologin + lock. Sunshine is already installed; it becomes excellent once
|
||||
the GPU lands (NVENC), replacing any xrdp idea.
|
||||
8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3).
|
||||
9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool,
|
||||
memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep
|
||||
window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
|
||||
Guest has no CPU governor to remove (KVM: host owns P-states; set host
|
||||
governor to performance once, done).
|
||||
|
||||
## 1. Measured current state
|
||||
|
||||
### Guest (ultix-streaming)
|
||||
| aspect | measured |
|
||||
|---|---|
|
||||
| CPU | 8 vCPU presented as 2 sockets × 4 cores × 1 thread; Xeon E5620 2.40GHz (Westmere-EP, LGA1366, 2010; no AVX); 1 NUMA node |
|
||||
| RAM | 47G usable + 8.8G swap on sda5 (priority -2, unused) |
|
||||
| Machine type | i440fx + SeaBIOS (blocks clean GPU passthrough; q35 preferred) |
|
||||
| Kernel/OS | Debian 13 trixie, 6.12.105, cgroup v2 unified, systemd |
|
||||
| Disks | sda 438G: sda1 root 279G ext4 (29% used), sda5 swap; sdb 400G ext4 /data2 (empty); sdc 200G ext4 /data1 (empty). MBR table. ALL show ROTA=1 |
|
||||
| Sched/mount | scheduler `none` on all disks (good), readahead 128K, relatime everywhere, fstrim.timer present |
|
||||
| Desktop | KDE + sddm, greeter idling at ~22% of a core since boot (118 min CPU); no autologin; Sunshine installed, unused; no xrdp; no nvidia userspace |
|
||||
| PSI now | cpu some avg60 ~1.2%, memory/io ~0 (idle baseline) |
|
||||
| Desktop default | readlink default.target empty (graphical) |
|
||||
|
||||
Key tunables (measured): swappiness 60, dirty_ratio 20/10 (percent-based),
|
||||
min_free_kbytes 66M, max_map_count 1048576 (fine), aio-max-nr 65536, inotify
|
||||
watches 386K / instances 128, somaxconn 4096 (fine), conntrack 262144 (fine),
|
||||
ip_local_port_range 32768-60999, cubic only (bbr module not loaded),
|
||||
rmem_max/wmem_max 212992, slow_start_after_idle=1, THP enabled=always,
|
||||
autogroup=1 (good, keep for screen sessions), pid_max fine. tuned not installed.
|
||||
|
||||
### Docker
|
||||
| aspect | measured |
|
||||
|---|---|
|
||||
| Version/driver | 29.7.2, overlayfs, systemd cgroup driver, cgroup v2 |
|
||||
| Config | NO daemon.json: unbounded json-file logs, live-restore false, data-root /var/lib/docker on root fs |
|
||||
| Footprint | 24 images 14.3G, build cache 9.3G (187 entries), 18 containers up, 6 volumes |
|
||||
| Networks | 12 total (9 user bridges). Default pool = 172.17-31/16, caps at ~15 networks |
|
||||
| Running | 2 gateway stacks + 2 postgres (prod+beta), 9 LSP, openwebui, cli-mred, 2 fakes |
|
||||
|
||||
### Fleet facts (from ukrrs tree)
|
||||
- 9 harness daemons planned on THIS host (harness/deploy/accounts.tsv, idx 0-8:
|
||||
reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO, reachableceo-offstage,
|
||||
COSRCEO-Personal, COSRCEO-Biz, COSWFO). NOTE: /home currently shows different
|
||||
names (COS-RCEO, COS-TSG, ...) so the account map is not final (Q7).
|
||||
- Gateway: fleet semaphore 15 parallel provider calls, policy ladder
|
||||
normal/peak/cooldown/hard recomputed every 30s, pacing cruise 90%.
|
||||
- Harness gate order: resources → quota → peak → soft-defer. Resource reads:
|
||||
loadavg, MemAvailable, PSI io only. Known gaps: harness still polls retired
|
||||
cop :8110 (#585), credits are estimates, no PSI cpu/mem.
|
||||
- Scale target: ~100 work streams, 24×7, 750 credits/hour.
|
||||
- `./dev.sh check` = pinned golang builder, go build+vet+test, NO -p/cpus flags
|
||||
→ each builder defaults to all 8 vCPUs. 42 MCP + 10 LSP always-on.
|
||||
- Beszel agent running (PID 940); prometheus with 30d TSDB already in harness
|
||||
compose (mopac-harness-prometheus).
|
||||
|
||||
## 2. Bottleneck model and the policy
|
||||
|
||||
Who fights whom on this box:
|
||||
|
||||
| workload | cpu | ram | disk | net | latency sensitivity |
|
||||
|---|---|---|---|---|---|
|
||||
| LLM harness loops ×9 | low | low | low | 9 × long streams | medium (streaming) |
|
||||
| gateway + 2× postgres | low-med | med (shared_buffers+cache) | WAL writes | all LLM traffic, via tailscale | HIGH |
|
||||
| docker pulls (burst) | med (gzip) | low | layer writes (NVMe) | burst 100s MB | low |
|
||||
| compiles (dev.sh check) | saturating, GOMAXPROCS=8 each | 2-6G each | cache reads (NVMe) | low | low |
|
||||
| ETL | med | page cache heavy | sequential r/w (SSD) | low | low |
|
||||
| LSP ×10 + MCP 42 | idle mostly | 0.3-1G total-ish | low | none | low (interactive bursts) |
|
||||
| SDR DSP (future) | 1-2 cores steady | low | low | low | HARD real-time |
|
||||
| Sunshine encode (future) | 2-4 cores CPU / ~0 with NVENC | low | low | LAN | medium |
|
||||
|
||||
Design in one sentence: schedule by intent, not by luck. Three CPU pools
|
||||
(interactive/gateway, batch/build, rt/sdr), per-account fair shares with cgroup
|
||||
v2 weights + memory fences, spindle separation per I/O class, and PSI as the
|
||||
universal pressure signal your harness already half-reads.
|
||||
|
||||
```
|
||||
+----------------- system.slice -----------------+
|
||||
| ukrrs-gateway.slice CPUWeight 900 (gateway+pg stacks)
|
||||
| ukrrs-pmo.slice CPUWeight 200 (9 PMO dispatch loops)
|
||||
| ukrrs-lsp.slice CPUWeight 150 (LSP + MCP tier)
|
||||
| ukrrs-batch.slice CPUWeight 25 AllowedCPUs pool A
|
||||
| ukrrs-acct-<9>.slice CPUWeight 75 AllowedCPUs pool A
|
||||
| ukrrs-rt.slice AllowedCPUs pool B (SDR, encode)
|
||||
| docker default (everything not tagged) = last
|
||||
+---------------------------------------------------+
|
||||
interim 8 vCPU: pool A = 0-5, pool B = 6-7
|
||||
post-upgrade 20: pool A = 0-15, pool B = 18-19 (gateway may use 0-19)
|
||||
```
|
||||
|
||||
Fairness math: with 9 accounts each compiling at weight 75 and gateway at 900,
|
||||
gateway keeps ~13% of CPU even under total compile storm; with weights alone an
|
||||
idle machine still lets ONE account use every core (burstable, no quotas). I/O
|
||||
weights are inert under the `none` scheduler, so fairness comes from separation
|
||||
(NVMe = images/builds, /data1 = databases, /data2 = ETL/backup/scratch) plus
|
||||
optional device throttles if a specific ETL goes rogue.
|
||||
|
||||
## 3. Host plan (pfv-tsys5)
|
||||
|
||||
### 3.1 Audit (do this first)
|
||||
My tooling cannot ssh (hard-banned), so run:
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
|
||||
```
|
||||
I parse the output and finalize §3.2-3.7 numbers. The script is read-only.
|
||||
|
||||
### 3.2 CPU upgrade (drop-in, LGA1366)
|
||||
| option | result | notes |
|
||||
|---|---|---|
|
||||
| add 2nd E5620 (~$10) | 8C/16T | cheapest, matches existing |
|
||||
| 2× X5675 (recommended) | 12C/24T @3.06/3.46 | 95W each, DDR3-1333, ~$25 each |
|
||||
| 2× X5690 | 12C/24T @3.46/3.73 | 130W each, hot, marginal gain, PSU risk with GPUs |
|
||||
| 2× L5640 | 12C/24T @2.27 | low power, slower |
|
||||
|
||||
Caveats the audit resolves: chassis must be dual-socket (T5500/T7500; T3500 is
|
||||
single), BIOS revision (A17 supports 5600-series), current DIMM layout, PSU
|
||||
rating. Westmere lacks AVX/AVX2: Go/Rust/CUDA fine, but some prebuilt binaries
|
||||
(newer Node native modules, llama.cpp CPU builds, some distro packages) assume
|
||||
AVX; plan on building from source or using GPU for inference.
|
||||
|
||||
### 3.3 RAM
|
||||
96G now is CPU0 DIMMs only (6 slots). Second CPU unlocks 6 more slots → 192G.
|
||||
Give the VM 128G, leave 64G for host + preprod VMs. Use DDR3 ECC RDIMM 1333
|
||||
(12800R); mixing 10600R works but clocks down. Balanced population per branch.
|
||||
|
||||
### 3.4 VM re-shape (one maintenance window, VM off)
|
||||
```
|
||||
qm set <id> --machine q35 # from i440fx; expect NIC rename in guest
|
||||
qm set <id> --cpu host --sockets 2 --cores 5 --threads 2 --numa 1 # 20 vCPU
|
||||
qm set <id> --memory 131072 --balloon 0
|
||||
qm set <id> --scsihw virtio-scsi-single
|
||||
# per disk (real disk ids from audit):
|
||||
qm set <id> -scsi0 <store>:vm-<id>-disk-0,ssd=1,discard=1,iothread=1 # repeat sdb sdc
|
||||
qm set <id> --net0 virtio=<mac>,bridge=<br>,multiqueue=4
|
||||
```
|
||||
q35 + NIC rename: prepare a systemd .link file or just let udev rename and
|
||||
restart networking; tailscale state survives. Guest then sets 4 RSS queues
|
||||
(ethtool -L, handled by a tiny oneshot unit, staged). If ballooning was on, off.
|
||||
i440fx→q35 is required for the GPU plan; do it once, with console access.
|
||||
|
||||
### 3.5 Storage backend
|
||||
Fixes: ssd=1 (kills the ROTA=1 lie the guest sees today), discard=on (with the
|
||||
fstrim.timer already present), virtio-scsi-single + iothread=1 (one iothread per
|
||||
disk spreads interrupts), cache=none on ZFS backend / writeback only on
|
||||
LVM-thin with UPS. Audit reports backend + link speeds. Known ceiling: if the
|
||||
two SSDs sit on the onboard ICH10R SATA2 ports they cap ~280MB/s each (3Gbps
|
||||
link); the NVMe on a PCIe2.0 x4 card caps ~1.7GB/s. Fine for this fleet, just
|
||||
set expectations for ETL throughput.
|
||||
|
||||
### 3.6 GPU passthrough
|
||||
Now: old NVIDIA on host. Soon: two newer GPUs. Plan:
|
||||
1. IOMMU: already active with clean per-device groups (audit section 8). No
|
||||
ACS override needed. Optional at next host reboot: add `iommu=pt` for
|
||||
cheaper DMA on passthrough devices.
|
||||
2. The Quadro 4000: skip entirely. Fermi has no support in any CUDA toolkit
|
||||
this decade (compute capability 2.x was dropped at CUDA 9) and no driver
|
||||
that builds on kernel 6.x. It stays as the host console card.
|
||||
3. The two newer GPUs are COMPUTE cards (per ruling 2026-08-31), which is
|
||||
easier than display passthrough: no dummy plug, usually no romfile, pass
|
||||
without x-vga (`qm set 5111 -hostpci0 0000:xx:00.0`), modern NVIDIA
|
||||
drivers (465+) officially support VMs. Remaining risk: this-era BIOS has
|
||||
no Above-4G-Decoding, so big-VRAM cards with large BARs may fail to map.
|
||||
Sweet spot for this box: 2× used RTX A2000 12GB (~$150-200 each): 75W
|
||||
slot power, no aux cables, no PSU change, 24G VRAM total, BARs fit. Larger
|
||||
cards (3060 12G / 4060Ti 16G class) need the PSU check + may hit the BAR
|
||||
wall. q35 recommended when they land; OVMF only if a UEFI-only vBIOS
|
||||
forces it.
|
||||
4. In-guest: nvidia driver + nvidia-container-toolkit; builders get `--gpus
|
||||
all` in the batch slice (house-style digest-pinned CUDA image).
|
||||
5. PSU budget if going bigger: 2× 95W CPU + 2× 200-250W GPUs + platform ≈
|
||||
750-850W sustained: needs the 875W PSU option (Q5 sticker check).
|
||||
6. Sunshine (already installed) still benefits even though the cards are
|
||||
compute-only: capture the KDE session via X11/KMS on the emulated VGA,
|
||||
encode on the GPU with NVENC headless. CPU encode (current state,
|
||||
2-4 Westmere cores at 1080p60) remains tolerable-but-temporary. No xrdp:
|
||||
it gives a second synthetic session with no accel; Sunshine mirrors the
|
||||
real session with audio. SSH stays break-glass.
|
||||
|
||||
### 3.7 SDR passthrough
|
||||
You have two PCI USB3 cards, SDRs load-balanced across them: pass the CARDS,
|
||||
never devices. `qm set <id> -hostpci1 <id1> -hostpci2 <id2>`. Guest gets native
|
||||
xHCI controllers with real MSI interrupts (emulated USB adds latency and jitter
|
||||
that ruins tuner sample streams). SDR processes run in ukrrs-rt.slice
|
||||
(AllowedCPUs pool B) so compile storms cannot preempt DSP. Bandwidth: tens of
|
||||
MB/s per SDR, trivial for USB3. libusb latency in a VM is fine for SDR work.
|
||||
|
||||
### 3.8 Proxmox control from the harness
|
||||
PVE API token (appendix B has exact pveum commands), then
|
||||
`staged/proxmox-ctl.sh vms|shutdown|start|snapshot` with a hard PROTECTED list
|
||||
containing this VM and anything untouchable. Natural uses: stand preprod VMs
|
||||
down during compile storms, snapshot before risky agent work, bring up scratch
|
||||
VMs. Scoping question Q12.
|
||||
|
||||
### 3.9 Host-side fleet arbitration (dev VM over sectestbed/preprod)
|
||||
Yes, the audit pulled every VM config (26 VMs: 24 running, 1 stopped, 1
|
||||
template). Two live levers, no reboots:
|
||||
```
|
||||
# priority: PVE maps cpuunits to cgroup v2 CPUWeight (clamped 1-10000)
|
||||
qm set 5111 --cpuunits 9000
|
||||
FLEET="5000 5101 5102 5103 5104 5105 5106 5107 5108 5109 51011 51012 51013 51014 51015 51016 515 53100 53101 53102 53103 53104 53105 53106 53107 53108"
|
||||
for id in $FLEET; do qm set $id --cpuunits 50; done
|
||||
# optional hard caps where the test fleet never needs burst:
|
||||
# qm set <id> --cpulimit 1
|
||||
# (VM 500 k8s-wnode was removed 2026-08-31: ultix-streaming becomes the k8s worker)
|
||||
```
|
||||
At equal demand the dev VM holds ~87% of CPU; the fleet still bursts to full
|
||||
idle capacity when the dev VM is quiet. Post-upgrade, physical partitioning
|
||||
on 24 threads: `qm set 5111 --affinity 0-19` and the fleet `--affinity 20-23`
|
||||
(so test VMs can never preempt dev cores at all). Add proxmox-ctl standing
|
||||
down idle sectestbed VMs entirely (frees host RAM too) and their disks are
|
||||
already separate (local-lvm spinner vs your dedicated SSDs), so no I/O
|
||||
arbitration needed. Optional host cmdline at next reboot: `iommu=pt`.
|
||||
|
||||
## 4. Guest plan (staged in ~/optimize/staged, apply via apply-guest.sh)
|
||||
|
||||
### 4.1 Kernel (60-ukrrs-vm.conf)
|
||||
| knob | now → set | why |
|
||||
|---|---|---|
|
||||
| dirty_background_bytes / dirty_bytes | % ratios → 256M / 1G | bytes-based caps writeback stalls (ETL) regardless of RAM size |
|
||||
| min_free_kbytes | 66M → 384M (768M post-upgrade) | survive bursty reclaim under compile+ETL |
|
||||
| aio-max-nr | 65536 → 1M | postgres AIO / io_uring era defaults |
|
||||
| inotify instances | 128 → 512; watches 386K → 1M | 9 harness daemons + LSPs + crush sessions watching repos |
|
||||
| ip_local_port_range | 32768-60999 → 10240-65535 | 9 accounts × long-lived streams + tailscale + docker NAT |
|
||||
| tcp_tw_reuse | 2 → 1 | outbound provider connection churn |
|
||||
| tcp_slow_start_after_idle | 1 → 0 | LLM turns idle minutes between bursts on live sockets |
|
||||
| rmem_max/wmem_max + tcp_r/wmem | 212K → 16M | large SSE/JSON streams through gateway |
|
||||
| congestion control | cubic → bbr + fq | module load staged; falls back to cubic if absent |
|
||||
| DefaultLimitNOFILE | 1024 soft → 65536 | harness daemons, many sockets/files |
|
||||
|
||||
Left alone deliberately: swappiness 60, overcommit 0, vfs_cache_pressure,
|
||||
pid_max, somaxconn 4096, conntrack 262144, autogroup 1, page-cluster.
|
||||
THP: always → madvise (oneshot unit, staged). Postgres dislikes always-THP;
|
||||
Go/builds get THP via madvise anyway where it matters.
|
||||
|
||||
### 4.2 Mounts and root growth to 500G (MBR shuffle, online except swapoff)
|
||||
Ruling 2026-08-31: storage operations are MANUAL ONLY, executed by the human,
|
||||
one command at a time. Nothing automatic, nothing boot-triggered. The
|
||||
step-by-step checklist with per-step verification is GROW-ROOT-RUNBOOK.md;
|
||||
the commands below are reference for what it does.
|
||||
Current MBR: sda1 root ends 279G, then extended sda2 holding sda5 swap 8.8G,
|
||||
then ~150G unallocated (disk 438G). Target: root 500G → grow disk at PVE first:
|
||||
`qm resize <id> scsi0 +80G` (→ 518G), then in-guest:
|
||||
```
|
||||
sfdisk --dump /dev/sda > /root/sda.sfdisk.bak; cp /etc/fstab /root/fstab.bak
|
||||
swapoff /dev/sda5
|
||||
sfdisk --delete /dev/sda 2 # removes extended + logical swap
|
||||
growpart /dev/sda 1 # cloud-guest-utils; grows root partition
|
||||
resize2fs /dev/sda1 # online ext4 grow → 505G
|
||||
parted /dev/sda mkpart primary linux-swap 505GB 100%; mkswap /dev/sdaX
|
||||
# fix fstab swap UUID (blkid), swapon -a; verify with lsblk + df -h /
|
||||
```
|
||||
Rollback: fstab + sfdisk dumps kept; original layout restorable offline. Also:
|
||||
add `noatime` to /, /data1, /data2 (remount online; relatime today).
|
||||
|
||||
### 4.3 Swap
|
||||
Keep sda5-style swap at disk tail (fresh 10G from the shuffle). Skip zram: old
|
||||
CPU (compression burns cores) and RAM is tripling soon. Optional: 32G low-pri
|
||||
swapfile on /data2 as OOM insurance post-upgrade (staged flag, default off).
|
||||
|
||||
### 4.4 Docker (daemon.json staged)
|
||||
live-restore true (daemon restarts stop killing agent containers; aligns with
|
||||
your "never bounce prod for convenience" rule), json-file logs capped 20m × 3
|
||||
(9 accounts × 42 MCP without caps = quiet-log rule violation waiting to happen),
|
||||
max-concurrent-downloads 6 (parallel pulls across accounts), address pool
|
||||
172.16/12 as /24s (you are at 12 of ~15 default networks; the fleet adds one
|
||||
per compose project), metrics on 127.0.0.1:9323 (experimental:true is required
|
||||
by docker for the engine metrics endpoint; drop both lines if unwanted).
|
||||
Weekly `docker builder prune --keep-storage 25GB` timer staged (9.3G today,
|
||||
unbounded growth otherwise). Applying daemon.json restarts docker once (do it in
|
||||
a quiet moment; live-restore protects future restarts only).
|
||||
|
||||
### 4.5 cgroup v2 architecture (the core deliverable)
|
||||
Static slices staged: gateway (CPUWeight 900, MemoryMin 2G, MemoryHigh 12G),
|
||||
pmo (200, High 1G/Max 1.5G; the dispatch control plane: cheap but weighted
|
||||
above workers so a compile storm can never starve dispatch), lsp (150, High
|
||||
6G/Max 8G), batch (25, High 12G/Max 16G, pool A cpus), rt (pool B cpus,
|
||||
weight 10000). Night profile overrides batch/gateway at runtime (§5.5). Per-account slices via `mkacct.sh <name> <uid>`:
|
||||
CPUWeight 75, TasksMax 4096, Memory fences (interim 48G: High 3G/Max 4.5G ×9;
|
||||
post-upgrade 128G: High 10G/Max 12G ×9), plus a matching user-<uid>.slice
|
||||
drop-in so the harness daemon process itself is fenced too.
|
||||
Containers join their account slice via compose `cgroup_parent:` (snippet
|
||||
printed by mkacct). Builders and ETL join ukrrs-batch via dev.sh/compose.
|
||||
Everything untagged lands in plain docker scope = effectively lowest priority.
|
||||
NOTE: with the `none` I/O scheduler, IOWeight is inert; fairness = separation
|
||||
across the three physical devices + (rare) per-container device rate limits.
|
||||
|
||||
### 4.6 Desktop
|
||||
Fix the greeter burn: sddm autologin into an UNLOCKED session (ruling
|
||||
2026-08-31: no autolock, ever; instant Jump/iPad re-attach outranks lock
|
||||
security on this box). `~/optimize/fix-kde.sh` does it (sudo, idempotent,
|
||||
optional --restart). Result: greeter stops rendering, session idles near 0%,
|
||||
an always-alive unlocked KDE session is attachable instantly. Delete sddm-greeter
|
||||
CPU cost entirely the day you stop wanting console KDE (multi-user.target) but
|
||||
that kills Sunshine's session, so default plan keeps KDE.
|
||||
|
||||
KDE vs XFCE verdict: do not switch for resource reasons. Locked/idle Plasma is
|
||||
~0% CPU and roughly 600-900M RSS vs XFCE ~350-450M; on a box going to 128G
|
||||
that delta is noise. The 22% greeter burn was a greeter-stuck artifact, not
|
||||
Plasma weight, and lightdm+XFCE can do the same trick. The one real argument
|
||||
for XFCE later: it is X11-only, and X11 is still the most battle-tested
|
||||
Sunshine capture path with the NVIDIA proprietary driver; Plasma Wayland
|
||||
capture on NVIDIA is good in 2026 but younger. Sequence: keep KDE now, fix
|
||||
autologin+lock, land the GPU, test Sunshine on Wayland+NVENC, switch to XFCE
|
||||
only if capture disappoints. Revisit when Agent Zero lands (Q15): if it ever
|
||||
runs headed-browser tasks, the desktop story matters more than RAM.
|
||||
|
||||
## 5. Signals and integrations
|
||||
|
||||
### 5.1 What the harness reads today and the minimal patch
|
||||
resources.go reads loadavg, MemAvailable, PSI-io avg60, disk free, with clean
|
||||
seams. Extend (same pattern, ~40 lines + tests, ticket per repo):
|
||||
- generalize readIODelay → readPressure(res) for cpu and memory
|
||||
- optional: read /sys/fs/cgroup/ukrrs-batch.slice/cpu.pressure to gate on "is
|
||||
the batch pool saturated" rather than host-wide load (loadavg counts 24
|
||||
post-upgrade vCPUs; PSI is the honest signal)
|
||||
- suggested defer thresholds: cpu some avg60 > 20, io avg60 > 25, mem some
|
||||
avg60 > 10 (tune with real data; the metric names already exist
|
||||
harness_resource_load_avg_1m / _io_delay_pct)
|
||||
|
||||
### 5.2 Metrics stack (your "super lightweight" ask)
|
||||
Nothing new to run on the host OS: node_exporter as a compose service
|
||||
(read-only binds of / and /var/lib/node_exporter/textfile, house-compliant,
|
||||
digest-pinned at adoption), scraped by the ALREADY RUNNING
|
||||
mopac-harness-prometheus (30d TSDB). PSI textfile script (staged) adds
|
||||
per-slice pressure gauges every 15s. Docker engine metrics land on
|
||||
127.0.0.1:9323 via daemon.json. Beszel (already installed) keeps the human
|
||||
dashboard role. Gateway choice (Q10): poll node_exporter directly in its 30s
|
||||
tick (no new dependency, ~30 lines) vs PromQL against existing prom. Either
|
||||
beats the retired cop :8110 (#585) it still polls.
|
||||
|
||||
### 5.3 k8s (yes, the APIs shed work cleanly)
|
||||
- `kubectl cordon ultix-streaming` stops new scheduling; `kubectl drain
|
||||
--ignore-daemonsets --delete-emptydir-data --grace-period=120` evicts.
|
||||
- Set all k8s workloads to a low PriorityClass + preemption policy, so burst
|
||||
pods die first under node pressure automatically.
|
||||
- kubelet eviction thresholds (memory.available<2Gi, imagefs.available<10Gi)
|
||||
auto-shed when builds eat disk/RAM. Plus --system-reserved/--kube-reserved.
|
||||
- Recommendation: join AFTER the host upgrade, as worker only, never control
|
||||
plane, kubelet+containerd units get CPUWeight 20 + MemoryHigh fences so the
|
||||
compose fleet always wins arbitration. Control plane location Q11.
|
||||
|
||||
### 5.4 Proxmox
|
||||
See §3.8; skeleton staged, token + PROTECTED list via Q12.
|
||||
|
||||
### 5.5 Day/night dynamic profile + PMO dispatch backpressure
|
||||
Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the
|
||||
z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset
|
||||
(gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect:
|
||||
22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO
|
||||
burn maxes, 05:00 everything returns to day bias.
|
||||
|
||||
What flips at night (systemctl set-property --runtime + sysctl, all
|
||||
non-persistent; a reboot lands safely in day mode; values in
|
||||
/etc/ukrrs/daynight.conf, staged):
|
||||
| object | day | night (48G interim / 128G post) |
|
||||
|---|---|---|
|
||||
| batch slice CPUWeight | 25 | 400 |
|
||||
| batch AllowedCPUs | 0-5 (post 0-15) | 0-6 (post 0-17) |
|
||||
| batch MemoryHigh | 12G (post 24G) | 30G (post 80G) |
|
||||
| gateway CPUWeight | 900 | 500 (still far above idle accounts) |
|
||||
| /data2 readahead | 256K | 1M (ETL sequential) |
|
||||
| dirty_bytes / background | 1G / 256M | 2G / 512M |
|
||||
Honest limit: the guest has no CPU governor to "take off"; KVM host owns
|
||||
P-states. Set the host governor to performance once (audit confirms current
|
||||
state); an always-busy 15-year-old Xeon saves nothing from dynamic P-states.
|
||||
Night is also the maintenance window: builder prune Sun 22:30, fstrim,
|
||||
postgres vacuum/reindex, image pre-pull for the morning fleet.
|
||||
|
||||
PMO backpressure: yes, gate at dispatch, it is the earliest and cheapest
|
||||
point. Each account's PMO loop claims from Redmine only when, in order (all
|
||||
zero-dependency reads, same pattern as resources.go):
|
||||
1. gateway /status policy mode is normal (else trickle: ladder already
|
||||
handles 01:00-05:00),
|
||||
2. /proc/pressure/{cpu,memory,io} avg60 under thresholds,
|
||||
3. ukrrs-batch.slice cpu/io pressure under threshold (worker pool headroom),
|
||||
4. wall clock: night = max parallel builders + bigger tickets (compiles,
|
||||
ETL, doc builds); day = fewer, interactive-sized tickets.
|
||||
The 9 PMO loops live in ukrrs-pmo.slice so worker storms cannot starve the
|
||||
dispatcher. This gate composes with the existing harness gate order
|
||||
(resources → quota → peak → soft-defer, harness/README.md:298-312) instead of
|
||||
replacing it.
|
||||
|
||||
Agent Zero: containerized python agent runtime. Placement: interactive
|
||||
agents → their account slice (or gateway-adjacent weights), background
|
||||
agents → ukrrs-batch.slice. Decide via Q15; no OS change either way, the
|
||||
slice architecture absorbs it.
|
||||
|
||||
## 6. Sequencing and rollback
|
||||
- Phase 0 (now, reversible, ~1h, no reboot): apply-guest.sh sysctl+thp+slices+
|
||||
noatime+timers+daynight+desktop; daemon.json in a quiet window (one docker
|
||||
restart).
|
||||
- Phase 1: host audit (one ssh line), order CPUs/RAM (Q2/Q3).
|
||||
- Phase 2 (window, VM off): q35 + resize + disk flags + multiqueue; then root
|
||||
growth to 500G; then GPU + USB3 passthrough; then VM vCPU/RAM at new values.
|
||||
- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in compose,
|
||||
resources.go PSI extension, gateway pressure input, #585 cop removal.
|
||||
- Phase 4: k8s join, proxmox-ctl wiring.
|
||||
Every phase independently revertible: sysctl.d and units are files, qm changes
|
||||
are per-flag, partition work has dumps. Never a bare `down`, per house rules.
|
||||
|
||||
## 7. Open items
|
||||
See questions-v1.md, answer inline; version the file if you want another round.
|
||||
|
||||
## 8. Host audit results (2026-08-31, parsed from host-audit.out)
|
||||
|
||||
### Confirmed
|
||||
- Dell Precision T7500 (baseboard 06FW8P A02), dual socket, **CPU2 EMPTY**.
|
||||
One E5620 4C/8T (HT on) serves 24 RUNNING VMs (~57G/96G RAM allocated);
|
||||
host load was 7.5 on 8 threads while mostly idle. Your VM gets all 8
|
||||
threads only because the rest are quiet. The 2× X5675 upgrade is now
|
||||
urgent, not optional: 3× threads. DMI quirk: "Upgrade: Socket LGA771" is a
|
||||
Dell string error (E5620 exists only in LGA1366); still verify visually
|
||||
before ordering.
|
||||
- RAM: 6× 16GB DDR3-1600 filling ALL six CPU1 slots. Second CPU unlocks six
|
||||
more: order 6× 16GB more → 192G. Runs at CPU max (1066 on E5620, 1333 on
|
||||
X5675).
|
||||
- IOMMU active with CLEAN per-device groups: Quadro alone (20), Renesas
|
||||
USB3 alone (5), NVMe alone (21), NIC ports separate (3/4). **No ACS
|
||||
override needed** (better than section 3.6 assumed).
|
||||
- No CPU frequency driver exists on this host (no /sys cpufreq at all, this
|
||||
kernel + 5520). There is no governor to tune anywhere, host or guest.
|
||||
Day/night dynamics are cgroup-only, exactly as staged in section 5.5.
|
||||
- Host storage: PVE root on 2TB Hitachi spinner via LVM-thin (local-lvm,
|
||||
where the 24 sectestbed VMs live); SMART PASSED on everything; temps fine.
|
||||
|
||||
### Corrected
|
||||
- Your "NVMe" = **Crucial P1 500G QLC (DRAM-less)** as qcow2 on ext4 dir
|
||||
storage (NVME:), dedicated to this VM, 59% used at storage level. /data1 =
|
||||
Samsung 860 PRO 256G, /data2 = SK hynix SC300B 512G, both dedicated, both
|
||||
on ICH10R SATA2 3Gbps links (~280MB/s ceiling each). None carry ssd=1 or
|
||||
discard (that is why the guest sees ROTA=1). Expect P1 sustained-write
|
||||
collapse under heavy builder churn (QLC, no DRAM): monitor; if it bites,
|
||||
the fix is a better NVMe, not config.
|
||||
- Old GPU = **Quadro 4000 (Fermi, 2GB)**. Verdict: skip passthrough. Fermi
|
||||
has no NVENC (starts with Kepler) and no driver that builds on kernel 6.x
|
||||
(390xx is dead): it buys display outputs only. Keep it as host console.
|
||||
Sunshine stays software-encode until the new GPUs arrive; their clean
|
||||
IOMMU groups are already waiting.
|
||||
- Only **ONE** Renesas uPD720201 USB3 card visible (24:00.0). You said two.
|
||||
Check the second: unseated, dead, or behind the (empty) Pericom PCI-X
|
||||
bridge at 01:00.0.
|
||||
- VM 5111 has **no onboot flag**: it will not auto-start after a host
|
||||
reboot. debian13.iso still attached to ide2 (nit).
|
||||
- k8s on this host consolidated (2026-08-31): VM 500 pfv-k8s-wnode-tsys5 was
|
||||
removed; ultix-streaming itself becomes the k8s worker on pfv-tsys5
|
||||
(12G RAM + 2 vCPU of host pressure freed; timing/control plane per Q11).
|
||||
- The 18-VM sectestbed fleet runs 24/7 on the same 8 threads and the 2TB
|
||||
spinner. proxmox-ctl.sh from this VM is the natural on/off switch (Q12).
|
||||
|
||||
### QM checklist for the next touch (any time, hot where noted)
|
||||
```
|
||||
qm set 5111 --onboot 1 --startup order=10,up=180
|
||||
qm set 5111 -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
||||
qm set 5111 -scsi1 ssd2:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=400G
|
||||
qm set 5111 -scsi2 SSD:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=200G
|
||||
qm set 5111 --net0 virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,multiqueue=4
|
||||
qm set 5111 --net1 virtio=BC:24:11:E3:32:D9,bridge=datanet,multiqueue=2
|
||||
qm set 5111 --delete ide2
|
||||
# q35 + 20 vCPU + 128G + sockets/threads reshape: only at the CPU/RAM window (3.4)
|
||||
# root growth to 500G: MANUAL only, GROW-ROOT-RUNBOOK.md (discard=on live first; fstrim reclaims qcow2)
|
||||
# net multiqueue needs a guest-side ethtool -L oneshot after reboot
|
||||
```
|
||||
|
||||
## Appendix A: staged file map
|
||||
```
|
||||
staged/60-ukrrs-vm.conf → /etc/sysctl.d/ (kernel)
|
||||
staged/modules-load.d/tcp_bbr.conf → /etc/modules-load.d/
|
||||
staged/systemd/*.slice|*.service|*.timer → /etc/systemd/system/
|
||||
staged/ukrrs-psi-textfile.sh → /usr/local/sbin/ (with .service/.timer)
|
||||
staged/ukrrs-daynight.sh → /usr/local/sbin/ + day/night profile timers
|
||||
+ /etc/ukrrs/daynight.conf (all staged values)
|
||||
staged/mkacct.sh → run per account: name uid [high] [max] [cpus]
|
||||
staged/apply-guest.sh → installer, RUN=1 to mutate, subcommands
|
||||
staged/docker/daemon.json → /etc/docker/ (restarts docker once)
|
||||
staged/proxmox-ctl.sh → PVE API wrapper, PROTECTED guard
|
||||
```
|
||||
|
||||
## Appendix B: repo patches (ticketed, not applied from here)
|
||||
1. compose builder caps (each repo with a builder/check svc):
|
||||
```
|
||||
x-ukrrs-batch: &ukrrs-batch
|
||||
cgroup_parent: ukrrs-batch.slice
|
||||
cpus: 4
|
||||
mem_limit: 4g
|
||||
services:
|
||||
check: { <<: *ukrrs-batch, ...existing... }
|
||||
```
|
||||
2. dev.sh one-shot builders: `docker run --rm --cgroup-parent ukrrs-batch.slice
|
||||
--cpus 4 --memory 4g ...` (GOMAXPROCS then follows cpuset/cpus).
|
||||
3. resources.go: generalize PSI read to cpu/mem + optional slice pressure
|
||||
(fields + gates + tests, mirrors readIODelay at resources.go:90).
|
||||
4. gateway pressure poller: in the 30s tick (gateway.go:802), GET
|
||||
127.0.0.1:9100/metrics, parse node_pressure_cpu_waiting_seconds... or the
|
||||
textfile gauges; feed the ladder as a new host-pressure input class.
|
||||
5. PVE token:
|
||||
```
|
||||
pveum user add ukrrs-infra@pam
|
||||
pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0
|
||||
pveum acl modify /pool/<preprod-pool> -user ukrrs-infra@pam -role PVEVMUser
|
||||
```
|
||||
6. GPU vBIOS dump (on host, card must be unused):
|
||||
```
|
||||
cd /sys/bus/pci/devices/0000:0X:00.0; echo 1 > remove
|
||||
cat rom > /root/gpu-<model>.rom; echo 0 > remove
|
||||
```
|
||||
7. node_exporter service (into harness deploy compose, digest pin at adoption):
|
||||
```
|
||||
node-exporter:
|
||||
image: prom/node-exporter@sha256:<pin>
|
||||
container_name: ukrrs-mopac-nodeexporter
|
||||
command: [--path.rootfs=/host, --collector.pressure,
|
||||
--collector.textfile.directory=/textfile]
|
||||
pid: host
|
||||
volumes: [/:/host:ro,rslave, /var/lib/node_exporter/textfile:/textfile:ro]
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
```
|
||||
@@ -0,0 +1,30 @@
|
||||
# Today: two commands, one reboot wave
|
||||
|
||||
**1. On the VM** (ultix-streaming, any time before step 2):
|
||||
```
|
||||
sudo ~/optimize/1-guest-prep.sh
|
||||
```
|
||||
Installs everything (kernel tunables, BBR, THP, cgroup slices, PSI collector,
|
||||
day/night timers, docker daemon.json, sddm autologin, no autolock). Docker
|
||||
restarts once. Touches NO storage.
|
||||
|
||||
**2. From your desk** (dry-run first if you like, then execute):
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
|
||||
```
|
||||
Graceful stop of 5111, ssd/discard/multiqueue/onboot flags, CPU priority over
|
||||
the sectestbed fleet, start 5111, then reboots every other running VM.
|
||||
No disk resizing, no host reboot (NFS role).
|
||||
|
||||
**3.** Say "done" here; I verify everything read-only and report.
|
||||
|
||||
**Separate, whenever you choose** (fully manual, step-by-step with checks):
|
||||
root growth 279G -> ~505G via `~/optimize/GROW-ROOT-RUNBOOK.md`. No script
|
||||
runs it; no boot triggers it; you type every command yourself.
|
||||
|
||||
Rollback: apply-guest backs up every file it touches; qm flags are
|
||||
individually reversible.
|
||||
|
||||
Friday still owns: CPU/RAM swap (host power-off -> plan the NFS outage for
|
||||
-02 first), second USB3 card reseat, q35 + 20 vCPU / 128G reshape, GPUs.
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# ultix-streaming optimization — tracking HUD
|
||||
|
||||
One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / blocked ⛔ / next ⏳
|
||||
|
||||
| id | lane | state | item |
|
||||
|---|---|---|---|
|
||||
| OPT-1 | guest | ✅ | recon: system + ukrrs fleet (REPORT §1) |
|
||||
| OPT-2 | guest | ✅ | REPORT.md + staged configs + questions-v1.md |
|
||||
| OPT-3 | host | ✅ | audit parsed → REPORT §8 (T7500, CPU2 empty, clean IOMMU, Q4000 skip) |
|
||||
| OPT-4 | host | ⏳ | Fri 2026-09-04 #601: 2× X5675 + 6× 16GB → 24T/192G (NFS outage for -02 first; USB3 reseat; q35/20c/128G) |
|
||||
| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue now in qm config (live at Friday's start) |
|
||||
| OPT-6 | guest | ✅ | guest prep applied + verified (bbr, THP, slices, timers, noatime, daemon.json, autologin) |
|
||||
| OPT-7 | guest | ⏳ | #603: root growth MANUAL ONLY, GROW-ROOT-RUNBOOK.md, user-executed |
|
||||
| OPT-8 | repos | ✅ | PMO backpressure design note in ukrrs/docs/harness; code = ticketed |
|
||||
| OPT-9 | integ | ⏳ | #604: node_exporter svc + gateway pressure scrape (Q10) |
|
||||
| OPT-10 | integ | ⏳ | #605: k8s join (Q11, blocked by #601) + proxmox-ctl token (Q12); GPUs #606 |
|
||||
|
||||
Inbox (mid-task interrupts): none.
|
||||
|
||||
Decisions log (latest wins):
|
||||
- 2026-08-31 16:5x: Redmine = record: perf pass + RCA filed as #602 (project 55, tracker Support, left New for human review); open items #601 (Fri window, Urgent, due 09-04) #603 root growth #604 metrics #605 k8s+proxmox (blocked by #601) #606 GPUs (relates #601). mred = docker exec wrapper (~/.local/bin/mred → mopac-cli-mred container).
|
||||
- 2026-08-31 16:40: net multiqueue CLOSED: queues=4/2 written to live qm config via crush ssh (user authorized non-disruptive host mods); activates at next VM start; guest oneshot enables channels on that boot. Earlier silent no-op explained: the 15:59 run used a pre-fix script copy.
|
||||
- 2026-08-31 16:3x: crush CAN ssh to the host from this workspace (old "hard-banned" note obsolete); still human-gated for disruptive ops.
|
||||
- 2026-08-31 (post-reboot validation): prod gateway down at boot ROOT-CAUSED twice over: (a) port bind to tailscale IP lost the dockerd-vs-tailscaled race (docker never retries failed starts), (b) live-restore dropped the container's network endpoint (host resolver → gateway-db unresolvable → crash loop). Recovered via compose --force-recreate (healthy, mode=normal). Permanent fix staged: ukrrs-gateway-ensure unit, BOTH lanes, MUST be enabled before Friday's bounces.
|
||||
- 2026-08-31: multiqueue host flags verified live (ROTA=0 + discard 1G on all 3 disks), but guest still runs 1 queue/NIC: ethtool not installed; 3-post-reboot-fixes.sh staged (installs ethtool + oneshot unit, sets 4/2 queues).
|
||||
- 2026-08-31: mopac-demo/mcli fake containers left DOWN (restart=no, exited at first bounce); prometheus/harness daemons not deployed on this box (pre-existing; belongs to OPT-9).
|
||||
- 2026-08-31: STORAGE OPS MANUAL ONLY: boot-time auto-grow service removed, qm resize removed from the one-shot, grow-root.sh deleted. Runbook = GROW-ROOT-RUNBOOK.md, every command typed by the human.
|
||||
- 2026-08-31: KDE autologin into UNLOCKED always-alive session (Relogin=true); autolock OFF (instant Jump/iPad attach outranks lock). XFCE only if Sunshine capture disappoints post-GPU.
|
||||
- 2026-08-31: streamlined to 1-guest-prep.sh + 2-host-one-shot.sh (one reboot wave; no post-reboot steps; no storage).
|
||||
- 2026-08-31: constraint: host pfv-tsys5 NEVER reboots casually (NFS server for -02); Friday CPU swap needs a planned NFS outage first.
|
||||
- 2026-08-31: GPUs are COMPUTE cards; Quadro 4000 skipped (Fermi: no NVENC, no kernel-6 driver, no modern CUDA); clean IOMMU groups verified.
|
||||
- 2026-08-31: day/night = runtime-only cgroup flips 22:00/05:00; NO cpufreq driver exists host or guest.
|
||||
- 2026-08-31: zram rejected; no docker data-root move; skip tuned (sysctl.d + units).
|
||||
- 2026-08-31: second USB3 SDR card missing on host (only one Renesas visible); Friday reseat/check.
|
||||
----
|
||||
@@ -0,0 +1,28 @@
|
||||
# crush.md — project preferences for crush sessions in ~/optimize
|
||||
|
||||
## Locale (ruling 2026-08-31)
|
||||
- Human + this host are in TEXAS: America/Chicago (Central Time; UTC-5 CDT in
|
||||
summer). "1638 CST" in chat means Central local time.
|
||||
- 2026-08-31 is a MONDAY. "Friday" in these docs = 2026-09-04.
|
||||
- Docker/ISO timestamps print UTC (trailing Z) = local +5h; never mislabel
|
||||
weekday or TZ when narrating logs.
|
||||
|
||||
## Host + no-Python rules (ruling 2026-08-31, also in global AGENTS.md)
|
||||
- No Python from agents: text/JSON/CSV = bash/sed/awk/perl/cut/jq. NEVER
|
||||
python one-liners. python in dev containers is fine.
|
||||
- Host stays clean: common shell commands fine, curl + jq + ripgrep etc
|
||||
ALLOWED (old curl ban retired). NO installs on the host (apt-get/pip/etc).
|
||||
- Missing CLI tool? docker pull it, standing permission — run via
|
||||
`docker run --rm <pinned image>`. Work in containers.
|
||||
|
||||
## Output format (user preference, 2026-08-31)
|
||||
- The crush TUI sidebar truncates long chat output. NEVER hand the user a long
|
||||
command in chat. ALWAYS write runnable commands as a numbered script in this
|
||||
directory (next free number, e.g. 5-foo.sh) and tell them only the one-liner
|
||||
to execute it. Scripts that produce diagnostics must tee output to a .out
|
||||
file in this directory so the next turn can read it.
|
||||
- Scripts that need root must SELF-ELEVATE (re-exec via `exec sudo bash "$0"`
|
||||
when EUID != 0) so the user never types sudo. Run-from-guest ssh-root scripts
|
||||
need no elevation; say so in the script header.
|
||||
- Keep chat replies short-line-width; details live in files (NEXT.md, REPORT.md,
|
||||
questions-v*.md, script headers).
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS KDE fix: sddm autologin into an UNLOCKED, always-alive session.
|
||||
# Deliberately NO autolock (instant Jump/iPad re-attach requirement).
|
||||
# Relogin=true: if the session ever exits/crashes, sddm logs straight back in,
|
||||
# so there is always a session to attach to.
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./fix-kde.sh config only; takes effect next boot/restart
|
||||
# sudo ./fix-kde.sh --restart also restart sddm NOW. WARNING: restart kills
|
||||
# any ACTIVE local GUI session. Right now only
|
||||
# the greeter runs, so it is safe; verify with
|
||||
# `loginctl list-sessions` if unsure.
|
||||
# Override user with: sudo DESKTOP_USER=someone ./fix-kde.sh
|
||||
set -euo pipefail
|
||||
[ "$(id -u)" = 0 ] || { echo "run with sudo" >&2; exit 1; }
|
||||
USER_NAME=${DESKTOP_USER:-reachableceo}
|
||||
CONF_DIR=/etc/sddm.conf.d
|
||||
CONF=$CONF_DIR/50-ukrrs-autologin.conf
|
||||
|
||||
mkdir -p "$CONF_DIR"
|
||||
[ -f "$CONF" ] && cp "$CONF" "$CONF.bak.$(date +%s)"
|
||||
|
||||
cat > "$CONF" <<EOF
|
||||
[Autologin]
|
||||
User=$USER_NAME
|
||||
Session=plasma.desktop
|
||||
Relogin=true
|
||||
EOF
|
||||
echo "wrote $CONF (user=$USER_NAME, session=plasma.desktop, unlocked, always-alive)"
|
||||
|
||||
if [ "${1:-}" = "--restart" ]; then
|
||||
systemctl restart sddm
|
||||
echo "sddm restarted: autologin session should be up now (check: loginctl)"
|
||||
else
|
||||
echo "takes effect at next sddm start/reboot; or rerun with --restart"
|
||||
fi
|
||||
+1221
File diff suppressed because it is too large
Load Diff
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS host audit for pfv-tsys5 (Proxmox VE). READ-ONLY, safe to run anytime.
|
||||
# Usage (from a machine that can ssh):
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s' < host-audit.sh > host-audit.out
|
||||
set -uo pipefail
|
||||
hdr() { echo; echo "########## $* ##########"; }
|
||||
|
||||
hdr "node"; hostname; uptime; pveversion 2>/dev/null || echo not-pve
|
||||
|
||||
hdr "dmi system+baseboard"
|
||||
dmidecode -t system 2>/dev/null | grep -E 'Manufacturer|Product Name|Version|Wake-up'
|
||||
dmidecode -t baseboard 2>/dev/null | grep -E 'Manufacturer|Product Name|Version'
|
||||
|
||||
hdr "processors"
|
||||
dmidecode -t processor 2>/dev/null | grep -E 'Socket Designation|Version|Max Speed|Current Speed|^Status|Core Count|Thread Count|Upgrade:'
|
||||
lscpu 2>/dev/null | grep -E 'Model name|^CPU\(s\)|Thread|Core|Socket|NUMA|MHz'
|
||||
|
||||
hdr "cpu frequency policy"
|
||||
for p in scaling_driver scaling_governor scaling_max_freq scaling_min_freq; do
|
||||
v=$(cat /sys/devices/system/cpu/cpu0/cpufreq/$p 2>/dev/null) && echo "$p = $v"
|
||||
done
|
||||
|
||||
hdr "memory"
|
||||
free -h; grep MemTotal /proc/meminfo
|
||||
dmidecode -t memory 2>/dev/null | grep -E '^\s*(Size|Speed|Type:|Locator|Form Factor)' | grep -v 'No Module'
|
||||
|
||||
hdr "storage: block devices"
|
||||
lsblk -o NAME,SIZE,TYPE,ROTA,FSTYPE,MOUNTPOINTS,MODEL 2>/dev/null
|
||||
|
||||
hdr "storage: pve"
|
||||
pvesm status 2>/dev/null
|
||||
|
||||
hdr "storage: zfs"
|
||||
if command -v zpool >/dev/null 2>&1; then
|
||||
zpool status 2>/dev/null
|
||||
zfs list -o name,used,avail,recordsize,compression 2>/dev/null | head -40
|
||||
grep -E '^(size|c_max|c_min) ' /proc/spl/kstat/zfs/arcstats 2>/dev/null
|
||||
else
|
||||
echo no-zfs
|
||||
fi
|
||||
|
||||
hdr "smart health"
|
||||
if command -v smartctl >/dev/null 2>&1; then
|
||||
for d in $(ls /dev | grep -E '^(sd[a-z]|nvme[0-9]+)$'); do
|
||||
echo "--- /dev/$d"
|
||||
smartctl -H /dev/$d 2>/dev/null | grep -iE 'result|overall|health'
|
||||
smartctl -i /dev/$d 2>/dev/null | grep -iE 'model|firmware|serial' | head -3
|
||||
done
|
||||
else
|
||||
echo no-smartctl
|
||||
fi
|
||||
|
||||
hdr "kernel + cmdline"
|
||||
uname -a
|
||||
cat /proc/cmdline
|
||||
[ -f /etc/kernel/cmdline ] && cat /etc/kernel/cmdline
|
||||
proxmox-boot-tool status 2>/dev/null | head -20
|
||||
|
||||
hdr "iommu/dmar"
|
||||
dmesg 2>/dev/null | grep -iE 'iommu|dmar|vtd' | head -25
|
||||
|
||||
hdr "lspci (all)"
|
||||
lspci -nn 2>/dev/null
|
||||
|
||||
hdr "iommu groups"
|
||||
if [ -d /sys/kernel/iommu_groups ]; then
|
||||
for g in /sys/kernel/iommu_groups/*; do
|
||||
[ -d "$g" ] || continue
|
||||
echo "group ${g##*/}:"
|
||||
for d in "$g"/devices/*; do
|
||||
printf ' '; lspci -s "${d##*/}" 2>/dev/null
|
||||
done
|
||||
done
|
||||
else
|
||||
echo "no iommu groups (intel_iommu not enabled?)"
|
||||
fi
|
||||
|
||||
hdr "pci link speeds (gpu/usb/nvme candidates)"
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
s=$(cat "$d/current_link_speed" 2>/dev/null); w=$(cat "$d/current_link_width" 2>/dev/null)
|
||||
[ -n "$s" ] && echo "${d##*/} $s x$w"
|
||||
done
|
||||
|
||||
hdr "network"
|
||||
ip -br a 2>/dev/null || cat /proc/net/dev
|
||||
bridge link 2>/dev/null
|
||||
|
||||
hdr "qemu vms"
|
||||
qm list 2>/dev/null
|
||||
|
||||
hdr "vm configs"
|
||||
for vm in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
echo "--- VM $vm"; qm config "$vm" 2>/dev/null
|
||||
done
|
||||
|
||||
hdr "lxc containers"
|
||||
pct list 2>/dev/null
|
||||
for ct in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
echo "--- CT $ct"; pct config "$ct" 2>/dev/null
|
||||
done
|
||||
|
||||
hdr "sensors"
|
||||
sensors 2>/dev/null || echo no-lm-sensors
|
||||
|
||||
hdr "guest agent check"
|
||||
qm agent 100 ping 2>/dev/null && echo "agent ok on vm 100" || true
|
||||
|
||||
echo
|
||||
echo "AUDIT DONE"
|
||||
@@ -0,0 +1,45 @@
|
||||
== qm config 5111 (relevant lines) ==
|
||||
cores: 4
|
||||
cpuunits: 9000
|
||||
memory: 50000
|
||||
net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
|
||||
net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
|
||||
onboot: 1
|
||||
scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
|
||||
scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
|
||||
scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1
|
||||
scsihw: virtio-scsi-single
|
||||
sockets: 2
|
||||
startup: order=10,up=180
|
||||
|
||||
== qm pending 5111 (unapplied staged changes) ==
|
||||
cur agent: 1
|
||||
cur balloon: 0
|
||||
cur boot: order=scsi0;net0
|
||||
cur cores: 4
|
||||
cur cpu: host,flags=+nested-virt
|
||||
cur cpuunits: 9000
|
||||
cur memory: 50000
|
||||
cur meta: creation-qemu=10.1.2,ctime=1784819584
|
||||
cur name: ultix-streaming
|
||||
cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
|
||||
cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
|
||||
cur numa: 0
|
||||
cur onboot: 1
|
||||
cur ostype: l26
|
||||
cur scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
|
||||
cur scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
|
||||
cur scsi2: SSD:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=200G,ssd=1
|
||||
cur scsihw: virtio-scsi-single
|
||||
cur smbios1: uuid=bc74eb91-b156-46e2-b946-808da9b4f037
|
||||
cur sockets: 2
|
||||
cur startup: order=10,up=180
|
||||
cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f
|
||||
|
||||
== last bounce log (/var/log/ukrrs-vm5111-bounce.log) ==
|
||||
bounce-start 2026-08-31T15:59:56-05:00
|
||||
CPU flag 'nested-virt' resolved to 'vmx'
|
||||
bounce-done 2026-08-31T16:01:41-05:00
|
||||
|
||||
== pve version ==
|
||||
pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve)
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
# questions-v1 — ultix-streaming optimization
|
||||
|
||||
Answer inline under each A:. Where I have a recommendation it is marked REC;
|
||||
"ok" is a sufficient answer. Version the file (v2) for another round.
|
||||
|
||||
## Q1. Host audit (required first, blocks host-side numbers)
|
||||
I cannot ssh from crush (tool policy), so run:
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
|
||||
```
|
||||
then just say "audit done". I read the file and finalize §3 of REPORT.md.
|
||||
A:
|
||||
|
||||
## Q2. CPU upgrade
|
||||
REC: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) → 24 threads.
|
||||
Cheaper alt: second E5620 (~$10) → 16 threads. Audit confirms the chassis is
|
||||
dual-socket before you buy.
|
||||
A:
|
||||
|
||||
## Q3. RAM
|
||||
REC: populate CPU2 DIMM slots to 192G total, give this VM 128G, ballooning off.
|
||||
Which DIMMs to order comes from the audit (current population + speed).
|
||||
A:
|
||||
|
||||
## Q4. Root growth to 500G
|
||||
REC: you run `qm resize <id> scsi0 +80G`, I do the in-guest MBR shuffle
|
||||
(online except a brief swapoff; backups of fstab + partition table first).
|
||||
When?
|
||||
A:
|
||||
|
||||
## Q5. GPUs
|
||||
Models + VRAM of the two incoming cards, and when? PSU wattage on the host
|
||||
(sticker)? Old card: passthrough to this VM now, or leave for preprod?
|
||||
Why it matters: no Above-4G-Decoding on this-era BIOS; big-VRAM cards may not
|
||||
map. UEFI-only vBIOS cards would force the OVMF migration.
|
||||
A:
|
||||
|
||||
## Q6. q35 switch
|
||||
REC: yes, one offline window, NIC rename expected in guest (tailscale survives).
|
||||
Needed for clean GPU/USB3 passthrough.
|
||||
A:
|
||||
|
||||
## Q7. Account → slice names
|
||||
accounts.tsv says: reachableceo, TSGBOD, TSGCOO, TSGCTO, TSGCCO,
|
||||
reachableceo-offstage, COSRCEO-Personal, COSRCEO-Biz, COSWFO.
|
||||
/home currently shows: COS-RCEO, COS-TSG, COS-WFO, ... Which 9 login names are
|
||||
final on THIS host? (I generate one slice + user drop-in per account.)
|
||||
A:
|
||||
|
||||
## Q8. Per-account memory fences
|
||||
REC interim (48G): each account MemoryHigh 3G / Max 4.5G.
|
||||
REC post-upgrade (128G): High 10G / Max 12G. CPU: weight 75, no hard quota
|
||||
(burstable). OK?
|
||||
A:
|
||||
|
||||
## Q9. Desktop
|
||||
REC: keep KDE, fix the greeter (autologin into a locked session), Sunshine +
|
||||
Moonlight after the GPU lands (NVENC). XFCE only if Sunshine capture on KDE
|
||||
Wayland disappoints. OK to apply the autologin+lock config?
|
||||
A:
|
||||
|
||||
## Q10. Metrics/pressure wiring
|
||||
REC: node_exporter as a compose service (house-style, digest-pinned) scraped
|
||||
by your existing mopac-harness prometheus; PSI textfile script for per-slice
|
||||
pressure; gateway polls node_exporter directly in its 30s tick (option A, no
|
||||
new dependency). Keep beszel for the human dashboard. Where does the beszel
|
||||
hub live?
|
||||
A:
|
||||
|
||||
## Q11. k8s
|
||||
Control plane: where/what (k3s? existing cluster?)? REC: join AFTER host
|
||||
upgrade, worker-only, low PriorityClass for burst workloads, kubelet eviction
|
||||
thresholds, kubelet+containerd fenced to CPUWeight 20 + MemoryHigh. OK?
|
||||
A:
|
||||
|
||||
## Q12. Proxmox control scope
|
||||
REC: dedicated API token (ukrrs-infra@pam harness) scoped to a preprod pool
|
||||
only. Which VMIDs may the harness stand down/snapshot, and which are
|
||||
PROTECTED (this VM certainly)? I hard-code the PROTECTED list into
|
||||
proxmox-ctl.sh.
|
||||
A:
|
||||
|
||||
## Q13. SDRs
|
||||
REC: passthrough both PCI USB3 controller cards whole (never per-device),
|
||||
SDR processes pinned to the rt CPU pool. Confirm: both cards → THIS VM,
|
||||
always-on? Any latency budget I should know about?
|
||||
A:
|
||||
|
||||
## Q14. Day/night windows
|
||||
REC: OS night profile 22:00-05:00 local; gateway z.ai peak ladder stays
|
||||
01:00-05:00 CST as configured. Confirm 22:00-05:00 (vs your words "2200 to
|
||||
0500") and the timezone the guest should key off (it currently sees system TZ).
|
||||
A:
|
||||
|
||||
## Q15. Agent Zero
|
||||
Which accounts host it, expected concurrent agents, docker-based? REC:
|
||||
background agents → ukrrs-batch.slice, interactive ones → their account slice.
|
||||
A:
|
||||
|
||||
## Q16. TCP BBR
|
||||
REC: load tcp_bbr module + bbr/fq for the long provider streams. If you
|
||||
prefer stock cubic, say so and I drop two lines.
|
||||
A:
|
||||
|
||||
## Audit update 2026-08-31 (host-audit.out parsed; details in REPORT.md section 8)
|
||||
- Q1 DONE. Q2 upgraded to URGENT: CPU2 socket confirmed empty; one E5620
|
||||
serves 24 running VMs. 2× X5675 triples threads. (DMI claims LGA771, a Dell
|
||||
string quirk; E5620 is LGA1366; verify visually before ordering.)
|
||||
- Q3 refined: 6× 16GB DDR3 already fill all CPU1 slots; order 6 more 16GB.
|
||||
- Q5 verdict: skip Quadro 4000 passthrough (Fermi: no NVENC, no modern CUDA
|
||||
driver). Still need: new GPU models/VRAM + PSU wattage sticker.
|
||||
- Q13 wrinkle: only ONE USB3 card visible on the host. Where is the second?
|
||||
- Q11 wrinkle: VM 500 pfv-k8s-wnode-tsys5 already exists here: join or
|
||||
consolidate?
|
||||
- IOMMU groups are clean (no ACS override needed). No cpufreq driver exists
|
||||
anywhere, so day/night stays pure cgroup (already the design).
|
||||
|
||||
## Q17. VM 5111 auto-start
|
||||
Audit shows onboot unset: after any host reboot your workstation stays DOWN
|
||||
until manually started. OK to set onboot=1 with startup order 10?
|
||||
A:
|
||||
|
||||
## Q18. Remote-desktop matrix (audit correction: xrdp IS running, sesman too)
|
||||
Confirmed live: xrdp + xrdp-sesman = your Jump RDP path (my earlier "no xrdp"
|
||||
was a bad probe, my error). Sunshine installed but not running. Two human
|
||||
accounts, both with always-alive KDE (reachableceo 1001, -offstage 1010).
|
||||
REC: keep xrdp for iPad quick-attach (verify /etc/xrdp/sesman.ini
|
||||
KillDisconnected=false so sessions persist); start Sunshine only after the
|
||||
compute GPUs land and use Moonlight for the video/audio editing sessions
|
||||
(NVENC + proper audio); sddm autologin keeps a warm console session for that.
|
||||
Confirm? And which account hosts the media-editing workload?
|
||||
A:
|
||||
|
||||
## Q11 partial (2026-08-31): consolidation decided
|
||||
VM 500 pfv-k8s-wnode-tsys5 removed; ultix-streaming will be THE k8s worker on
|
||||
this host. Still open: control plane location/type (k3s?), and timing (rec:
|
||||
join after Friday's CPU/RAM resize, worker-only, kubelet+containerd fenced at
|
||||
CPUWeight 20 + MemoryHigh, low PriorityClass + eviction thresholds per REPORT
|
||||
5.3).
|
||||
A (control plane + timing):
|
||||
@@ -0,0 +1,34 @@
|
||||
# /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile
|
||||
# Measured-before values and rationale: ~/optimize/REPORT.md §4.1.
|
||||
# Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure,
|
||||
# pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3).
|
||||
|
||||
# Predictable writeback under ETL bursts (bytes-based, RAM-size independent).
|
||||
# Night profile (ukrrs-daynight.sh) raises these to 2G/512M.
|
||||
vm.dirty_background_bytes = 268435456
|
||||
vm.dirty_bytes = 1073741824
|
||||
|
||||
# Survive bursty reclaim when compiles+ETL hit at once. 768M post-upgrade.
|
||||
vm.min_free_kbytes = 393216
|
||||
|
||||
# Postgres AIO / io_uring era; 9 harness daemons + LSPs watching many repos.
|
||||
fs.aio-max-nr = 1048576
|
||||
fs.inotify.max_user_watches = 1048576
|
||||
fs.inotify.max_user_instances = 512
|
||||
|
||||
# 9 accounts x long-lived provider streams + tailscale + docker NAT.
|
||||
net.ipv4.ip_local_port_range = 10240 65535
|
||||
net.ipv4.tcp_tw_reuse = 1
|
||||
|
||||
# LLM turns idle minutes between bursts on live sockets; large SSE/JSON.
|
||||
net.ipv4.tcp_slow_start_after_idle = 0
|
||||
net.core.rmem_max = 16777216
|
||||
net.core.wmem_max = 16777216
|
||||
net.ipv4.tcp_rmem = 4096 131072 16777216
|
||||
net.ipv4.tcp_wmem = 4096 65536 16777216
|
||||
net.core.netdev_max_backlog = 8192
|
||||
|
||||
# BBR requires the module: staged modules-load.d/tcp_bbr.conf loads it.
|
||||
# If bbr is unavailable, comment the last two lines out (cubic is fine).
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
net.core.default_qdisc = fq
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first.
|
||||
# sudo RUN=1 ./apply-guest.sh <step>
|
||||
# RUN=1 required to mutate anything (default: plan only).
|
||||
# daemonjson and desktop additionally require CONFIRM=1 (one-time disruption).
|
||||
# Steps: sysctl modules thp slices psi prune nofile noatime daemonjson desktop
|
||||
# daynight all
|
||||
set -euo pipefail
|
||||
RUN=${RUN:-0}
|
||||
CONFIRM=${CONFIRM:-0}
|
||||
DESKTOP_USER=${DESKTOP_USER:-reachableceo}
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
say() { echo "[apply-guest] $*"; }
|
||||
die() { echo "[apply-guest] $*" >&2; exit 2; }
|
||||
need_root() { [ "$(id -u)" = 0 ] || die "needs root"; }
|
||||
gate() {
|
||||
if [ "$RUN" != 1 ]; then say "DRY: would $* (rerun with RUN=1)"; return 1; fi
|
||||
need_root; return 0
|
||||
}
|
||||
inst() { # src dst mode
|
||||
install -m "$3" "$1" "$2" && say "installed $2"
|
||||
}
|
||||
|
||||
step_sysctl() {
|
||||
gate "install sysctl.d profile" || return 0
|
||||
inst "$here/60-ukrrs-vm.conf" /etc/sysctl.d/60-ukrrs-vm.conf 0644
|
||||
sysctl --system >/dev/null && say "sysctl applied (bbr lines need the module: step modules)"
|
||||
}
|
||||
step_modules() {
|
||||
gate "install tcp_bbr module load" || return 0
|
||||
inst "$here/modules-load.d/tcp_bbr.conf" /etc/modules-load.d/tcp_bbr.conf 0644
|
||||
modprobe tcp_bbr 2>/dev/null || say "tcp_bbr not loaded now (will load at boot)"
|
||||
}
|
||||
step_thp() {
|
||||
gate "install THP madvise unit" || return 0
|
||||
inst "$here/systemd/ukrrs-thp-madvise.service" /etc/systemd/system/ukrrs-thp-madvise.service 0644
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-thp-madvise.service
|
||||
say "THP=$(cat /sys/kernel/mm/transparent_hugepage/enabled)"
|
||||
}
|
||||
step_slices() {
|
||||
gate "install ukrrs slices" || return 0
|
||||
for s in gateway pmo lsp batch rt; do
|
||||
inst "$here/systemd/ukrrs-$s.slice" "/etc/systemd/system/ukrrs-$s.slice" 0644
|
||||
done
|
||||
systemctl daemon-reload
|
||||
say "slices ready; containers opt in via cgroup_parent=ukrrs-<x>.slice"
|
||||
}
|
||||
step_psi() {
|
||||
gate "install PSI textfile collector" || return 0
|
||||
inst "$here/ukrrs-psi-textfile.sh" /usr/local/sbin/ukrrs-psi-textfile.sh 0755
|
||||
mkdir -p /var/lib/node_exporter/textfile && chmod 755 /var/lib/node_exporter /var/lib/node_exporter/textfile
|
||||
inst "$here/systemd/ukrrs-psi-textfile.service" /etc/systemd/system/ukrrs-psi-textfile.service 0644
|
||||
inst "$here/systemd/ukrrs-psi-textfile.timer" /etc/systemd/system/ukrrs-psi-textfile.timer 0644
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-psi-textfile.timer
|
||||
say "pressure gauges: /var/lib/node_exporter/textfile/ukrrs_pressure.prom"
|
||||
}
|
||||
step_prune() {
|
||||
gate "install builder-prune timer" || return 0
|
||||
inst "$here/systemd/ukrrs-builder-prune.service" /etc/systemd/system/ukrrs-builder-prune.service 0644
|
||||
inst "$here/systemd/ukrrs-builder-prune.timer" /etc/systemd/system/ukrrs-builder-prune.timer 0644
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-builder-prune.timer
|
||||
}
|
||||
step_nofile() {
|
||||
gate "raise DefaultLimitNOFILE" || return 0
|
||||
mkdir -p /etc/systemd/system.conf.d
|
||||
cat > /etc/systemd/system.conf.d/50-ukrrs.conf <<'EOF'
|
||||
[Manager]
|
||||
DefaultLimitNOFILE=65536:1048576
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
say "DefaultLimitNOFILE raised (new sessions)"
|
||||
}
|
||||
step_noatime() {
|
||||
gate "add noatime to /, /data1, /data2" || return 0
|
||||
cp /etc/fstab "/etc/fstab.bak.ukrrs.$(date +%s)"
|
||||
for mnt in / /data1 /data2; do
|
||||
opts=$(findmnt -n -o OPTIONS "$mnt") || continue
|
||||
case ",$opts," in *,noatime,*) say "$mnt already noatime"; continue ;; esac
|
||||
mount -o remount,noatime "$mnt"
|
||||
awk -v m="$mnt" 'BEGIN{FS=OFS=" "} $2==m && $3=="ext4" { if ($4 !~ /(^|,)noatime(,|$)/) $4=$4",noatime" } 1' \
|
||||
/etc/fstab > /etc/fstab.ukrrs.new && mv /etc/fstab.ukrrs.new /etc/fstab
|
||||
say "$mnt: remounted noatime + fstab updated"
|
||||
done
|
||||
}
|
||||
step_daemonjson() {
|
||||
[ "$CONFIRM" = 1 ] || die "daemonjson needs CONFIRM=1: restarts docker ONCE (live-restore protects future restarts)"
|
||||
gate "install daemon.json" || return 0
|
||||
[ -f /etc/docker/daemon.json ] && cp /etc/docker/daemon.json "/etc/docker/daemon.json.bak.ukrrs.$(date +%s)"
|
||||
inst "$here/docker/daemon.json" /etc/docker/daemon.json 0644
|
||||
systemctl restart docker
|
||||
say "docker restarted with new config; check: docker info | grep -E 'Live|Logging'"
|
||||
}
|
||||
step_desktop() {
|
||||
[ "$CONFIRM" = 1 ] || die "desktop needs CONFIRM=1: enables sddm autologin for $DESKTOP_USER"
|
||||
gate "configure sddm autologin+lock" || return 0
|
||||
mkdir -p /etc/sddm.conf.d
|
||||
cat > /etc/sddm.conf.d/50-ukrrs-autologin.conf <<EOF
|
||||
[Autologin]
|
||||
User=$DESKTOP_USER
|
||||
Session=plasma.desktop
|
||||
Relogin=true
|
||||
EOF
|
||||
say "sddm autologin configured (UNLOCKED session; autolock deliberately off"
|
||||
say "per 2026-08-31 ruling: instant Jump/iPad re-attach outranks lock)"
|
||||
}
|
||||
step_daynight() {
|
||||
gate "install day/night profile timers" || return 0
|
||||
inst "$here/ukrrs-daynight.sh" /usr/local/sbin/ukrrs-daynight.sh 0755
|
||||
mkdir -p /etc/ukrrs
|
||||
inst "$here/etc-ukrrs-daynight.conf" /etc/ukrrs/daynight.conf 0644
|
||||
for u in nightprofile.service nightprofile.timer dayprofile.service dayprofile.timer; do
|
||||
inst "$here/systemd/ukrrs-$u" "/etc/systemd/system/ukrrs-$u" 0644
|
||||
done
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer
|
||||
say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night"
|
||||
}
|
||||
step_all() {
|
||||
step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune
|
||||
step_nofile; step_noatime; step_daynight
|
||||
if [ "$CONFIRM" = 1 ]; then step_daemonjson; step_desktop
|
||||
else say "skipped daemonjson + desktop (need CONFIRM=1)"; fi
|
||||
}
|
||||
|
||||
step=${1:-}
|
||||
case "$step" in
|
||||
sysctl|modules|thp|slices|psi|prune|nofile|noatime|daemonjson|desktop|daynight|all)
|
||||
"step_$step" ;;
|
||||
*) die "unknown step: $step" ;;
|
||||
esac
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"live-restore": true,
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "20m",
|
||||
"max-file": "3"
|
||||
},
|
||||
"max-concurrent-downloads": 6,
|
||||
"max-concurrent-uploads": 4,
|
||||
"metrics-addr": "127.0.0.1:9323",
|
||||
"experimental": true,
|
||||
"default-address-pools": [
|
||||
{
|
||||
"base": "172.16.0.0/12",
|
||||
"size": 24
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# /etc/ukrrs/daynight.conf — overrides for ukrrs-daynight.sh
|
||||
# Interim values (8 vCPU / 48G) are the script defaults; only set what differs.
|
||||
|
||||
DATA2_DEV=/dev/sdb
|
||||
|
||||
# Post-upgrade values (20 vCPU / 128G), uncomment after the VM resize:
|
||||
# BATCH_CPUS_DAY=0-15
|
||||
# BATCH_CPUS_NIGHT=0-17
|
||||
# BATCH_HIGH_DAY=24G
|
||||
# BATCH_HIGH_NIGHT=80G
|
||||
# Also edit ukrrs-rt.slice AllowedCPUs to 18-19 and account slices (mkacct.sh).
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS: create one account lane = cgroup slice + matching user-slice drop-in.
|
||||
# usage: mkacct.sh <account> <uid> [mem_high] [mem_max] [allowed_cpus] [tier]
|
||||
#
|
||||
# tier=agent (default): CPUWeight=75, pinned to a CPU pool — the 9 PMO/worker
|
||||
# lanes. interim rec: 3G 4.5G 0-5 post-upgrade: 10G 12G 0-15
|
||||
# tier=human: CPUWeight=600, NO cpu restriction, generous memory — the two
|
||||
# human interactive accounts (KDE/CAD/EDA/video via xrdp live in these).
|
||||
# interim rec: 12G 16G all post-upgrade: 32G 40G all
|
||||
#
|
||||
# Examples (uids verified 2026-08-31):
|
||||
# sudo RUN=1 ./mkacct.sh reachableceo 1001 12G 16G all human
|
||||
# sudo RUN=1 ./mkacct.sh reachableceo-offstage 1010 12G 16G all human
|
||||
# sudo RUN=1 ./mkacct.sh TSGBOD <uid> 3G 4.5G 0-5 agent
|
||||
#
|
||||
# RUN=1 installs (needs root); default prints the plan + snippets only.
|
||||
set -euo pipefail
|
||||
acct=${1:?account}; uid=${2:?uid}
|
||||
high=${3:-3G}; max=${4:-4.5G}; cpus=${5:-0-5}; tier=${6:-agent}
|
||||
RUN=${RUN:-0}
|
||||
|
||||
case "$tier" in
|
||||
human) weight=600; cpuline="" ;;
|
||||
agent) weight=75; cpuline="AllowedCPUs=$cpus" ;;
|
||||
*) echo "tier must be human or agent" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
slice="/etc/systemd/system/ukrrs-acct-$acct.slice"
|
||||
userdrop="/etc/systemd/system/user-$uid.slice.d/50-ukrrs.conf"
|
||||
|
||||
cat <<EOF
|
||||
plan ($tier tier):
|
||||
$slice
|
||||
CPUWeight=$weight ${cpuline:+$cpuline }MemoryHigh=$high MemoryMax=$max TasksMax=4096
|
||||
$userdrop
|
||||
CPUWeight=$weight MemoryHigh=$high
|
||||
|
||||
compose snippet (this account's agent projects):
|
||||
x-ukrrs-acct: &ukrrs_acct
|
||||
cgroup_parent: ukrrs-acct-$acct.slice
|
||||
services:
|
||||
anything: { <<: *ukrrs_acct }
|
||||
|
||||
dev.sh one-shot builders belong in the batch pool, not the account slice:
|
||||
docker run --rm --cgroup-parent ukrrs-batch.slice --cpus 4 --memory 4g ...
|
||||
EOF
|
||||
|
||||
if [ "$RUN" = 1 ]; then
|
||||
[ "$(id -u)" = 0 ] || { echo "RUN=1 needs root" >&2; exit 1; }
|
||||
cat > "$slice" <<EOF
|
||||
[Unit]
|
||||
Description=UKRRS account lane ($tier): $acct
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
CPUWeight=$weight
|
||||
${cpuline}
|
||||
MemoryHigh=$high
|
||||
MemoryMax=$max
|
||||
TasksMax=4096
|
||||
EOF
|
||||
mkdir -p "$(dirname "$userdrop")"
|
||||
cat > "$userdrop" <<EOF
|
||||
[Slice]
|
||||
CPUWeight=$weight
|
||||
MemoryHigh=$high
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
# push live: drop-ins alone don't re-apply to already-existing user slices
|
||||
systemctl set-property "user-$uid.slice" CPUWeight="$weight" MemoryHigh="$high"
|
||||
echo "installed + daemon-reload + live-applied ok: $acct ($tier)"
|
||||
fi
|
||||
@@ -0,0 +1,2 @@
|
||||
# Load tcp_bbr at boot so sysctl.d/60-ukrrs-vm.conf can set bbr.
|
||||
tcp_bbr
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS Proxmox control wrapper for harness/human use. HARD RULES baked in:
|
||||
# - vmids listed in PROTECTED are never touched (list THIS VM first).
|
||||
# - graceful shutdown only; no reset, no hard stop, no bare down (house rule).
|
||||
# Env:
|
||||
# PVE_LOCAL=1 run ON the pve host itself (pvesh direct)
|
||||
# or PVE_HOST + PVE_TOKEN remote, token format: user@realm!tokenid=secret
|
||||
# PVE_NODE node name (required for vm ops)
|
||||
# PROTECTED required, comma-separated vmids
|
||||
# Usage:
|
||||
# proxmox-ctl.sh vms
|
||||
# proxmox-ctl.sh shutdown <vmid>
|
||||
# proxmox-ctl.sh start <vmid>
|
||||
# proxmox-ctl.sh snapshot <vmid> <name>
|
||||
# Token setup (on the pve host, see REPORT.md appendix B):
|
||||
# pveum user add ukrrs-infra@pam
|
||||
# pveum user token add ukrrs-infra@pam harness -privsep 0 -expire 0
|
||||
# pveum acl modify /pool/<preprod-pool> -user ukrrs-infra@pam -role PVEVMUser
|
||||
set -euo pipefail
|
||||
: "${PROTECTED:?set PROTECTED=vmid1,vmid2,... (this VM must be in the list)}"
|
||||
|
||||
pvesh_() {
|
||||
if [ "${PVE_LOCAL:-0}" = 1 ]; then
|
||||
command pvesh "$@"
|
||||
else
|
||||
: "${PVE_HOST:?}" "${PVE_TOKEN:?}"
|
||||
command pvesh --host "$PVE_HOST" --api-token "$PVE_TOKEN" "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
guard() {
|
||||
case ",$PROTECTED," in
|
||||
*",$1,"*) echo "REFUSED: vmid $1 is PROTECTED" >&2; exit 3 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd=${1:-}; shift || true
|
||||
case "$cmd" in
|
||||
vms)
|
||||
pvesh_ get /cluster/resources --type vm ;;
|
||||
shutdown)
|
||||
[ $# = 1 ] || { echo "usage: $0 shutdown <vmid>" >&2; exit 2; }
|
||||
guard "$1"; : "${PVE_NODE:?}"
|
||||
pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/shutdown" --timeout 120 ;;
|
||||
start)
|
||||
[ $# = 1 ] || { echo "usage: $0 start <vmid>" >&2; exit 2; }
|
||||
guard "$1"; : "${PVE_NODE:?}"
|
||||
pvesh_ create "/nodes/$PVE_NODE/qemu/$1/status/start" ;;
|
||||
snapshot)
|
||||
[ $# = 2 ] || { echo "usage: $0 snapshot <vmid> <name>" >&2; exit 2; }
|
||||
guard "$1"; : "${PVE_NODE:?}"
|
||||
pvesh_ create "/nodes/$PVE_NODE/qemu/$1/snapshot" snapname="$2" ;;
|
||||
*)
|
||||
echo "usage: $0 vms|shutdown <vmid>|start <vmid>|snapshot <vmid> <name>" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Day bias: lowest weight, restricted to pool A. The night profile timer
|
||||
# (ukrrs-nightprofile.service) raises weight/MemoryHigh and widens AllowedCPUs
|
||||
# via `systemctl set-property --runtime`; reboot lands back in day mode.
|
||||
# IOWeight deliberately absent: inert under the `none` I/O scheduler.
|
||||
CPUWeight=25
|
||||
AllowedCPUs=0-5
|
||||
MemoryHigh=12884901888
|
||||
MemoryMax=17179869184
|
||||
TasksMax=16384
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=UKRRS: weekly docker build-cache prune (25G ceiling)
|
||||
After=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/docker builder prune --force --keep-storage 25GB
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=UKRRS: build-cache prune, Sunday night window
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 22:30:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS: switch back to day profile (interactive bias)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/ukrrs-daynight.sh day
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=UKRRS: day profile at 05:00
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 05:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=ukrrs gateway ensure-up (tailscale port race + live-restore endpoint loss)
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/ukrrs-gateway-ensure.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta)
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Latency-sensitive: all provider traffic + spend accounting. Weight beats any
|
||||
# single account even under total compile storm (see REPORT section 2 math).
|
||||
CPUWeight=900
|
||||
# Hard floor so accounts can never squeeze the accounting DB into reclaim.
|
||||
MemoryMin=2147483648
|
||||
MemoryHigh=12884901888
|
||||
TasksMax=infinity
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads)
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
CPUWeight=150
|
||||
MemoryHigh=6442450944
|
||||
MemoryMax=8589934592
|
||||
TasksMax=8192
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Enable virtio NIC multiqueue (host offers 4 queues on ens18, 2 on ens19)
|
||||
After=network-pre.target
|
||||
Before=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/sh -c '/usr/sbin/ethtool -L ens18 combined 4 || true; /usr/sbin/ethtool -L ens19 combined 2 || true; true'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/ukrrs-daynight.sh night
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=UKRRS: night profile at 22:00
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 22:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=UKRRS PMO dispatch loops (one per account): control plane
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Dispatcher must never starve behind worker storms: cheap but weighted above
|
||||
# batch and idle accounts. Night profile leaves this untouched.
|
||||
CPUWeight=200
|
||||
MemoryHigh=1073741824
|
||||
MemoryMax=1610612736
|
||||
TasksMax=512
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=UKRRS: PSI textfile collector tick
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/ukrrs-psi-textfile.sh
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=UKRRS: PSI textfile collector (15s)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=15
|
||||
AccuracySec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores,
|
||||
# so anything placed here keeps headroom even under full compile storm.
|
||||
# Interim 8 vCPU: 6-7. Post-upgrade 20 vCPU: 18-19 (edit after resize).
|
||||
CPUWeight=10000
|
||||
AllowedCPUs=6-7
|
||||
MemoryHigh=2147483648
|
||||
MemoryMax=4294967296
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=UKRRS: transparent hugepages to madvise (postgres-friendly)
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/sh -c 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS day/night resource profile (REPORT.md section 5.5).
|
||||
# Runtime-only changes (set-property --runtime + sysctl): a reboot always
|
||||
# lands safely in day mode (slice unit files carry the day defaults).
|
||||
set -euo pipefail
|
||||
mode=${1:?usage: ukrrs-daynight.sh day|night}
|
||||
CONF=/etc/ukrrs/daynight.conf
|
||||
[ -r "$CONF" ] && . "$CONF"
|
||||
|
||||
: "${DATA2_DEV:=/dev/sdb}"
|
||||
: "${BATCH_WEIGHT_DAY:=25}"; : "${BATCH_WEIGHT_NIGHT:=400}"
|
||||
: "${BATCH_CPUS_DAY:=0-5}"; : "${BATCH_CPUS_NIGHT:=0-6}"
|
||||
: "${BATCH_HIGH_DAY:=12G}"; : "${BATCH_HIGH_NIGHT:=30G}"
|
||||
: "${GW_WEIGHT_DAY:=900}"; : "${GW_WEIGHT_NIGHT:=500}"
|
||||
: "${DATA2_RA_DAY:=256}"; : "${DATA2_RA_NIGHT:=2048}"
|
||||
: "${DIRTY_DAY:=1073741824}"; : "${DIRTY_NIGHT:=2147483648}"
|
||||
: "${DIRTY_BG_DAY:=268435456}"; : "${DIRTY_BG_NIGHT:=536870912}"
|
||||
|
||||
log() { echo "[ukrrs-daynight] $*"; }
|
||||
setprop() { systemctl set-property --runtime "$@"; }
|
||||
setra() { blockdev --setra "$1" "$2" 2>/dev/null || log "readahead skip: $2"; }
|
||||
|
||||
case "$mode" in
|
||||
night)
|
||||
setprop ukrrs-batch.slice \
|
||||
CPUWeight="$BATCH_WEIGHT_NIGHT" \
|
||||
AllowedCPUs="$BATCH_CPUS_NIGHT" \
|
||||
MemoryHigh="$BATCH_HIGH_NIGHT"
|
||||
setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_NIGHT"
|
||||
setra "$DATA2_RA_NIGHT" "$DATA2_DEV"; setra "$DATA2_RA_NIGHT" "${DATA2_DEV}1"
|
||||
sysctl -q -w vm.dirty_bytes="$DIRTY_NIGHT" vm.dirty_background_bytes="$DIRTY_BG_NIGHT"
|
||||
;;
|
||||
day)
|
||||
setprop ukrrs-batch.slice \
|
||||
CPUWeight="$BATCH_WEIGHT_DAY" \
|
||||
AllowedCPUs="$BATCH_CPUS_DAY" \
|
||||
MemoryHigh="$BATCH_HIGH_DAY"
|
||||
setprop ukrrs-gateway.slice CPUWeight="$GW_WEIGHT_DAY"
|
||||
setra "$DATA2_RA_DAY" "$DATA2_DEV"; setra "$DATA2_RA_DAY" "${DATA2_DEV}1"
|
||||
sysctl -q -w vm.dirty_bytes="$DIRTY_DAY" vm.dirty_background_bytes="$DIRTY_BG_DAY"
|
||||
;;
|
||||
*)
|
||||
echo "unknown mode: $mode" >&2; exit 2 ;;
|
||||
esac
|
||||
log "profile $mode applied $(date -Is)"
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Boots are racy for the gateway containers (both root-caused 2026-08-31):
|
||||
# - prod binds :4000/:9090 to the tailscale IP; if dockerd wins the race over
|
||||
# tailscaled, the bind fails and docker NEVER retries failed starts;
|
||||
# - live-restore can drop a container's network endpoint across a boot,
|
||||
# leaving it started with the host resolver (gateway-db unresolvable,
|
||||
# crash loop). A plain `docker start` does NOT reattach the endpoint.
|
||||
# Waits for the tailscale IP, then recreates any lane not actually serving.
|
||||
# Idempotent; recreating is safe at boot time (nothing in flight).
|
||||
set -u
|
||||
GW_DIR=/home/reachableceo/projects/ukrrs/gateway
|
||||
TS_IP=100.101.187.119
|
||||
|
||||
for _ in $(seq 1 60); do
|
||||
/usr/sbin/ip -o addr | grep -q "$TS_IP/" && break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
ensure_lane() { # name host:port project compose_file
|
||||
local name=$1 hostport=$2 project=$3 file=$4
|
||||
if timeout 5 bash -c "</dev/tcp/$hostport" 2>/dev/null; then
|
||||
echo "$name: serving on $hostport"
|
||||
return 0
|
||||
fi
|
||||
echo "$name: NOT serving on $hostport; recreating"
|
||||
docker compose -p "$project" -f "$GW_DIR/$file" up -d --no-deps --force-recreate gateway
|
||||
}
|
||||
|
||||
ensure_lane prod "$TS_IP:4000" mopac-gateway docker-compose.yml
|
||||
ensure_lane beta "127.0.0.1:4002" mopac-gateway-beta compose.beta.yaml
|
||||
exit 0
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Per-slice + host PSI (pressure stall info) avg60 -> node_exporter textfile.
|
||||
# Consumed by the existing mopac-harness prometheus; gateway may also read it.
|
||||
set -u
|
||||
OUT_DIR=${UKRRS_TEXTFILE_DIR:-/var/lib/node_exporter/textfile}
|
||||
CGROOT=${UKRRS_CGROOT:-/sys/fs/cgroup}
|
||||
mkdir -p "$OUT_DIR"
|
||||
tmp=$(mktemp "$OUT_DIR/.ukrrs.XXXXXX")
|
||||
|
||||
some60() { # file -> prints avg60 value or nothing
|
||||
awk '$1=="some"{for(i=2;i<=NF;i++) if($i ~ /^avg60=/){sub("avg60=","",$i); print $i; exit}}' "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
emit() { # name cgroup_path
|
||||
local res v
|
||||
for res in cpu memory io; do
|
||||
[ -r "$2/$res.pressure" ] || continue
|
||||
v=$(some60 "$2/$res.pressure")
|
||||
[ -n "${v:-}" ] && echo "ukrrs_slice_pressure_some60{slice=\"$1\",res=\"$res\"} $v"
|
||||
done
|
||||
}
|
||||
|
||||
echo "# HELP ukrrs_slice_pressure_some60 PSI some avg60 (percent) for ukrrs slices" >>"$tmp"
|
||||
echo "# TYPE ukrrs_slice_pressure_some60 gauge" >>"$tmp"
|
||||
for cg in "$CGROOT"/system.slice/ukrrs-*.slice; do
|
||||
[ -d "$cg" ] && emit "${cg##*/}" "$cg"
|
||||
done
|
||||
[ -d "$CGROOT/user.slice" ] && emit "user.slice" "$CGROOT/user.slice"
|
||||
|
||||
echo "# HELP ukrrs_host_pressure_some60 PSI some avg60 (percent) host-wide" >>"$tmp"
|
||||
echo "# TYPE ukrrs_host_pressure_some60 gauge" >>"$tmp"
|
||||
for res in cpu memory io; do
|
||||
[ -r "/proc/pressure/$res" ] || continue
|
||||
v=$(some60 "/proc/pressure/$res")
|
||||
[ -n "${v:-}" ] && echo "ukrrs_host_pressure_some60{res=\"$res\"} $v"
|
||||
done
|
||||
|
||||
mv "$tmp" "$OUT_DIR/ukrrs_pressure.prom"
|
||||
chmod 0644 "$OUT_DIR/ukrrs_pressure.prom"
|
||||
Reference in New Issue
Block a user