docs sync post-pass: REPORT v1.1 status, Oct replan runbook, repo path fix
REPORT.md gets a post-pass status banner + per-item STATUS tags + phase/QM checklist truth; RUNBOOK-TODAY amended for the Oct window replan (Friday is troubleshooting-only #609); all ~/optimize paths updated after the repo move to ~/projects/ultix. Adds the 22:00 night-flip watcher log and 6-remove-agent-stacks.sh (boot-time screen/crush relauncher teardown with backups to removed-agent-stacks/). 💘 Generated with Crush Assisted-by: Crush:glm-5.2 [#602]
This commit is contained in:
+3
-3
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE-SHOT guest prep (idempotent; safe to re-run). Run ON ultix-streaming:
|
||||
# sudo ~/optimize/1-guest-prep.sh
|
||||
# sudo ~/projects/ultix/1-guest-prep.sh
|
||||
# 1. All staged configs (docker restarts once) + sddm autologin.
|
||||
# 2. Human-tier slices for the two human accounts (KDE/CAD/video priority).
|
||||
# NO storage operations (root growth is manual: GROW-ROOT-RUNBOOK.md).
|
||||
@@ -18,6 +18,6 @@ RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo-offstage 1010 12G 16G all human
|
||||
echo "== [3/3] done =="
|
||||
echo "GUEST READY. Next, the host one-shot (safe to run from THIS VM; the"
|
||||
echo "VM bounce is detached on the host and survives your ssh dying):"
|
||||
echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/optimize/2-host-one-shot.sh"
|
||||
echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/projects/ultix/2-host-one-shot.sh"
|
||||
echo
|
||||
echo "Root growth to ~505G stays MANUAL: ~/optimize/GROW-ROOT-RUNBOOK.md"
|
||||
echo "Root growth to ~505G stays MANUAL: ~/projects/ultix/GROW-ROOT-RUNBOOK.md"
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111.
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/projects/ultix/2-host-one-shot.sh
|
||||
# ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ... # optional
|
||||
#
|
||||
# 1. VM 5111's disk/net/onboot flags are written as PENDING config: they
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Post-reboot validation fallout (2026-08-31). Run with sudo, once:
|
||||
# sudo ~/optimize/3-post-reboot-fixes.sh
|
||||
# sudo ~/projects/ultix/3-post-reboot-fixes.sh
|
||||
# 1. Installs ethtool + enables virtio multiqueue (host side now offers
|
||||
# 4 queues on ens18, 2 on ens19; the guest defaults to 1 until told).
|
||||
# 2. Installs the gateway ensure-up unit (prod boot race on the tailscale-IP
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Read-only host check: why did the net multiqueue flags not land on VM 5111?
|
||||
# Run from the guest: ~/optimize/4-host-netcheck.sh
|
||||
# Everything is saved to ~/optimize/host-netcheck.out for crush to read.
|
||||
# Run from the guest: ~/projects/ultix/4-host-netcheck.sh
|
||||
# Everything is saved to ~/projects/ultix/host-netcheck.out for crush to read.
|
||||
set -euo pipefail
|
||||
OUT="$(cd "$(dirname "$0")" && pwd)/host-netcheck.out"
|
||||
timeout 120 ssh root@pfv-tsys5.knel.net '
|
||||
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# 6-remove-agent-stacks.sh — remove ALL boot-time screen/crush stack relaunchers.
|
||||
# Target state: no screen/crush sessions auto-start on reboot, any account.
|
||||
# - reachableceo-agent-stack.service (boot respawn RCEO-PMO/RCEO-Work + prompt injection)
|
||||
# - tsg-supervisor.timer/.service (5-min self-heal respawning TSG PMO/Work screens)
|
||||
# - tsg-agent-stacks.service (dead unit: ExecStart script already missing)
|
||||
# - launcher scripts moved to backup so nothing re-triggers them by accident
|
||||
# Everything is backed up to removed-agent-stacks/ in this repo; rollback =
|
||||
# restore files + systemctl enable. Also quits the leftover ROOT screen
|
||||
# 'ukrrs-gateway-final' (perf-pass leftover). Does NOT touch screen 'boot'
|
||||
# (may host the live crush session; it dies at the multiqueue bounce anyway).
|
||||
# Self-elevating. Run from the guest: ~/projects/ultix/6-remove-agent-stacks.sh
|
||||
set -euo pipefail
|
||||
[ "$EUID" -eq 0 ] || exec sudo bash "$0"
|
||||
|
||||
OUT=/home/reachableceo/projects/ultix/6-remove-agent-stacks.out
|
||||
exec > >(tee "$OUT") 2>&1
|
||||
echo "== remove agent stacks $(date -Is) =="
|
||||
|
||||
BK=/home/reachableceo/projects/ultix/removed-agent-stacks
|
||||
mkdir -p "$BK"
|
||||
|
||||
systemctl disable --now tsg-supervisor.timer 2>&1 || true
|
||||
systemctl disable --now tsg-supervisor.service tsg-agent-stacks.service \
|
||||
reachableceo-agent-stack.service 2>&1 || true
|
||||
|
||||
for u in tsg-supervisor.timer tsg-supervisor.service tsg-agent-stacks.service \
|
||||
reachableceo-agent-stack.service; do
|
||||
if [ -e "/etc/systemd/system/$u" ]; then
|
||||
cp -a "/etc/systemd/system/$u" "$BK/"
|
||||
rm -f "/etc/systemd/system/$u"
|
||||
echo "removed unit: $u"
|
||||
fi
|
||||
find /etc/systemd/system -maxdepth 2 -lname "*/$u" -print -delete 2>/dev/null || true
|
||||
done
|
||||
systemctl daemon-reload
|
||||
systemctl reset-failed 2>/dev/null || true
|
||||
|
||||
for f in /usr/local/sbin/launch-rceo-stack.sh \
|
||||
/usr/local/sbin/setup-rceo-stack.sh \
|
||||
/home/reachableceo/launch-all-tsg-stacks.sh; do
|
||||
if [ -e "$f" ]; then mv "$f" "$BK/" && echo "moved launcher: $f"; fi
|
||||
done
|
||||
|
||||
screen -S ukrrs-gateway-final -X quit 2>/dev/null \
|
||||
&& echo "quit leftover root screen: ukrrs-gateway-final" \
|
||||
|| echo "root screen ukrrs-gateway-final: not present"
|
||||
|
||||
echo "-- cron cross-check (expect clean) --"
|
||||
crontab -l 2>/dev/null | grep -Ei 'supervisor|stack|screen|pmo' || echo "root crontab: clean"
|
||||
grep -rEl 'supervisor\.sh|launch-rceo|launch-all-tsg' /etc/cron* 2>/dev/null || echo "cron dirs: clean"
|
||||
|
||||
echo "-- verify: units gone, timer list, remaining screens --"
|
||||
ls /etc/systemd/system/ | grep -Ei 'stack|supervisor' || echo "no stack/supervisor units remain"
|
||||
screen -ls 2>/dev/null || true
|
||||
echo "== done $(date -Is) — rollback: restore $BK + re-enable units =="
|
||||
@@ -6,9 +6,10 @@
|
||||
order=10, cpuunits 9000/50, ide2 gone.
|
||||
- Net multiqueue: queues=4/2 now IN live qm config (staged 16:40 by crush
|
||||
via ssh); activates at the next VM start — ONE reboot still owed, human-
|
||||
timed: run ~/optimize/5-activate-multiqueue.sh whenever convenient
|
||||
(kills crush sessions, ~4 min). Guest oneshot ukrrs-net-multiqueue
|
||||
auto-runs ethtool -L on that boot.
|
||||
timed: run ~/projects/ultix/5-activate-multiqueue.sh (kills crush
|
||||
sessions, ~4 min). HUMAN 2026-08-31 eve: nothing else is running on this
|
||||
system — bounce THIS SESSION, right after the remaining session tasks.
|
||||
Guest oneshot ukrrs-net-multiqueue auto-runs ethtool -L on that boot.
|
||||
- Gateway boot-race + live-restore endpoint loss: permanently fixed by
|
||||
ukrrs-gateway-ensure unit (enabled, both lanes).
|
||||
- Root cause of the 15:59 no-op: the script copy executed then predates the
|
||||
@@ -30,7 +31,7 @@
|
||||
recovered via compose --force-recreate; /status = mode normal, serving ✅
|
||||
|
||||
## Nothing owed. Optional sanity check any time (crush can ssh now):
|
||||
~/optimize/4-host-netcheck.sh
|
||||
~/projects/ultix/4-host-netcheck.sh
|
||||
|
||||
## Noted, no action taken
|
||||
- mopac-demo/mcli fake containers: restart=no, exited at the first bounce,
|
||||
|
||||
@@ -8,7 +8,9 @@ are manual-only by ruling.
|
||||
Services" — open work: #601 (hardware window ~Oct 2026; presume
|
||||
8 vCPU/48G until then), #603 root growth, #604 metrics/PSI wiring,
|
||||
#605 k8s + proxmox token (blocked by #601), #606 GPU passthrough,
|
||||
#607 account map/mkacct; pass record: #602.
|
||||
#607 account map/mkacct, #608 compose cgroup_parent wiring (gateway
|
||||
starvation fix on 8 vCPU), #609 Fri 09-04 troubleshooting-only outage;
|
||||
pass record: #602.
|
||||
https://projects.knownelement.com/projects/55
|
||||
- Discourse doc: pending (house cross-link rule: create at next doc pass).
|
||||
- Docs live in-repo (runbook exception per house rules):
|
||||
@@ -18,6 +20,9 @@ are manual-only by ruling.
|
||||
- GROW-ROOT-RUNBOOK.md — manual root-growth checklist (#603)
|
||||
- RUNBOOK-TODAY.md, questions-v1.md
|
||||
- Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh,
|
||||
3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh);
|
||||
staged/ holds the gated configs those scripts install.
|
||||
3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh),
|
||||
5-activate-multiqueue.sh (the one remaining reboot), 6-remove-agent-stacks.sh
|
||||
(kill boot-time screen/crush relaunchers); staged/ holds the gated configs
|
||||
those scripts install. removed-agent-stacks/ holds unit/launcher backups
|
||||
from 6-.
|
||||
- crush.md = session preferences for agents working in this repo.
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# ultix-streaming optimization report v1 (2026-08-31)
|
||||
# ultix-streaming optimization report v1.1 (2026-08-31)
|
||||
|
||||
> **POST-PASS STATUS (2026-08-31 evening).** The guest plan (§4) is applied
|
||||
> and verified live; the hot host flags (§8 checklist) are applied; net
|
||||
> multiqueue is in the qm config and activates at the next VM start (one
|
||||
> human-timed bounce: `~/projects/ultix/5-activate-multiqueue.sh`). §1 below
|
||||
> is the PRE-PASS baseline, kept for reference. Hardware work moved to the
|
||||
> ~Oct 2026 window (#601); Fri 2026-09-04 is troubleshooting-only (#609).
|
||||
> Living state of record: NEXT.md + TRACKING.md. Repo moved: ~/optimize →
|
||||
> ~/projects/ultix (paths updated throughout).
|
||||
|
||||
Scope: the ultix-streaming KVM guest on pfv-tsys5, tuned for ~9 autonomous agent
|
||||
accounts + gateway fleet + builds + ETL + SDR + GPU, all concurrent. Inputs:
|
||||
@@ -10,9 +19,9 @@ Companion files: `questions-v1.md` (answer inline), `host-audit.sh` (run on PVE)
|
||||
|
||||
One 2010-era 4-core Xeon currently serves this entire host: your dev
|
||||
workstation plus 24 other running VMs, with zero resource limits anywhere in
|
||||
the fleet, and the workstation does not even auto-start after a host reboot.
|
||||
The fix is roughly $300 of used parts, one maintenance window, and the
|
||||
configurations already staged in this directory.
|
||||
the fleet. [2026-08-31: the software half of the fix is LIVE — onboot=1,
|
||||
ssd=1/discard/iothread, cpuunits 9000/50, all guest tuning, gateway
|
||||
boot-race fix; the hardware half (~$300 parts) moved to ~Oct 2026, #601.]
|
||||
|
||||
| spend item | rough cost (used, 2026) | effect |
|
||||
|---|---|---|
|
||||
@@ -35,36 +44,54 @@ backpressure wired end to end. All software pieces are staged and $0.
|
||||
including 24 running VMs (load 7.5/8 at idle). CPU2 socket is empty.
|
||||
Drop-in fix: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) = 24
|
||||
threads, plus CPU2 DIMM slots unlock 192G. See section 8.
|
||||
[STATUS: parts + window deferred to ~Oct 2026, #601; Fri 09-04 =
|
||||
troubleshooting-only, #609.]
|
||||
2. One maintenance window fixes VM shape: machine q35 (i440fx today, blocks clean
|
||||
GPU passthrough), 20 vCPU / 128G / ballooning off, virtio-scsi-single +
|
||||
iothread + ssd=1 + discard=on per disk (guest currently sees ALL disks as
|
||||
ROTATIONAL: the SSD hint is not passed through), net0 multiqueue.
|
||||
[STATUS: iothread/ssd/discard LIVE + verified (ROTA=0, discard 4K/1G);
|
||||
multiqueue in qm config, activates at next VM start; q35/20c/128G → Oct
|
||||
window #601.]
|
||||
3. No resource limits exist anywhere in the fleet: 18 live containers, zero
|
||||
cpus/mem/cgroup_parent in any compose, and `dev.sh check` builders default to
|
||||
GOMAXPROCS=8 on an 8-vCPU box. Staged: cgroup v2 slice architecture
|
||||
(ukrrs-{gateway,pmo,lsp,batch,rt} + per-account slices via mkacct.sh).
|
||||
[STATUS: LANDED — slice units installed+enabled; compose cgroup_parent
|
||||
wiring = #608.]
|
||||
4. Docker daemon is 100% stock: unbounded json-file logs, live-restore off,
|
||||
default address pool (~15 networks; you already have 12), no metrics socket.
|
||||
Staged daemon.json + weekly builder-prune timer (build cache is 9.3G/187 entries).
|
||||
[STATUS: LANDED + verified live 2026-08-31.]
|
||||
5. Guest kernel: THP=always (bad for the two Postgres), 66M min_free_kbytes,
|
||||
tiny socket buffers (212K) for long LLM streams, dirty ratios in % of 48G
|
||||
(multi-second writeback stalls under ETL). Staged sysctl.d + THP=madvise.
|
||||
[STATUS: LANDED + verified post-reboot; day/night runtime flips live.]
|
||||
6. Signals: harness already reads loadavg/MemAvailable/PSI-io
|
||||
(harness/internal/quota/resources.go:24). Missing: PSI cpu+memory, per-slice
|
||||
pressure, docker awareness. Cheapest full stack: node_exporter as a compose
|
||||
service into your existing mopac prometheus + PSI textfile script. Beszel
|
||||
agent already runs (PID 940). k8s and Proxmox both have clean shedding APIs.
|
||||
[STATUS: pending, #604.]
|
||||
7. sddm-greeter burns ~22% of one core 24/7 (stuck on login screen). Fix:
|
||||
autologin + lock. Sunshine is already installed; it becomes excellent once
|
||||
the GPU lands (NVENC), replacing any xrdp idea.
|
||||
[STATUS: FIXED — sddm autologin (Relogin=true) verified, greeter burn
|
||||
gone. xrdp stays as the Jump/iPad quick-attach path per Q18; Sunshine
|
||||
waits for the compute GPUs.]
|
||||
8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3).
|
||||
[STATUS: pending — manual runbook, #603.]
|
||||
9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool,
|
||||
memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep
|
||||
window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
|
||||
Guest has no CPU governor to remove (KVM: host owns P-states; set host
|
||||
governor to performance once, done).
|
||||
[STATUS: LANDED — day/night timers live; interim values are the operating
|
||||
config until the Oct window; no cpufreq driver exists (audit §8), so the
|
||||
profile is cgroup-only exactly as designed.]
|
||||
|
||||
## 1. Measured current state
|
||||
## 1. Measured baseline (PRE-PASS, 2026-08-31 morning — kept for reference;
|
||||
"now/measured" below means that morning, before anything was applied)
|
||||
|
||||
### Guest (ultix-streaming)
|
||||
| aspect | measured |
|
||||
@@ -154,10 +181,11 @@ optional device throttles if a specific ETL goes rogue.
|
||||
|
||||
## 3. Host plan (pfv-tsys5)
|
||||
|
||||
### 3.1 Audit (do this first)
|
||||
My tooling cannot ssh (hard-banned), so run:
|
||||
### 3.1 Audit (DONE 2026-08-31 — parsed into §8; the ssh tool-ban was lifted
|
||||
later that day, crush reaches the host now)
|
||||
Original instructions, for the record:
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/host-audit.sh > ~/projects/ultix/host-audit.out
|
||||
```
|
||||
I parse the output and finalize §3.2-3.7 numbers. The script is read-only.
|
||||
|
||||
@@ -268,7 +296,7 @@ down idle sectestbed VMs entirely (frees host RAM too) and their disks are
|
||||
already separate (local-lvm spinner vs your dedicated SSDs), so no I/O
|
||||
arbitration needed. Optional host cmdline at next reboot: `iommu=pt`.
|
||||
|
||||
## 4. Guest plan (staged in ~/optimize/staged, apply via apply-guest.sh)
|
||||
## 4. Guest plan (staged in ~/projects/ultix/staged, apply via apply-guest.sh)
|
||||
|
||||
### 4.1 Kernel (60-ukrrs-vm.conf)
|
||||
| knob | now → set | why |
|
||||
@@ -344,7 +372,7 @@ across the three physical devices + (rare) per-container device rate limits.
|
||||
### 4.6 Desktop
|
||||
Fix the greeter burn: sddm autologin into an UNLOCKED session (ruling
|
||||
2026-08-31: no autolock, ever; instant Jump/iPad re-attach outranks lock
|
||||
security on this box). `~/optimize/fix-kde.sh` does it (sudo, idempotent,
|
||||
security on this box). `~/projects/ultix/fix-kde.sh` does it (sudo, idempotent,
|
||||
optional --restart). Result: greeter stops rendering, session idles near 0%,
|
||||
an always-alive unlocked KDE session is attachable instantly. Delete sddm-greeter
|
||||
CPU cost entirely the day you stop wanting console KDE (multi-user.target) but
|
||||
@@ -415,7 +443,7 @@ non-persistent; a reboot lands safely in day mode; values in
|
||||
| batch AllowedCPUs | 0-5 (post 0-15) | 0-6 (post 0-17) |
|
||||
| batch MemoryHigh | 12G (post 24G) | 30G (post 80G) |
|
||||
| gateway CPUWeight | 900 | 500 (still far above idle accounts) |
|
||||
| /data2 readahead | 256K | 1M (ETL sequential) |
|
||||
| /data2 readahead | 128K (256 sectors) | 1M (2048 sectors, ETL sequential) |
|
||||
| dirty_bytes / background | 1G / 256M | 2G / 512M |
|
||||
Honest limit: the guest has no CPU governor to "take off"; KVM host owns
|
||||
P-states. Set the host governor to performance once (audit confirms current
|
||||
@@ -443,15 +471,18 @@ agents → ukrrs-batch.slice. Decide via Q15; no OS change either way, the
|
||||
slice architecture absorbs it.
|
||||
|
||||
## 6. Sequencing and rollback
|
||||
- Phase 0 (now, reversible, ~1h, no reboot): apply-guest.sh sysctl+thp+slices+
|
||||
noatime+timers+daynight+desktop; daemon.json in a quiet window (one docker
|
||||
restart).
|
||||
- Phase 1: host audit (one ssh line), order CPUs/RAM (Q2/Q3).
|
||||
- Phase 2 (window, VM off): q35 + resize + disk flags + multiqueue; then root
|
||||
growth to 500G; then GPU + USB3 passthrough; then VM vCPU/RAM at new values.
|
||||
- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in compose,
|
||||
resources.go PSI extension, gateway pressure input, #585 cop removal.
|
||||
- Phase 4: k8s join, proxmox-ctl wiring.
|
||||
- Phase 0 DONE 2026-08-31 (apply-guest.sh: sysctl+thp+slices+noatime+timers+
|
||||
daynight+desktop; daemon.json; all verified live post-reboot).
|
||||
- Phase 1 DONE 2026-08-31 (host audit parsed, §8; CPU/RAM ordering deferred
|
||||
with the window slip).
|
||||
- Phase 2 SPLIT by the Oct replan: disk flags + onboot + cpuunits DONE hot
|
||||
2026-08-31; multiqueue in qm config, activates at next VM start; q35 +
|
||||
resize + root growth (#603, manual) + GPU/USB3 + 20 vCPU/128G → ~Oct 2026
|
||||
(#601/#606).
|
||||
- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in
|
||||
compose (#608), resources.go PSI extension (#604), gateway pressure input,
|
||||
#585 cop removal.
|
||||
- Phase 4: k8s join + proxmox-ctl wiring (#605, blocked by #601).
|
||||
Every phase independently revertible: sysctl.d and units are files, qm changes
|
||||
are per-flag, partition work has dumps. Never a bare `down`, per house rules.
|
||||
|
||||
@@ -496,15 +527,15 @@ See questions-v1.md, answer inline; version the file if you want another round.
|
||||
- Only **ONE** Renesas uPD720201 USB3 card visible (24:00.0). You said two.
|
||||
Check the second: unseated, dead, or behind the (empty) Pericom PCI-X
|
||||
bridge at 01:00.0.
|
||||
- VM 5111 has **no onboot flag**: it will not auto-start after a host
|
||||
reboot. debian13.iso still attached to ide2 (nit).
|
||||
- VM 5111 had **no onboot flag** and ide2 still held the debian13.iso.
|
||||
[FIXED 2026-08-31: onboot=1, startup order=10, ide2 deleted.]
|
||||
- k8s on this host consolidated (2026-08-31): VM 500 pfv-k8s-wnode-tsys5 was
|
||||
removed; ultix-streaming itself becomes the k8s worker on pfv-tsys5
|
||||
(12G RAM + 2 vCPU of host pressure freed; timing/control plane per Q11).
|
||||
- The 18-VM sectestbed fleet runs 24/7 on the same 8 threads and the 2TB
|
||||
spinner. proxmox-ctl.sh from this VM is the natural on/off switch (Q12).
|
||||
|
||||
### QM checklist for the next touch (any time, hot where noted)
|
||||
### QM checklist (APPLIED 2026-08-31, every line except the two deferred notes)
|
||||
```
|
||||
qm set 5111 --onboot 1 --startup order=10,up=180
|
||||
qm set 5111 -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G
|
||||
|
||||
+32
-25
@@ -1,30 +1,37 @@
|
||||
# Today: two commands, one reboot wave
|
||||
# Runbook 2026-08-31 — COMPLETE (amended after the Oct replan)
|
||||
|
||||
**1. On the VM** (ultix-streaming, any time before step 2):
|
||||
```
|
||||
sudo ~/optimize/1-guest-prep.sh
|
||||
```
|
||||
Installs everything (kernel tunables, BBR, THP, cgroup slices, PSI collector,
|
||||
day/night timers, docker daemon.json, sddm autologin, no autolock). Docker
|
||||
restarts once. Touches NO storage.
|
||||
Steps 1-3 below all landed and were verified read-only the same day (see
|
||||
REPORT.md status banner + §8, and NEXT.md). Kept for the record. Paths
|
||||
updated: repo moved ~/optimize → ~/projects/ultix.
|
||||
|
||||
**2. From your desk** (dry-run first if you like, then execute):
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh
|
||||
```
|
||||
Graceful stop of 5111, ssd/discard/multiqueue/onboot flags, CPU priority over
|
||||
the sectestbed fleet, start 5111, then reboots every other running VM.
|
||||
No disk resizing, no host reboot (NFS role).
|
||||
**Remaining perf actions (as of 2026-08-31 evening):**
|
||||
1. Net multiqueue activation — ONE graceful bounce of VM 5111:
|
||||
`~/projects/ultix/5-activate-multiqueue.sh` (~4 min, kills every crush
|
||||
session on this VM). Human ruling 08-31 eve: system idle tonight — good
|
||||
window after the session's remaining tasks. Verify after:
|
||||
`ethtool -l ens18` → Combined 4/4, `ethtool -l ens19` → 2/2.
|
||||
2. Root growth 279G → ~505G — fully manual, step-by-step with checks:
|
||||
`~/projects/ultix/GROW-ROOT-RUNBOOK.md` (#603). No script runs it; you
|
||||
type every command yourself.
|
||||
|
||||
**3.** Say "done" here; I verify everything read-only and report.
|
||||
**Friday 2026-09-04 (#609) is TROUBLESHOOTING-ONLY**: USB SDR reseat +
|
||||
zero-cost socket/PSU/BIOS checks. NO CPU/RAM swap, NO upgrades. All hardware
|
||||
moved to the ~Oct 2026 window (#601: 2× X5675 + 6× 16GB → 192G; NFS outage
|
||||
for -02 planned first; then q35 + 20 vCPU/128G + GPUs #606). Until then the
|
||||
interim fences (accounts 3G/4.5G) and daynight interim values ARE the
|
||||
operating config.
|
||||
|
||||
**Separate, whenever you choose** (fully manual, step-by-step with checks):
|
||||
root growth 279G -> ~505G via `~/optimize/GROW-ROOT-RUNBOOK.md`. No script
|
||||
runs it; no boot triggers it; you type every command yourself.
|
||||
Original steps, for the record:
|
||||
1. `sudo ~/projects/ultix/1-guest-prep.sh` — kernel tunables, BBR, THP,
|
||||
cgroup slices, PSI collector, day/night timers, docker daemon.json, sddm
|
||||
autologin. DONE + verified.
|
||||
2. `ssh root@pfv-tsys5.knel.net 'bash -s -- --go' <
|
||||
~/projects/ultix/2-host-one-shot.sh` — graceful stop of 5111,
|
||||
ssd/discard/multiqueue-config/onboot flags, cpuunits 9000 over the
|
||||
sectestbed fleet, start 5111. DONE + verified (16:01 bounce).
|
||||
3. Verification — DONE (16:04-16:15 pass). Follow-up root cause: gateway
|
||||
prod was DOWN at boot (bind race + live-restore endpoint loss);
|
||||
permanently fixed same day by the ukrrs-gateway-ensure unit, both lanes.
|
||||
|
||||
Rollback: apply-guest backs up every file it touches; qm flags are
|
||||
individually reversible.
|
||||
|
||||
Friday still owns: CPU/RAM swap (host power-off -> plan the NFS outage for
|
||||
-02 first), second USB3 card reseat, q35 + 20 vCPU / 128G reshape, GPUs.
|
||||
Rollback notes unchanged: apply-guest backs up every file it touches; qm
|
||||
flags are individually reversible.
|
||||
|
||||
+2
-1
@@ -8,7 +8,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 /
|
||||
| OPT-2 | guest | ✅ | REPORT.md + staged configs + questions-v1.md |
|
||||
| OPT-3 | host | ✅ | audit parsed → REPORT §8 (T7500, CPU2 empty, clean IOMMU, Q4000 skip) |
|
||||
| OPT-4 | host | ⏳ | #601 ~OCT 2026: 2× X5675 + 6× 16GB → 24T/192G (NFS outage for -02 first; USB3 reseat; q35/20c/128G). PRESUME 8 vCPU/48G until then |
|
||||
| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue now in qm config (live at Friday's start) |
|
||||
| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue in qm config, activates at next VM start (bounce approved for tonight) |
|
||||
| OPT-6 | guest | ✅ | guest prep applied + verified (bbr, THP, slices, timers, noatime, daemon.json, autologin) |
|
||||
| OPT-7 | guest | ⏳ | #603: root growth MANUAL ONLY, GROW-ROOT-RUNBOOK.md, user-executed |
|
||||
| OPT-8 | repos | ✅ | PMO backpressure design note in ukrrs/docs/harness; code = ticketed |
|
||||
@@ -18,6 +18,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 /
|
||||
Inbox (mid-task interrupts): none.
|
||||
|
||||
Decisions log (latest wins):
|
||||
- 2026-08-31 (night): docs synced post-pass: REPORT v1.1 (status banner, per-item STATUS tags, §6/§8 truth), RUNBOOK amended for the Oct replan, all paths ~/optimize → ~/projects/ultix (repo moved). Human ruling: system idle tonight → multiqueue bounce approved for this session, after remaining tasks. Agent-stack teardown prepped as 6-remove-agent-stacks.sh (kills boot-time RCEO/TSG screen respawn + the 5-min supervisor self-heal; unit/launcher backups to removed-agent-stacks/; root also quits leftover root screen ukrrs-gateway-final, leaves 'boot' alone).
|
||||
- 2026-08-31 (late): Redmine replan complete: #601 Oct window (due 10-31, High, version october-2026-window; Redmine API refuses start_date on update — Oct-1 start recorded in the note instead), NEW #608 cgroup_parent wiring (High; THE starvation fix on 8 vCPU), NEW #609 Fri 09-04 troubleshooting-only outage (USB SDR reseat + socket/PSU/BIOS checks; NO upgrades; host down → this VM auto-starts via onboot=1, gateway auto-recovers). #605/#606 noted. docs repo pushed (main 9eb324e).
|
||||
- 2026-08-31 (eve): HARDWARE WINDOW → ~OCT 2026; presume 8 vCPU/48G operating values; cpuset reservation deferred → cgroup_parent wiring = the starvation fix; one VM-5111 reboot owed for multiqueue (5-activate-multiqueue.sh, human-timed; crush may roll OTHER VMs on pfv-tsys5 anytime).
|
||||
- 2026-08-31 (eve): PUSH AS YOU GO standing permission (human monitors gitea/redmine/discourse, not ssh/crush). House rules updated + pushed (86d8565).
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# crush.md — project preferences for crush sessions in ~/optimize
|
||||
# crush.md — project preferences for crush sessions in ~/projects/ultix
|
||||
|
||||
## Locale (ruling 2026-08-31)
|
||||
- Human + this host are in TEXAS: America/Chicago (Central Time; UTC-5 CDT in
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
== pre-flip baseline 2026-08-31 21:44 CDT ==
|
||||
dirty_bytes=1073741824 dirty_bg=268435456 sdb_ra_kb=128 sdc_ra_kb=128
|
||||
ukrrs cgroup dirs: NONE (batch/gateway slices enabled but never started)
|
||||
+1
-1
@@ -6,7 +6,7 @@ Answer inline under each A:. Where I have a recommendation it is marked REC;
|
||||
## Q1. Host audit (required first, blocks host-side numbers)
|
||||
I cannot ssh from crush (tool policy), so run:
|
||||
```
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out
|
||||
ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/host-audit.sh > ~/projects/ultix/host-audit.out
|
||||
```
|
||||
then just say "audit done". I read the file and finalize §3 of REPORT.md.
|
||||
A:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile
|
||||
# Measured-before values and rationale: ~/optimize/REPORT.md §4.1.
|
||||
# Measured-before values and rationale: ~/projects/ultix/REPORT.md §4.1.
|
||||
# Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure,
|
||||
# pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3).
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first.
|
||||
# UKRRS guest optimizer installer. Review ~/projects/ultix/REPORT.md section 4 first.
|
||||
# sudo RUN=1 ./apply-guest.sh <step>
|
||||
# RUN=1 required to mutate anything (default: plan only).
|
||||
# daemonjson and desktop additionally require CONFIRM=1 (one-time disruption).
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ if [ "$RUN" = 1 ]; then
|
||||
cat > "$slice" <<EOF
|
||||
[Unit]
|
||||
Description=UKRRS account lane ($tier): $acct
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
CPUWeight=$weight
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS batch pool: dev.sh builders, ETL, doc builds, background agents
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Day bias: lowest weight, restricted to pool A. The night profile timer
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS gateway stack: LLM traffic + accounting postgres (prod+beta)
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Latency-sensitive: all provider traffic + spend accounting. Weight beats any
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS LSP fleet + MCP connector tier (mostly idle, bursty reads)
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
CPUWeight=150
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS PMO dispatch loops (one per account): control plane
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Dispatcher must never starve behind worker storms: cheap but weighted above
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=UKRRS realtime pool: SDR DSP, video encode, latency-hard work
|
||||
Documentation=file:///home/reachableceo/optimize/REPORT.md
|
||||
Documentation=file:///home/reachableceo/projects/ultix/REPORT.md
|
||||
|
||||
[Slice]
|
||||
# Exclusive-ish by exclusion: account+batch slices are pinned OFF these cores,
|
||||
|
||||
Reference in New Issue
Block a user