diff --git a/1-guest-prep.sh b/1-guest-prep.sh index 735aaaa..5be2c04 100755 --- a/1-guest-prep.sh +++ b/1-guest-prep.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # ONE-SHOT guest prep (idempotent; safe to re-run). Run ON ultix-streaming: -# sudo ~/optimize/1-guest-prep.sh +# sudo ~/projects/ultix/1-guest-prep.sh # 1. All staged configs (docker restarts once) + sddm autologin. # 2. Human-tier slices for the two human accounts (KDE/CAD/video priority). # NO storage operations (root growth is manual: GROW-ROOT-RUNBOOK.md). @@ -18,6 +18,6 @@ RUN=1 bash "$DIR/staged/mkacct.sh" reachableceo-offstage 1010 12G 16G all human echo "== [3/3] done ==" echo "GUEST READY. Next, the host one-shot (safe to run from THIS VM; the" echo "VM bounce is detached on the host and survives your ssh dying):" -echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/optimize/2-host-one-shot.sh" +echo " ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ~/projects/ultix/2-host-one-shot.sh" echo -echo "Root growth to ~505G stays MANUAL: ~/optimize/GROW-ROOT-RUNBOOK.md" +echo "Root growth to ~505G stays MANUAL: ~/projects/ultix/GROW-ROOT-RUNBOOK.md" diff --git a/2-host-one-shot.sh b/2-host-one-shot.sh index 3f1f10b..f9797d8 100755 --- a/2-host-one-shot.sh +++ b/2-host-one-shot.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ONE-SHOT host pass v3 — does EVERYTHING except rebooting VM 5111. -# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh -# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh +# ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/2-host-one-shot.sh +# ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/projects/ultix/2-host-one-shot.sh # ssh root@pfv-tsys5.knel.net 'bash -s -- --go --with-vm-restart' < ... # optional # # 1. VM 5111's disk/net/onboot flags are written as PENDING config: they diff --git a/3-post-reboot-fixes.sh b/3-post-reboot-fixes.sh index 7eefdc5..4c607f4 100755 --- a/3-post-reboot-fixes.sh +++ b/3-post-reboot-fixes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Post-reboot validation fallout (2026-08-31). Run with sudo, once: -# sudo ~/optimize/3-post-reboot-fixes.sh +# sudo ~/projects/ultix/3-post-reboot-fixes.sh # 1. Installs ethtool + enables virtio multiqueue (host side now offers # 4 queues on ens18, 2 on ens19; the guest defaults to 1 until told). # 2. Installs the gateway ensure-up unit (prod boot race on the tailscale-IP diff --git a/4-host-netcheck.sh b/4-host-netcheck.sh index 482505b..70eb5a2 100755 --- a/4-host-netcheck.sh +++ b/4-host-netcheck.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Read-only host check: why did the net multiqueue flags not land on VM 5111? -# Run from the guest: ~/optimize/4-host-netcheck.sh -# Everything is saved to ~/optimize/host-netcheck.out for crush to read. +# Run from the guest: ~/projects/ultix/4-host-netcheck.sh +# Everything is saved to ~/projects/ultix/host-netcheck.out for crush to read. set -euo pipefail OUT="$(cd "$(dirname "$0")" && pwd)/host-netcheck.out" timeout 120 ssh root@pfv-tsys5.knel.net ' diff --git a/6-remove-agent-stacks.sh b/6-remove-agent-stacks.sh new file mode 100755 index 0000000..4325ed9 --- /dev/null +++ b/6-remove-agent-stacks.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# 6-remove-agent-stacks.sh — remove ALL boot-time screen/crush stack relaunchers. +# Target state: no screen/crush sessions auto-start on reboot, any account. +# - reachableceo-agent-stack.service (boot respawn RCEO-PMO/RCEO-Work + prompt injection) +# - tsg-supervisor.timer/.service (5-min self-heal respawning TSG PMO/Work screens) +# - tsg-agent-stacks.service (dead unit: ExecStart script already missing) +# - launcher scripts moved to backup so nothing re-triggers them by accident +# Everything is backed up to removed-agent-stacks/ in this repo; rollback = +# restore files + systemctl enable. Also quits the leftover ROOT screen +# 'ukrrs-gateway-final' (perf-pass leftover). Does NOT touch screen 'boot' +# (may host the live crush session; it dies at the multiqueue bounce anyway). +# Self-elevating. Run from the guest: ~/projects/ultix/6-remove-agent-stacks.sh +set -euo pipefail +[ "$EUID" -eq 0 ] || exec sudo bash "$0" + +OUT=/home/reachableceo/projects/ultix/6-remove-agent-stacks.out +exec > >(tee "$OUT") 2>&1 +echo "== remove agent stacks $(date -Is) ==" + +BK=/home/reachableceo/projects/ultix/removed-agent-stacks +mkdir -p "$BK" + +systemctl disable --now tsg-supervisor.timer 2>&1 || true +systemctl disable --now tsg-supervisor.service tsg-agent-stacks.service \ + reachableceo-agent-stack.service 2>&1 || true + +for u in tsg-supervisor.timer tsg-supervisor.service tsg-agent-stacks.service \ + reachableceo-agent-stack.service; do + if [ -e "/etc/systemd/system/$u" ]; then + cp -a "/etc/systemd/system/$u" "$BK/" + rm -f "/etc/systemd/system/$u" + echo "removed unit: $u" + fi + find /etc/systemd/system -maxdepth 2 -lname "*/$u" -print -delete 2>/dev/null || true +done +systemctl daemon-reload +systemctl reset-failed 2>/dev/null || true + +for f in /usr/local/sbin/launch-rceo-stack.sh \ + /usr/local/sbin/setup-rceo-stack.sh \ + /home/reachableceo/launch-all-tsg-stacks.sh; do + if [ -e "$f" ]; then mv "$f" "$BK/" && echo "moved launcher: $f"; fi +done + +screen -S ukrrs-gateway-final -X quit 2>/dev/null \ + && echo "quit leftover root screen: ukrrs-gateway-final" \ + || echo "root screen ukrrs-gateway-final: not present" + +echo "-- cron cross-check (expect clean) --" +crontab -l 2>/dev/null | grep -Ei 'supervisor|stack|screen|pmo' || echo "root crontab: clean" +grep -rEl 'supervisor\.sh|launch-rceo|launch-all-tsg' /etc/cron* 2>/dev/null || echo "cron dirs: clean" + +echo "-- verify: units gone, timer list, remaining screens --" +ls /etc/systemd/system/ | grep -Ei 'stack|supervisor' || echo "no stack/supervisor units remain" +screen -ls 2>/dev/null || true +echo "== done $(date -Is) — rollback: restore $BK + re-enable units ==" diff --git a/NEXT.md b/NEXT.md index b680b81..8968413 100644 --- a/NEXT.md +++ b/NEXT.md @@ -6,9 +6,10 @@ order=10, cpuunits 9000/50, ide2 gone. - Net multiqueue: queues=4/2 now IN live qm config (staged 16:40 by crush via ssh); activates at the next VM start — ONE reboot still owed, human- - timed: run ~/optimize/5-activate-multiqueue.sh whenever convenient - (kills crush sessions, ~4 min). Guest oneshot ukrrs-net-multiqueue - auto-runs ethtool -L on that boot. + timed: run ~/projects/ultix/5-activate-multiqueue.sh (kills crush + sessions, ~4 min). HUMAN 2026-08-31 eve: nothing else is running on this + system — bounce THIS SESSION, right after the remaining session tasks. + Guest oneshot ukrrs-net-multiqueue auto-runs ethtool -L on that boot. - Gateway boot-race + live-restore endpoint loss: permanently fixed by ukrrs-gateway-ensure unit (enabled, both lanes). - Root cause of the 15:59 no-op: the script copy executed then predates the @@ -30,7 +31,7 @@ recovered via compose --force-recreate; /status = mode normal, serving ✅ ## Nothing owed. Optional sanity check any time (crush can ssh now): - ~/optimize/4-host-netcheck.sh + ~/projects/ultix/4-host-netcheck.sh ## Noted, no action taken - mopac-demo/mcli fake containers: restart=no, exited at the first bounce, diff --git a/README.md b/README.md index 87d0531..bf64035 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ are manual-only by ruling. Services" — open work: #601 (hardware window ~Oct 2026; presume 8 vCPU/48G until then), #603 root growth, #604 metrics/PSI wiring, #605 k8s + proxmox token (blocked by #601), #606 GPU passthrough, - #607 account map/mkacct; pass record: #602. + #607 account map/mkacct, #608 compose cgroup_parent wiring (gateway + starvation fix on 8 vCPU), #609 Fri 09-04 troubleshooting-only outage; + pass record: #602. https://projects.knownelement.com/projects/55 - Discourse doc: pending (house cross-link rule: create at next doc pass). - Docs live in-repo (runbook exception per house rules): @@ -18,6 +20,9 @@ are manual-only by ruling. - GROW-ROOT-RUNBOOK.md — manual root-growth checklist (#603) - RUNBOOK-TODAY.md, questions-v1.md - Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh, - 3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh); - staged/ holds the gated configs those scripts install. + 3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh), + 5-activate-multiqueue.sh (the one remaining reboot), 6-remove-agent-stacks.sh + (kill boot-time screen/crush relaunchers); staged/ holds the gated configs + those scripts install. removed-agent-stacks/ holds unit/launcher backups + from 6-. - crush.md = session preferences for agents working in this repo. diff --git a/REPORT.md b/REPORT.md index 3457662..20b8297 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,4 +1,13 @@ -# ultix-streaming optimization report v1 (2026-08-31) +# ultix-streaming optimization report v1.1 (2026-08-31) + +> **POST-PASS STATUS (2026-08-31 evening).** The guest plan (§4) is applied +> and verified live; the hot host flags (§8 checklist) are applied; net +> multiqueue is in the qm config and activates at the next VM start (one +> human-timed bounce: `~/projects/ultix/5-activate-multiqueue.sh`). §1 below +> is the PRE-PASS baseline, kept for reference. Hardware work moved to the +> ~Oct 2026 window (#601); Fri 2026-09-04 is troubleshooting-only (#609). +> Living state of record: NEXT.md + TRACKING.md. Repo moved: ~/optimize → +> ~/projects/ultix (paths updated throughout). Scope: the ultix-streaming KVM guest on pfv-tsys5, tuned for ~9 autonomous agent accounts + gateway fleet + builds + ETL + SDR + GPU, all concurrent. Inputs: @@ -10,9 +19,9 @@ Companion files: `questions-v1.md` (answer inline), `host-audit.sh` (run on PVE) One 2010-era 4-core Xeon currently serves this entire host: your dev workstation plus 24 other running VMs, with zero resource limits anywhere in -the fleet, and the workstation does not even auto-start after a host reboot. -The fix is roughly $300 of used parts, one maintenance window, and the -configurations already staged in this directory. +the fleet. [2026-08-31: the software half of the fix is LIVE — onboot=1, +ssd=1/discard/iothread, cpuunits 9000/50, all guest tuning, gateway +boot-race fix; the hardware half (~$300 parts) moved to ~Oct 2026, #601.] | spend item | rough cost (used, 2026) | effect | |---|---|---| @@ -35,36 +44,54 @@ backpressure wired end to end. All software pieces are staged and $0. including 24 running VMs (load 7.5/8 at idle). CPU2 socket is empty. Drop-in fix: 2× Xeon X5675 (LGA1366, 6C/12T each, 95W, ~$25 each used) = 24 threads, plus CPU2 DIMM slots unlock 192G. See section 8. + [STATUS: parts + window deferred to ~Oct 2026, #601; Fri 09-04 = + troubleshooting-only, #609.] 2. One maintenance window fixes VM shape: machine q35 (i440fx today, blocks clean GPU passthrough), 20 vCPU / 128G / ballooning off, virtio-scsi-single + iothread + ssd=1 + discard=on per disk (guest currently sees ALL disks as ROTATIONAL: the SSD hint is not passed through), net0 multiqueue. + [STATUS: iothread/ssd/discard LIVE + verified (ROTA=0, discard 4K/1G); + multiqueue in qm config, activates at next VM start; q35/20c/128G → Oct + window #601.] 3. No resource limits exist anywhere in the fleet: 18 live containers, zero cpus/mem/cgroup_parent in any compose, and `dev.sh check` builders default to GOMAXPROCS=8 on an 8-vCPU box. Staged: cgroup v2 slice architecture (ukrrs-{gateway,pmo,lsp,batch,rt} + per-account slices via mkacct.sh). + [STATUS: LANDED — slice units installed+enabled; compose cgroup_parent + wiring = #608.] 4. Docker daemon is 100% stock: unbounded json-file logs, live-restore off, default address pool (~15 networks; you already have 12), no metrics socket. Staged daemon.json + weekly builder-prune timer (build cache is 9.3G/187 entries). + [STATUS: LANDED + verified live 2026-08-31.] 5. Guest kernel: THP=always (bad for the two Postgres), 66M min_free_kbytes, tiny socket buffers (212K) for long LLM streams, dirty ratios in % of 48G (multi-second writeback stalls under ETL). Staged sysctl.d + THP=madvise. + [STATUS: LANDED + verified post-reboot; day/night runtime flips live.] 6. Signals: harness already reads loadavg/MemAvailable/PSI-io (harness/internal/quota/resources.go:24). Missing: PSI cpu+memory, per-slice pressure, docker awareness. Cheapest full stack: node_exporter as a compose service into your existing mopac prometheus + PSI textfile script. Beszel agent already runs (PID 940). k8s and Proxmox both have clean shedding APIs. + [STATUS: pending, #604.] 7. sddm-greeter burns ~22% of one core 24/7 (stuck on login screen). Fix: autologin + lock. Sunshine is already installed; it becomes excellent once the GPU lands (NVENC), replacing any xrdp idea. + [STATUS: FIXED — sddm autologin (Relogin=true) verified, greeter burn + gone. xrdp stays as the Jump/iPad quick-attach path per Q18; Sunshine + waits for the compute GPUs.] 8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3). + [STATUS: pending — manual runbook, #603.] 9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool, memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST). Guest has no CPU governor to remove (KVM: host owns P-states; set host governor to performance once, done). + [STATUS: LANDED — day/night timers live; interim values are the operating + config until the Oct window; no cpufreq driver exists (audit §8), so the + profile is cgroup-only exactly as designed.] -## 1. Measured current state +## 1. Measured baseline (PRE-PASS, 2026-08-31 morning — kept for reference; +"now/measured" below means that morning, before anything was applied) ### Guest (ultix-streaming) | aspect | measured | @@ -154,10 +181,11 @@ optional device throttles if a specific ETL goes rogue. ## 3. Host plan (pfv-tsys5) -### 3.1 Audit (do this first) -My tooling cannot ssh (hard-banned), so run: +### 3.1 Audit (DONE 2026-08-31 — parsed into §8; the ssh tool-ban was lifted +later that day, crush reaches the host now) +Original instructions, for the record: ``` -ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out +ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/host-audit.sh > ~/projects/ultix/host-audit.out ``` I parse the output and finalize §3.2-3.7 numbers. The script is read-only. @@ -268,7 +296,7 @@ down idle sectestbed VMs entirely (frees host RAM too) and their disks are already separate (local-lvm spinner vs your dedicated SSDs), so no I/O arbitration needed. Optional host cmdline at next reboot: `iommu=pt`. -## 4. Guest plan (staged in ~/optimize/staged, apply via apply-guest.sh) +## 4. Guest plan (staged in ~/projects/ultix/staged, apply via apply-guest.sh) ### 4.1 Kernel (60-ukrrs-vm.conf) | knob | now → set | why | @@ -344,7 +372,7 @@ across the three physical devices + (rare) per-container device rate limits. ### 4.6 Desktop Fix the greeter burn: sddm autologin into an UNLOCKED session (ruling 2026-08-31: no autolock, ever; instant Jump/iPad re-attach outranks lock -security on this box). `~/optimize/fix-kde.sh` does it (sudo, idempotent, +security on this box). `~/projects/ultix/fix-kde.sh` does it (sudo, idempotent, optional --restart). Result: greeter stops rendering, session idles near 0%, an always-alive unlocked KDE session is attachable instantly. Delete sddm-greeter CPU cost entirely the day you stop wanting console KDE (multi-user.target) but @@ -415,7 +443,7 @@ non-persistent; a reboot lands safely in day mode; values in | batch AllowedCPUs | 0-5 (post 0-15) | 0-6 (post 0-17) | | batch MemoryHigh | 12G (post 24G) | 30G (post 80G) | | gateway CPUWeight | 900 | 500 (still far above idle accounts) | -| /data2 readahead | 256K | 1M (ETL sequential) | +| /data2 readahead | 128K (256 sectors) | 1M (2048 sectors, ETL sequential) | | dirty_bytes / background | 1G / 256M | 2G / 512M | Honest limit: the guest has no CPU governor to "take off"; KVM host owns P-states. Set the host governor to performance once (audit confirms current @@ -443,15 +471,18 @@ agents → ukrrs-batch.slice. Decide via Q15; no OS change either way, the slice architecture absorbs it. ## 6. Sequencing and rollback -- Phase 0 (now, reversible, ~1h, no reboot): apply-guest.sh sysctl+thp+slices+ - noatime+timers+daynight+desktop; daemon.json in a quiet window (one docker - restart). -- Phase 1: host audit (one ssh line), order CPUs/RAM (Q2/Q3). -- Phase 2 (window, VM off): q35 + resize + disk flags + multiqueue; then root - growth to 500G; then GPU + USB3 passthrough; then VM vCPU/RAM at new values. -- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in compose, - resources.go PSI extension, gateway pressure input, #585 cop removal. -- Phase 4: k8s join, proxmox-ctl wiring. +- Phase 0 DONE 2026-08-31 (apply-guest.sh: sysctl+thp+slices+noatime+timers+ + daynight+desktop; daemon.json; all verified live post-reboot). +- Phase 1 DONE 2026-08-31 (host audit parsed, §8; CPU/RAM ordering deferred + with the window slip). +- Phase 2 SPLIT by the Oct replan: disk flags + onboot + cpuunits DONE hot + 2026-08-31; multiqueue in qm config, activates at next VM start; q35 + + resize + root growth (#603, manual) + GPU/USB3 + 20 vCPU/128G → ~Oct 2026 + (#601/#606). +- Phase 3 (repo tickets, independent): builder caps + cgroup_parent in + compose (#608), resources.go PSI extension (#604), gateway pressure input, + #585 cop removal. +- Phase 4: k8s join + proxmox-ctl wiring (#605, blocked by #601). Every phase independently revertible: sysctl.d and units are files, qm changes are per-flag, partition work has dumps. Never a bare `down`, per house rules. @@ -496,15 +527,15 @@ See questions-v1.md, answer inline; version the file if you want another round. - Only **ONE** Renesas uPD720201 USB3 card visible (24:00.0). You said two. Check the second: unseated, dead, or behind the (empty) Pericom PCI-X bridge at 01:00.0. -- VM 5111 has **no onboot flag**: it will not auto-start after a host - reboot. debian13.iso still attached to ide2 (nit). +- VM 5111 had **no onboot flag** and ide2 still held the debian13.iso. + [FIXED 2026-08-31: onboot=1, startup order=10, ide2 deleted.] - k8s on this host consolidated (2026-08-31): VM 500 pfv-k8s-wnode-tsys5 was removed; ultix-streaming itself becomes the k8s worker on pfv-tsys5 (12G RAM + 2 vCPU of host pressure freed; timing/control plane per Q11). - The 18-VM sectestbed fleet runs 24/7 on the same 8 threads and the 2TB spinner. proxmox-ctl.sh from this VM is the natural on/off switch (Q12). -### QM checklist for the next touch (any time, hot where noted) +### QM checklist (APPLIED 2026-08-31, every line except the two deferred notes) ``` qm set 5111 --onboot 1 --startup order=10,up=180 qm set 5111 -scsi0 NVME:5111/vm-5111-disk-0.qcow2,iothread=1,ssd=1,discard=on,size=438G diff --git a/RUNBOOK-TODAY.md b/RUNBOOK-TODAY.md index 71770db..4e75f77 100644 --- a/RUNBOOK-TODAY.md +++ b/RUNBOOK-TODAY.md @@ -1,30 +1,37 @@ -# Today: two commands, one reboot wave +# Runbook 2026-08-31 — COMPLETE (amended after the Oct replan) -**1. On the VM** (ultix-streaming, any time before step 2): -``` -sudo ~/optimize/1-guest-prep.sh -``` -Installs everything (kernel tunables, BBR, THP, cgroup slices, PSI collector, -day/night timers, docker daemon.json, sddm autologin, no autolock). Docker -restarts once. Touches NO storage. +Steps 1-3 below all landed and were verified read-only the same day (see +REPORT.md status banner + §8, and NEXT.md). Kept for the record. Paths +updated: repo moved ~/optimize → ~/projects/ultix. -**2. From your desk** (dry-run first if you like, then execute): -``` -ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/2-host-one-shot.sh -ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < ~/optimize/2-host-one-shot.sh -``` -Graceful stop of 5111, ssd/discard/multiqueue/onboot flags, CPU priority over -the sectestbed fleet, start 5111, then reboots every other running VM. -No disk resizing, no host reboot (NFS role). +**Remaining perf actions (as of 2026-08-31 evening):** +1. Net multiqueue activation — ONE graceful bounce of VM 5111: + `~/projects/ultix/5-activate-multiqueue.sh` (~4 min, kills every crush + session on this VM). Human ruling 08-31 eve: system idle tonight — good + window after the session's remaining tasks. Verify after: + `ethtool -l ens18` → Combined 4/4, `ethtool -l ens19` → 2/2. +2. Root growth 279G → ~505G — fully manual, step-by-step with checks: + `~/projects/ultix/GROW-ROOT-RUNBOOK.md` (#603). No script runs it; you + type every command yourself. -**3.** Say "done" here; I verify everything read-only and report. +**Friday 2026-09-04 (#609) is TROUBLESHOOTING-ONLY**: USB SDR reseat + +zero-cost socket/PSU/BIOS checks. NO CPU/RAM swap, NO upgrades. All hardware +moved to the ~Oct 2026 window (#601: 2× X5675 + 6× 16GB → 192G; NFS outage +for -02 planned first; then q35 + 20 vCPU/128G + GPUs #606). Until then the +interim fences (accounts 3G/4.5G) and daynight interim values ARE the +operating config. -**Separate, whenever you choose** (fully manual, step-by-step with checks): -root growth 279G -> ~505G via `~/optimize/GROW-ROOT-RUNBOOK.md`. No script -runs it; no boot triggers it; you type every command yourself. +Original steps, for the record: +1. `sudo ~/projects/ultix/1-guest-prep.sh` — kernel tunables, BBR, THP, + cgroup slices, PSI collector, day/night timers, docker daemon.json, sddm + autologin. DONE + verified. +2. `ssh root@pfv-tsys5.knel.net 'bash -s -- --go' < + ~/projects/ultix/2-host-one-shot.sh` — graceful stop of 5111, + ssd/discard/multiqueue-config/onboot flags, cpuunits 9000 over the + sectestbed fleet, start 5111. DONE + verified (16:01 bounce). +3. Verification — DONE (16:04-16:15 pass). Follow-up root cause: gateway + prod was DOWN at boot (bind race + live-restore endpoint loss); + permanently fixed same day by the ukrrs-gateway-ensure unit, both lanes. -Rollback: apply-guest backs up every file it touches; qm flags are -individually reversible. - -Friday still owns: CPU/RAM swap (host power-off -> plan the NFS outage for --02 first), second USB3 card reseat, q35 + 20 vCPU / 128G reshape, GPUs. +Rollback notes unchanged: apply-guest backs up every file it touches; qm +flags are individually reversible. diff --git a/TRACKING.md b/TRACKING.md index f0678b2..8e1a77e 100644 --- a/TRACKING.md +++ b/TRACKING.md @@ -8,7 +8,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / | OPT-2 | guest | ✅ | REPORT.md + staged configs + questions-v1.md | | OPT-3 | host | ✅ | audit parsed → REPORT §8 (T7500, CPU2 empty, clean IOMMU, Q4000 skip) | | OPT-4 | host | ⏳ | #601 ~OCT 2026: 2× X5675 + 6× 16GB → 24T/192G (NFS outage for -02 first; USB3 reseat; q35/20c/128G). PRESUME 8 vCPU/48G until then | -| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue now in qm config (live at Friday's start) | +| OPT-5 | host | ✅ | one-shot landed + verified post-reboot; net multiqueue in qm config, activates at next VM start (bounce approved for tonight) | | OPT-6 | guest | ✅ | guest prep applied + verified (bbr, THP, slices, timers, noatime, daemon.json, autologin) | | OPT-7 | guest | ⏳ | #603: root growth MANUAL ONLY, GROW-ROOT-RUNBOOK.md, user-executed | | OPT-8 | repos | ✅ | PMO backpressure design note in ukrrs/docs/harness; code = ticketed | @@ -18,6 +18,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / Inbox (mid-task interrupts): none. Decisions log (latest wins): +- 2026-08-31 (night): docs synced post-pass: REPORT v1.1 (status banner, per-item STATUS tags, §6/§8 truth), RUNBOOK amended for the Oct replan, all paths ~/optimize → ~/projects/ultix (repo moved). Human ruling: system idle tonight → multiqueue bounce approved for this session, after remaining tasks. Agent-stack teardown prepped as 6-remove-agent-stacks.sh (kills boot-time RCEO/TSG screen respawn + the 5-min supervisor self-heal; unit/launcher backups to removed-agent-stacks/; root also quits leftover root screen ukrrs-gateway-final, leaves 'boot' alone). - 2026-08-31 (late): Redmine replan complete: #601 Oct window (due 10-31, High, version october-2026-window; Redmine API refuses start_date on update — Oct-1 start recorded in the note instead), NEW #608 cgroup_parent wiring (High; THE starvation fix on 8 vCPU), NEW #609 Fri 09-04 troubleshooting-only outage (USB SDR reseat + socket/PSU/BIOS checks; NO upgrades; host down → this VM auto-starts via onboot=1, gateway auto-recovers). #605/#606 noted. docs repo pushed (main 9eb324e). - 2026-08-31 (eve): HARDWARE WINDOW → ~OCT 2026; presume 8 vCPU/48G operating values; cpuset reservation deferred → cgroup_parent wiring = the starvation fix; one VM-5111 reboot owed for multiqueue (5-activate-multiqueue.sh, human-timed; crush may roll OTHER VMs on pfv-tsys5 anytime). - 2026-08-31 (eve): PUSH AS YOU GO standing permission (human monitors gitea/redmine/discourse, not ssh/crush). House rules updated + pushed (86d8565). diff --git a/crush.md b/crush.md index 2df66aa..7678082 100644 --- a/crush.md +++ b/crush.md @@ -1,4 +1,4 @@ -# crush.md — project preferences for crush sessions in ~/optimize +# crush.md — project preferences for crush sessions in ~/projects/ultix ## Locale (ruling 2026-08-31) - Human + this host are in TEXAS: America/Chicago (Central Time; UTC-5 CDT in diff --git a/night-flip-20260831.out b/night-flip-20260831.out new file mode 100644 index 0000000..b534ad5 --- /dev/null +++ b/night-flip-20260831.out @@ -0,0 +1,3 @@ +== pre-flip baseline 2026-08-31 21:44 CDT == +dirty_bytes=1073741824 dirty_bg=268435456 sdb_ra_kb=128 sdc_ra_kb=128 +ukrrs cgroup dirs: NONE (batch/gateway slices enabled but never started) diff --git a/questions-v1.md b/questions-v1.md index 6ddbf28..f0a66b7 100644 --- a/questions-v1.md +++ b/questions-v1.md @@ -6,7 +6,7 @@ Answer inline under each A:. Where I have a recommendation it is marked REC; ## Q1. Host audit (required first, blocks host-side numbers) I cannot ssh from crush (tool policy), so run: ``` -ssh root@pfv-tsys5.knel.net 'bash -s' < ~/optimize/host-audit.sh > ~/optimize/host-audit.out +ssh root@pfv-tsys5.knel.net 'bash -s' < ~/projects/ultix/host-audit.sh > ~/projects/ultix/host-audit.out ``` then just say "audit done". I read the file and finalize §3 of REPORT.md. A: diff --git a/staged/60-ukrrs-vm.conf b/staged/60-ukrrs-vm.conf index 4ee13c9..9cc190f 100644 --- a/staged/60-ukrrs-vm.conf +++ b/staged/60-ukrrs-vm.conf @@ -1,5 +1,5 @@ # /etc/sysctl.d/60-ukrrs-vm.conf — ultix-streaming mixed-workload profile -# Measured-before values and rationale: ~/optimize/REPORT.md §4.1. +# Measured-before values and rationale: ~/projects/ultix/REPORT.md §4.1. # Deliberately NOT touched: swappiness(60), overcommit(0), vfs_cache_pressure, # pid_max, somaxconn(4096), conntrack(262144), autogroup(1), page-cluster(3). diff --git a/staged/apply-guest.sh b/staged/apply-guest.sh index 9db12c6..b3acf09 100755 --- a/staged/apply-guest.sh +++ b/staged/apply-guest.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# UKRRS guest optimizer installer. Review ~/optimize/REPORT.md section 4 first. +# UKRRS guest optimizer installer. Review ~/projects/ultix/REPORT.md section 4 first. # sudo RUN=1 ./apply-guest.sh # RUN=1 required to mutate anything (default: plan only). # daemonjson and desktop additionally require CONFIRM=1 (one-time disruption). diff --git a/staged/mkacct.sh b/staged/mkacct.sh index 964941b..88edb3b 100755 --- a/staged/mkacct.sh +++ b/staged/mkacct.sh @@ -50,7 +50,7 @@ if [ "$RUN" = 1 ]; then cat > "$slice" <