diff --git a/10-distribute-openwebui-creds.sh b/10-distribute-openwebui-creds.sh new file mode 100755 index 0000000..2257bf2 --- /dev/null +++ b/10-distribute-openwebui-creds.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# 10-distribute-openwebui-creds.sh — per-account openwebui.creds (#610). +# For each of the 9 Open Terminal accounts: write THAT account's own +# URL + API key into /.creds/openwebui.creds (mkdir -p .creds), +# 0600, owned by the account. Source of truth = the per-user TOML in +# /etc/ukrrs/open-terminal/ (keys read from there, NEVER echoed here, +# never on a command line, never in this repo or the .out log). +# ~/projects/ultix/10-distribute-openwebui-creds.sh +set -euo pipefail +[ "$EUID" -eq 0 ] || exec sudo bash "$0" + +OUT=/home/reachableceo/projects/ultix/10-distribute-openwebui-creds.out +exec > >(tee "$OUT") 2>&1 +echo "== openwebui.creds distribution $(date -Is) ==" + +TOML_DIR=/etc/ukrrs/open-terminal + +while read -r USER; do + [ -z "$USER" ] && continue + HOME_DIR=$(getent passwd "$USER" | cut -d: -f6) + TOML="$TOML_DIR/$USER.toml" + if [ ! -r "$TOML" ]; then + echo "SKIP $USER (no TOML at $TOML)" + continue + fi + + PORT=$(sed -n 's/^port = //p' "$TOML" | tr -d ' ') + KEY=$(sed -n 's/^api_key = "\(.*\)"$/\1/p' "$TOML") + + mkdir -p "$HOME_DIR/.creds" + chown "$USER:$USER" "$HOME_DIR/.creds" + chmod 700 "$HOME_DIR/.creds" + + CREDS="$HOME_DIR/.creds/openwebui.creds" + cat > "$CREDS" < Admin -> Integrations -> Open Terminal -> "+" +OPENWEBUI_TERMINAL_URL=http://100.101.187.119:$PORT +OPENWEBUI_TERMINAL_API_KEY=$KEY +OPENWEBUI_TERMINAL_AUTH=bearer +EOF + chown "$USER:$USER" "$CREDS" + chmod 600 "$CREDS" + + HEALTH=$(curl -s -m 5 "http://100.101.187.119:$PORT/health" || echo unreachable) + MODE=$(stat -c '%U:%G %a' "$CREDS") + printf '%-24s port %s %s %s health: %s\n' "$USER" "$PORT" "$CREDS" "$MODE" "$HEALTH" +done <<'LIST' +reachableceo +TSGCOO +TSGCCO +TSGCTO +TSGBOD +COS-RCEO +COS-WFO +COS-TSG +reachableceo-offstage +LIST + +chown reachableceo:reachableceo "$OUT" 2>/dev/null || true +echo "== done $(date -Is) ==" diff --git a/11-shift-day-flip-0700.out b/11-shift-day-flip-0700.out new file mode 100644 index 0000000..25709a7 --- /dev/null +++ b/11-shift-day-flip-0700.out @@ -0,0 +1,22 @@ +== day flip 05:00 -> 07:00 2026-09-01T07:29:27-05:00 == +-- live timer now -- +[Unit] +Description=UKRRS: day profile at 07:00 + +[Timer] +OnCalendar=*-*-* 07:00:00 +Persistent=true + +[Install] +WantedBy=timers.target +-- armed timers -- +NEXT LEFT LAST PASSED UNIT ACTIVATES +Tue 2026-09-01 22:00:00 CDT 14h Mon 2026-08-31 22:00:06 CDT - ukrrs-nightprofile.timer ukrrs-nightprofile.service +Wed 2026-09-02 07:00:00 CDT 23h Tue 2026-09-01 07:29:28 CDT 122ms ago ukrrs-dayprofile.timer ukrrs-dayprofile.service + +2 timers listed. +Pass --all to see loaded but inactive timers, too. +-- current profile markers -- +vm.dirty_bytes = 1073741824 +vm.dirty_background_bytes = 268435456 +== done 2026-09-01T07:29:28-05:00 == diff --git a/11-shift-day-flip-0700.sh b/11-shift-day-flip-0700.sh new file mode 100755 index 0000000..256b570 --- /dev/null +++ b/11-shift-day-flip-0700.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# 11-shift-day-flip-0700.sh — night-profile window 22:00-05:00 -> 22:00-07:00 +# (#610 session, human ruling 2026-09-01: "Charles isn't using the VM" +# hours now end 07:00). Moves the LIVE day-flip timer to 07:00 and +# re-arms it. Repo staged copy + docs synced in the same commit. +# The z.ai gateway peak ladder (01:00-05:00) is NOT touched. +# ~/projects/ultix/11-shift-day-flip-0700.sh +set -euo pipefail +[ "$EUID" -eq 0 ] || exec sudo bash "$0" + +OUT=/home/reachableceo/projects/ultix/11-shift-day-flip-0700.out +exec > >(tee "$OUT") 2>&1 +echo "== day flip 05:00 -> 07:00 $(date -Is) ==" + +LIVE=/etc/systemd/system/ukrrs-dayprofile.timer +sed -i 's/05:00:00/07:00:00/; s/day profile at 05:00/day profile at 07:00/' "$LIVE" +systemctl daemon-reload +systemctl restart ukrrs-dayprofile.timer + +echo "-- live timer now --" +cat "$LIVE" +echo "-- armed timers --" +systemctl list-timers 'ukrrs-*profile*' --no-pager +echo "-- current profile markers --" +sysctl vm.dirty_bytes vm.dirty_background_bytes 2>/dev/null || true + +chown reachableceo:reachableceo "$OUT" 2>/dev/null || true +echo "== done $(date -Is) ==" diff --git a/NEXT.md b/NEXT.md index 61a5c83..d193f3f 100644 --- a/NEXT.md +++ b/NEXT.md @@ -11,11 +11,10 @@ - HUMAN OWED: wire production OpenWebUI per OPEN-TERMINAL.md. The openwebui CONTAINER on this box (v0.3.10, :3000) is too old — upgrade to 0.11.x first. Cloudron prod (0.11.1) is fine. -- LAST ACTION of the session: multiqueue bounce via - ~/projects/ultix/5-activate-multiqueue.sh (approved; kills crush - sessions ~4 min). After reboot verify: - ethtool -l ens18 -> Combined 4/4; ens19 -> 2/2; then - ~/projects/ultix/4-host-netcheck.sh for the full sweep. +- Multiqueue bounce DONE + VERIFIED (2026-09-01 morning audit): ens18 + Combined 4/4, ens19 2/2; 4-host-netcheck.sh sweep clean (qm config + live, pending empty, bounce 22:51->22:53). Full stack healthy post-boot. +- 2026-09-01: night window now 22:00-07:00 (day-flip timer moved to 07:00). ## Final state — nothing owed (perf pass) - All guest tuning live and verified (see 16:04-16:15 pass below). diff --git a/REPORT.md b/REPORT.md index 20b8297..f0f54de 100644 --- a/REPORT.md +++ b/REPORT.md @@ -33,7 +33,7 @@ boot-race fix; the hardware half (~$300 parts) moved to ~Oct 2026, #601.] | 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) | Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes, -gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn +gateway and PMO dispatch permanently protected, nightly 22:00-07:00 batch burn window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven backpressure wired end to end. All software pieces are staged and $0. @@ -82,8 +82,8 @@ backpressure wired end to end. All software pieces are staged and $0. 8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3). [STATUS: pending — manual runbook, #603.] 9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool, - memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep - window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST). + memory fences and /data2 readahead at 22:00 and 07:00, matching your sleep + window (22:00-07:00) with the gateway's z.ai peak ladder (01:00-05:00 CST). Guest has no CPU governor to remove (KVM: host owns P-states; set host governor to performance once, done). [STATUS: LANDED — day/night timers live; interim values are the operating @@ -428,11 +428,11 @@ beats the retired cop :8110 (#585) it still polls. See §3.8; skeleton staged, token + PROTECTED list via Q12. ### 5.5 Day/night dynamic profile + PMO dispatch backpressure -Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the +Windows (Q14): OS night profile 22:00-07:00 local (your sleep window); the z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset (gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect: 22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO -burn maxes, 05:00 everything returns to day bias. +burn maxes, 07:00 everything returns to day bias. What flips at night (systemctl set-property --runtime + sysctl, all non-persistent; a reboot lands safely in day mode; values in diff --git a/TRACKING.md b/TRACKING.md index 47ae066..987b8b5 100644 --- a/TRACKING.md +++ b/TRACKING.md @@ -19,6 +19,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / Inbox (mid-task interrupts): none. Decisions log (latest wins): +- 2026-09-01 (morning): POST-REBOOT AUDIT PASS (boot 22:53 = multiqueue bounce): ens18 Combined 4/4 + ens19 2/2 LIVE; qm pending empty; bounce log 22:51->22:53 clean; gw prod (ts :4000) + beta (:4002) both 200/healthy; postgres x2, openwebui :3000, 12 LSPs, all 9 open-terminal ports bound on 100.101.187.119; PSI textfile fresh; no relauncher respawn. Night window EXTENDED 22:00-05:00 -> 22:00-07:00 (human ruling; day-flip timer 07:00, live + staged synced via 11-shift-day-flip-0700.sh; z.ai peak ladder 01:00-05:00 untouched). openwebui.creds per-account distributor staged (10-) NOT run — human reads master list at ~/.creds/open-terminal.env. - 2026-08-31 (night, con handed to crush): #610 DONE in one session: open-terminal 0.11.34 bare-metal (HUMAN EXCEPTION to no-host-install rule, recorded), template unit open-terminal@.service, per-user TOML keys (never in ps/repo; ~/.creds holds copies), bind TAILSCALE-ONLY 100.101.187.119 per human ruling (0.0.0.0 rejected); all 8 other accounts + docker group; UAT via throwaway openwebui v0.11.1 (matches Cloudron prod) ALL PASS incl. hallucination-guarded chat round trip (model ran ls through OpenWebUI's terminal proxy, answered 9/9 real names). NOTE: prod openwebui container (v0.3.10, :3000) is too old for Open Terminal — upgrade to 0.11.x before wiring; Cloudron 0.11.1 is fine. Learned: raw API callers execute tool_calls client-side; the UI path runs the server-side loop. - 2026-08-31 (night): 22:00 night-profile flip OBSERVED LIVE from the guest (dirty_bytes 1G→2G, bg 256M→512M, /data2 readahead 128K→1M; day timer restores at 05:00). First runtime-only flip since install. - 2026-08-31 (night): agent-stack relaunchers REMOVED by 6-remove-agent-stacks.sh (reachableceo-agent-stack.service, tsg-supervisor.timer/.service 5-min self-heal, dead tsg-agent-stacks.service; launchers + units backed up to removed-agent-stacks/). No screen/crush sessions auto-start on reboot anymore; human: stacks no longer used for pmo/work. diff --git a/host-netcheck.out b/host-netcheck.out index 0547e79..6c46419 100644 --- a/host-netcheck.out +++ b/host-netcheck.out @@ -2,8 +2,8 @@ cores: 4 cpuunits: 9000 memory: 50000 -net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0 -net1: virtio=BC:24:11:E3:32:D9,bridge=datanet +net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4 +net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2 onboot: 1 scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1 scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1 @@ -22,8 +22,8 @@ cur cpuunits: 9000 cur memory: 50000 cur meta: creation-qemu=10.1.2,ctime=1784819584 cur name: ultix-streaming -cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0 -cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet +cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4 +cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2 cur numa: 0 cur onboot: 1 cur ostype: l26 @@ -37,9 +37,9 @@ cur startup: order=10,up=180 cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f == last bounce log (/var/log/ukrrs-vm5111-bounce.log) == -bounce-start 2026-08-31T15:59:56-05:00 +bounce-start 2026-08-31T22:51:32-05:00 CPU flag 'nested-virt' resolved to 'vmx' -bounce-done 2026-08-31T16:01:41-05:00 +bounce-done 2026-08-31T22:53:16-05:00 == pve version == pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve) diff --git a/staged/apply-guest.sh b/staged/apply-guest.sh index b3acf09..bd40b46 100755 --- a/staged/apply-guest.sh +++ b/staged/apply-guest.sh @@ -117,7 +117,7 @@ step_daynight() { done systemctl daemon-reload systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer - say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night" + say "night profile flips at 22:00, day at 07:00; test: ukrrs-daynight.sh night" } step_all() { step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune diff --git a/staged/systemd/ukrrs-dayprofile.timer b/staged/systemd/ukrrs-dayprofile.timer index fbca759..fb46e2d 100644 --- a/staged/systemd/ukrrs-dayprofile.timer +++ b/staged/systemd/ukrrs-dayprofile.timer @@ -1,8 +1,8 @@ [Unit] -Description=UKRRS: day profile at 05:00 +Description=UKRRS: day profile at 07:00 [Timer] -OnCalendar=*-*-* 05:00:00 +OnCalendar=*-*-* 07:00:00 Persistent=true [Install] diff --git a/staged/systemd/ukrrs-nightprofile.service b/staged/systemd/ukrrs-nightprofile.service index 95b58c0..efd8d13 100644 --- a/staged/systemd/ukrrs-nightprofile.service +++ b/staged/systemd/ukrrs-nightprofile.service @@ -1,5 +1,5 @@ [Unit] -Description=UKRRS: switch to night profile (batch burn window 22:00-05:00) +Description=UKRRS: switch to night profile (batch burn window 22:00-07:00) [Service] Type=oneshot