Post-reboot audit PASS + night window extended to 22:00-07:00 (#610)

Verified after the 22:53 multiqueue bounce: NIC queues live at 4/4 and
2/2 (the whole point of the reboot), PVE config clean with nothing
pending, both gateway lanes healthy, all 9 Open Terminal ports bound on
the tailscale IP, PSI metrics fresh, and no removed relaunchers
respawned. Moved the day-profile flip from 05:00 to 07:00 per human
ruling (live timer re-armed + staged copy and docs synced; the gateway
z.ai peak ladder is untouched). Also staged a ready-to-run script that
drops each account's own Open Terminal key into its ~/.creds as
openwebui.creds - not run yet; the master key list stays at
~/.creds/open-terminal.env.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2

[#610]

10-distribute-openwebui-creds.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# 10-distribute-openwebui-creds.sh — per-account openwebui.creds (#610).
+# For each of the 9 Open Terminal accounts: write THAT account's own
+# URL + API key into <home>/.creds/openwebui.creds (mkdir -p .creds),
+# 0600, owned by the account. Source of truth = the per-user TOML in
+# /etc/ukrrs/open-terminal/ (keys read from there, NEVER echoed here,
+# never on a command line, never in this repo or the .out log).
+#   ~/projects/ultix/10-distribute-openwebui-creds.sh
+set -euo pipefail
+[ "$EUID" -eq 0 ] || exec sudo bash "$0"
+
+OUT=/home/reachableceo/projects/ultix/10-distribute-openwebui-creds.out
+exec > >(tee "$OUT") 2>&1
+echo "== openwebui.creds distribution $(date -Is) =="
+
+TOML_DIR=/etc/ukrrs/open-terminal
+
+while read -r USER; do
+  [ -z "$USER" ] && continue
+  HOME_DIR=$(getent passwd "$USER" | cut -d: -f6)
+  TOML="$TOML_DIR/$USER.toml"
+  if [ ! -r "$TOML" ]; then
+    echo "SKIP $USER (no TOML at $TOML)"
+    continue
+  fi
+
+  PORT=$(sed -n 's/^port = //p' "$TOML" | tr -d ' ')
+  KEY=$(sed -n 's/^api_key = "\(.*\)"$/\1/p' "$TOML")
+
+  mkdir -p "$HOME_DIR/.creds"
+  chown "$USER:$USER" "$HOME_DIR/.creds"
+  chmod 700 "$HOME_DIR/.creds"
+
+  CREDS="$HOME_DIR/.creds/openwebui.creds"
+  cat > "$CREDS" <<EOF
+# OpenWebUI Open Terminal wiring for $USER (#610). 0600, never repo/chat.
+# Admin UI: Settings -> Admin -> Integrations -> Open Terminal -> "+"
+OPENWEBUI_TERMINAL_URL=http://100.101.187.119:$PORT
+OPENWEBUI_TERMINAL_API_KEY=$KEY
+OPENWEBUI_TERMINAL_AUTH=bearer
+EOF
+  chown "$USER:$USER" "$CREDS"
+  chmod 600 "$CREDS"
+
+  HEALTH=$(curl -s -m 5 "http://100.101.187.119:$PORT/health" || echo unreachable)
+  MODE=$(stat -c '%U:%G %a' "$CREDS")
+  printf '%-24s port %s  %s  %s  health: %s\n' "$USER" "$PORT" "$CREDS" "$MODE" "$HEALTH"
+done <<'LIST'
+reachableceo
+TSGCOO
+TSGCCO
+TSGCTO
+TSGBOD
+COS-RCEO
+COS-WFO
+COS-TSG
+reachableceo-offstage
+LIST
+
+chown reachableceo:reachableceo "$OUT" 2>/dev/null || true
+echo "== done $(date -Is) =="

11-shift-day-flip-0700.out

@@ -0,0 +1,22 @@
+== day flip 05:00 -> 07:00 2026-09-01T07:29:27-05:00 ==
+-- live timer now --
+[Unit]
+Description=UKRRS: day profile at 07:00
+
+[Timer]
+OnCalendar=*-*-* 07:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+-- armed timers --
+NEXT                        LEFT LAST                           PASSED UNIT                     ACTIVATES
+Tue 2026-09-01 22:00:00 CDT  14h Mon 2026-08-31 22:00:06 CDT         - ukrrs-nightprofile.timer ukrrs-nightprofile.service
+Wed 2026-09-02 07:00:00 CDT  23h Tue 2026-09-01 07:29:28 CDT 122ms ago ukrrs-dayprofile.timer   ukrrs-dayprofile.service
+
+2 timers listed.
+Pass --all to see loaded but inactive timers, too.
+-- current profile markers --
+vm.dirty_bytes = 1073741824
+vm.dirty_background_bytes = 268435456
+== done 2026-09-01T07:29:28-05:00 ==

11-shift-day-flip-0700.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# 11-shift-day-flip-0700.sh — night-profile window 22:00-05:00 -> 22:00-07:00
+# (#610 session, human ruling 2026-09-01: "Charles isn't using the VM"
+# hours now end 07:00). Moves the LIVE day-flip timer to 07:00 and
+# re-arms it. Repo staged copy + docs synced in the same commit.
+# The z.ai gateway peak ladder (01:00-05:00) is NOT touched.
+#   ~/projects/ultix/11-shift-day-flip-0700.sh
+set -euo pipefail
+[ "$EUID" -eq 0 ] || exec sudo bash "$0"
+
+OUT=/home/reachableceo/projects/ultix/11-shift-day-flip-0700.out
+exec > >(tee "$OUT") 2>&1
+echo "== day flip 05:00 -> 07:00 $(date -Is) =="
+
+LIVE=/etc/systemd/system/ukrrs-dayprofile.timer
+sed -i 's/05:00:00/07:00:00/; s/day profile at 05:00/day profile at 07:00/' "$LIVE"
+systemctl daemon-reload
+systemctl restart ukrrs-dayprofile.timer
+
+echo "-- live timer now --"
+cat "$LIVE"
+echo "-- armed timers --"
+systemctl list-timers 'ukrrs-*profile*' --no-pager
+echo "-- current profile markers --"
+sysctl vm.dirty_bytes vm.dirty_background_bytes 2>/dev/null || true
+
+chown reachableceo:reachableceo "$OUT" 2>/dev/null || true
+echo "== done $(date -Is) =="

NEXT.md

@@ -11,11 +11,10 @@
 - HUMAN OWED: wire production OpenWebUI per OPEN-TERMINAL.md. The
   openwebui CONTAINER on this box (v0.3.10, :3000) is too old — upgrade to
   0.11.x first. Cloudron prod (0.11.1) is fine.
-- LAST ACTION of the session: multiqueue bounce via
+- Multiqueue bounce DONE + VERIFIED (2026-09-01 morning audit): ens18
-  ~/projects/ultix/5-activate-multiqueue.sh (approved; kills crush
+  Combined 4/4, ens19 2/2; 4-host-netcheck.sh sweep clean (qm config
-  sessions ~4 min). After reboot verify:
+  live, pending empty, bounce 22:51->22:53). Full stack healthy post-boot.
-  ethtool -l ens18 -> Combined 4/4; ens19 -> 2/2; then
+- 2026-09-01: night window now 22:00-07:00 (day-flip timer moved to 07:00).
-  ~/projects/ultix/4-host-netcheck.sh for the full sweep.
 
 ## Final state — nothing owed (perf pass)
 - All guest tuning live and verified (see 16:04-16:15 pass below).

REPORT.md

@@ -33,7 +33,7 @@ boot-race fix; the hardware half (~$300 parts) moved to ~Oct 2026, #601.]
 | 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) |
 
 Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes,
-gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn
+gateway and PMO dispatch permanently protected, nightly 22:00-07:00 batch burn
 window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by
 host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven
 backpressure wired end to end. All software pieces are staged and $0.
@@ -82,8 +82,8 @@ backpressure wired end to end. All software pieces are staged and $0.
 8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3).
    [STATUS: pending — manual runbook, #603.]
 9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool,
-   memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep
+   memory fences and /data2 readahead at 22:00 and 07:00, matching your sleep
-   window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
+   window (22:00-07:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
    Guest has no CPU governor to remove (KVM: host owns P-states; set host
    governor to performance once, done).
    [STATUS: LANDED — day/night timers live; interim values are the operating
@@ -428,11 +428,11 @@ beats the retired cop :8110 (#585) it still polls.
 See §3.8; skeleton staged, token + PROTECTED list via Q12.
 
 ### 5.5 Day/night dynamic profile + PMO dispatch backpressure
-Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the
+Windows (Q14): OS night profile 22:00-07:00 local (your sleep window); the
 z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset
 (gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect:
 22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO
-burn maxes, 05:00 everything returns to day bias.
+burn maxes, 07:00 everything returns to day bias.
 
 What flips at night (systemctl set-property --runtime + sysctl, all
 non-persistent; a reboot lands safely in day mode; values in

TRACKING.md

@@ -19,6 +19,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 /
 Inbox (mid-task interrupts): none.
 
 Decisions log (latest wins):
+- 2026-09-01 (morning): POST-REBOOT AUDIT PASS (boot 22:53 = multiqueue bounce): ens18 Combined 4/4 + ens19 2/2 LIVE; qm pending empty; bounce log 22:51->22:53 clean; gw prod (ts :4000) + beta (:4002) both 200/healthy; postgres x2, openwebui :3000, 12 LSPs, all 9 open-terminal ports bound on 100.101.187.119; PSI textfile fresh; no relauncher respawn. Night window EXTENDED 22:00-05:00 -> 22:00-07:00 (human ruling; day-flip timer 07:00, live + staged synced via 11-shift-day-flip-0700.sh; z.ai peak ladder 01:00-05:00 untouched). openwebui.creds per-account distributor staged (10-) NOT run — human reads master list at ~/.creds/open-terminal.env.
 - 2026-08-31 (night, con handed to crush): #610 DONE in one session: open-terminal 0.11.34 bare-metal (HUMAN EXCEPTION to no-host-install rule, recorded), template unit open-terminal@.service, per-user TOML keys (never in ps/repo; ~/.creds holds copies), bind TAILSCALE-ONLY 100.101.187.119 per human ruling (0.0.0.0 rejected); all 8 other accounts + docker group; UAT via throwaway openwebui v0.11.1 (matches Cloudron prod) ALL PASS incl. hallucination-guarded chat round trip (model ran ls through OpenWebUI's terminal proxy, answered 9/9 real names). NOTE: prod openwebui container (v0.3.10, :3000) is too old for Open Terminal — upgrade to 0.11.x before wiring; Cloudron 0.11.1 is fine. Learned: raw API callers execute tool_calls client-side; the UI path runs the server-side loop.
 - 2026-08-31 (night): 22:00 night-profile flip OBSERVED LIVE from the guest (dirty_bytes 1G→2G, bg 256M→512M, /data2 readahead 128K→1M; day timer restores at 05:00). First runtime-only flip since install.
 - 2026-08-31 (night): agent-stack relaunchers REMOVED by 6-remove-agent-stacks.sh (reachableceo-agent-stack.service, tsg-supervisor.timer/.service 5-min self-heal, dead tsg-agent-stacks.service; launchers + units backed up to removed-agent-stacks/). No screen/crush sessions auto-start on reboot anymore; human: stacks no longer used for pmo/work.

host-netcheck.out

@@ -2,8 +2,8 @@
 cores: 4
 cpuunits: 9000
 memory: 50000
-net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
+net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
-net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
+net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
 onboot: 1
 scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
 scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
@@ -22,8 +22,8 @@ cur cpuunits: 9000
 cur memory: 50000
 cur meta: creation-qemu=10.1.2,ctime=1784819584
 cur name: ultix-streaming
-cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
+cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
-cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
+cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
 cur numa: 0
 cur onboot: 1
 cur ostype: l26
@@ -37,9 +37,9 @@ cur startup: order=10,up=180
 cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f
 
 == last bounce log (/var/log/ukrrs-vm5111-bounce.log) ==
-bounce-start 2026-08-31T15:59:56-05:00
+bounce-start 2026-08-31T22:51:32-05:00
 CPU flag 'nested-virt' resolved to 'vmx'
-bounce-done 2026-08-31T16:01:41-05:00
+bounce-done 2026-08-31T22:53:16-05:00
 
 == pve version ==
 pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve)

staged/apply-guest.sh

@@ -117,7 +117,7 @@ step_daynight() {
   done
   systemctl daemon-reload
   systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer
-  say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night"
+  say "night profile flips at 22:00, day at 07:00; test: ukrrs-daynight.sh night"
 }
 step_all() {
   step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune

staged/systemd/ukrrs-dayprofile.timer

@@ -1,8 +1,8 @@
 [Unit]
-Description=UKRRS: day profile at 05:00
+Description=UKRRS: day profile at 07:00
 
 [Timer]
-OnCalendar=*-*-* 05:00:00
+OnCalendar=*-*-* 07:00:00
 Persistent=true
 
 [Install]

staged/systemd/ukrrs-nightprofile.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
+Description=UKRRS: switch to night profile (batch burn window 22:00-07:00)
 
 [Service]
 Type=oneshot