Post-reboot audit PASS + night window extended to 22:00-07:00 (#610)
Verified after the 22:53 multiqueue bounce: NIC queues live at 4/4 and 2/2 (the whole point of the reboot), PVE config clean with nothing pending, both gateway lanes healthy, all 9 Open Terminal ports bound on the tailscale IP, PSI metrics fresh, and no removed relaunchers respawned. Moved the day-profile flip from 05:00 to 07:00 per human ruling (live timer re-armed + staged copy and docs synced; the gateway z.ai peak ladder is untouched). Also staged a ready-to-run script that drops each account's own Open Terminal key into its ~/.creds as openwebui.creds - not run yet; the master key list stays at ~/.creds/open-terminal.env. 💘 Generated with Crush Assisted-by: Crush:glm-5.2 [#610]
This commit is contained in:
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
# 10-distribute-openwebui-creds.sh — per-account openwebui.creds (#610).
|
||||
# For each of the 9 Open Terminal accounts: write THAT account's own
|
||||
# URL + API key into <home>/.creds/openwebui.creds (mkdir -p .creds),
|
||||
# 0600, owned by the account. Source of truth = the per-user TOML in
|
||||
# /etc/ukrrs/open-terminal/ (keys read from there, NEVER echoed here,
|
||||
# never on a command line, never in this repo or the .out log).
|
||||
# ~/projects/ultix/10-distribute-openwebui-creds.sh
|
||||
set -euo pipefail
|
||||
[ "$EUID" -eq 0 ] || exec sudo bash "$0"
|
||||
|
||||
OUT=/home/reachableceo/projects/ultix/10-distribute-openwebui-creds.out
|
||||
exec > >(tee "$OUT") 2>&1
|
||||
echo "== openwebui.creds distribution $(date -Is) =="
|
||||
|
||||
TOML_DIR=/etc/ukrrs/open-terminal
|
||||
|
||||
while read -r USER; do
|
||||
[ -z "$USER" ] && continue
|
||||
HOME_DIR=$(getent passwd "$USER" | cut -d: -f6)
|
||||
TOML="$TOML_DIR/$USER.toml"
|
||||
if [ ! -r "$TOML" ]; then
|
||||
echo "SKIP $USER (no TOML at $TOML)"
|
||||
continue
|
||||
fi
|
||||
|
||||
PORT=$(sed -n 's/^port = //p' "$TOML" | tr -d ' ')
|
||||
KEY=$(sed -n 's/^api_key = "\(.*\)"$/\1/p' "$TOML")
|
||||
|
||||
mkdir -p "$HOME_DIR/.creds"
|
||||
chown "$USER:$USER" "$HOME_DIR/.creds"
|
||||
chmod 700 "$HOME_DIR/.creds"
|
||||
|
||||
CREDS="$HOME_DIR/.creds/openwebui.creds"
|
||||
cat > "$CREDS" <<EOF
|
||||
# OpenWebUI Open Terminal wiring for $USER (#610). 0600, never repo/chat.
|
||||
# Admin UI: Settings -> Admin -> Integrations -> Open Terminal -> "+"
|
||||
OPENWEBUI_TERMINAL_URL=http://100.101.187.119:$PORT
|
||||
OPENWEBUI_TERMINAL_API_KEY=$KEY
|
||||
OPENWEBUI_TERMINAL_AUTH=bearer
|
||||
EOF
|
||||
chown "$USER:$USER" "$CREDS"
|
||||
chmod 600 "$CREDS"
|
||||
|
||||
HEALTH=$(curl -s -m 5 "http://100.101.187.119:$PORT/health" || echo unreachable)
|
||||
MODE=$(stat -c '%U:%G %a' "$CREDS")
|
||||
printf '%-24s port %s %s %s health: %s\n' "$USER" "$PORT" "$CREDS" "$MODE" "$HEALTH"
|
||||
done <<'LIST'
|
||||
reachableceo
|
||||
TSGCOO
|
||||
TSGCCO
|
||||
TSGCTO
|
||||
TSGBOD
|
||||
COS-RCEO
|
||||
COS-WFO
|
||||
COS-TSG
|
||||
reachableceo-offstage
|
||||
LIST
|
||||
|
||||
chown reachableceo:reachableceo "$OUT" 2>/dev/null || true
|
||||
echo "== done $(date -Is) =="
|
||||
@@ -0,0 +1,22 @@
|
||||
== day flip 05:00 -> 07:00 2026-09-01T07:29:27-05:00 ==
|
||||
-- live timer now --
|
||||
[Unit]
|
||||
Description=UKRRS: day profile at 07:00
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 07:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
-- armed timers --
|
||||
NEXT LEFT LAST PASSED UNIT ACTIVATES
|
||||
Tue 2026-09-01 22:00:00 CDT 14h Mon 2026-08-31 22:00:06 CDT - ukrrs-nightprofile.timer ukrrs-nightprofile.service
|
||||
Wed 2026-09-02 07:00:00 CDT 23h Tue 2026-09-01 07:29:28 CDT 122ms ago ukrrs-dayprofile.timer ukrrs-dayprofile.service
|
||||
|
||||
2 timers listed.
|
||||
Pass --all to see loaded but inactive timers, too.
|
||||
-- current profile markers --
|
||||
vm.dirty_bytes = 1073741824
|
||||
vm.dirty_background_bytes = 268435456
|
||||
== done 2026-09-01T07:29:28-05:00 ==
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# 11-shift-day-flip-0700.sh — night-profile window 22:00-05:00 -> 22:00-07:00
|
||||
# (#610 session, human ruling 2026-09-01: "Charles isn't using the VM"
|
||||
# hours now end 07:00). Moves the LIVE day-flip timer to 07:00 and
|
||||
# re-arms it. Repo staged copy + docs synced in the same commit.
|
||||
# The z.ai gateway peak ladder (01:00-05:00) is NOT touched.
|
||||
# ~/projects/ultix/11-shift-day-flip-0700.sh
|
||||
set -euo pipefail
|
||||
[ "$EUID" -eq 0 ] || exec sudo bash "$0"
|
||||
|
||||
OUT=/home/reachableceo/projects/ultix/11-shift-day-flip-0700.out
|
||||
exec > >(tee "$OUT") 2>&1
|
||||
echo "== day flip 05:00 -> 07:00 $(date -Is) =="
|
||||
|
||||
LIVE=/etc/systemd/system/ukrrs-dayprofile.timer
|
||||
sed -i 's/05:00:00/07:00:00/; s/day profile at 05:00/day profile at 07:00/' "$LIVE"
|
||||
systemctl daemon-reload
|
||||
systemctl restart ukrrs-dayprofile.timer
|
||||
|
||||
echo "-- live timer now --"
|
||||
cat "$LIVE"
|
||||
echo "-- armed timers --"
|
||||
systemctl list-timers 'ukrrs-*profile*' --no-pager
|
||||
echo "-- current profile markers --"
|
||||
sysctl vm.dirty_bytes vm.dirty_background_bytes 2>/dev/null || true
|
||||
|
||||
chown reachableceo:reachableceo "$OUT" 2>/dev/null || true
|
||||
echo "== done $(date -Is) =="
|
||||
@@ -11,11 +11,10 @@
|
||||
- HUMAN OWED: wire production OpenWebUI per OPEN-TERMINAL.md. The
|
||||
openwebui CONTAINER on this box (v0.3.10, :3000) is too old — upgrade to
|
||||
0.11.x first. Cloudron prod (0.11.1) is fine.
|
||||
- LAST ACTION of the session: multiqueue bounce via
|
||||
~/projects/ultix/5-activate-multiqueue.sh (approved; kills crush
|
||||
sessions ~4 min). After reboot verify:
|
||||
ethtool -l ens18 -> Combined 4/4; ens19 -> 2/2; then
|
||||
~/projects/ultix/4-host-netcheck.sh for the full sweep.
|
||||
- Multiqueue bounce DONE + VERIFIED (2026-09-01 morning audit): ens18
|
||||
Combined 4/4, ens19 2/2; 4-host-netcheck.sh sweep clean (qm config
|
||||
live, pending empty, bounce 22:51->22:53). Full stack healthy post-boot.
|
||||
- 2026-09-01: night window now 22:00-07:00 (day-flip timer moved to 07:00).
|
||||
|
||||
## Final state — nothing owed (perf pass)
|
||||
- All guest tuning live and verified (see 16:04-16:15 pass below).
|
||||
|
||||
@@ -33,7 +33,7 @@ boot-race fix; the hardware half (~$300 parts) moved to ~Oct 2026, #601.]
|
||||
| 2× compute GPU (models pending, Q5) | ~$300-400 if 2× RTX A2000 12GB | CUDA pool, no PSU change (75W each) |
|
||||
|
||||
Expected end state: dev VM at 20 vCPU / 128G with cgroup-fenced account lanes,
|
||||
gateway and PMO dispatch permanently protected, nightly 22:00-05:00 batch burn
|
||||
gateway and PMO dispatch permanently protected, nightly 22:00-07:00 batch burn
|
||||
window aligned with the z.ai trickle ladder, sectestbed fleet subordinate by
|
||||
host-side CPU weights, SDRs on their own controller + RT cores, and PSI-driven
|
||||
backpressure wired end to end. All software pieces are staged and $0.
|
||||
@@ -82,8 +82,8 @@ backpressure wired end to end. All software pieces are staged and $0.
|
||||
8. Root growth to 500G is a safe online operation (MBR shuffle, §4.3).
|
||||
[STATUS: pending — manual runbook, #603.]
|
||||
9. Day/night dynamic profile (§5.5): staged timers flip batch weight, CPU pool,
|
||||
memory fences and /data2 readahead at 22:00 and 05:00, matching your sleep
|
||||
window (22:00-05:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
|
||||
memory fences and /data2 readahead at 22:00 and 07:00, matching your sleep
|
||||
window (22:00-07:00) with the gateway's z.ai peak ladder (01:00-05:00 CST).
|
||||
Guest has no CPU governor to remove (KVM: host owns P-states; set host
|
||||
governor to performance once, done).
|
||||
[STATUS: LANDED — day/night timers live; interim values are the operating
|
||||
@@ -428,11 +428,11 @@ beats the retired cop :8110 (#585) it still polls.
|
||||
See §3.8; skeleton staged, token + PROTECTED list via Q12.
|
||||
|
||||
### 5.5 Day/night dynamic profile + PMO dispatch backpressure
|
||||
Windows (Q14): OS night profile 22:00-05:00 local (your sleep window); the
|
||||
Windows (Q14): OS night profile 22:00-07:00 local (your sleep window); the
|
||||
z.ai peak ladder stays 01:00-05:00 CST as the LLM-trickle subset
|
||||
(gateway/config.yaml:21-26, harness mirror config.go:235-244). Net effect:
|
||||
22:00 the system opens up for batch, 01:00 LLM drops to trickle while CPU/IO
|
||||
burn maxes, 05:00 everything returns to day bias.
|
||||
burn maxes, 07:00 everything returns to day bias.
|
||||
|
||||
What flips at night (systemctl set-property --runtime + sysctl, all
|
||||
non-persistent; a reboot lands safely in day mode; values in
|
||||
|
||||
@@ -19,6 +19,7 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 /
|
||||
Inbox (mid-task interrupts): none.
|
||||
|
||||
Decisions log (latest wins):
|
||||
- 2026-09-01 (morning): POST-REBOOT AUDIT PASS (boot 22:53 = multiqueue bounce): ens18 Combined 4/4 + ens19 2/2 LIVE; qm pending empty; bounce log 22:51->22:53 clean; gw prod (ts :4000) + beta (:4002) both 200/healthy; postgres x2, openwebui :3000, 12 LSPs, all 9 open-terminal ports bound on 100.101.187.119; PSI textfile fresh; no relauncher respawn. Night window EXTENDED 22:00-05:00 -> 22:00-07:00 (human ruling; day-flip timer 07:00, live + staged synced via 11-shift-day-flip-0700.sh; z.ai peak ladder 01:00-05:00 untouched). openwebui.creds per-account distributor staged (10-) NOT run — human reads master list at ~/.creds/open-terminal.env.
|
||||
- 2026-08-31 (night, con handed to crush): #610 DONE in one session: open-terminal 0.11.34 bare-metal (HUMAN EXCEPTION to no-host-install rule, recorded), template unit open-terminal@.service, per-user TOML keys (never in ps/repo; ~/.creds holds copies), bind TAILSCALE-ONLY 100.101.187.119 per human ruling (0.0.0.0 rejected); all 8 other accounts + docker group; UAT via throwaway openwebui v0.11.1 (matches Cloudron prod) ALL PASS incl. hallucination-guarded chat round trip (model ran ls through OpenWebUI's terminal proxy, answered 9/9 real names). NOTE: prod openwebui container (v0.3.10, :3000) is too old for Open Terminal — upgrade to 0.11.x before wiring; Cloudron 0.11.1 is fine. Learned: raw API callers execute tool_calls client-side; the UI path runs the server-side loop.
|
||||
- 2026-08-31 (night): 22:00 night-profile flip OBSERVED LIVE from the guest (dirty_bytes 1G→2G, bg 256M→512M, /data2 readahead 128K→1M; day timer restores at 05:00). First runtime-only flip since install.
|
||||
- 2026-08-31 (night): agent-stack relaunchers REMOVED by 6-remove-agent-stacks.sh (reachableceo-agent-stack.service, tsg-supervisor.timer/.service 5-min self-heal, dead tsg-agent-stacks.service; launchers + units backed up to removed-agent-stacks/). No screen/crush sessions auto-start on reboot anymore; human: stacks no longer used for pmo/work.
|
||||
|
||||
+6
-6
@@ -2,8 +2,8 @@
|
||||
cores: 4
|
||||
cpuunits: 9000
|
||||
memory: 50000
|
||||
net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
|
||||
net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
|
||||
net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
||||
net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
||||
onboot: 1
|
||||
scsi0: NVME:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=438G,ssd=1
|
||||
scsi1: ssd2:5111/vm-5111-disk-0.qcow2,discard=on,iothread=1,size=400G,ssd=1
|
||||
@@ -22,8 +22,8 @@ cur cpuunits: 9000
|
||||
cur memory: 50000
|
||||
cur meta: creation-qemu=10.1.2,ctime=1784819584
|
||||
cur name: ultix-streaming
|
||||
cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0
|
||||
cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet
|
||||
cur net0: virtio=BC:24:11:1A:8F:6F,bridge=vmbr0,queues=4
|
||||
cur net1: virtio=BC:24:11:E3:32:D9,bridge=datanet,queues=2
|
||||
cur numa: 0
|
||||
cur onboot: 1
|
||||
cur ostype: l26
|
||||
@@ -37,9 +37,9 @@ cur startup: order=10,up=180
|
||||
cur vmgenid: 06fc08de-c644-497a-a1e9-72304fefd32f
|
||||
|
||||
== last bounce log (/var/log/ukrrs-vm5111-bounce.log) ==
|
||||
bounce-start 2026-08-31T15:59:56-05:00
|
||||
bounce-start 2026-08-31T22:51:32-05:00
|
||||
CPU flag 'nested-virt' resolved to 'vmx'
|
||||
bounce-done 2026-08-31T16:01:41-05:00
|
||||
bounce-done 2026-08-31T22:53:16-05:00
|
||||
|
||||
== pve version ==
|
||||
pve-manager/9.2.5/20242970da7fbcef (running kernel: 7.0.14-6-pve)
|
||||
|
||||
@@ -117,7 +117,7 @@ step_daynight() {
|
||||
done
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now ukrrs-nightprofile.timer ukrrs-dayprofile.timer
|
||||
say "night profile flips at 22:00, day at 05:00; test: ukrrs-daynight.sh night"
|
||||
say "night profile flips at 22:00, day at 07:00; test: ukrrs-daynight.sh night"
|
||||
}
|
||||
step_all() {
|
||||
step_sysctl; step_modules; step_thp; step_slices; step_psi; step_prune
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
[Unit]
|
||||
Description=UKRRS: day profile at 05:00
|
||||
Description=UKRRS: day profile at 07:00
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 05:00:00
|
||||
OnCalendar=*-*-* 07:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=UKRRS: switch to night profile (batch burn window 22:00-05:00)
|
||||
Description=UKRRS: switch to night profile (batch burn window 22:00-07:00)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
|
||||
Reference in New Issue
Block a user