From c8109422390985ed23d152f86e16294b5065dde1 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Mon, 31 Aug 2026 22:50:53 -0500 Subject: [PATCH] Open Terminal fleet for all accounts + OpenWebUI UAT pass (#610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit open-terminal 0.11.34 bare-metal (human-authorized host install) with a systemd template unit: reachableceo on tailscale-only :30000 (cwd ~/projects) and all 8 other accounts on :30001-30008, each with its own key kept out of ps in per-user TOMLs, all users added to the docker group. Keys live in ~/.creds + /etc/ukrrs/open-terminal, never in this repo. UAT: throwaway OpenWebUI v0.11.1 (matched to the human's Cloudron prod) driven purely over its HTTP API against the beta gateway lane - connection verify/config PASS, direct exec as uid 1001 PASS, and the chat round trip PASS: the model emitted run_command, we executed it through OpenWebUI's terminal proxy (server holds the key), and the final answer named 9/9 real project directories; earlier negative runs prove the test catches hallucinated output. Ops note OPEN-TERMINAL.md documents production wiring (prod openwebui container v0.3.10 is too old and needs an upgrade first). Also tonight: agent-stack relaunchers removed (6-, backups kept) so no screen/crush sessions auto-start on reboot; 22:00 night-profile flip observed live; docs synced for the earlier teardown. ๐Ÿ’˜ Generated with Crush Assisted-by: Crush:glm-5.2 [#610] --- .gitignore | 1 + 6-remove-agent-stacks.out | 19 ++ 7-open-terminal-install.out | 82 +++++++ 7-open-terminal-install.sh | 86 +++++++ 8-open-terminal-users.out | 18 ++ 8-open-terminal-users.sh | 68 ++++++ 9-uat-openwebui.out | 50 +++++ 9-uat-openwebui.sh | 170 ++++++++++++++ NEXT.md | 21 +- OPEN-TERMINAL.md | 81 +++++++ README.md | 9 +- TRACKING.md | 4 + night-flip-20260831.out | 10 + removed-agent-stacks/launch-rceo-stack.sh | 41 ++++ .../reachableceo-agent-stack.service | 14 ++ removed-agent-stacks/setup-rceo-stack.sh | 209 ++++++++++++++++++ removed-agent-stacks/tsg-agent-stacks.service | 14 ++ removed-agent-stacks/tsg-supervisor.service | 7 + removed-agent-stacks/tsg-supervisor.timer | 10 + uat-sse-last.log | 6 + uat/docker-compose.yml | 25 +++ 21 files changed, 940 insertions(+), 5 deletions(-) create mode 100644 6-remove-agent-stacks.out create mode 100644 7-open-terminal-install.out create mode 100755 7-open-terminal-install.sh create mode 100644 8-open-terminal-users.out create mode 100755 8-open-terminal-users.sh create mode 100644 9-uat-openwebui.out create mode 100755 9-uat-openwebui.sh create mode 100644 OPEN-TERMINAL.md create mode 100755 removed-agent-stacks/launch-rceo-stack.sh create mode 100644 removed-agent-stacks/reachableceo-agent-stack.service create mode 100755 removed-agent-stacks/setup-rceo-stack.sh create mode 100644 removed-agent-stacks/tsg-agent-stacks.service create mode 100644 removed-agent-stacks/tsg-supervisor.service create mode 100644 removed-agent-stacks/tsg-supervisor.timer create mode 100644 uat-sse-last.log create mode 100644 uat/docker-compose.yml diff --git a/.gitignore b/.gitignore index 0e494a4..f2e5913 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .crush/ +uat/.env diff --git a/6-remove-agent-stacks.out b/6-remove-agent-stacks.out new file mode 100644 index 0000000..10f9331 --- /dev/null +++ b/6-remove-agent-stacks.out @@ -0,0 +1,19 @@ +== remove agent stacks 2026-08-31T21:58:26-05:00 == +Removed '/etc/systemd/system/timers.target.wants/tsg-supervisor.timer'. +Removed '/etc/systemd/system/multi-user.target.wants/reachableceo-agent-stack.service'. +removed unit: tsg-supervisor.timer +removed unit: tsg-supervisor.service +removed unit: tsg-agent-stacks.service +removed unit: reachableceo-agent-stack.service +moved launcher: /usr/local/sbin/launch-rceo-stack.sh +moved launcher: /usr/local/sbin/setup-rceo-stack.sh +No screen session found. +root screen ukrrs-gateway-final: not present +-- cron cross-check (expect clean) -- +root crontab: clean +cron dirs: clean +-- verify: units gone, timer list, remaining screens -- +no stack/supervisor units remain +No Sockets found in /run/screen/S-root. + +== done 2026-08-31T21:58:29-05:00 โ€” rollback: restore /home/reachableceo/projects/ultix/removed-agent-stacks + re-enable units == diff --git a/7-open-terminal-install.out b/7-open-terminal-install.out new file mode 100644 index 0000000..4709dd7 --- /dev/null +++ b/7-open-terminal-install.out @@ -0,0 +1,82 @@ +== open-terminal install 2026-08-31T22:04:33-05:00 == +Downloading cpython-3.12.14-linux-x86_64-gnu (download) (32.5MiB) + Downloaded cpython-3.12.14-linux-x86_64-gnu (download) +Ignoring existing environment for `open-terminal`: the requested Python interpreter does not match the environment interpreter +Resolved 71 packages in 25ms +Installed 71 packages in 228ms + + aiofiles==25.1.0 + + annotated-doc==0.0.5 + + annotated-types==0.8.0 + + anyio==4.14.2 + + asttokens==3.0.2 + + attrs==26.1.0 + + certifi==2026.7.22 + + click==8.5.0 + + comm==0.2.3 + + debugpy==1.8.21 + + et-xmlfile==2.0.0 + + executing==2.2.1 + + fastapi==0.141.1 + + fastjsonschema==2.22.2 + + h11==0.16.0 + + httpcore==1.0.9 + + httptools==0.8.0 + + httpx==0.28.1 + + idna==3.19 + + ipykernel==7.3.0 + + ipython==9.17.0 + + ipython-pygments-lexers==1.1.1 + + jedi==0.20.0 + + jsonschema==4.26.0 + + jsonschema-specifications==2025.9.1 + + jupyter-client==8.10.0 + + jupyter-core==5.9.1 + + lxml==6.1.2 + + matplotlib-inline==0.2.2 + + nbclient==0.11.0 + + nbformat==5.11.1 + + nest-asyncio2==1.7.2 + + open-terminal==0.11.34 + + openpyxl==3.1.5 + + packaging==26.3 + + parso==0.8.7 + + pexpect==4.9.0 + + pillow==12.3.0 + + platformdirs==4.11.5 + + prompt-toolkit==3.0.53 + + psutil==7.2.2 + + ptyprocess==0.7.0 + + pure-eval==0.2.3 + + pydantic==2.13.5 + + pydantic-core==2.46.5 + + pygments==2.21.0 + + pypdf==6.16.2 + + python-dateutil==2.9.0.post0 + + python-docx==1.2.0 + + python-dotenv==1.2.3 + + python-multipart==0.0.32 + + python-pptx==1.0.2 + + pyyaml==6.0.3 + + pyzmq==27.2.0 + + referencing==0.37.0 + + rpds-py==2026.6.3 + + six==1.17.0 + + stack-data==0.6.3 + + starlette==1.6.0 + + striprtf==0.0.33 + + tornado==6.5.8 + + traitlets==5.16.1 + + typing-extensions==4.16.0 + + typing-inspection==0.4.4 + + uvicorn==0.52.4 + + uvloop==0.22.1 + + watchfiles==1.2.0 + + wcwidth==0.8.3 + + websockets==17.1 + + xlrd==2.0.2 + + xlsxwriter==3.2.9 +Installed 1 executable: open-terminal +open-terminal 0.11.34 ok +-- verify -- +unit: active +== done 2026-08-31T22:04:42-05:00 == diff --git a/7-open-terminal-install.sh b/7-open-terminal-install.sh new file mode 100755 index 0000000..95a2b8e --- /dev/null +++ b/7-open-terminal-install.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# 7-open-terminal-install.sh โ€” OpenWebUI Open Terminal for reachableceo (#610). +# Bare-metal (human-authorized one-case host install, ruling 2026-08-31): +# - binary via uv tool install (pinned open-terminal==0.11.34, managed py3.12) +# โ†’ /usr/local/bin/open-terminal, venvs under /opt/uv-tools +# - per-user TOML config keeps the API key OUT of ps/cmdline (docs' advice) +# - binds TAILSCALE ONLY (100.101.187.119, human ruling 2026-08-31 night): +# reachable from openwebui containers + tailscale clients, never the LAN +# - systemd template unit open-terminal@.service โ†’ instance per account +# - this script creates the reachableceo instance: port 30000, cwd ~/projects +# - key copy for the human lands in ~/.creds/open-terminal.env (0600) +# Self-elevating. Run: ~/projects/ultix/7-open-terminal-install.sh +set -euo pipefail +[ "$EUID" -eq 0 ] || exec sudo bash "$0" + +OUT=/home/reachableceo/projects/ultix/7-open-terminal-install.out +exec > >(tee "$OUT") 2>&1 +echo "== open-terminal install $(date -Is) ==" + +UV=/home/reachableceo/.local/bin/uv +VER=0.11.34 +UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ + UV_PYTHON_INSTALL_DIR=/opt/uv-python \ + $UV tool install --python 3.12 "open-terminal==$VER" \ + || UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ + UV_PYTHON_INSTALL_DIR=/opt/uv-python $UV tool install --reinstall --python 3.12 "open-terminal==$VER" +chmod -R a+rX /opt/uv-tools /opt/uv-python +/usr/local/bin/open-terminal --help >/dev/null && echo "open-terminal $VER ok" + +mkdir -p /etc/ukrrs/open-terminal /etc/systemd/system + +KEY=$(openssl rand -hex 32) +cat > /etc/ukrrs/open-terminal/reachableceo.toml < /etc/systemd/system/open-terminal@.service <<'EOF' +[Unit] +Description=Open Terminal (%i) +Documentation=https://docs.openwebui.com/features/open-terminal/ +After=network-online.target tailscaled.service +Wants=network-online.target + +[Service] +Type=simple +User=%i +EnvironmentFile=/etc/ukrrs/open-terminal/%i.env +ExecStart=/usr/local/bin/open-terminal run --config /etc/ukrrs/open-terminal/%i.toml --cwd ${OPEN_TERMINAL_CWD} +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +printf 'OPEN_TERMINAL_CWD=/home/reachableceo/projects\n' \ + > /etc/ukrrs/open-terminal/reachableceo.env +chown root:reachableceo /etc/ukrrs/open-terminal/reachableceo.env +chmod 640 /etc/ukrrs/open-terminal/reachableceo.env + +systemctl daemon-reload +systemctl enable --now open-terminal@reachableceo + +CREDS=/home/reachableceo/.creds/open-terminal.env +{ + echo "# Open Terminal fleet โ€” added 2026-08-31 (#610). 0600, never repo." + echo "REACHABLECEO_URL=http://100.101.187.119:30000" + echo "REACHABLECEO_API_KEY=$KEY" +} > "$CREDS" +chown reachableceo:reachableceo "$CREDS" +chmod 600 "$CREDS" + +sleep 2 +for _ in 1 2 3 4 5 6 7 8; do + STATE=$(systemctl is-active open-terminal@reachableceo || true) + [ "$STATE" = active ] && break + sleep 2 +done +echo "-- verify --" +echo "unit: $STATE" +curl -s -m 10 http://100.101.187.119:30000/health && echo +echo "== done $(date -Is) ==" diff --git a/8-open-terminal-users.out b/8-open-terminal-users.out new file mode 100644 index 0000000..6a27946 --- /dev/null +++ b/8-open-terminal-users.out @@ -0,0 +1,18 @@ +== open-terminal per-user install 2026-08-31T22:05:03-05:00 == +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@TSGCOO.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +TSGCOO port 30001 cwd /home/TSGCOO/projects active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@TSGCCO.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +TSGCCO port 30002 cwd /home/TSGCCO active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@TSGCTO.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +TSGCTO port 30003 cwd /home/TSGCTO/projects active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@TSGBOD.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +TSGBOD port 30004 cwd /home/TSGBOD active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@COS-RCEO.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +COS-RCEO port 30005 cwd /home/COS-RCEO active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@COS-WFO.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +COS-WFO port 30006 cwd /home/COS-WFO active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@COS-TSG.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +COS-TSG port 30007 cwd /home/COS-TSG active unreachable +Created symlink '/etc/systemd/system/multi-user.target.wants/open-terminal@reachableceo-offstage.service' โ†’ '/etc/systemd/system/open-terminal@.service'. +reachableceo-offstage port 30008 cwd /home/reachableceo-offstage/projects active unreachable +== done 2026-08-31T22:05:18-05:00 == diff --git a/8-open-terminal-users.sh b/8-open-terminal-users.sh new file mode 100755 index 0000000..40b71bf --- /dev/null +++ b/8-open-terminal-users.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# 8-open-terminal-users.sh โ€” per-user Open Terminal instances (#610). +# For each account: docker group membership (human ruling 2026-08-31) + +# own open-terminal instance on 30001+, own API key, TAILSCALE-ONLY bind +# (100.101.187.119, human ruling 2026-08-31 night), cwd ~/projects if it +# exists else $HOME. Wired for reboot via open-terminal@.service. +# Ports: TSGCOO 30001 TSGCCO 30002 TSGCTO 30003 TSGBOD 30004 +# COS-RCEO 30005 COS-WFO 30006 COS-TSG 30007 reachableceo-offstage 30008 +# Keys append to ~/.creds/open-terminal.env (reachableceo, 0600). +# Run AFTER 7-open-terminal-install.sh. Self-elevating. +# ~/projects/ultix/8-open-terminal-users.sh +set -euo pipefail +[ "$EUID" -eq 0 ] || exec sudo bash "$0" + +OUT=/home/reachableceo/projects/ultix/8-open-terminal-users.out +exec > >(tee "$OUT") 2>&1 +echo "== open-terminal per-user install $(date -Is) ==" + +CREDS=/home/reachableceo/.creds/open-terminal.env + +while read -r USER PORT; do + [ -z "$USER" ] && continue + if ! getent passwd "$USER" >/dev/null; then + echo "SKIP $USER (no such account)" + continue + fi + HOME_DIR=$(getent passwd "$USER" | cut -d: -f6) + if [ -d "$HOME_DIR/projects" ]; then CWD="$HOME_DIR/projects"; else CWD="$HOME_DIR"; fi + + usermod -aG docker "$USER" + + KEY=$(openssl rand -hex 32) + cat > "/etc/ukrrs/open-terminal/$USER.toml" < "/etc/ukrrs/open-terminal/$USER.env" + chown "root:$USER" "/etc/ukrrs/open-terminal/$USER.env" + chmod 640 "/etc/ukrrs/open-terminal/$USER.env" + + systemctl enable --now "open-terminal@$USER" + + URLVAR=$(echo "$USER" | tr '[:lower:]-' '[:upper:]_') + echo "${URLVAR}_URL=http://100.101.187.119:$PORT" >> "$CREDS" + echo "${URLVAR}_API_KEY=$KEY" >> "$CREDS" + + sleep 1 + STATE=$(systemctl is-active "open-terminal@$USER" || true) + HEALTH=$(curl -s -m 5 "http://100.101.187.119:$PORT/health" || echo unreachable) + printf '%-24s port %s cwd %s %s %s\n' "$USER" "$PORT" "$CWD" "$STATE" "$HEALTH" +done <<'LIST' +TSGCOO 30001 +TSGCCO 30002 +TSGCTO 30003 +TSGBOD 30004 +COS-RCEO 30005 +COS-WFO 30006 +COS-TSG 30007 +reachableceo-offstage 30008 +LIST + +chown reachableceo:reachableceo "$CREDS" +chmod 600 "$CREDS" +echo "== done $(date -Is) ==" diff --git a/9-uat-openwebui.out b/9-uat-openwebui.out new file mode 100644 index 0000000..95287da --- /dev/null +++ b/9-uat-openwebui.out @@ -0,0 +1,50 @@ +[22:47:32] P0 uat/.env written (beta gateway http://127.0.0.1:4002/v1, model glm-4.7) + Container ukrrs-openwebui-uat Stopping + Container ukrrs-openwebui-uat Stopped + Container ukrrs-openwebui-uat Removing + Container ukrrs-openwebui-uat Removed + Volume openwebui-uat_openwebui-uat Removing + Volume openwebui-uat_openwebui-uat Removed + Volume openwebui-uat_openwebui-uat Creating + Volume openwebui-uat_openwebui-uat Creating + Volume openwebui-uat_openwebui-uat Created + Volume openwebui-uat_openwebui-uat Created + Container ukrrs-openwebui-uat Creating + Container ukrrs-openwebui-uat Created + Container ukrrs-openwebui-uat Starting + Container ukrrs-openwebui-uat Started +[22:48:29] P1 PASS: openwebui v0.11.1 healthy on 127.0.0.1:8081 +[22:48:29] P1 PASS: glm-4.7 reachable via beta gateway +[22:48:30] P2 PASS: admin signup (role admin) +[22:48:30] P3 verify: {"status":true,"type":"terminal"} +[22:48:30] P3 PASS: terminal server verified (terminal) +[22:48:30] P3 set connections: {"TERMINAL_SERVER_CONNECTIONS":[{"id":"ultix-rceo","name":"ultix-rceo","enabled":true,"url":"http://100.101.187.119:30000","path":"/openapi.json","key":"***","auth_type":"bearer","config":null,"server_type":null,"policy_id":null}]} +[22:48:30] P3 user terminal list: [{"id":"ultix-rceo","url":"http://100.101.187.119:30000","name":"ultix-rceo","contexts":{},"config":{"chat_uploads":"default"}}] +[22:48:30] P3 PASS: connection ultix-rceo visible to users +[22:48:30] P4 exec result: {"id":"20260831-224830-489a48","command":"id && ls -1 /home/reachableceo/projects","status":"done","exit_code":0,"output":[{"type":"output","data":"uid=1001(reachableceo) gid=1001(reachableceo) groups=1001(reachableceo),100(users),118(libvirt),988(docker)\r\n"},{"type":"output","data":"dotfiles\r\nKNEL\r\nPFVCluster\r\nreference\r\nSITER-Solar\r\nTSYSDevStack-SupportStack-LocalWorkstation\r\nukrrs\r\nultix\r\nWorkstationStack\r\n"}],"truncated":false,"next_offset":2,"log_path":"/home/reachableceo/.local/state/open-terminal/logs/processes/20260831-224830-489a48.jsonl"} +[22:48:30] P4 PASS: ran as reachableceo +[22:48:30] P4 PASS: sees ~/projects/ultix +[22:48:33] P5 turn1: model emitted 1 tool call(s): [{"id":"call_-7296378025417892855","name":"run_command"}] +[22:48:33] P5 executing via openwebui terminal proxy: ls -1 /home/reachableceo/projects +[22:48:33] P5 proxy exec: {"id":"20260831-224833-94630c","command":"ls -1 /home/reachableceo/projects\n","status":"done","exit_code":0,"output":[{"type":"output","data":"dotfiles\r\nKNEL\r\nPFVCluster\r\nreference\r\nSITER-Solar\r\nTSYSDevStack-SupportStack-LocalWorkstation\r\nukrrs\r\nultix\r\nWorkstationStack\r\n"}],"truncated":false,"next_offset":1,"log_path":"/home/reachableceo/.local/state/open-terminal/logs/processes/20260831-224833-94630c.jsonl"} +[22:48:33] P5 PASS: proxy-executed command returned real ~/projects content +[22:48:33] P5 turn2 body: valid JSON +[22:49:01] P5 turn2 raw (first 400): {"id":"2026090111483398401ef2c2a6440b","object":"chat.completion","created":1788234541,"model":"glm-4.7","choices":[{"index":0,"message":{"role":"assistant","content":"dotfiles\nKNEL\nPFVCluster\nreference\nSITER-Solar\nTSYSDevStack-SupportStack-LocalWorkstation\nukrrs\nultix\nWorkstationStack","reasoning_content":"The user wants me to run a specific command in a terminal-like environment (simulat +[22:49:01] P5 final answer: dotfiles +KNEL +PFVCluster +reference +SITER-Solar +TSYSDevStack-SupportStack-LocalWorkstation +ukrrs +ultix +WorkstationStack +[22:49:01] P5 PASS: final model answer contains 9/9 real directory names (full round trip: tool_call -> proxy exec on terminal -> real output -> answer) + Container ukrrs-openwebui-uat Stopping + Container ukrrs-openwebui-uat Stopped + Container ukrrs-openwebui-uat Removing + Container ukrrs-openwebui-uat Removed + Volume openwebui-uat_openwebui-uat Removing + Volume openwebui-uat_openwebui-uat Removed +[22:49:05] teardown complete (project+volume removed) +[22:49:05] EXIT=0 ALL PHASES PASSED diff --git a/9-uat-openwebui.sh b/9-uat-openwebui.sh new file mode 100755 index 0000000..fc1a380 --- /dev/null +++ b/9-uat-openwebui.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# 9-uat-openwebui.sh โ€” OpenWebUI <-> Open Terminal UAT (#610). +# Throwaway OpenWebUI v0.11.1 (matches the human's Cloudron prod), driven +# entirely over its real HTTP API: +# P1 stack up + gateway models visible (beta lane) +# P2 admin signup +# P3 terminal connection: POST /api/v1/configs/terminal_servers + +# /verify + user list via /api/v1/terminals/ (v0.11.1 contracts) +# P4 DIRECT open-terminal API: async /execute + poll -> id shows +# uid=1001(reachableceo), ls shows ~/projects entries +# P5 CHAT: /api/chat/completions with metadata.terminal_id set; the model +# must RUN the read-only ls through the terminal and answer with REAL +# directory names (hallucination = FAIL) +# Teardown at end (KEEP=1 leaves it up). No sudo needed. +# Run: ~/projects/ultix/9-uat-openwebui.sh +set -uo pipefail + +cd "$(dirname "$0")" +OUT=$PWD/9-uat-openwebui.out +: > "$OUT" +log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$OUT"; } +fail() { log "FAIL: $*"; EXIT=1; } +EXIT=0 + +GW_URL=http://127.0.0.1:4002/v1 +GW_KEY=$(grep -oP '(?<=--api-key ")[^"]+' ~/.config/crush-gw-beta/crushrc | head -1) +OT_URL=http://100.101.187.119:30000 +OT_KEY=$(grep -oP '(?<=REACHABLECEO_API_KEY=).*' ~/.creds/open-terminal.env) +EXPECTED_DIRS=$(ls -1 /home/reachableceo/projects | grep -E '^[a-zA-Z0-9._-]+$' | sort -u) +CONN_ID=ultix-rceo +[ -n "$GW_KEY" ] && [ -n "$OT_KEY" ] || { log "FAIL: missing creds"; exit 1; } + +WEBUI=http://127.0.0.1:8081 +EMAIL=uat@ultix.local +ADMIN_PASS=$(openssl rand -hex 12) + +# ---- P0/P1 ---- +umask 077 +cat > uat/.env <>"$OUT" 2>&1 || true +docker compose -p openwebui-uat -f uat/docker-compose.yml up -d >>"$OUT" 2>&1 +HEALTH=no +for i in $(seq 1 60); do + sleep 2 + [ "$(timeout 5 curl -s -o /dev/null -w '%{http_code}' $WEBUI/health 2>/dev/null)" = 200 ] && { HEALTH=yes; break; } +done +[ "$HEALTH" = yes ] || { fail "P1 openwebui never healthy"; docker logs ukrrs-openwebui-uat --tail 40 >>"$OUT" 2>&1; exit 1; } +log "P1 PASS: openwebui v0.11.1 healthy on 127.0.0.1:8081" +MODELS=$(timeout 20 curl -s -H "Authorization: Bearer $GW_KEY" "$GW_URL/models" | jq -r '[.data[].id] | join(",")' 2>/dev/null) +echo "$MODELS" | grep -q glm-4.7 && log "P1 PASS: glm-4.7 reachable via beta gateway" || fail "P1 glm-4.7 missing: $MODELS" + +# ---- P2 ---- +SIGNUP=$(timeout 20 curl -s -X POST $WEBUI/api/v1/auths/signup \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"UAT\",\"email\":\"$EMAIL\",\"password\":\"$ADMIN_PASS\"}") +TOKEN=$(echo "$SIGNUP" | jq -r .token 2>/dev/null) +{ [ -n "$TOKEN" ] && [ "$TOKEN" != null ]; } || { fail "P2 signup failed: $(echo "$SIGNUP" | head -c 200)"; exit 1; } +log "P2 PASS: admin signup (role $(echo "$SIGNUP" | jq -r .role))" + +# ---- P3 ---- +VERIFY=$(timeout 20 curl -s -X POST $WEBUI/api/v1/configs/terminal_servers/verify \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d "{\"url\":\"$OT_URL\",\"key\":\"$OT_KEY\"}") +log "P3 verify: $VERIFY" +echo "$VERIFY" | jq -e '.status == true' >/dev/null 2>&1 && log "P3 PASS: terminal server verified ($(echo "$VERIFY" | jq -r .type))" || fail "P3 verify did not return status:true" + +SETCONN=$(timeout 20 curl -s -X POST $WEBUI/api/v1/configs/terminal_servers \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d "{\"TERMINAL_SERVER_CONNECTIONS\":[{\"id\":\"$CONN_ID\",\"name\":\"$CONN_ID\",\"url\":\"$OT_URL\",\"key\":\"$OT_KEY\",\"auth_type\":\"bearer\",\"enabled\":true}]}" \ + | sed -E 's/"key":"[^"]*"/"key":"***"/g') +log "P3 set connections: $(echo "$SETCONN" | head -c 400)" + +LIST=$(timeout 20 curl -s $WEBUI/api/v1/terminals/ -H "Authorization: Bearer $TOKEN") +log "P3 user terminal list: $(echo "$LIST" | head -c 300)" +echo "$LIST" | jq -e --arg id "$CONN_ID" '[.[] | select(.id == $id)] | length == 1' >/dev/null 2>&1 \ + && log "P3 PASS: connection $CONN_ID visible to users" || fail "P3 connection not listed: $LIST" + +# ---- P4: direct async exec through open-terminal's own API ---- +EXEC=$(timeout 60 curl -s -X POST "$OT_URL/execute?wait=30" \ + -H "Authorization: Bearer $OT_KEY" -H 'Content-Type: application/json' \ + -d '{"command":"id && ls -1 /home/reachableceo/projects"}') +log "P4 exec result: $(echo "$EXEC" | head -c 900)" +OUTTXT=$(echo "$EXEC" | jq -r '[.output[]?.data] | join("")' 2>/dev/null | tr -d '\r') +PID=$(echo "$EXEC" | jq -r .id 2>/dev/null) +if [ -z "$OUTTXT" ] && [ -n "${PID:-}" ] && [ "$PID" != null ]; then + ST2=$(timeout 40 curl -s "$OT_URL/execute/$PID/status?wait=30&offset=0" -H "Authorization: Bearer $OT_KEY") + OUTTXT=$(echo "$ST2" | jq -r '[.output[]?.data] | join("")' 2>/dev/null) + log "P4 status fallback: $(echo "$ST2" | head -c 900)" +fi +echo "$OUTTXT" | grep -q 'uid=1001(reachableceo)' && log "P4 PASS: ran as reachableceo" || fail "P4 no uid=1001(reachableceo) in: $OUTTXT" +echo "$OUTTXT" | grep -qx ultix && log "P4 PASS: sees ~/projects/ultix" || fail "P4 no ultix line in: $OUTTXT" + +# ---- P5: chat must run a read-only terminal command and return REAL output. +# OpenWebUI contract (v0.11.1): for API callers the tool loop runs client-side +# (server-side loop needs a socket session). So: 1) chat w/ terminal_id -> +# model emits run_command tool_call; 2) WE execute it through OpenWebUI's own +# terminal proxy (server holds the key); 3) feed the tool result back; 4) the +# model's final answer must name REAL ~/projects entries. +CHAT_ID=$(cat /proc/sys/kernel/random/uuid) +PROMPT="You have a terminal available (already selected for this chat). Use it to run exactly this read-only command: ls -1 /home/reachableceo/projects โ€” then reply with ONLY the directory names you saw, one per line. Do not guess; if the tool fails, say so." +TURN1=$(timeout 180 curl -sN -X POST $WEBUI/api/chat/completions \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d "{\"model\":\"glm-4.7\",\"stream\":true,\"terminal_id\":\"$CONN_ID\",\"chat_id\":\"$CHAT_ID\",\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rs . <<<"$PROMPT")}]}" \ + | sed -E 's/"key":"[^"]*"/"key":"***"/g') +TC_JSON=$(echo "$TURN1" | grep '^data: ' | sed 's/^data: //' | grep -v '^\[DONE\]' \ + | jq -rs 'map(try (.choices[0].delta.tool_calls // empty) // empty) | flatten | map(select(.function.name))' 2>/dev/null) +TC_COUNT=$(echo "$TC_JSON" | jq 'length' 2>/dev/null) +log "P5 turn1: model emitted $TC_COUNT tool call(s): $(echo "$TC_JSON" | jq -c '[.[] | {id, name: .function.name}]' 2>/dev/null)" +[ "${TC_COUNT:-0}" -ge 1 ] || { fail "P5 turn1 produced no tool call (model: $(echo "$TURN1" | grep -o '"content":"[^"]*"' | head -c 300))"; } + +TOOL_RESULTS="[]" +if [ "${TC_COUNT:-0}" -ge 1 ]; then + CMD=$(echo "$TC_JSON" | jq -r '.[0].function.arguments' | jq -r '.command') + TCID=$(echo "$TC_JSON" | jq -r '.[0].id') + log "P5 executing via openwebui terminal proxy: $CMD" + PROXY=$(timeout 60 curl -s -X POST "$WEBUI/api/v1/terminals/$CONN_ID/execute?wait=30" \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d "{\"command\":$(jq -Rs . <<<"$CMD")}" ) + log "P5 proxy exec: $(echo "$PROXY" | head -c 500)" + TOOL_OUT=$(echo "$PROXY" | jq -r '[.output[]?.data] | join("")' 2>/dev/null | tr -d '\r') + echo "$TOOL_OUT" | grep -q ultix && log "P5 PASS: proxy-executed command returned real ~/projects content" || fail "P5 proxy execution did not return real output: $TOOL_OUT" + TOOL_RESULTS=$(jq -n --arg id "$TCID" --arg out "$TOOL_OUT" '[{role:"tool", tool_call_id:$id, content:$out}]') +fi + +if [ "${TC_COUNT:-0}" -ge 1 ]; then + TCID=$(echo "$TC_JSON" | jq -r '.[0].id') + TCNAME=$(echo "$TC_JSON" | jq -r '.[0].function.name') + TCARGS=$(echo "$TC_JSON" | jq -c '.[0].function.arguments') + BODY=$(cat </dev/null 2>&1 && log "P5 turn2 body: valid JSON" || log "P5 turn2 body: INVALID: $(echo "$BODY" | head -c 500)" + TURN2=$(timeout 180 curl -s -X POST $WEBUI/api/chat/completions \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d "$BODY" \ + | sed -E 's/"key":"[^"]*"/"key":"***"/g') + log "P5 turn2 raw (first 400): $(echo "$TURN2" | head -c 400)" + CONTENT=$(echo "$TURN2" | jq -r '.choices[0].message.content // empty' 2>/dev/null) + log "P5 final answer: $(echo "$CONTENT" | head -c 500)" + HITS=0; MISSED="" + for d in $EXPECTED_DIRS; do + if echo "$CONTENT" | grep -q "$d"; then HITS=$((HITS+1)); else MISSED="$MISSED $d"; fi + done + if [ -n "$CONTENT" ] && [ "$HITS" -ge 3 ]; then + log "P5 PASS: final model answer contains $HITS/$(echo $EXPECTED_DIRS | wc -w) real directory names (full round trip: tool_call -> proxy exec on terminal -> real output -> answer)" + else + fail "P5 final answer not grounded (hits=$HITS missed:$MISSED)" + log "P5 last openwebui log lines follow" + docker logs ukrrs-openwebui-uat --tail 25 >>"$OUT" 2>&1 + fi +fi + +# ---- teardown ---- +if [ "${KEEP:-0}" = 1 ]; then + log "KEEP=1 โ€” stack up at http://127.0.0.1:8081 (login $EMAIL / pass in .out if logged)" + echo "ADMIN_PASS=$ADMIN_PASS" >> "$OUT" +else + docker compose -p openwebui-uat -f uat/docker-compose.yml down -v >>"$OUT" 2>&1 + log "teardown complete (project+volume removed)" +fi +log "EXIT=$EXIT $([ $EXIT = 0 ] && echo 'ALL PHASES PASSED' || echo 'SEE FAIL LINES')" +exit $EXIT diff --git a/NEXT.md b/NEXT.md index 8968413..61a5c83 100644 --- a/NEXT.md +++ b/NEXT.md @@ -1,6 +1,23 @@ -# NEXT โ€” perf-opt closeout COMPLETE (2026-08-31 16:4x) +# NEXT โ€” perf-opt closeout COMPLETE (2026-08-31); #610 open-terminal fleet DONE -## Final state โ€” nothing owed +## Tonight (2026-08-31 late, crush had the con) +- Agent-stack relaunchers REMOVED (6-): no screen/crush sessions auto-start + on reboot, any account. Backups: removed-agent-stacks/. +- Open Terminal fleet LIVE (#610): 9 accounts, tailscale-only + 100.101.187.119:30000-30008, systemd open-terminal@, keys in + /etc/ukrrs/open-terminal/ + ~/.creds/open-terminal.env. Ops note: + OPEN-TERMINAL.md. UAT PASSED (9-uat-openwebui.out). +- 22:00 night flip verified live (see TRACKING decisions). +- HUMAN OWED: wire production OpenWebUI per OPEN-TERMINAL.md. The + openwebui CONTAINER on this box (v0.3.10, :3000) is too old โ€” upgrade to + 0.11.x first. Cloudron prod (0.11.1) is fine. +- LAST ACTION of the session: multiqueue bounce via + ~/projects/ultix/5-activate-multiqueue.sh (approved; kills crush + sessions ~4 min). After reboot verify: + ethtool -l ens18 -> Combined 4/4; ens19 -> 2/2; then + ~/projects/ultix/4-host-netcheck.sh for the full sweep. + +## Final state โ€” nothing owed (perf pass) - All guest tuning live and verified (see 16:04-16:15 pass below). - Host flags live: ssd=1/discard (ROTA=0), iothread, onboot=1, startup order=10, cpuunits 9000/50, ide2 gone. diff --git a/OPEN-TERMINAL.md b/OPEN-TERMINAL.md new file mode 100644 index 0000000..3ecdf06 --- /dev/null +++ b/OPEN-TERMINAL.md @@ -0,0 +1,81 @@ +# Open Terminal fleet โ€” ops note (#610, deployed 2026-08-31) + +OpenWebUI "Open Terminal" for every account on ultix-streaming, in computer +mode (bare metal, real shell). Wired to start on reboot. UAT passed end to +end the same night (see 9-uat-openwebui.out). + +## Layout + +| account | port | bind | cwd | unit | +|---|---|---|---|---| +| reachableceo | 30000 | 100.101.187.119 (tailscale ONLY) | ~/projects | open-terminal@reachableceo | +| TSGCOO | 30001 | ใ€ƒ | ~/projects | open-terminal@TSGCOO | +| TSGCCO | 30002 | ใ€ƒ | ~ (no ~/projects yet) | open-terminal@TSGCCO | +| TSGCTO | 30003 | ใ€ƒ | ~/projects | open-terminal@TSGCTO | +| TSGBOD | 30004 | ใ€ƒ | ~ | open-terminal@TSGBOD | +| COS-RCEO | 30005 | ใ€ƒ | ~ | open-terminal@COS-RCEO | +| COS-WFO | 30006 | ใ€ƒ | ~ | open-terminal@COS-WFO | +| COS-TSG | 30007 | ใ€ƒ | ~ | open-terminal@COS-TSG | +| reachableceo-offstage | 30008 | ใ€ƒ | ~/projects | open-terminal@reachableceo-offstage | + +- Binary: /usr/local/bin/open-terminal (open-terminal==0.11.34 via uv tool, + managed python at /opt/uv-python, venvs at /opt/uv-tools; world-readable + on purpose โ€” non-root units exec it). Host-install exception authorized by + human 2026-08-31 ("pip/uv directly on my host... mandatory"). +- Template unit: /etc/systemd/system/open-terminal@.service (After + tailscaled; Restart=always). Per-user TOML at + /etc/ukrrs/open-terminal/.toml (key lives THERE, not on any command + line โ€” nothing leaks via ps). Per-user cwd env alongside it. +- All 8 non-founder accounts were also added to the docker group (human + ruling 2026-08-31). +- Bind is 100.101.187.119 ONLY (tailscale interface; human ruling + 2026-08-31 night: no LAN, no 0.0.0.0). Containers reach it fine (bridge โ†’ + host IP routing); so does anything over tailscale. + +## API keys (NEVER in this repo) + +~/.creds/open-terminal.env (reachableceo, 0600): URL + key per account in +`_URL` / `_API_KEY` form. Rotate a key with: + sudo sed -i "s/^api_key = .*/api_key = \"$(openssl rand -hex 32)\"/" \ + /etc/ukrrs/open-terminal/.toml && sudo systemctl restart open-terminal@ +then update ~/.creds/open-terminal.env to match. + +## Wiring a production OpenWebUI (admin, server-side key) + +Your Cloudron prod runs Open WebUI 0.11.1 โ€” new enough. Admin UI: +Settings โ†’ Admin โ†’ Integrations โ†’ Open Terminal โ†’ "+": + URL: http://100.101.187.119:3000X (per-account port from the table) + API key: from ~/.creds/open-terminal.env + Auth: Bearer. Save โ†’ green "Connected". +API-only equivalent (what the UAT does): + POST /api/v1/configs/terminal_servers/verify {"url":โ€ฆ,"key":โ€ฆ} + POST /api/v1/configs/terminal_servers {"TERMINAL_SERVER_CONNECTIONS":[{"id":โ€ฆ,"url":โ€ฆ,"key":โ€ฆ,"auth_type":"bearer","enabled":true}]} +Scope per-user access via the connection's access control (each user gets +THEIR OWN port; do not share one terminal across accounts โ€” one shell each). +In chat: pick the terminal (cloud icon) once per conversation; native +function calling is already the default in 0.11.x. + +## UAT record (2026-08-31 22:48, all passed) + +Throwaway openwebui:v0.11.1 (matched to Cloudron prod) on 127.0.0.1:8081, +LLM via the beta gateway lane: +- P3 connection verify+list OK; P4 direct exec runs as uid 1001 with real + ~/projects output; P5 model emitted run_command, WE executed it through + OpenWebUI's terminal proxy (server holds the key), fed the result back, + and the model answered with 9/9 REAL directory names (no hallucination โ€” + earlier runs without the tool attached fabricated names, proving the test + catches it). Full log: 9-uat-openwebui.out. + +Gotcha learned: for raw API callers OpenWebUI returns tool_calls for the +CALLER to execute (the server-side loop needs a socket session โ€” that's the +UI path). UI chats run the loop automatically. + +## Hygiene notes + +- open-terminal keeps per-process logs in + ~/.local/state/open-terminal/logs/processes/ per user (unbounded; watch + when agents go wild). +- 0.0.0.0 was rejected by ruling; if tailscale is down at boot the units + retry (Restart=always) until the IP exists. +- Not in any ukrrs slice yet; fold into per-account slices when mkacct + lands (#607). diff --git a/README.md b/README.md index bf64035..ca39392 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,10 @@ are manual-only by ruling. - Scripts (human-run unless noted): 1-guest-prep.sh, 2-host-one-shot.sh, 3-post-reboot-fixes.sh, 4-host-netcheck.sh (crush may run via ssh), 5-activate-multiqueue.sh (the one remaining reboot), 6-remove-agent-stacks.sh - (kill boot-time screen/crush relaunchers); staged/ holds the gated configs - those scripts install. removed-agent-stacks/ holds unit/launcher backups - from 6-. + (boot-time screen/crush relauncher teardown; backups in + removed-agent-stacks/), 7-open-terminal-install.sh + 8-open-terminal-users.sh + (Open Terminal fleet, #610), 9-uat-openwebui.sh (OpenWebUI<->terminal UAT + harness; uat/ holds its compose). staged/ holds the gated configs. +- OPEN-TERMINAL.md โ€” ops note for the terminal fleet: ports, keys location, + production OpenWebUI wiring, UAT record. - crush.md = session preferences for agents working in this repo. diff --git a/TRACKING.md b/TRACKING.md index 8e1a77e..47ae066 100644 --- a/TRACKING.md +++ b/TRACKING.md @@ -14,10 +14,14 @@ One table, updated in place at each checkpoint. States: done โœ… / doing ๐Ÿ”„ / | OPT-8 | repos | โœ… | PMO backpressure design note in ukrrs/docs/harness; code = ticketed | | OPT-9 | integ | โณ | #604: node_exporter svc + gateway pressure scrape (Q10) | | OPT-10 | integ | โณ | #605: k8s join (Q11, blocked by #601) + proxmox-ctl token (Q12); GPUs #606 | +| #610 | fleet | โœ… | Open Terminal 9-account fleet (tailscale-only :30000-30008) + OpenWebUI UAT PASS | Inbox (mid-task interrupts): none. Decisions log (latest wins): +- 2026-08-31 (night, con handed to crush): #610 DONE in one session: open-terminal 0.11.34 bare-metal (HUMAN EXCEPTION to no-host-install rule, recorded), template unit open-terminal@.service, per-user TOML keys (never in ps/repo; ~/.creds holds copies), bind TAILSCALE-ONLY 100.101.187.119 per human ruling (0.0.0.0 rejected); all 8 other accounts + docker group; UAT via throwaway openwebui v0.11.1 (matches Cloudron prod) ALL PASS incl. hallucination-guarded chat round trip (model ran ls through OpenWebUI's terminal proxy, answered 9/9 real names). NOTE: prod openwebui container (v0.3.10, :3000) is too old for Open Terminal โ€” upgrade to 0.11.x before wiring; Cloudron 0.11.1 is fine. Learned: raw API callers execute tool_calls client-side; the UI path runs the server-side loop. +- 2026-08-31 (night): 22:00 night-profile flip OBSERVED LIVE from the guest (dirty_bytes 1Gโ†’2G, bg 256Mโ†’512M, /data2 readahead 128Kโ†’1M; day timer restores at 05:00). First runtime-only flip since install. +- 2026-08-31 (night): agent-stack relaunchers REMOVED by 6-remove-agent-stacks.sh (reachableceo-agent-stack.service, tsg-supervisor.timer/.service 5-min self-heal, dead tsg-agent-stacks.service; launchers + units backed up to removed-agent-stacks/). No screen/crush sessions auto-start on reboot anymore; human: stacks no longer used for pmo/work. - 2026-08-31 (night): docs synced post-pass: REPORT v1.1 (status banner, per-item STATUS tags, ยง6/ยง8 truth), RUNBOOK amended for the Oct replan, all paths ~/optimize โ†’ ~/projects/ultix (repo moved). Human ruling: system idle tonight โ†’ multiqueue bounce approved for this session, after remaining tasks. Agent-stack teardown prepped as 6-remove-agent-stacks.sh (kills boot-time RCEO/TSG screen respawn + the 5-min supervisor self-heal; unit/launcher backups to removed-agent-stacks/; root also quits leftover root screen ukrrs-gateway-final, leaves 'boot' alone). - 2026-08-31 (late): Redmine replan complete: #601 Oct window (due 10-31, High, version october-2026-window; Redmine API refuses start_date on update โ€” Oct-1 start recorded in the note instead), NEW #608 cgroup_parent wiring (High; THE starvation fix on 8 vCPU), NEW #609 Fri 09-04 troubleshooting-only outage (USB SDR reseat + socket/PSU/BIOS checks; NO upgrades; host down โ†’ this VM auto-starts via onboot=1, gateway auto-recovers). #605/#606 noted. docs repo pushed (main 9eb324e). - 2026-08-31 (eve): HARDWARE WINDOW โ†’ ~OCT 2026; presume 8 vCPU/48G operating values; cpuset reservation deferred โ†’ cgroup_parent wiring = the starvation fix; one VM-5111 reboot owed for multiqueue (5-activate-multiqueue.sh, human-timed; crush may roll OTHER VMs on pfv-tsys5 anytime). diff --git a/night-flip-20260831.out b/night-flip-20260831.out index b534ad5..b811f7e 100644 --- a/night-flip-20260831.out +++ b/night-flip-20260831.out @@ -1,3 +1,13 @@ == pre-flip baseline 2026-08-31 21:44 CDT == dirty_bytes=1073741824 dirty_bg=268435456 sdb_ra_kb=128 sdc_ra_kb=128 ukrrs cgroup dirs: NONE (batch/gateway slices enabled but never started) + +== sample 1: 2026-08-31T22:00:20-05:00 == +dirty_bytes=2147483648 dirty_bg=536870912 +sdb_ra_kb=1024 sdc_ra_kb=128 +/sys/fs/cgroup/ukrrs.slice + +== sample 2: 2026-08-31T22:01:05-05:00 == +dirty_bytes=2147483648 dirty_bg=536870912 +sdb_ra_kb=1024 sdc_ra_kb=128 +/sys/fs/cgroup/ukrrs.slice diff --git a/removed-agent-stacks/launch-rceo-stack.sh b/removed-agent-stacks/launch-rceo-stack.sh new file mode 100755 index 0000000..2e467f3 --- /dev/null +++ b/removed-agent-stacks/launch-rceo-stack.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Boot-time restore for the reachableceo PMO/Work stack (systemd unit: +# reachableceo-agent-stack.service). Self-sufficient at boot: re-preps +# /run/screen + screen setuid (tmpfs resets each boot), then respawns any +# dead screens via the spawner and injects intro prompts into fresh +# spawns only. Idempotent. +set -uo pipefail + +SETUP=/usr/local/sbin/setup-rceo-stack.sh +USR=reachableceo +PREFIX=RCEO +FOUNDER_ACCT=reachableceo +SRC_CRUSH=/home/TSGCOO/.local/share/crush + +mkdir -p /run/screen +chmod 755 /run/screen +[[ -u /usr/bin/screen ]] || chmod u+s /usr/bin/screen + +out="$(bash "$SETUP" "$USR" "$FOUNDER_ACCT" "$SRC_CRUSH" "$PREFIX" 2>&1)" || true +echo "$out" + +declare -A SPAWNED +for s in "${PREFIX}-PMO" "${PREFIX}-Work"; do + [[ "$out" == *"spawned: $s"* ]] && SPAWNED["$s"]=1 +done + +if [[ ${#SPAWNED[@]} -eq 0 ]]; then + echo "no sessions needed respawn" + exit 0 +fi + +echo "settling 20s before intro injection..." +sleep 20 +for s in "${!SPAWNED[@]}"; do + if [[ "$s" == *-PMO ]]; then intro=INTRO-PMO.md; else intro=INTRO-WORK.md; fi + msg="Please read ~/.coordinate/prompts/${intro} and treat it as your introduction from the founder, then follow it."$'\r' + runuser -u "$USR" -- env HOME="/home/$USR" TERM=xterm-256color \ + screen -S "$s" -X stuff "$msg" \ + && echo "injected: $USR/$s ($intro)" \ + || echo "INJECT FAILED: $USR/$s" +done diff --git a/removed-agent-stacks/reachableceo-agent-stack.service b/removed-agent-stacks/reachableceo-agent-stack.service new file mode 100644 index 0000000..3480aa0 --- /dev/null +++ b/removed-agent-stacks/reachableceo-agent-stack.service @@ -0,0 +1,14 @@ +[Unit] +Description=reachableceo PMO/Work agent stack (screen/crush) +After=local-fs.target network-online.target systemd-user-sessions.service +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +KillMode=process +TimeoutStartSec=180 +ExecStart=/usr/bin/bash /usr/local/sbin/launch-rceo-stack.sh + +[Install] +WantedBy=multi-user.target diff --git a/removed-agent-stacks/setup-rceo-stack.sh b/removed-agent-stacks/setup-rceo-stack.sh new file mode 100755 index 0000000..0d99cbf --- /dev/null +++ b/removed-agent-stacks/setup-rceo-stack.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# setup-rceo-stack.sh โ€” per-account PMO/Work stack spawner for the +# reachableceo stack on ULTIX-STREAMING. ROOT-run, idempotent, PROTOCOL v2 +# (headless dispatch, founder gate, doorbell-only stuff). +# +# Coordinate files are generated ONCE and never overwritten afterwards +# (they are founder/agent-owned); a file still carrying __TEMPLATE__ +# placeholders is debris from a failed run and gets regenerated. +# +# Usage: setup-rceo-stack.sh [PREFIX] +# PREFIX defaults to USER. Screens: -PMO / -Work. +set -euo pipefail + +USR="${1:?usage: setup-rceo-stack.sh [PREFIX]}" +FOUNDER_ACCT="${2:?founder account for screen acl}" +SRC_CRUSH="${3:?source dir with crush.json/providers.json}" +PREFIX="${4:-$USR}" +CWD="/home/${USR}" +HOME_U="$CWD" +PMO_SCREEN="${PREFIX}-PMO" +WORKSCREEN="${PREFIX}-Work" + +id "$USR" >/dev/null 2>&1 || { echo "no such user: $USR"; exit 1; } + +if ! id -nG "$USR" | tr ' ' '\n' | grep -qx docker; then + usermod -aG docker "$USR" + echo "groups: added docker" +fi + +rc="$HOME_U/.screenrc" +[[ -f "$rc" ]] && cp -a "$rc" "${rc}.bak-$(date +%Y%m%d-%H%M%S)" +touch "$rc" +grep -q '^multiuser on' "$rc" || printf 'multiuser on\n' >> "$rc" +grep -q "^acladd ${FOUNDER_ACCT}" "$rc" || printf 'acladd %s\n' "$FOUNDER_ACCT" >> "$rc" +grep -q '^defscrollback 10000' "$rc" || printf 'defscrollback 10000\n' >> "$rc" +chown "${USR}:${USR}" "$rc" +chmod 644 "$rc" +echo "screenrc: merged, never overwritten (backup kept if pre-existing)" + +DST_CRUSH="$HOME_U/.local/share/crush" +mkdir -p "$DST_CRUSH" +for f in crush.json providers.json; do + if [[ ! -f "$DST_CRUSH/$f" && -f "$SRC_CRUSH/$f" ]]; then + cp "$SRC_CRUSH/$f" "$DST_CRUSH/$f" + chown "${USR}:${USR}" "$DST_CRUSH/$f" + chmod 600 "$DST_CRUSH/$f" + echo "crush: copied $f" + fi +done + +CC="$HOME_U/.coordinate" +mkdir -p "$CC"/{inbox-work,inbox-pmo,archive,prompts,logs} + +fresh() { + [[ ! -f "$1" ]] || grep -qE '__(USR|PREFIX|WORKSCREEN|FOUNDER|DATE)__' "$1" +} +sub() { + sed -i -e "s/__USR__/${USR}/g" -e "s/__PREFIX__/${PREFIX}/g" \ + -e "s/__WORKSCREEN__/${WORKSCREEN}/g" \ + -e "s/__FOUNDER__/${FOUNDER_ACCT}/g" \ + -e "s/__DATE__/$(date +%F)/g" "$1" +} + +if fresh "$CC/PROTOCOL.md"; then +cat > "$CC/PROTOCOL.md" <<'EOF' +# __USR__ .coordinate โ€” PMO/Work scratch channel + +Local scratch under $HOME: not git-tracked, not durable. Durable artifacts +(tickets/code/docs/decisions) go to the systems of record the founder +attaches to this stack; until then they stay in ~/.coordinate. + +Layout: +- inbox-work/ PMO writes TASK---.md (CANCEL- to drop). +- inbox-pmo/ Work writes REPORT-*.md, QUESTION-*.md, BLOCKED-*.md. +- archive/ handled items; prune >30 days. +- BOARD.md roster + standing state. log.md append-only journal + (date -Is | WHO | what). prompts/ intros. logs/ worker output. + +TASK fields: Objective / Context / Constraints / Deliverables / Priority. + +Turn-start checklist (both agents, every turn): read BOARD.md, scan your +inbox, tail log.md. + +Rules: +- PMO plans, prioritizes, writes TASKs, reports to the founder on drop-in. +- FOUNDER GATE: PMO never dispatches without explicit founder approval of + that TASK. Draft -> present -> founder says go -> dispatch. Every time. +- Work executes in priority order; one turn = do work, file REPORT, archive + the TASK. Never self-assign scope (QUESTION instead). +- Empty inbox when pinged: say so, stop. No watchers/polling loops. +- No sudo for agents; root needs go BLOCKED -> PMO -> founder. +- Never kill or attach to the other agent's screen. + +Dispatch (headless; never screen-stuff task text; ping success-gated): + screen -dmS work-turn bash -c 'crush run --quiet --session \ + "$(cat ~/.coordinate/inbox-work/TASK-*.md)" > ~/.coordinate/logs/work-turn.log 2>&1 \ + && screen -S __PREFIX__-PMO -X stuff "Work turn done OK - read inbox-pmo.\r" \ + || screen -S __PREFIX__-PMO -X stuff "Work turn FAILED - read log.\r"' +First dispatch (or after a kill orphans the lineage): omit --session, then +capture id via `crush session list --json` into BOARD.md. + +screen -X stuff is DOORBELL-ONLY (short fixed lines to an idle screen): a +stuffed line can answer an open dialog; a dead pty eats it silently. +Founder contract: founder observes Work read-only; founder writes in PMO; +Work turns come only from PMO dispatch. +EOF +sub "$CC/PROTOCOL.md" +fi + +if fresh "$CC/BOARD.md"; then +cat > "$CC/BOARD.md" <<'EOF' +# Board โ€” __USR__ +- PMO (screen __PREFIX__-PMO): manager. Founder liaison, planning, TASKs. +- Work (screen __WORKSCREEN__): IC. Executes PMO headless dispatches; files + REPORT/QUESTION/BLOCKED in inbox-pmo. +- Founder (__FOUNDER__): authority + sudo. Observes Work read-only; writes + only in PMO; approves every dispatch (founder gate). + +Current focus: (PMO maintains this line) + +Session ids: PMO interactive=(), Work interactive=(), Work turn lineage=(). +EOF +sub "$CC/BOARD.md" +fi + +if [[ ! -f "$CC/log.md" ]]; then + printf '%s | founder-channel | .coordinate created\n' "$(date -Is)" > "$CC/log.md" +fi + +if fresh "$CC/prompts/INTRO-PMO.md"; then +cat > "$CC/prompts/INTRO-PMO.md" <<'EOF' +INTRODUCTION (from founder, __DATE__) + +You are __USR__-PMO โ€” manager of this stack. Direct report: +__USR__-Work (screen __WORKSCREEN__), a crush instance as the same user. +The founder (__FOUNDER__) interacts with you day-to-day and observes Work +READ-ONLY; all founder writes happen here with you. + +Read ~/.coordinate/PROTOCOL.md and follow it exactly: TASK files to +inbox-work/, consume REPORT/QUESTION/BLOCKED from inbox-pmo/, maintain +BOARD.md + log.md. Dispatch = headless crush-run wrappers (PROTOCOL); +never dispatch without the FOUNDER GATE. Monitor headless turns by reading +logs/, never by attaching. + +CAPABILITIES: no sudo (root needs go to the founder). Headless workers: + env HOME=/home/__USR__ TERM=xterm-256color screen -dmS bash -c \ + 'crush run --quiet "$(cat /path/to/prompt.md)"' \ + > /home/__USR__/.coordinate/logs/.log 2>&1 +Use crush sub-agent tooling instead of long sequential exploration. + +YOUR MISSION: <<>> +YOUR SYSTEMS OF RECORD: <<>> + +TOKEN DISCIPLINE: terse (<=5 line replies); cite file paths; batch reads; +never paste large files; findings go to ~/.coordinate files. + +NOW: read PROTOCOL.md, append intro line to log.md, set BOARD.md +current-focus, greet the founder in 5 lines max. Then wait. +EOF +sub "$CC/prompts/INTRO-PMO.md" +fi + +if fresh "$CC/prompts/INTRO-WORK.md"; then +cat > "$CC/prompts/INTRO-WORK.md" <<'EOF' +INTRODUCTION (from founder, __DATE__) + +You are __USR__-Work (screen __WORKSCREEN__) โ€” executor of this stack, +running ~99% headless. Humans rarely talk to you directly. __USR__-PMO is +your manager: turns arrive ONLY as headless crush-run TASK files from +PMO (~/.coordinate/inbox-work/). Founder (__FOUNDER__) is final authority, +read-only observer. + +YOUR LOOP โ€” every turn, before anything else: +1. PROTOCOL checklist: BOARD.md, inbox-work/ (P1 first; CANCEL drops), + log.md tail. +2. Execute TASKs; obey Constraints; deliver Deliverables. Parallelizable + subtasks may go to headless workers (logs/.log; verify before + reporting done). +3. File REPORT/QUESTION/BLOCKED in inbox-pmo/, archive the TASK, append + log line. Chat replies are one-liners; substance goes in files. + +RULES: never self-assign scope (QUESTION instead). Empty inbox when pinged += one line, stop. No watchers/polling loops. No sudo โ€” root needs go +BLOCKED to PMO. Never kill the other session's screens. If PROTOCOL.md is +missing, say so and wait โ€” do not recreate. + +NOW: read PROTOCOL.md, append your online line to log.md, post +inbox-pmo/REPORT-ready.md ("__USR__-Work online, awaiting first TASK"). +One chat line only. Then wait. +EOF +sub "$CC/prompts/INTRO-WORK.md" +fi + +chown -R "${USR}:${USR}" "$CC" "$DST_CRUSH" + +spawn() { + local s="$1" + if compgen -G "/run/screen/S-${USR}/.*.${s}" > /dev/null; then + echo "already running: $s" + else + runuser -u "$USR" -- env HOME="$HOME_U" TERM=xterm-256color \ + screen -dmS "$s" bash -c "cd ${CWD} && exec crush --yolo" + echo "spawned: $s" + fi +} +spawn "$PMO_SCREEN" +spawn "$WORKSCREEN" + +echo "OK: stack for ${USR} (cwd=${CWD}, pmo=${PMO_SCREEN}, work=${WORKSCREEN})" diff --git a/removed-agent-stacks/tsg-agent-stacks.service b/removed-agent-stacks/tsg-agent-stacks.service new file mode 100644 index 0000000..14668c7 --- /dev/null +++ b/removed-agent-stacks/tsg-agent-stacks.service @@ -0,0 +1,14 @@ +[Unit] +Description=TSG agent stacks (screen/crush PMO+Work per executive account) +After=local-fs.target network-online.target systemd-user-sessions.service +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +KillMode=process +TimeoutStartSec=180 +ExecStart=/usr/bin/bash /home/reachableceo/launch-all-tsg-stacks.sh + +[Install] +WantedBy=multi-user.target diff --git a/removed-agent-stacks/tsg-supervisor.service b/removed-agent-stacks/tsg-supervisor.service new file mode 100644 index 0000000..a46ba30 --- /dev/null +++ b/removed-agent-stacks/tsg-supervisor.service @@ -0,0 +1,7 @@ +[Unit] +Description=TSG agent-stack supervisor tick (metrics, self-heal, alerts) +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/bash /home/_crossfeed/tooling/agent-stack/supervisor.sh diff --git a/removed-agent-stacks/tsg-supervisor.timer b/removed-agent-stacks/tsg-supervisor.timer new file mode 100644 index 0000000..f120c2b --- /dev/null +++ b/removed-agent-stacks/tsg-supervisor.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run TSG supervisor every 5 minutes + +[Timer] +OnBootSec=3min +OnUnitActiveSec=5min +AccuracySec=30s + +[Install] +WantedBy=timers.target diff --git a/uat-sse-last.log b/uat-sse-last.log new file mode 100644 index 0000000..3285ad7 --- /dev/null +++ b/uat-sse-last.log @@ -0,0 +1,6 @@ +data: {"choices":[{"delta":{"content":"","reasoning_content":"The user wants me to run a specific command and return only the directory names, one per line. The command is: `ls -1 /home/reachableceo/projects`\n\nLet me run this command and see what it returns.","role":"assistant","tool_calls":[{"index":0,"id":"call_-7296376513589405675","type":"function","function":{"name":"run_command","arguments":"{\"command\":\"ls -1 /home/reachableceo/projects\"}"}}]},"finish_reason":null,"index":0}],"created":1788233502,"id":"20260901113140eae7bb7a7ca94820","model":"glm-4.7","object":"chat.completion.chunk"} + +data: {"choices":[{"delta":{},"finish_reason":"tool_calls","index":0}],"created":1788233502,"id":"20260901113140eae7bb7a7ca94820","model":"glm-4.7","object":"chat.completion.chunk","usage":{"prompt_tokens":2487,"completion_tokens":66,"total_tokens":2553}} + +data: [DONE] + diff --git a/uat/docker-compose.yml b/uat/docker-compose.yml new file mode 100644 index 0000000..dcf97fc --- /dev/null +++ b/uat/docker-compose.yml @@ -0,0 +1,25 @@ +# UAT harness for Open Terminal (#610). Throwaway OpenWebUI matched to the +# human's Cloudron prod version (v0.11.1). Host networking: reaches the beta +# gateway (127.0.0.1:4002) and the open-terminal fleet (100.101.187.119) in +# one hop. Loopback-only bind; torn down by 9-uat-openwebui.sh when done. +# Secrets come from uat/.env (gitignored; written by 9-uat-openwebui.sh). +services: + openwebui-uat: + image: ghcr.io/open-webui/open-webui:v0.11.1 + container_name: ukrrs-openwebui-uat + network_mode: host + environment: + - WEBUI_NAME=OpenWebUI UAT (open-terminal) + - PORT=8081 + - UVICORN_HOST=127.0.0.1 + - ENABLE_SIGNUP=true + - WEBUI_SECRET_KEY=${WEBUI_SECRET_KEY} + - OPENAI_API_BASE_URL=${OPENAI_API_BASE_URL} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - DEFAULT_MODELS=${DEFAULT_MODELS} + - ENABLE_VERSION_UPDATE_CHECK=false + volumes: + - openwebui-uat:/app/backend/data + restart: "no" +volumes: + openwebui-uat: