mrcharles
c810942239
open-terminal 0.11.34 bare-metal (human-authorized host install) with a systemd template unit: reachableceo on tailscale-only :30000 (cwd ~/projects) and all 8 other accounts on :30001-30008, each with its own key kept out of ps in per-user TOMLs, all users added to the docker group. Keys live in ~/.creds + /etc/ukrrs/open-terminal, never in this repo. UAT: throwaway OpenWebUI v0.11.1 (matched to the human's Cloudron prod) driven purely over its HTTP API against the beta gateway lane - connection verify/config PASS, direct exec as uid 1001 PASS, and the chat round trip PASS: the model emitted run_command, we executed it through OpenWebUI's terminal proxy (server holds the key), and the final answer named 9/9 real project directories; earlier negative runs prove the test catches hallucinated output. Ops note OPEN-TERMINAL.md documents production wiring (prod openwebui container v0.3.10 is too old and needs an upgrade first). Also tonight: agent-stack relaunchers removed (6-, backups kept) so no screen/crush sessions auto-start on reboot; 22:00 night-profile flip observed live; docs synced for the earlier teardown. 💘 Generated with Crush Assisted-by: Crush:glm-5.2 [#610]
171 lines
9.7 KiB
Bash
Executable File
171 lines
9.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# 9-uat-openwebui.sh — OpenWebUI <-> Open Terminal UAT (#610).
|
|
# Throwaway OpenWebUI v0.11.1 (matches the human's Cloudron prod), driven
|
|
# entirely over its real HTTP API:
|
|
# P1 stack up + gateway models visible (beta lane)
|
|
# P2 admin signup
|
|
# P3 terminal connection: POST /api/v1/configs/terminal_servers +
|
|
# /verify + user list via /api/v1/terminals/ (v0.11.1 contracts)
|
|
# P4 DIRECT open-terminal API: async /execute + poll -> id shows
|
|
# uid=1001(reachableceo), ls shows ~/projects entries
|
|
# P5 CHAT: /api/chat/completions with metadata.terminal_id set; the model
|
|
# must RUN the read-only ls through the terminal and answer with REAL
|
|
# directory names (hallucination = FAIL)
|
|
# Teardown at end (KEEP=1 leaves it up). No sudo needed.
|
|
# Run: ~/projects/ultix/9-uat-openwebui.sh
|
|
set -uo pipefail
|
|
|
|
cd "$(dirname "$0")"
|
|
OUT=$PWD/9-uat-openwebui.out
|
|
: > "$OUT"
|
|
log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$OUT"; }
|
|
fail() { log "FAIL: $*"; EXIT=1; }
|
|
EXIT=0
|
|
|
|
GW_URL=http://127.0.0.1:4002/v1
|
|
GW_KEY=$(grep -oP '(?<=--api-key ")[^"]+' ~/.config/crush-gw-beta/crushrc | head -1)
|
|
OT_URL=http://100.101.187.119:30000
|
|
OT_KEY=$(grep -oP '(?<=REACHABLECEO_API_KEY=).*' ~/.creds/open-terminal.env)
|
|
EXPECTED_DIRS=$(ls -1 /home/reachableceo/projects | grep -E '^[a-zA-Z0-9._-]+$' | sort -u)
|
|
CONN_ID=ultix-rceo
|
|
[ -n "$GW_KEY" ] && [ -n "$OT_KEY" ] || { log "FAIL: missing creds"; exit 1; }
|
|
|
|
WEBUI=http://127.0.0.1:8081
|
|
EMAIL=uat@ultix.local
|
|
ADMIN_PASS=$(openssl rand -hex 12)
|
|
|
|
# ---- P0/P1 ----
|
|
umask 077
|
|
cat > uat/.env <<EOF
|
|
WEBUI_SECRET_KEY=$(openssl rand -hex 24)
|
|
OPENAI_API_BASE_URL=$GW_URL
|
|
OPENAI_API_KEY=$GW_KEY
|
|
DEFAULT_MODELS=glm-4.7
|
|
EOF
|
|
log "P0 uat/.env written (beta gateway $GW_URL, model glm-4.7)"
|
|
|
|
docker compose -p openwebui-uat -f uat/docker-compose.yml down -v >>"$OUT" 2>&1 || true
|
|
docker compose -p openwebui-uat -f uat/docker-compose.yml up -d >>"$OUT" 2>&1
|
|
HEALTH=no
|
|
for i in $(seq 1 60); do
|
|
sleep 2
|
|
[ "$(timeout 5 curl -s -o /dev/null -w '%{http_code}' $WEBUI/health 2>/dev/null)" = 200 ] && { HEALTH=yes; break; }
|
|
done
|
|
[ "$HEALTH" = yes ] || { fail "P1 openwebui never healthy"; docker logs ukrrs-openwebui-uat --tail 40 >>"$OUT" 2>&1; exit 1; }
|
|
log "P1 PASS: openwebui v0.11.1 healthy on 127.0.0.1:8081"
|
|
MODELS=$(timeout 20 curl -s -H "Authorization: Bearer $GW_KEY" "$GW_URL/models" | jq -r '[.data[].id] | join(",")' 2>/dev/null)
|
|
echo "$MODELS" | grep -q glm-4.7 && log "P1 PASS: glm-4.7 reachable via beta gateway" || fail "P1 glm-4.7 missing: $MODELS"
|
|
|
|
# ---- P2 ----
|
|
SIGNUP=$(timeout 20 curl -s -X POST $WEBUI/api/v1/auths/signup \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"UAT\",\"email\":\"$EMAIL\",\"password\":\"$ADMIN_PASS\"}")
|
|
TOKEN=$(echo "$SIGNUP" | jq -r .token 2>/dev/null)
|
|
{ [ -n "$TOKEN" ] && [ "$TOKEN" != null ]; } || { fail "P2 signup failed: $(echo "$SIGNUP" | head -c 200)"; exit 1; }
|
|
log "P2 PASS: admin signup (role $(echo "$SIGNUP" | jq -r .role))"
|
|
|
|
# ---- P3 ----
|
|
VERIFY=$(timeout 20 curl -s -X POST $WEBUI/api/v1/configs/terminal_servers/verify \
|
|
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
|
-d "{\"url\":\"$OT_URL\",\"key\":\"$OT_KEY\"}")
|
|
log "P3 verify: $VERIFY"
|
|
echo "$VERIFY" | jq -e '.status == true' >/dev/null 2>&1 && log "P3 PASS: terminal server verified ($(echo "$VERIFY" | jq -r .type))" || fail "P3 verify did not return status:true"
|
|
|
|
SETCONN=$(timeout 20 curl -s -X POST $WEBUI/api/v1/configs/terminal_servers \
|
|
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
|
-d "{\"TERMINAL_SERVER_CONNECTIONS\":[{\"id\":\"$CONN_ID\",\"name\":\"$CONN_ID\",\"url\":\"$OT_URL\",\"key\":\"$OT_KEY\",\"auth_type\":\"bearer\",\"enabled\":true}]}" \
|
|
| sed -E 's/"key":"[^"]*"/"key":"***"/g')
|
|
log "P3 set connections: $(echo "$SETCONN" | head -c 400)"
|
|
|
|
LIST=$(timeout 20 curl -s $WEBUI/api/v1/terminals/ -H "Authorization: Bearer $TOKEN")
|
|
log "P3 user terminal list: $(echo "$LIST" | head -c 300)"
|
|
echo "$LIST" | jq -e --arg id "$CONN_ID" '[.[] | select(.id == $id)] | length == 1' >/dev/null 2>&1 \
|
|
&& log "P3 PASS: connection $CONN_ID visible to users" || fail "P3 connection not listed: $LIST"
|
|
|
|
# ---- P4: direct async exec through open-terminal's own API ----
|
|
EXEC=$(timeout 60 curl -s -X POST "$OT_URL/execute?wait=30" \
|
|
-H "Authorization: Bearer $OT_KEY" -H 'Content-Type: application/json' \
|
|
-d '{"command":"id && ls -1 /home/reachableceo/projects"}')
|
|
log "P4 exec result: $(echo "$EXEC" | head -c 900)"
|
|
OUTTXT=$(echo "$EXEC" | jq -r '[.output[]?.data] | join("")' 2>/dev/null | tr -d '\r')
|
|
PID=$(echo "$EXEC" | jq -r .id 2>/dev/null)
|
|
if [ -z "$OUTTXT" ] && [ -n "${PID:-}" ] && [ "$PID" != null ]; then
|
|
ST2=$(timeout 40 curl -s "$OT_URL/execute/$PID/status?wait=30&offset=0" -H "Authorization: Bearer $OT_KEY")
|
|
OUTTXT=$(echo "$ST2" | jq -r '[.output[]?.data] | join("")' 2>/dev/null)
|
|
log "P4 status fallback: $(echo "$ST2" | head -c 900)"
|
|
fi
|
|
echo "$OUTTXT" | grep -q 'uid=1001(reachableceo)' && log "P4 PASS: ran as reachableceo" || fail "P4 no uid=1001(reachableceo) in: $OUTTXT"
|
|
echo "$OUTTXT" | grep -qx ultix && log "P4 PASS: sees ~/projects/ultix" || fail "P4 no ultix line in: $OUTTXT"
|
|
|
|
# ---- P5: chat must run a read-only terminal command and return REAL output.
|
|
# OpenWebUI contract (v0.11.1): for API callers the tool loop runs client-side
|
|
# (server-side loop needs a socket session). So: 1) chat w/ terminal_id ->
|
|
# model emits run_command tool_call; 2) WE execute it through OpenWebUI's own
|
|
# terminal proxy (server holds the key); 3) feed the tool result back; 4) the
|
|
# model's final answer must name REAL ~/projects entries.
|
|
CHAT_ID=$(cat /proc/sys/kernel/random/uuid)
|
|
PROMPT="You have a terminal available (already selected for this chat). Use it to run exactly this read-only command: ls -1 /home/reachableceo/projects — then reply with ONLY the directory names you saw, one per line. Do not guess; if the tool fails, say so."
|
|
TURN1=$(timeout 180 curl -sN -X POST $WEBUI/api/chat/completions \
|
|
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
|
-d "{\"model\":\"glm-4.7\",\"stream\":true,\"terminal_id\":\"$CONN_ID\",\"chat_id\":\"$CHAT_ID\",\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rs . <<<"$PROMPT")}]}" \
|
|
| sed -E 's/"key":"[^"]*"/"key":"***"/g')
|
|
TC_JSON=$(echo "$TURN1" | grep '^data: ' | sed 's/^data: //' | grep -v '^\[DONE\]' \
|
|
| jq -rs 'map(try (.choices[0].delta.tool_calls // empty) // empty) | flatten | map(select(.function.name))' 2>/dev/null)
|
|
TC_COUNT=$(echo "$TC_JSON" | jq 'length' 2>/dev/null)
|
|
log "P5 turn1: model emitted $TC_COUNT tool call(s): $(echo "$TC_JSON" | jq -c '[.[] | {id, name: .function.name}]' 2>/dev/null)"
|
|
[ "${TC_COUNT:-0}" -ge 1 ] || { fail "P5 turn1 produced no tool call (model: $(echo "$TURN1" | grep -o '"content":"[^"]*"' | head -c 300))"; }
|
|
|
|
TOOL_RESULTS="[]"
|
|
if [ "${TC_COUNT:-0}" -ge 1 ]; then
|
|
CMD=$(echo "$TC_JSON" | jq -r '.[0].function.arguments' | jq -r '.command')
|
|
TCID=$(echo "$TC_JSON" | jq -r '.[0].id')
|
|
log "P5 executing via openwebui terminal proxy: $CMD"
|
|
PROXY=$(timeout 60 curl -s -X POST "$WEBUI/api/v1/terminals/$CONN_ID/execute?wait=30" \
|
|
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
|
-d "{\"command\":$(jq -Rs . <<<"$CMD")}" )
|
|
log "P5 proxy exec: $(echo "$PROXY" | head -c 500)"
|
|
TOOL_OUT=$(echo "$PROXY" | jq -r '[.output[]?.data] | join("")' 2>/dev/null | tr -d '\r')
|
|
echo "$TOOL_OUT" | grep -q ultix && log "P5 PASS: proxy-executed command returned real ~/projects content" || fail "P5 proxy execution did not return real output: $TOOL_OUT"
|
|
TOOL_RESULTS=$(jq -n --arg id "$TCID" --arg out "$TOOL_OUT" '[{role:"tool", tool_call_id:$id, content:$out}]')
|
|
fi
|
|
|
|
if [ "${TC_COUNT:-0}" -ge 1 ]; then
|
|
TCID=$(echo "$TC_JSON" | jq -r '.[0].id')
|
|
TCNAME=$(echo "$TC_JSON" | jq -r '.[0].function.name')
|
|
TCARGS=$(echo "$TC_JSON" | jq -c '.[0].function.arguments')
|
|
BODY=$(cat <<EOF
|
|
{"model":"glm-4.7","stream":false,"chat_id":"$CHAT_ID","messages":[{"role":"user","content":"Use your terminal. Run the command from the tool call and reply ONLY the directory names you saw."},{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"$TCID","type":"function","function":{"name":"$TCNAME","arguments":$TCARGS}}]},{"role":"tool","tool_call_id":"$TCID","content":$(jq -Rs . <<<"$TOOL_OUT")}]}
|
|
EOF
|
|
)
|
|
echo "$BODY" | jq -e . >/dev/null 2>&1 && log "P5 turn2 body: valid JSON" || log "P5 turn2 body: INVALID: $(echo "$BODY" | head -c 500)"
|
|
TURN2=$(timeout 180 curl -s -X POST $WEBUI/api/chat/completions \
|
|
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
|
-d "$BODY" \
|
|
| sed -E 's/"key":"[^"]*"/"key":"***"/g')
|
|
log "P5 turn2 raw (first 400): $(echo "$TURN2" | head -c 400)"
|
|
CONTENT=$(echo "$TURN2" | jq -r '.choices[0].message.content // empty' 2>/dev/null)
|
|
log "P5 final answer: $(echo "$CONTENT" | head -c 500)"
|
|
HITS=0; MISSED=""
|
|
for d in $EXPECTED_DIRS; do
|
|
if echo "$CONTENT" | grep -q "$d"; then HITS=$((HITS+1)); else MISSED="$MISSED $d"; fi
|
|
done
|
|
if [ -n "$CONTENT" ] && [ "$HITS" -ge 3 ]; then
|
|
log "P5 PASS: final model answer contains $HITS/$(echo $EXPECTED_DIRS | wc -w) real directory names (full round trip: tool_call -> proxy exec on terminal -> real output -> answer)"
|
|
else
|
|
fail "P5 final answer not grounded (hits=$HITS missed:$MISSED)"
|
|
log "P5 last openwebui log lines follow"
|
|
docker logs ukrrs-openwebui-uat --tail 25 >>"$OUT" 2>&1
|
|
fi
|
|
fi
|
|
|
|
# ---- teardown ----
|
|
if [ "${KEEP:-0}" = 1 ]; then
|
|
log "KEEP=1 — stack up at http://127.0.0.1:8081 (login $EMAIL / pass in .out if logged)"
|
|
echo "ADMIN_PASS=$ADMIN_PASS" >> "$OUT"
|
|
else
|
|
docker compose -p openwebui-uat -f uat/docker-compose.yml down -v >>"$OUT" 2>&1
|
|
log "teardown complete (project+volume removed)"
|
|
fi
|
|
log "EXIT=$EXIT $([ $EXIT = 0 ] && echo 'ALL PHASES PASSED' || echo 'SEE FAIL LINES')"
|
|
exit $EXIT
|