ops(scripts): dispatch, queue gating, heartbeat, quota probe

dispatch-turn.sh (headless turn launcher with TURN_MODEL routing),
queue-after.sh (serial gate chain + quota-wall guard), pmo-heartbeat.sh
(PMO wake loop), quota-probe-then-chain.sh (post-wall relaunch).
This commit is contained in:
2026-08-29 05:22:43 -05:00
parent 458d92831e
commit 79d93b6c49
18 changed files with 825 additions and 0 deletions
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
# agent-metrics.sh — harvest per-vertical crush usage into the org ledger.
# Appends one JSONL line per session seen, deduped by session id via
# ledger/seen index. Idempotent; safe to run every timer tick.
#
# Ledger: /home/_crossfeed/metrics/ledger.jsonl
# Event: {"ts","vertical","role","session","model","prompt_tokens",
# "completion_tokens","total_tokens","cost"}
# Role heuristic: title contains 'PMO intro' -> pmo; 'INTRO-WORK' -> work;
# 'headless' -> workturn; else 'other'.
set -uo pipefail
METRICS=${METRICS:-/home/_crossfeed/metrics}
LEDGER="$METRICS/ledger.jsonl"
SEEN="$METRICS/.seen-ids"
VERTICALS=(TSGBOD TSGCTO TSGCCO TSGCOO reachableceo)
RUNAS=${RUNAS:-sudo} # agent-metrics runs from the supervisor service (root)
mkdir -p "$METRICS"; touch "$LEDGER" "$SEEN"
emit() { printf '%s\n' "$1" >> "$LEDGER"; }
for user in "${VERTICALS[@]}"; do
home=$(getent passwd "$user" | cut -d: -f6)
[ -n "$home" ] || continue
# shellcheck disable=SC2016
json=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
bash -c "cd '$home' && crush session list --json 2>/dev/null" || true)
[ -n "$json" ] || continue
# one session id per line; token/cost fields only exist in `session show`
while IFS= read -r sid; do
[ -n "$sid" ] || continue
grep -q "$sid" "$SEEN" && continue
obj=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
bash -c "cd '$home' && crush session show '$sid' --json 2>/dev/null | head -c 4000" || true)
[ -n "$obj" ] || continue
title=$(printf '%s' "$obj" | grep -o '"title":"[^"]*"' | head -1 | cut -d'"' -f4)
role=other
case "$title" in
*PMO*intro*) role=pmo ;;
*INTRO-WORK*) role=work ;;
*headless*|*TSG*-Work*executing*) role=workturn ;;
esac
model=$(printf '%s' "$obj" | grep -o '"model":"[^"]*"' | head -1 | cut -d'"' -f4)
pt=$(printf '%s' "$obj" | grep -o '"prompt_tokens":[0-9]*' | head -1 | cut -d: -f2)
ct=$(printf '%s' "$obj" | grep -o '"completion_tokens":[0-9]*' | head -1 | cut -d: -f2)
tt=$(printf '%s' "$obj" | grep -o '"total_tokens":[0-9]*' | head -1 | cut -d: -f2)
cost=$(printf '%s' "$obj" | grep -o '"cost":[0-9.e-]*' | head -1 | cut -d: -f2)
ts=$(date -Is)
emit "{\"ts\":\"$ts\",\"vertical\":\"$user\",\"role\":\"$role\",\"session\":\"$sid\",\"model\":\"${model:-?}\",\"prompt_tokens\":${pt:-0},\"completion_tokens\":${ct:-0},\"total_tokens\":${tt:-0},\"cost\":${cost:-0}}"
echo "$sid" >> "$SEEN"
done < <(printf '%s' "$json" | grep -o '"id":"[0-9a-f]*"' | cut -d'"' -f4)
done
# rolling summary
python3 - "$LEDGER" <<'PYEOF' > "$METRICS/SUMMARY.md" 2>/dev/null || true
import json, sys, collections, datetime
tot = collections.Counter(); cost = collections.Counter(); n = collections.Counter()
day = collections.Counter()
now = datetime.datetime.now().astimezone()
for line in open(sys.argv[1]):
try: e = json.loads(line)
except Exception: continue
v = e.get("vertical","?"); t = e.get("total_tokens",0)
tot[v] += t; cost[v] += e.get("cost",0.0); n[v] += 1
try:
ts = datetime.datetime.fromisoformat(e["ts"])
if (now-ts).total_seconds() < 86400: day[v] += t
except Exception: pass
print(f"# Agent usage summary — {now.isoformat()}")
print(f"- sessions ledgered total: {sum(n.values())}")
print("- all-time by vertical:")
for v in sorted(tot):
print(f" {v:<13} n={n[v]:<4} tokens={tot[v]:<10} cost={cost[v]:.4f} (24h tokens={day[v]})")
PYEOF
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
# dispatch-turn.sh <slug> <prompt-file> — headless work-turn launcher.
# Semaphore-gated (org concurrency cap), FRESH session by default (small
# context = quota-friendly; the turn self-bootstraps from the prompt file),
# success/fail doorbell to the vertical PMO screen. PMO usage:
# screen -dmS work-turn-<slug> \
# ~/.coordinate/scripts/dispatch-turn.sh <slug> <prompt-file>
set -uo pipefail
SLUG=${1:?slug}; PROMPT=${2:?prompt-file}
TOOLING=${TOOLING:-/home/_crossfeed/tooling}
SEMI="$TOOLING/agent-stack/semaphore.sh"
LOG="$HOME/.coordinate/logs/$SLUG.log"
TAG="$USER-$SLUG"
WAIT=${WAIT_SECS:-300}
[ -f "$PROMPT" ] || { echo "no prompt file: $PROMPT" >&2; exit 1; }
semirc=0
bash "$SEMI" try "$TAG" "$WAIT" >/dev/null 2>&1 || semirc=$?
if [ "$semirc" = 99 ]; then
echo "$(date -Is) semaphore gate unavailable - proceeding UNGATED (install metrics dirs)" >> "$LOG"
elif [ "$semirc" != 0 ]; then
echo "$(date -Is) semaphore rc=$semirc after ${WAIT}s budget - not launched" >> "$LOG"
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn DEFERRED (concurrency cap) - rerun later: %s.\r' "$SLUG")" || true
exit 3
fi
GATED=1; [ "$semirc" = 0 ] || GATED=0
cd "$HOME"
rc=0
# 2026-08-29 ~06:30: Charles upgraded to MAX plan, all buckets zero.
# Quality-first: default back to glm-5.2; TURN_MODEL=glm-4.7-flash for
# grind/admin turns when wanted.
TURN_MODEL=${TURN_MODEL:-litellm/glm-5.2}
env HOME="$HOME" TERM=xterm-256color \
crush run --quiet --model "$TURN_MODEL" "$(cat "$PROMPT")" >> "$LOG" 2>&1 || rc=$?
bash "$SEMI" release "$TAG" >/dev/null 2>&1 || true
if [ "$rc" = 0 ]; then
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn done OK - read inbox-pmo + logs/%s.\r' "$SLUG")" || true
else
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn FAILED (rc=%s) - read logs/%s.\r' "$rc" "$SLUG")" || true
fi
exit "$rc"
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# founder-crossfeed-inboxes.sh — FOUNDER-RUN as root.
# Cross-vertical inbox interconnect under /home/_crossfeed/inbox/.
# Semantics per dir: owner=<vertical acct>, group=users, mode 3775
# (setgid+sticky): any vertical may CREATE files in any inbox; only the
# file's author or the inbox OWNER may move/delete. Homes stay 700.
set -uo pipefail
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
declare -A VMAP=( [bod]=TSGBOD [cco]=TSGCCO [cto]=TSGCTO [coo]=TSGCOO [founder]=reachableceo )
echo "== preflight =="
for v in "${!VMAP[@]}"; do
acct=${VMAP[$v]}; id "$acct" >/dev/null 2>&1 || { echo "missing acct $acct" >&2; exit 1; }
done
echo "ok: all five accounts exist"
echo "== create inbox tree =="
for v in bod cco cto coo founder; do
acct=${VMAP[$v]}
d=/home/_crossfeed/inbox/$v
mkdir -p "$d"
chown "$acct":users "$d"
chmod 3775 "$d"
echo " $d -> $acct:users 3775"
done
echo "== update crossfeed README =="
grep -q 'inbox/' /home/_crossfeed/README.md || cat >> /home/_crossfeed/README.md <<'EOF'
## inbox/ — cross-vertical direct messaging (2026-08-28)
/home/_crossfeed/inbox/<bod|cco|cto|coo|founder>/ — mode 3775, owner is the
receiving vertical, group users. Any vertical may DROP a file in any inbox
(name: MSG-<from>-<HHMM>-<slug>.md). Only the author or the receiving
vertical consumes/moves/deletes. This is for lightweight cross-vertical
coordination when a Redmine/Discourse round trip is overkill; requests
with teeth still go via the founder or systems of record, never MSG files.
Receiving PMO scans own inbox at turn start (alongside peers' STATUS).
EOF
echo " README updated"
echo "== drop NOTICE into each vertical's inbox-pmo =="
for v in bod cco cto coo; do
acct=${VMAP[$v]}
home=$(getent passwd "$acct" | cut -d: -f6)
[ -d "$home/.coordinate/inbox-pmo" ] || continue
cat > "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md" <<'EOF'
Founder directive: /home/_crossfeed/inbox/<bod|cco|cto|coo|founder>/ is live.
Cross-vertical direct messaging: any PMO may drop
MSG-<from>-<HHMM>-<slug>.md in another vertical's inbox (dirs 3775: anyone
creates, only author or inbox owner removes). Scan YOUR inbox at turn
start. Lightweight coordination only — requests with teeth go via the
founder or systems of record. Archive this notice after reading.
EOF
chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md"
echo " NOTICE -> $acct"
done
echo "== verify =="
ls -la /home/_crossfeed/inbox/
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env bash
# founder-full-reset.sh — FOUNDER-RUN as reachableceo (sudo required).
# Org-wide clean reset of all FIVE verticals: TSGBOD TSGCCO TSGCTO TSGCOO RCEO.
# 1) kill every vertical screen (old 07:02 + new 11:13 generations alike)
# 2) clean stale /run/screen sockets
# 3) relaunch the four TSG stacks via /tmp/tsg-stack-relaunch.sh --yes
# (no live screens -> no duplicate names -> multiuser+acladd succeeds)
# 4) give RCEO the same wrapper pattern: pmo-loop.sh + pinned brains
# 5) verify + attach cheat sheet
# Does NOT touch AZHost/AgentZero or any other screens. RCEO-PMO/RCEO-Work
# (agent screens) die mid-run and return pinned (conversations preserved).
# Blank-slate lever (NOT default): before running, founder may
# touch ~/.coordinate/RELAUNCH-FRESH ~/.coordinate/RELAUNCH-FRESH-WORK
# but fresh sessions get NO intro injected by the wrapper — skip unless
# you plan to hand-feed the INTRO prompts after.
set -uo pipefail
[ "$(id -un)" = "reachableceo" ] || { echo "run as reachableceo"; exit 1; }
sudo -v || { echo "sudo required"; exit 1; }
USERS=(TSGBOD TSGCCO TSGCTO TSGCOO)
echo "== 1/5 kill all vertical screens (both generations) =="
pat='SCREEN -dmS (TSGBOD-|TSGCCO-|TSGCTO-|TSGCOO-|RCEO-)'
pids=$(pgrep -f "$pat" || true)
if [ -n "$pids" ]; then sudo kill $pids 2>/dev/null; fi
sleep 2
pids=$(pgrep -f "$pat" || true)
if [ -n "$pids" ]; then sudo kill -9 $pids 2>/dev/null; fi
for u in "${USERS[@]}" reachableceo; do
sudo pkill -u "$u" -f pmo-loop.sh 2>/dev/null
sudo pkill -u "$u" -f 'crush --' 2>/dev/null
sudo pkill -u "$u" -fx 'crush' 2>/dev/null
done
sleep 1
echo "== 2/5 clean stale sockets =="
for u in "${USERS[@]}" reachableceo; do
sudo rm -f "/run/screen/S-$u"/* 2>/dev/null
done
echo "== 3/5 relaunch TSG stacks (wrappers + existing pins) =="
if [ -f /tmp/tsg-stack-relaunch.sh ]; then
bash /tmp/tsg-stack-relaunch.sh --yes || echo "WARN: relaunch script failed — inspect its output"
else
echo "WARN: /tmp/tsg-stack-relaunch.sh missing — TSG stacks NOT relaunched"
fi
echo "== 4/5 relaunch RCEO stack (wrapper parity) =="
mkdir -p "$HOME/.coordinate/scripts" "$HOME/.coordinate/logs"
cat > "$HOME/.coordinate/scripts/pmo-loop.sh" <<'WRAP'
#!/usr/bin/env bash
# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen.
ROLE="${1:-pmo}"
case "$ROLE" in
pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;;
work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;;
*) echo "usage: $0 [pmo|work]" >&2; exit 1 ;;
esac
LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log"
cd "$HOME"
export TERM="${TERM:-xterm-256color}"
export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
while true; do
started=$(date +%s)
if [ -f "$FRESH" ]; then
rm -f "$FRESH"
echo "$(date -Is) relaunch FRESH" >> "$LOG"
crush
elif [ -s "$PIN" ]; then
sid=$(cat "$PIN")
echo "$(date -Is) relaunch PINNED $sid" >> "$LOG"
crush --session "$sid"
else
echo "$(date -Is) relaunch CONTINUE" >> "$LOG"
crush --continue
fi
rt=$(( $(date +%s) - started ))
if [ "$rt" -lt 60 ]; then
echo "$(date -Is) short run (${rt}s) - backoff 60s" >> "$LOG"
sleep 60
else
sleep 2
fi
done
WRAP
chmod 755 "$HOME/.coordinate/scripts/pmo-loop.sh"
echo 78cf63b8a72f4282 > "$HOME/.coordinate/PMO-SESSION-ID"
echo e1079fafb8c37d60 > "$HOME/.coordinate/WORK-SESSION-ID"
env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-PMO \
/home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo
env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-Work \
/home/reachableceo/.coordinate/scripts/pmo-loop.sh work
echo "== 5/5 verify =="
sleep 3
screen -ls
for u in "${USERS[@]}"; do
echo "-- $u:"
sudo -u "$u" env TERM=dumb screen -ls 2>/dev/null | grep -E 'PMO|Work' || echo " NONE (check relaunch output)"
done
echo
echo "Attach: TSGxx-PMO / TSGxx-work aliases; RCEO: RCEO-PMO | RCEO-work"
echo "Reboot bring-up units (enabled): reachableceo-agent-stack.service,"
echo "tsg-agent-stacks.service — safe to reboot-test after this."
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# founder-tooling-and-yolo.sh — FOUNDER-RUN: sudo bash ~/.coordinate/scripts/founder-tooling-and-yolo.sh
# 1) publish /home/_crossfeed/tooling (3 group-shared git repos) from PMO draft
# 2) fix org-wide --yolo regression: install canonical wrapper into the four
# TSG verticals + restart their screens (8 agents currently non-yolo)
# 3) deprecate /tmp/tsg-stack-relaunch.sh (v1, source of both regressions)
# 4) restart RCEO-PMO with canonical wrapper (PMO still non-yolo), then
# doorbell the PMO to finish live-repo verification + publish STATUS
set -uo pipefail
DRAFT=/home/reachableceo/.coordinate/tooling-draft
TOOLING=/home/_crossfeed/tooling
FOUNDER=reachableceo
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
echo "== 1/4 publish tooling repos =="
mkdir -p "$TOOLING"
cp -a "$DRAFT"/. "$TOOLING"/
chown -R $FOUNDER:users "$TOOLING"
chmod 2775 "$TOOLING"
find "$TOOLING" -type d -exec chmod g+s {} +
find "$TOOLING" -type f -exec chmod g+rw {} +
chmod -R o-rwx "$TOOLING"
ls -la "$TOOLING"
echo "== 2/4 TSG stacks: canonical wrapper (--yolo) + restart =="
bash "$TOOLING/agent-stack/tsg-stack-relaunch-v2.sh"
echo "== 3/4 deprecate /tmp v1 relaunch =="
for f in /tmp/tsg-stack-relaunch.sh; do
[ -f "$f" ] && mv "$f" "$f.deprecated-20260828" && echo "moved: $f -> $f.deprecated-20260828"
done
echo "== 4/4 RCEO-PMO restart with canonical wrapper + doorbell =="
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
screen -S RCEO-PMO -X quit 2>/dev/null || true
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
screen -dmS RCEO-PMO /home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo
echo "waiting 20s for PMO crush to settle..."
sleep 20
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
screen -S RCEO-PMO -X stuff $'Tooling live + yolo fix applied. Next turn: verify /home/_crossfeed/tooling, log + publish STATUS.\r' \
&& echo "doorbelled RCEO-PMO" || echo "doorbell failed - ping PMO manually"
echo "== VERIFY =="
for u in TSGBOD TSGCCO TSGCTO TSGCOO $FOUNDER; do
echo " $u sockets: $(ls /run/screen/S-$u 2>/dev/null | tr '\n' ' ')"
done
echo "yolo agents: $(pgrep -af 'crush --yolo' | wc -l) / expected 10"
echo "Attach: RCEO-PMO | RCEO-work; TSGxx-PMO | TSGxx-work aliases."
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# founder-wire-tooling.sh — FOUNDER-RUN as root.
# Wire the tooling repo (agent-stack @ current HEAD) into all four TSG
# verticals: canonical pmo-loop.sh (metrics-ledgered), dispatch-turn.sh,
# semaphore.sh, agent-metrics.sh into ~/.coordinate/scripts/. RCEO already
# wired itself. Supervisor units (tsg-supervisor.*) already org-wide.
# Stamped copies per provenance rule; fix forward in the repo, reinstall.
set -uo pipefail
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
TOOLING=/home/_crossfeed/tooling
SRC=$TOOLING/agent-stack
SHA=$(git -C "$SRC" rev-parse --short HEAD)
FILES="pmo-loop.sh dispatch-turn.sh semaphore.sh agent-metrics.sh"
echo "== preflight =="
for f in $FILES; do [ -f "$SRC/$f" ] || { echo "missing $SRC/$f" >&2; exit 1; }; done
echo "ok: source files present @ $SHA"
for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do
home=$(getent passwd "$acct" | cut -d: -f6)
echo "== $acct =="
mkdir -p "$home/.coordinate/scripts" "$home/.coordinate/logs"
for f in $FILES; do
install -o "$acct" -g "$acct" -m755 "$SRC/$f" "$home/.coordinate/scripts/$f"
done
sed -i "1a # installed: agent-stack @ $SHA ($(date -I))" "$home/.coordinate/scripts/pmo-loop.sh"
cat > "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md" <<EOF
Founder directive: your ~/.coordinate/scripts now carries agent-stack @ $SHA
(repo: /home/_crossfeed/tooling/agent-stack). Dispatch work-turns with:
screen -dmS work-turn-<slug> ~/.coordinate/scripts/dispatch-turn.sh <slug> <prompt-file>
(semaphore-gated, fresh session, doorbell back to your PMO screen; see
GOVERNANCE.md in the repo: capture free, execution metered; founder gate
on every TASK still applies). Wrapper now logs launches to metrics. Fix
bugs IN THE REPO (commit as your uid via tooling/commit.sh), then rerun
this installer. Archive this notice after reading.
EOF
chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md"
echo " wired + NOTICE"
done
echo "== verify =="
for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do
home=$(getent passwd "$acct" | cut -d: -f6)
echo " $acct: $(ls "$home/.coordinate/scripts/" 2>/dev/null | tr '\n' ' ')"
done
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# pmo-heartbeat.sh — wake the PMO session while the overnight chain runs.
# Self-terminates when inbox-work has no TASKs AND no semaphore slots are held.
INBOX=/home/reachableceo/.coordinate/inbox-work
SLOTS=/home/_crossfeed/metrics/active
LOG=/home/reachableceo/.coordinate/logs/heartbeat.log
INTERVAL=3600
while true; do
pending=$(ls -1 "$INBOX"/TASK-*.md 2>/dev/null | wc -l)
active=$(ls -1 "$SLOTS"/*.pid 2>/dev/null | wc -l)
if [ "$pending" -eq 0 ] && [ "$active" -eq 0 ]; then
echo "$(date -Is) chain drained — heartbeat exiting" >> "$LOG"
exit 0
fi
screen -S RCEO-PMO -X stuff "hb: chain check — verify reports, archive, mirror redmine, redispatch flakes, dispatch next$(printf '\r')"
echo "$(date -Is) wake sent (pending=$pending active=$active)" >> "$LOG"
sleep "$INTERVAL"
done
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen.
ROLE="${1:-pmo}"
case "$ROLE" in
pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;;
work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;;
*) echo "usage: $0 [pmo|work]" >&2; exit 1 ;;
esac
LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log"
LEDGER=/home/_crossfeed/metrics/launches.jsonl
ledger() { printf '%s\n' "$1" >> "$LEDGER" 2>/dev/null || true; }
cd "$HOME"
export TERM="${TERM:-xterm-256color}"
export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
while true; do
started=$(date +%s)
mode=continue
if [ -f "$FRESH" ]; then
rm -f "$FRESH"
mode=fresh
echo "$(date -Is) relaunch FRESH" >> "$LOG"
crush --yolo
elif [ -s "$PIN" ]; then
sid=$(cat "$PIN")
mode=pinned:$sid
echo "$(date -Is) relaunch PINNED $sid" >> "$LOG"
crush --yolo --session "$sid"
else
echo "$(date -Is) relaunch CONTINUE" >> "$LOG"
crush --yolo --continue
fi
rc=$?
rt=$(( $(date +%s) - started ))
ledger "{\"ts\":\"$(date -Is)\",\"user\":\"$USER\",\"role\":\"$ROLE\",\"mode\":\"$mode\",\"rc\":$rc,\"runtime_s\":$rt}"
if [ "$rt" -lt 60 ]; then
echo "$(date -Is) short run (${rt}s rc=$rc) - backoff 60s" >> "$LOG"
sleep 60
else
sleep 2
fi
done
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# queue-after.sh <gate-screen> <slug> <task-file>
# Wait until <gate-screen> no longer exists, then dispatch the task turn.
# Keeps repo-touching turns serialized when two queue screens would overlap.
set -uo pipefail
GATE="$1"; SLUG="$2"; TASK="$3"
LOGDIR="$HOME/.coordinate/logs"
while screen -ls 2>/dev/null | grep -q "\.${GATE}[[:space:]]"; do
sleep 60
done
# QUOTA WALL GUARD: if any recent turn log shows the z.ai usage wall,
# halt the chain instead of burning the next turn into the same wall.
WALLLOG=$(ls -t "$LOGDIR"/q-*.log 2>/dev/null | head -1)
if [ -n "$WALLLOG" ] && grep -q "Usage limit reached" "$WALLLOG" 2>/dev/null; then
echo "$(date -Is) QUOTA WALL detected in $WALLLOG — chain halted before $SLUG" >> "$LOGDIR/queue.status"
screen -S RCEO-PMO -X stuff "QUOTA WALL - chain halted before $SLUG. Do not dispatch until quota verified.$(printf '\r')" || true
exit 0
fi
echo "$(date -Is) GATE $GATE cleared — RUN $SLUG" >> "$LOGDIR/queue.status"
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$SLUG" "$TASK" \
>> "$LOGDIR/queue-$SLUG.log" 2>&1
rc=$?
echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo "FAIL(rc=$rc)") $SLUG" >> "$LOGDIR/queue.status"
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# queue-next.sh — bash-only slot watcher: run queued TASK turns as slots free.
# rc=3 from dispatch = DEFERRED (cap) -> keep waiting on same task, don't exit.
set -uo pipefail
LOGDIR="$HOME/.coordinate/logs"
i=1
while [ $i -le $# ]; do
task=${!i}
i=$((i+1))
slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//')
while true; do
n=$(ls /home/_crossfeed/metrics/active/*.pid 2>/dev/null | wc -l)
[ "$n" -lt 2 ] && break
sleep 60
done
echo "$(date -Is) RUN $slug" >> "$LOGDIR/queue.status"
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$slug" "$task" \
>> "$LOGDIR/queue-$slug.log" 2>&1
rc=$?
if [ "$rc" = 3 ]; then
echo "$(date -Is) DEFER $slug (cap) - re-queueing" >> "$LOGDIR/queue.status"
i=$((i-1)); sleep 30; continue
fi
echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo FAIL) $slug rc=$rc" >> "$LOGDIR/queue.status"
[ $rc -ne 0 ] && exit $rc
done
echo "$(date -Is) queue complete" >> "$LOGDIR/queue.status"
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# quota-probe-then-chain.sh — wait for z.ai quota window to reset, then
# launch the serial build chain. Probe: 8-token glm-4.7-flash ping via the
# harness vkey. On first 200 -> fire work-q1..q4 chain and exit.
set -uo pipefail
VKEY_FILE="$HOME/.coordinate/secrets/mopac-harness-vkey.env"
LOG="$HOME/.coordinate/logs/queue.status"
PROBE_LOG="$HOME/.coordinate/logs/quota-probe.log"
while true; do
ok=$(bash -c "set -a; source '$VKEY_FILE'; set +a; python3 - <<EOF
import os, json, urllib.request, urllib.error
body = json.dumps({'model': 'glm-4.7-flash', 'messages': [{'role': 'user', 'content': 'ping'}], 'max_tokens': 8}).encode()
req = urllib.request.Request('http://192.168.3.78:4001/v1/chat/completions', data=body, headers={'Authorization': 'Bearer ' + os.environ['HARNESS_LITELLM_KEY'], 'Content-Type': 'application/json'})
try:
urllib.request.urlopen(req, timeout=30)
print('OK')
except urllib.error.HTTPError as e:
print('HTTP', e.code)
except Exception as e:
print('ERR', type(e).__name__)
EOF" 2>/dev/null)
echo "$(date -Is) probe: $ok" >> "$PROBE_LOG"
if [ "$ok" = "OK" ]; then
echo "$(date -Is) QUOTA BACK — launching build chain" >> "$LOG"
W="$HOME/.coordinate/scripts"
T="$HOME/.coordinate/inbox-work"
screen -dmS work-q1 bash "$W/queue-after.sh" NONE-GATE docs-standards "$T/TASK-20260828-2100-docs-standards.md"
screen -dmS work-q2 bash "$W/queue-after.sh" work-q1 events-receiver "$T/TASK-20260828-1975-events-receiver.md"
screen -dmS work-q3 bash "$W/queue-after.sh" work-q2 keyproxy-v0 "$T/TASK-20260828-2130-keyproxy-v0.md"
screen -dmS work-q4 bash "$W/queue-after.sh" work-q3 selfhost-core "$T/TASK-20260828-2131-selfhost-core.md"
echo "$(date -Is) chain armed: docs -> events -> keyproxy-v0 -> selfhost-core" >> "$LOG"
exit 0
fi
sleep 1200
done
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# relaunch-screen.sh <name> <pmo|work> [fuse] — restart an RCEO screen so it
# re-reads ~/.screenrc (termcapinfo scroll fix). Fuse >0 schedules via helper
# screen (used to restart RCEO-PMO from inside itself). Owner-run; no root.
set -euo pipefail
name="${1:?screen name}"; role="${2:?pmo|work}"; fuse="${3:-0}"
SELF="$HOME/.coordinate/scripts/relaunch-screen.sh"
if [ "$fuse" -gt 0 ]; then
screen -S RCEO-Relaunch -X quit 2>/dev/null || true
screen -dmS RCEO-Relaunch bash -c "sleep $fuse; $SELF $name $role 0"
echo "scheduled: $name restart in ${fuse}s"
exit 0
fi
screen -S "$name" -X quit 2>/dev/null || true
sleep 2
screen -dmS "$name" "$HOME/.coordinate/scripts/pmo-loop.sh" "$role"
sleep 5
if screen -ls 2>/dev/null | grep -q "\.$name"; then
echo "OK: $name relaunched"
else
echo "FAILED: $name not present after relaunch" >&2; exit 1
fi
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env bash
# root-archive-tmp-artifacts.sh — archive /tmp agent artifacts into _crossfeed.
# meta (stack launchers/coordinate/crossfeed tooling) -> tooling/agent-stack/artifacts-archive/
# everything else (ops/tickets/monitoring) -> coo/artifacts-archive/
# Usage: sudo bash root-archive-tmp-artifacts.sh [--dry-run]
set -euo pipefail
DRY=0; [ "${1:-}" = "--dry-run" ] && DRY=1
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
SRC=/tmp
META_ROOT=/home/_crossfeed/tooling/agent-stack/artifacts-archive
COO_ROOT=/home/_crossfeed/coo/artifacts-archive
STAMP=$(date +%Y%m%d-%H%M%S)
META_DEST="$META_ROOT/$STAMP"
COO_DEST="$COO_ROOT/$STAMP"
# Never touch: system entries + .crush (live TSGCOO crush session data)
is_system() {
case "$1" in
systemd-private-*|.X11-unix|.ICE-unix|.font-unix|.Test-unix|.XIM-unix|.IMAP-unix|\
sddm*|xauth_*|ssh-*|scoped_dir*|.crush) return 0 ;;
*) return 1 ;;
esac
}
# Meta-scoped: stack launchers, coordinate/crossfeed setup, prompts, boot captures, standdown kits
is_meta() {
case "$1" in
launch-all-tsg-stacks.sh|launch-tsgcoo-stack.sh|launch-coordinate.sh|\
setup-tsg-stack.sh|setup-tsgcoo-coordinate.sh|\
tsg-pmo-autorelaunch.sh|tsg-stack-relaunch.sh.deprecated-20260828|tsg-crossfeed-setup.sh|\
tsg-agent-stacks.service|tsgcoo-bootstrap.sh|tsgcoo-cleanup.sh|tsgcoo-crush-prompt.md|\
tsgcoo-crush-prompt.oneline|tsgcoo-paste.sh|tsgcoo-pmo2.sh|tsgcoo-preflight.sh|\
tsgcoo-screen-setup.sh|tsgcoo-stack-fix.sh|tsgcoo-stack-fix2.sh|\
mk-coordinate.sh|rm-coordinate.sh|local-pmo.sh|pmo-check.sh|pmo-prompt.md|pmo-prompt.oneline|\
prompt-knel-basis.md|prompt-legacy-fold.md|legacy-fold.log|\
seed-intros.sh|run-seed.sh|run-setup.sh|\
LIFTSHIFT-MANIFEST.md|liftshift-copy.sh|liftshift-copy2.sh|liftshift-manifest|\
tsgcap1.txt|localpmo2.txt|localpmo3.txt|localpmo-after.txt|localpmo-boot.txt|\
crushboot.txt|crushafter.txt|crushafter2.txt|hb1.txt|hb2.txt|yolotest.log|\
tsgpmo2.txt|tsgpmo3.txt|tsgpmo-after.txt|tsgpmo-boot.txt|\
pmo-before.txt|pmo-after.txt|pmo-yolo.txt|\
work-before.txt|work-after.txt|work-screen.txt|work-screen2.txt|work-yolo.txt|\
bodw2.txt|cco-pmo.txt|cco-work2.txt|\
TSGBOD-pmo.txt|TSGBOD-work.txt|TSGCCO-pmo.txt|TSGCTO-pmo.txt|TSGCTO-work.txt|\
CTOHandoff.md|COO-PMO-RESUME.md|human-feed-hud.txt|\
offstage-kit|cos-pkg|cos-pkg-v2|\
w.sh|w1.sh|tl.sh|logs.sh|fix.sh|verify.sh|restart.sh|provision.sh|dbg.sh|\
inspect.sh|inspect-body.sh|inspect-wrap.sh|\
install-unit.sh|install-body.sh|final2.sh|final-check.sh|final-topology.sh|\
ssh-diag.sh|wrapper-extract.sh|w-extract.sh) return 0 ;;
*) return 1 ;;
esac
}
for d in /home/_crossfeed/coo "$META_ROOT" "$COO_ROOT"; do
[ -d "$(dirname "$d")" ] || { echo "ERROR: missing $d"; exit 1; }
done
if [ "$DRY" -eq 0 ]; then
mkdir -p "$META_DEST" "$COO_DEST"
fi
META_MANIFEST="${META_DEST:-/tmp}/MANIFEST.txt"; COO_MANIFEST="${COO_DEST:-/tmp}/MANIFEST.txt"
[ "$DRY" -eq 1 ] && { META_MANIFEST=/dev/null; COO_MANIFEST=/dev/null; }
meta_n=0; coo_n=0; sys_n=0
for f in "$SRC"/* "$SRC"/.[!.]*; do
[ -e "$f" ] || continue
name=$(basename "$f")
if is_system "$name"; then
echo "SKIP(system) $name"; sys_n=$((sys_n+1)); continue
fi
meta=$(stat -c '%U:%G:%s:%y' "$f")
if is_meta "$name"; then
printf '%s|%s\n' "$name" "$meta" >> "$META_MANIFEST"
echo "META $name ($meta)"
[ "$DRY" -eq 0 ] && mv -- "$f" "$META_DEST/"
meta_n=$((meta_n+1))
else
printf '%s|%s\n' "$name" "$meta" >> "$COO_MANIFEST"
echo "COO $name ($meta)"
[ "$DRY" -eq 0 ] && mv -- "$f" "$COO_DEST/"
coo_n=$((coo_n+1))
fi
done
if [ "$DRY" -eq 0 ]; then
chgrp -R users "$META_ROOT" "$COO_ROOT"
chmod -R g+rX "$META_ROOT" "$COO_ROOT"
echo "DONE: meta=$meta_n -> $META_DEST | coo=$coo_n -> $COO_DEST | skipped(system)=$sys_n"
else
echo "DRY RUN: meta=$meta_n coo=$coo_n skipped=$sys_n — nothing moved"
fi
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# root-fix-tooling-perms.sh — make _crossfeed/tooling uniformly group-writable
# (collaboration across agent accounts) while keeping credentials/ locked down.
# Usage: sudo bash ~/.coordinate/scripts/root-fix-tooling-perms.sh
set -euo pipefail
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
T=/home/_crossfeed/tooling
[ -d "$T" ] || { echo "ERROR: $T missing"; exit 1; }
chgrp -R users "$T"
# group rw, but NEVER widen credentials/ (skip symlinks; dangling links OK)
find "$T" -path "$T/credentials" -prune -o ! -type l -print0 | xargs -0 -r chmod g+rwX || true
find "$T" -path "$T/credentials" -prune -o -type d -print0 | xargs -0 -r chmod g+s || true
# lock credentials to owner only
chgrp -R reachableceo "$T/credentials" 2>/dev/null || true
chmod -R o-rwx,g-rwx "$T/credentials" || true
echo "OK: tooling group-writable (setgid dirs), credentials owner-only"
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
# root-migrate-coordinate.sh — move ~/.coordinate into _crossfeed/state/rceo (symlink back).
# Run as root, when RCEO agents are idle (no work-turn screens, PMO/Work between turns):
# sudo bash ~/.coordinate/scripts/root-migrate-coordinate.sh
set -euo pipefail
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
SRC=/home/reachableceo/.coordinate
STATE_ROOT=/home/_crossfeed/state
DEST=$STATE_ROOT/rceo
LINK=/home/reachableceo/.coordinate
if screen -ls 2>/dev/null | grep -q work-turn; then
echo "ERROR: work-turn screens still running — let them finish first (or --force)"
[ "${1:-}" = "--force" ] || exit 1
fi
if [ -L "$LINK" ]; then
echo "Already migrated: $LINK -> $(readlink "$LINK")"; exit 0
fi
[ -d "$SRC" ] || { echo "ERROR: $SRC missing"; exit 1; }
[ -e "$DEST" ] && { echo "ERROR: $DEST already exists — resolve manually"; exit 1; }
mkdir -p "$STATE_ROOT"
mv "$SRC" "$DEST"
ln -s "$DEST" "$LINK"
chgrp -R users "$DEST"
chmod -R g+rX "$DEST"
chgrp users "$STATE_ROOT"; chmod g+rx "$STATE_ROOT"
echo "OK: state now at $DEST (owner reachableceo, group users read-only)"
echo "NOTE: ~/projects symlink + all scripts keep working via $LINK"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# root-rename-inbox.sh — rename /home/_crossfeed/inbox/founder -> inbox/rceo
# (Charles dislikes "founder" as a name for himself.) Idempotent. Root runs:
# sudo bash ~/.coordinate/scripts/root-rename-inbox.sh
set -euo pipefail
[[ $EUID -eq 0 ]] || { echo "run as root: sudo bash $0"; exit 1; }
src=/home/_crossfeed/inbox/founder
dst=/home/_crossfeed/inbox/rceo
if [[ -d $src && ! -e $dst ]]; then
mv "$src" "$dst"
elif [[ -d $dst ]]; then
echo "already renamed"
else
echo "source missing: $src"; ls -la /home/_crossfeed/inbox/; exit 1
fi
chown reachableceo:users "$dst"
chmod 3775 "$dst"
ls -ld "$dst"
echo "OK. Remind PMO to update PROTOCOL.md path references."
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
# semaphore.sh — org-wide agent-turn concurrency gate (z.ai plan: 3
# simultaneous API calls; we approximate by limiting concurrent ACTIVE
# agent processes: headless work-turns + freshly-dispatched turns).
#
# Usage:
# semaphore.sh status -> print active count / max
# semaphore.sh try <tag> <secs> -> exit 0 + hold slot if free, else exit 1
# semaphore.sh release <tag> -> release slot held by <tag>
# semaphore.sh clean -> drop stale holds (>MaxAge, default 30m)
#
# Slot dir: /home/_crossfeed/metrics/active/ (one file per active turn,
# filename = <tag>.pid; content = ISO start). Not a hard API-call gate —
# it bounds AGENT TURN concurrency, the controllable unit.
set -uo pipefail
METRICS=${METRICS:-/home/_crossfeed/metrics}
ACTIVE="$METRICS/active"
MAX=${MAX_CONCURRENT:-2} # headroom of 1 below z.ai cap for interactive PMO use
STALE_SECS=${STALE_SECS:-1800}
mkdir -p "$ACTIVE" 2>/dev/null || true
# graceful degrade: no writable metrics dir -> gate unavailable (exit 99)
[ -w "$ACTIVE" ] || { [ "${1:-}" = status ] && echo "gate=unavailable"; exit 99; }
active_count() {
local n=0 f
for f in "$ACTIVE"/*; do
[ -f "$f" ] || continue
if [ "$STALE_SECS" -gt 0 ]; then
local age=$(( $(date +%s) - $(stat -c %Y "$f") ))
[ "$age" -gt "$STALE_SECS" ] && continue
fi
n=$((n+1))
done
echo "$n"
}
cmd=${1:-status}
case "$cmd" in
status)
echo "active=$(active_count)/$MAX"
[ "$(active_count)" -lt "$MAX" ] && exit 0 || exit 1
;;
try)
tag=${2:?tag}; [ "$#" -ge 3 ] || { echo "usage: try <tag> <max-wait-secs>" >&2; exit 2; }
wait_secs=$3; waited=0
while true; do
if [ "$(active_count)" -lt "$MAX" ]; then
date -Is > "$ACTIVE/$tag.pid" 2>/dev/null && { echo "acquired: $tag"; exit 0; }
echo "ERROR: cannot write $ACTIVE (perms?)" >&2; exit 2
fi
[ "$waited" -ge "$wait_secs" ] && { echo "timeout: no slot after ${wait_secs}s (active=$(active_count)/$MAX)" >&2; exit 1; }
sleep 10; waited=$((waited+10))
done
;;
release)
tag=${2:?tag}
rm -f "$ACTIVE/$tag.pid" && echo "released: $tag"
;;
clean)
for f in "$ACTIVE"/*; do
[ -f "$f" ] || continue
age=$(( $(date +%s) - $(stat -c %Y "$f") ))
[ "$age" -gt "$STALE_SECS" ] && { rm -f "$f"; echo "dropped stale: $(basename "$f")"; }
done
;;
*)
echo "usage: semaphore.sh {status|try <tag> <secs>|release <tag>|clean}" >&2; exit 2
;;
esac
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
# work-chain.sh <chain-name> <task-file>... — sequential overnight work driver.
# Runs dispatch-turn.sh per TASK in order. Skips tasks already REPORTed.
# Stops on first failure; status -> ~/.coordinate/logs/<chain-name>.status
set -uo pipefail
CHAIN=${1:?chain name}; shift
LOGDIR="$HOME/.coordinate/logs"
STATUS="$LOGDIR/$CHAIN.status"
inbox="$HOME/.coordinate/inbox-pmo"
echo "$(date -Is) chain start: $*" > "$STATUS"
for task in "$@"; do
slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//')
if ls "$inbox"/REPORT-*"$slug"* >/dev/null 2>&1; then
echo "$(date -Is) SKIP $slug (reported)" >> "$STATUS"; continue
fi
if [ ! -f "$task" ]; then
echo "$(date -Is) FAIL $slug (task file missing) — chain halted" >> "$STATUS"
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED missing task $slug.\r"
exit 2
fi
echo "$(date -Is) RUN $slug" >> "$STATUS"
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "$CHAIN-$slug" "$task" \
>> "$LOGDIR/$CHAIN-$slug.log" 2>&1
rc=$?
if [ $rc -ne 0 ]; then
echo "$(date -Is) FAIL $slug rc=$rc — chain halted" >> "$STATUS"
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED at $slug - read $STATUS.\r"
exit $rc
fi
echo "$(date -Is) OK $slug" >> "$STATUS"
done
echo "$(date -Is) chain COMPLETE" >> "$STATUS"
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN COMPLETE - read status + inbox-pmo.\r"