ops(scripts): dispatch, queue gating, heartbeat, quota probe
dispatch-turn.sh (headless turn launcher with TURN_MODEL routing), queue-after.sh (serial gate chain + quota-wall guard), pmo-heartbeat.sh (PMO wake loop), quota-probe-then-chain.sh (post-wall relaunch).
This commit is contained in:
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
|
||||
# agent-metrics.sh — harvest per-vertical crush usage into the org ledger.
|
||||
# Appends one JSONL line per session seen, deduped by session id via
|
||||
# ledger/seen index. Idempotent; safe to run every timer tick.
|
||||
#
|
||||
# Ledger: /home/_crossfeed/metrics/ledger.jsonl
|
||||
# Event: {"ts","vertical","role","session","model","prompt_tokens",
|
||||
# "completion_tokens","total_tokens","cost"}
|
||||
# Role heuristic: title contains 'PMO intro' -> pmo; 'INTRO-WORK' -> work;
|
||||
# 'headless' -> workturn; else 'other'.
|
||||
set -uo pipefail
|
||||
|
||||
METRICS=${METRICS:-/home/_crossfeed/metrics}
|
||||
LEDGER="$METRICS/ledger.jsonl"
|
||||
SEEN="$METRICS/.seen-ids"
|
||||
VERTICALS=(TSGBOD TSGCTO TSGCCO TSGCOO reachableceo)
|
||||
RUNAS=${RUNAS:-sudo} # agent-metrics runs from the supervisor service (root)
|
||||
|
||||
mkdir -p "$METRICS"; touch "$LEDGER" "$SEEN"
|
||||
|
||||
emit() { printf '%s\n' "$1" >> "$LEDGER"; }
|
||||
|
||||
for user in "${VERTICALS[@]}"; do
|
||||
home=$(getent passwd "$user" | cut -d: -f6)
|
||||
[ -n "$home" ] || continue
|
||||
# shellcheck disable=SC2016
|
||||
json=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
|
||||
bash -c "cd '$home' && crush session list --json 2>/dev/null" || true)
|
||||
[ -n "$json" ] || continue
|
||||
# one session id per line; token/cost fields only exist in `session show`
|
||||
while IFS= read -r sid; do
|
||||
[ -n "$sid" ] || continue
|
||||
grep -q "$sid" "$SEEN" && continue
|
||||
obj=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
|
||||
bash -c "cd '$home' && crush session show '$sid' --json 2>/dev/null | head -c 4000" || true)
|
||||
[ -n "$obj" ] || continue
|
||||
title=$(printf '%s' "$obj" | grep -o '"title":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||||
role=other
|
||||
case "$title" in
|
||||
*PMO*intro*) role=pmo ;;
|
||||
*INTRO-WORK*) role=work ;;
|
||||
*headless*|*TSG*-Work*executing*) role=workturn ;;
|
||||
esac
|
||||
model=$(printf '%s' "$obj" | grep -o '"model":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||||
pt=$(printf '%s' "$obj" | grep -o '"prompt_tokens":[0-9]*' | head -1 | cut -d: -f2)
|
||||
ct=$(printf '%s' "$obj" | grep -o '"completion_tokens":[0-9]*' | head -1 | cut -d: -f2)
|
||||
tt=$(printf '%s' "$obj" | grep -o '"total_tokens":[0-9]*' | head -1 | cut -d: -f2)
|
||||
cost=$(printf '%s' "$obj" | grep -o '"cost":[0-9.e-]*' | head -1 | cut -d: -f2)
|
||||
ts=$(date -Is)
|
||||
emit "{\"ts\":\"$ts\",\"vertical\":\"$user\",\"role\":\"$role\",\"session\":\"$sid\",\"model\":\"${model:-?}\",\"prompt_tokens\":${pt:-0},\"completion_tokens\":${ct:-0},\"total_tokens\":${tt:-0},\"cost\":${cost:-0}}"
|
||||
echo "$sid" >> "$SEEN"
|
||||
done < <(printf '%s' "$json" | grep -o '"id":"[0-9a-f]*"' | cut -d'"' -f4)
|
||||
done
|
||||
|
||||
# rolling summary
|
||||
python3 - "$LEDGER" <<'PYEOF' > "$METRICS/SUMMARY.md" 2>/dev/null || true
|
||||
import json, sys, collections, datetime
|
||||
tot = collections.Counter(); cost = collections.Counter(); n = collections.Counter()
|
||||
day = collections.Counter()
|
||||
now = datetime.datetime.now().astimezone()
|
||||
for line in open(sys.argv[1]):
|
||||
try: e = json.loads(line)
|
||||
except Exception: continue
|
||||
v = e.get("vertical","?"); t = e.get("total_tokens",0)
|
||||
tot[v] += t; cost[v] += e.get("cost",0.0); n[v] += 1
|
||||
try:
|
||||
ts = datetime.datetime.fromisoformat(e["ts"])
|
||||
if (now-ts).total_seconds() < 86400: day[v] += t
|
||||
except Exception: pass
|
||||
print(f"# Agent usage summary — {now.isoformat()}")
|
||||
print(f"- sessions ledgered total: {sum(n.values())}")
|
||||
print("- all-time by vertical:")
|
||||
for v in sorted(tot):
|
||||
print(f" {v:<13} n={n[v]:<4} tokens={tot[v]:<10} cost={cost[v]:.4f} (24h tokens={day[v]})")
|
||||
PYEOF
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
|
||||
# dispatch-turn.sh <slug> <prompt-file> — headless work-turn launcher.
|
||||
# Semaphore-gated (org concurrency cap), FRESH session by default (small
|
||||
# context = quota-friendly; the turn self-bootstraps from the prompt file),
|
||||
# success/fail doorbell to the vertical PMO screen. PMO usage:
|
||||
# screen -dmS work-turn-<slug> \
|
||||
# ~/.coordinate/scripts/dispatch-turn.sh <slug> <prompt-file>
|
||||
set -uo pipefail
|
||||
|
||||
SLUG=${1:?slug}; PROMPT=${2:?prompt-file}
|
||||
TOOLING=${TOOLING:-/home/_crossfeed/tooling}
|
||||
SEMI="$TOOLING/agent-stack/semaphore.sh"
|
||||
LOG="$HOME/.coordinate/logs/$SLUG.log"
|
||||
TAG="$USER-$SLUG"
|
||||
WAIT=${WAIT_SECS:-300}
|
||||
|
||||
[ -f "$PROMPT" ] || { echo "no prompt file: $PROMPT" >&2; exit 1; }
|
||||
|
||||
semirc=0
|
||||
bash "$SEMI" try "$TAG" "$WAIT" >/dev/null 2>&1 || semirc=$?
|
||||
if [ "$semirc" = 99 ]; then
|
||||
echo "$(date -Is) semaphore gate unavailable - proceeding UNGATED (install metrics dirs)" >> "$LOG"
|
||||
elif [ "$semirc" != 0 ]; then
|
||||
echo "$(date -Is) semaphore rc=$semirc after ${WAIT}s budget - not launched" >> "$LOG"
|
||||
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn DEFERRED (concurrency cap) - rerun later: %s.\r' "$SLUG")" || true
|
||||
exit 3
|
||||
fi
|
||||
GATED=1; [ "$semirc" = 0 ] || GATED=0
|
||||
|
||||
cd "$HOME"
|
||||
rc=0
|
||||
# 2026-08-29 ~06:30: Charles upgraded to MAX plan, all buckets zero.
|
||||
# Quality-first: default back to glm-5.2; TURN_MODEL=glm-4.7-flash for
|
||||
# grind/admin turns when wanted.
|
||||
TURN_MODEL=${TURN_MODEL:-litellm/glm-5.2}
|
||||
env HOME="$HOME" TERM=xterm-256color \
|
||||
crush run --quiet --model "$TURN_MODEL" "$(cat "$PROMPT")" >> "$LOG" 2>&1 || rc=$?
|
||||
|
||||
bash "$SEMI" release "$TAG" >/dev/null 2>&1 || true
|
||||
|
||||
if [ "$rc" = 0 ]; then
|
||||
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn done OK - read inbox-pmo + logs/%s.\r' "$SLUG")" || true
|
||||
else
|
||||
screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn FAILED (rc=%s) - read logs/%s.\r' "$rc" "$SLUG")" || true
|
||||
fi
|
||||
exit "$rc"
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# founder-crossfeed-inboxes.sh — FOUNDER-RUN as root.
|
||||
# Cross-vertical inbox interconnect under /home/_crossfeed/inbox/.
|
||||
# Semantics per dir: owner=<vertical acct>, group=users, mode 3775
|
||||
# (setgid+sticky): any vertical may CREATE files in any inbox; only the
|
||||
# file's author or the inbox OWNER may move/delete. Homes stay 700.
|
||||
set -uo pipefail
|
||||
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
|
||||
|
||||
declare -A VMAP=( [bod]=TSGBOD [cco]=TSGCCO [cto]=TSGCTO [coo]=TSGCOO [founder]=reachableceo )
|
||||
|
||||
echo "== preflight =="
|
||||
for v in "${!VMAP[@]}"; do
|
||||
acct=${VMAP[$v]}; id "$acct" >/dev/null 2>&1 || { echo "missing acct $acct" >&2; exit 1; }
|
||||
done
|
||||
echo "ok: all five accounts exist"
|
||||
|
||||
echo "== create inbox tree =="
|
||||
for v in bod cco cto coo founder; do
|
||||
acct=${VMAP[$v]}
|
||||
d=/home/_crossfeed/inbox/$v
|
||||
mkdir -p "$d"
|
||||
chown "$acct":users "$d"
|
||||
chmod 3775 "$d"
|
||||
echo " $d -> $acct:users 3775"
|
||||
done
|
||||
|
||||
echo "== update crossfeed README =="
|
||||
grep -q 'inbox/' /home/_crossfeed/README.md || cat >> /home/_crossfeed/README.md <<'EOF'
|
||||
|
||||
## inbox/ — cross-vertical direct messaging (2026-08-28)
|
||||
/home/_crossfeed/inbox/<bod|cco|cto|coo|founder>/ — mode 3775, owner is the
|
||||
receiving vertical, group users. Any vertical may DROP a file in any inbox
|
||||
(name: MSG-<from>-<HHMM>-<slug>.md). Only the author or the receiving
|
||||
vertical consumes/moves/deletes. This is for lightweight cross-vertical
|
||||
coordination when a Redmine/Discourse round trip is overkill; requests
|
||||
with teeth still go via the founder or systems of record, never MSG files.
|
||||
Receiving PMO scans own inbox at turn start (alongside peers' STATUS).
|
||||
EOF
|
||||
echo " README updated"
|
||||
|
||||
echo "== drop NOTICE into each vertical's inbox-pmo =="
|
||||
for v in bod cco cto coo; do
|
||||
acct=${VMAP[$v]}
|
||||
home=$(getent passwd "$acct" | cut -d: -f6)
|
||||
[ -d "$home/.coordinate/inbox-pmo" ] || continue
|
||||
cat > "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md" <<'EOF'
|
||||
Founder directive: /home/_crossfeed/inbox/<bod|cco|cto|coo|founder>/ is live.
|
||||
Cross-vertical direct messaging: any PMO may drop
|
||||
MSG-<from>-<HHMM>-<slug>.md in another vertical's inbox (dirs 3775: anyone
|
||||
creates, only author or inbox owner removes). Scan YOUR inbox at turn
|
||||
start. Lightweight coordination only — requests with teeth go via the
|
||||
founder or systems of record. Archive this notice after reading.
|
||||
EOF
|
||||
chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md"
|
||||
echo " NOTICE -> $acct"
|
||||
done
|
||||
|
||||
echo "== verify =="
|
||||
ls -la /home/_crossfeed/inbox/
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# founder-full-reset.sh — FOUNDER-RUN as reachableceo (sudo required).
|
||||
# Org-wide clean reset of all FIVE verticals: TSGBOD TSGCCO TSGCTO TSGCOO RCEO.
|
||||
# 1) kill every vertical screen (old 07:02 + new 11:13 generations alike)
|
||||
# 2) clean stale /run/screen sockets
|
||||
# 3) relaunch the four TSG stacks via /tmp/tsg-stack-relaunch.sh --yes
|
||||
# (no live screens -> no duplicate names -> multiuser+acladd succeeds)
|
||||
# 4) give RCEO the same wrapper pattern: pmo-loop.sh + pinned brains
|
||||
# 5) verify + attach cheat sheet
|
||||
# Does NOT touch AZHost/AgentZero or any other screens. RCEO-PMO/RCEO-Work
|
||||
# (agent screens) die mid-run and return pinned (conversations preserved).
|
||||
# Blank-slate lever (NOT default): before running, founder may
|
||||
# touch ~/.coordinate/RELAUNCH-FRESH ~/.coordinate/RELAUNCH-FRESH-WORK
|
||||
# but fresh sessions get NO intro injected by the wrapper — skip unless
|
||||
# you plan to hand-feed the INTRO prompts after.
|
||||
set -uo pipefail
|
||||
|
||||
[ "$(id -un)" = "reachableceo" ] || { echo "run as reachableceo"; exit 1; }
|
||||
sudo -v || { echo "sudo required"; exit 1; }
|
||||
|
||||
USERS=(TSGBOD TSGCCO TSGCTO TSGCOO)
|
||||
|
||||
echo "== 1/5 kill all vertical screens (both generations) =="
|
||||
pat='SCREEN -dmS (TSGBOD-|TSGCCO-|TSGCTO-|TSGCOO-|RCEO-)'
|
||||
pids=$(pgrep -f "$pat" || true)
|
||||
if [ -n "$pids" ]; then sudo kill $pids 2>/dev/null; fi
|
||||
sleep 2
|
||||
pids=$(pgrep -f "$pat" || true)
|
||||
if [ -n "$pids" ]; then sudo kill -9 $pids 2>/dev/null; fi
|
||||
for u in "${USERS[@]}" reachableceo; do
|
||||
sudo pkill -u "$u" -f pmo-loop.sh 2>/dev/null
|
||||
sudo pkill -u "$u" -f 'crush --' 2>/dev/null
|
||||
sudo pkill -u "$u" -fx 'crush' 2>/dev/null
|
||||
done
|
||||
sleep 1
|
||||
|
||||
echo "== 2/5 clean stale sockets =="
|
||||
for u in "${USERS[@]}" reachableceo; do
|
||||
sudo rm -f "/run/screen/S-$u"/* 2>/dev/null
|
||||
done
|
||||
|
||||
echo "== 3/5 relaunch TSG stacks (wrappers + existing pins) =="
|
||||
if [ -f /tmp/tsg-stack-relaunch.sh ]; then
|
||||
bash /tmp/tsg-stack-relaunch.sh --yes || echo "WARN: relaunch script failed — inspect its output"
|
||||
else
|
||||
echo "WARN: /tmp/tsg-stack-relaunch.sh missing — TSG stacks NOT relaunched"
|
||||
fi
|
||||
|
||||
echo "== 4/5 relaunch RCEO stack (wrapper parity) =="
|
||||
mkdir -p "$HOME/.coordinate/scripts" "$HOME/.coordinate/logs"
|
||||
cat > "$HOME/.coordinate/scripts/pmo-loop.sh" <<'WRAP'
|
||||
#!/usr/bin/env bash
|
||||
# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen.
|
||||
ROLE="${1:-pmo}"
|
||||
case "$ROLE" in
|
||||
pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;;
|
||||
work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;;
|
||||
*) echo "usage: $0 [pmo|work]" >&2; exit 1 ;;
|
||||
esac
|
||||
LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log"
|
||||
cd "$HOME"
|
||||
export TERM="${TERM:-xterm-256color}"
|
||||
export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
|
||||
while true; do
|
||||
started=$(date +%s)
|
||||
if [ -f "$FRESH" ]; then
|
||||
rm -f "$FRESH"
|
||||
echo "$(date -Is) relaunch FRESH" >> "$LOG"
|
||||
crush
|
||||
elif [ -s "$PIN" ]; then
|
||||
sid=$(cat "$PIN")
|
||||
echo "$(date -Is) relaunch PINNED $sid" >> "$LOG"
|
||||
crush --session "$sid"
|
||||
else
|
||||
echo "$(date -Is) relaunch CONTINUE" >> "$LOG"
|
||||
crush --continue
|
||||
fi
|
||||
rt=$(( $(date +%s) - started ))
|
||||
if [ "$rt" -lt 60 ]; then
|
||||
echo "$(date -Is) short run (${rt}s) - backoff 60s" >> "$LOG"
|
||||
sleep 60
|
||||
else
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
WRAP
|
||||
chmod 755 "$HOME/.coordinate/scripts/pmo-loop.sh"
|
||||
echo 78cf63b8a72f4282 > "$HOME/.coordinate/PMO-SESSION-ID"
|
||||
echo e1079fafb8c37d60 > "$HOME/.coordinate/WORK-SESSION-ID"
|
||||
env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-PMO \
|
||||
/home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo
|
||||
env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-Work \
|
||||
/home/reachableceo/.coordinate/scripts/pmo-loop.sh work
|
||||
|
||||
echo "== 5/5 verify =="
|
||||
sleep 3
|
||||
screen -ls
|
||||
for u in "${USERS[@]}"; do
|
||||
echo "-- $u:"
|
||||
sudo -u "$u" env TERM=dumb screen -ls 2>/dev/null | grep -E 'PMO|Work' || echo " NONE (check relaunch output)"
|
||||
done
|
||||
echo
|
||||
echo "Attach: TSGxx-PMO / TSGxx-work aliases; RCEO: RCEO-PMO | RCEO-work"
|
||||
echo "Reboot bring-up units (enabled): reachableceo-agent-stack.service,"
|
||||
echo "tsg-agent-stacks.service — safe to reboot-test after this."
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# founder-tooling-and-yolo.sh — FOUNDER-RUN: sudo bash ~/.coordinate/scripts/founder-tooling-and-yolo.sh
|
||||
# 1) publish /home/_crossfeed/tooling (3 group-shared git repos) from PMO draft
|
||||
# 2) fix org-wide --yolo regression: install canonical wrapper into the four
|
||||
# TSG verticals + restart their screens (8 agents currently non-yolo)
|
||||
# 3) deprecate /tmp/tsg-stack-relaunch.sh (v1, source of both regressions)
|
||||
# 4) restart RCEO-PMO with canonical wrapper (PMO still non-yolo), then
|
||||
# doorbell the PMO to finish live-repo verification + publish STATUS
|
||||
set -uo pipefail
|
||||
DRAFT=/home/reachableceo/.coordinate/tooling-draft
|
||||
TOOLING=/home/_crossfeed/tooling
|
||||
FOUNDER=reachableceo
|
||||
|
||||
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
|
||||
|
||||
echo "== 1/4 publish tooling repos =="
|
||||
mkdir -p "$TOOLING"
|
||||
cp -a "$DRAFT"/. "$TOOLING"/
|
||||
chown -R $FOUNDER:users "$TOOLING"
|
||||
chmod 2775 "$TOOLING"
|
||||
find "$TOOLING" -type d -exec chmod g+s {} +
|
||||
find "$TOOLING" -type f -exec chmod g+rw {} +
|
||||
chmod -R o-rwx "$TOOLING"
|
||||
ls -la "$TOOLING"
|
||||
|
||||
echo "== 2/4 TSG stacks: canonical wrapper (--yolo) + restart =="
|
||||
bash "$TOOLING/agent-stack/tsg-stack-relaunch-v2.sh"
|
||||
|
||||
echo "== 3/4 deprecate /tmp v1 relaunch =="
|
||||
for f in /tmp/tsg-stack-relaunch.sh; do
|
||||
[ -f "$f" ] && mv "$f" "$f.deprecated-20260828" && echo "moved: $f -> $f.deprecated-20260828"
|
||||
done
|
||||
|
||||
echo "== 4/4 RCEO-PMO restart with canonical wrapper + doorbell =="
|
||||
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
|
||||
screen -S RCEO-PMO -X quit 2>/dev/null || true
|
||||
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
|
||||
screen -dmS RCEO-PMO /home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo
|
||||
echo "waiting 20s for PMO crush to settle..."
|
||||
sleep 20
|
||||
runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \
|
||||
screen -S RCEO-PMO -X stuff $'Tooling live + yolo fix applied. Next turn: verify /home/_crossfeed/tooling, log + publish STATUS.\r' \
|
||||
&& echo "doorbelled RCEO-PMO" || echo "doorbell failed - ping PMO manually"
|
||||
|
||||
echo "== VERIFY =="
|
||||
for u in TSGBOD TSGCCO TSGCTO TSGCOO $FOUNDER; do
|
||||
echo " $u sockets: $(ls /run/screen/S-$u 2>/dev/null | tr '\n' ' ')"
|
||||
done
|
||||
echo "yolo agents: $(pgrep -af 'crush --yolo' | wc -l) / expected 10"
|
||||
echo "Attach: RCEO-PMO | RCEO-work; TSGxx-PMO | TSGxx-work aliases."
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# founder-wire-tooling.sh — FOUNDER-RUN as root.
|
||||
# Wire the tooling repo (agent-stack @ current HEAD) into all four TSG
|
||||
# verticals: canonical pmo-loop.sh (metrics-ledgered), dispatch-turn.sh,
|
||||
# semaphore.sh, agent-metrics.sh into ~/.coordinate/scripts/. RCEO already
|
||||
# wired itself. Supervisor units (tsg-supervisor.*) already org-wide.
|
||||
# Stamped copies per provenance rule; fix forward in the repo, reinstall.
|
||||
set -uo pipefail
|
||||
[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; }
|
||||
|
||||
TOOLING=/home/_crossfeed/tooling
|
||||
SRC=$TOOLING/agent-stack
|
||||
SHA=$(git -C "$SRC" rev-parse --short HEAD)
|
||||
FILES="pmo-loop.sh dispatch-turn.sh semaphore.sh agent-metrics.sh"
|
||||
|
||||
echo "== preflight =="
|
||||
for f in $FILES; do [ -f "$SRC/$f" ] || { echo "missing $SRC/$f" >&2; exit 1; }; done
|
||||
echo "ok: source files present @ $SHA"
|
||||
|
||||
for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do
|
||||
home=$(getent passwd "$acct" | cut -d: -f6)
|
||||
echo "== $acct =="
|
||||
mkdir -p "$home/.coordinate/scripts" "$home/.coordinate/logs"
|
||||
for f in $FILES; do
|
||||
install -o "$acct" -g "$acct" -m755 "$SRC/$f" "$home/.coordinate/scripts/$f"
|
||||
done
|
||||
sed -i "1a # installed: agent-stack @ $SHA ($(date -I))" "$home/.coordinate/scripts/pmo-loop.sh"
|
||||
cat > "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md" <<EOF
|
||||
Founder directive: your ~/.coordinate/scripts now carries agent-stack @ $SHA
|
||||
(repo: /home/_crossfeed/tooling/agent-stack). Dispatch work-turns with:
|
||||
screen -dmS work-turn-<slug> ~/.coordinate/scripts/dispatch-turn.sh <slug> <prompt-file>
|
||||
(semaphore-gated, fresh session, doorbell back to your PMO screen; see
|
||||
GOVERNANCE.md in the repo: capture free, execution metered; founder gate
|
||||
on every TASK still applies). Wrapper now logs launches to metrics. Fix
|
||||
bugs IN THE REPO (commit as your uid via tooling/commit.sh), then rerun
|
||||
this installer. Archive this notice after reading.
|
||||
EOF
|
||||
chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md"
|
||||
echo " wired + NOTICE"
|
||||
done
|
||||
|
||||
echo "== verify =="
|
||||
for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do
|
||||
home=$(getent passwd "$acct" | cut -d: -f6)
|
||||
echo " $acct: $(ls "$home/.coordinate/scripts/" 2>/dev/null | tr '\n' ' ')"
|
||||
done
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# pmo-heartbeat.sh — wake the PMO session while the overnight chain runs.
|
||||
# Self-terminates when inbox-work has no TASKs AND no semaphore slots are held.
|
||||
|
||||
INBOX=/home/reachableceo/.coordinate/inbox-work
|
||||
SLOTS=/home/_crossfeed/metrics/active
|
||||
LOG=/home/reachableceo/.coordinate/logs/heartbeat.log
|
||||
INTERVAL=3600
|
||||
|
||||
while true; do
|
||||
pending=$(ls -1 "$INBOX"/TASK-*.md 2>/dev/null | wc -l)
|
||||
active=$(ls -1 "$SLOTS"/*.pid 2>/dev/null | wc -l)
|
||||
if [ "$pending" -eq 0 ] && [ "$active" -eq 0 ]; then
|
||||
echo "$(date -Is) chain drained — heartbeat exiting" >> "$LOG"
|
||||
exit 0
|
||||
fi
|
||||
screen -S RCEO-PMO -X stuff "hb: chain check — verify reports, archive, mirror redmine, redispatch flakes, dispatch next$(printf '\r')"
|
||||
echo "$(date -Is) wake sent (pending=$pending active=$active)" >> "$LOG"
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen.
|
||||
ROLE="${1:-pmo}"
|
||||
case "$ROLE" in
|
||||
pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;;
|
||||
work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;;
|
||||
*) echo "usage: $0 [pmo|work]" >&2; exit 1 ;;
|
||||
esac
|
||||
LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log"
|
||||
LEDGER=/home/_crossfeed/metrics/launches.jsonl
|
||||
ledger() { printf '%s\n' "$1" >> "$LEDGER" 2>/dev/null || true; }
|
||||
cd "$HOME"
|
||||
export TERM="${TERM:-xterm-256color}"
|
||||
export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
|
||||
while true; do
|
||||
started=$(date +%s)
|
||||
mode=continue
|
||||
if [ -f "$FRESH" ]; then
|
||||
rm -f "$FRESH"
|
||||
mode=fresh
|
||||
echo "$(date -Is) relaunch FRESH" >> "$LOG"
|
||||
crush --yolo
|
||||
elif [ -s "$PIN" ]; then
|
||||
sid=$(cat "$PIN")
|
||||
mode=pinned:$sid
|
||||
echo "$(date -Is) relaunch PINNED $sid" >> "$LOG"
|
||||
crush --yolo --session "$sid"
|
||||
else
|
||||
echo "$(date -Is) relaunch CONTINUE" >> "$LOG"
|
||||
crush --yolo --continue
|
||||
fi
|
||||
rc=$?
|
||||
rt=$(( $(date +%s) - started ))
|
||||
ledger "{\"ts\":\"$(date -Is)\",\"user\":\"$USER\",\"role\":\"$ROLE\",\"mode\":\"$mode\",\"rc\":$rc,\"runtime_s\":$rt}"
|
||||
if [ "$rt" -lt 60 ]; then
|
||||
echo "$(date -Is) short run (${rt}s rc=$rc) - backoff 60s" >> "$LOG"
|
||||
sleep 60
|
||||
else
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# queue-after.sh <gate-screen> <slug> <task-file>
|
||||
# Wait until <gate-screen> no longer exists, then dispatch the task turn.
|
||||
# Keeps repo-touching turns serialized when two queue screens would overlap.
|
||||
set -uo pipefail
|
||||
GATE="$1"; SLUG="$2"; TASK="$3"
|
||||
LOGDIR="$HOME/.coordinate/logs"
|
||||
while screen -ls 2>/dev/null | grep -q "\.${GATE}[[:space:]]"; do
|
||||
sleep 60
|
||||
done
|
||||
# QUOTA WALL GUARD: if any recent turn log shows the z.ai usage wall,
|
||||
# halt the chain instead of burning the next turn into the same wall.
|
||||
WALLLOG=$(ls -t "$LOGDIR"/q-*.log 2>/dev/null | head -1)
|
||||
if [ -n "$WALLLOG" ] && grep -q "Usage limit reached" "$WALLLOG" 2>/dev/null; then
|
||||
echo "$(date -Is) QUOTA WALL detected in $WALLLOG — chain halted before $SLUG" >> "$LOGDIR/queue.status"
|
||||
screen -S RCEO-PMO -X stuff "QUOTA WALL - chain halted before $SLUG. Do not dispatch until quota verified.$(printf '\r')" || true
|
||||
exit 0
|
||||
fi
|
||||
echo "$(date -Is) GATE $GATE cleared — RUN $SLUG" >> "$LOGDIR/queue.status"
|
||||
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$SLUG" "$TASK" \
|
||||
>> "$LOGDIR/queue-$SLUG.log" 2>&1
|
||||
rc=$?
|
||||
echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo "FAIL(rc=$rc)") $SLUG" >> "$LOGDIR/queue.status"
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# queue-next.sh — bash-only slot watcher: run queued TASK turns as slots free.
|
||||
# rc=3 from dispatch = DEFERRED (cap) -> keep waiting on same task, don't exit.
|
||||
set -uo pipefail
|
||||
LOGDIR="$HOME/.coordinate/logs"
|
||||
i=1
|
||||
while [ $i -le $# ]; do
|
||||
task=${!i}
|
||||
i=$((i+1))
|
||||
slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//')
|
||||
while true; do
|
||||
n=$(ls /home/_crossfeed/metrics/active/*.pid 2>/dev/null | wc -l)
|
||||
[ "$n" -lt 2 ] && break
|
||||
sleep 60
|
||||
done
|
||||
echo "$(date -Is) RUN $slug" >> "$LOGDIR/queue.status"
|
||||
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$slug" "$task" \
|
||||
>> "$LOGDIR/queue-$slug.log" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" = 3 ]; then
|
||||
echo "$(date -Is) DEFER $slug (cap) - re-queueing" >> "$LOGDIR/queue.status"
|
||||
i=$((i-1)); sleep 30; continue
|
||||
fi
|
||||
echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo FAIL) $slug rc=$rc" >> "$LOGDIR/queue.status"
|
||||
[ $rc -ne 0 ] && exit $rc
|
||||
done
|
||||
echo "$(date -Is) queue complete" >> "$LOGDIR/queue.status"
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# quota-probe-then-chain.sh — wait for z.ai quota window to reset, then
|
||||
# launch the serial build chain. Probe: 8-token glm-4.7-flash ping via the
|
||||
# harness vkey. On first 200 -> fire work-q1..q4 chain and exit.
|
||||
set -uo pipefail
|
||||
VKEY_FILE="$HOME/.coordinate/secrets/mopac-harness-vkey.env"
|
||||
LOG="$HOME/.coordinate/logs/queue.status"
|
||||
PROBE_LOG="$HOME/.coordinate/logs/quota-probe.log"
|
||||
|
||||
while true; do
|
||||
ok=$(bash -c "set -a; source '$VKEY_FILE'; set +a; python3 - <<EOF
|
||||
import os, json, urllib.request, urllib.error
|
||||
body = json.dumps({'model': 'glm-4.7-flash', 'messages': [{'role': 'user', 'content': 'ping'}], 'max_tokens': 8}).encode()
|
||||
req = urllib.request.Request('http://192.168.3.78:4001/v1/chat/completions', data=body, headers={'Authorization': 'Bearer ' + os.environ['HARNESS_LITELLM_KEY'], 'Content-Type': 'application/json'})
|
||||
try:
|
||||
urllib.request.urlopen(req, timeout=30)
|
||||
print('OK')
|
||||
except urllib.error.HTTPError as e:
|
||||
print('HTTP', e.code)
|
||||
except Exception as e:
|
||||
print('ERR', type(e).__name__)
|
||||
EOF" 2>/dev/null)
|
||||
echo "$(date -Is) probe: $ok" >> "$PROBE_LOG"
|
||||
if [ "$ok" = "OK" ]; then
|
||||
echo "$(date -Is) QUOTA BACK — launching build chain" >> "$LOG"
|
||||
W="$HOME/.coordinate/scripts"
|
||||
T="$HOME/.coordinate/inbox-work"
|
||||
screen -dmS work-q1 bash "$W/queue-after.sh" NONE-GATE docs-standards "$T/TASK-20260828-2100-docs-standards.md"
|
||||
screen -dmS work-q2 bash "$W/queue-after.sh" work-q1 events-receiver "$T/TASK-20260828-1975-events-receiver.md"
|
||||
screen -dmS work-q3 bash "$W/queue-after.sh" work-q2 keyproxy-v0 "$T/TASK-20260828-2130-keyproxy-v0.md"
|
||||
screen -dmS work-q4 bash "$W/queue-after.sh" work-q3 selfhost-core "$T/TASK-20260828-2131-selfhost-core.md"
|
||||
echo "$(date -Is) chain armed: docs -> events -> keyproxy-v0 -> selfhost-core" >> "$LOG"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1200
|
||||
done
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# relaunch-screen.sh <name> <pmo|work> [fuse] — restart an RCEO screen so it
|
||||
# re-reads ~/.screenrc (termcapinfo scroll fix). Fuse >0 schedules via helper
|
||||
# screen (used to restart RCEO-PMO from inside itself). Owner-run; no root.
|
||||
set -euo pipefail
|
||||
name="${1:?screen name}"; role="${2:?pmo|work}"; fuse="${3:-0}"
|
||||
SELF="$HOME/.coordinate/scripts/relaunch-screen.sh"
|
||||
if [ "$fuse" -gt 0 ]; then
|
||||
screen -S RCEO-Relaunch -X quit 2>/dev/null || true
|
||||
screen -dmS RCEO-Relaunch bash -c "sleep $fuse; $SELF $name $role 0"
|
||||
echo "scheduled: $name restart in ${fuse}s"
|
||||
exit 0
|
||||
fi
|
||||
screen -S "$name" -X quit 2>/dev/null || true
|
||||
sleep 2
|
||||
screen -dmS "$name" "$HOME/.coordinate/scripts/pmo-loop.sh" "$role"
|
||||
sleep 5
|
||||
if screen -ls 2>/dev/null | grep -q "\.$name"; then
|
||||
echo "OK: $name relaunched"
|
||||
else
|
||||
echo "FAILED: $name not present after relaunch" >&2; exit 1
|
||||
fi
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# root-archive-tmp-artifacts.sh — archive /tmp agent artifacts into _crossfeed.
|
||||
# meta (stack launchers/coordinate/crossfeed tooling) -> tooling/agent-stack/artifacts-archive/
|
||||
# everything else (ops/tickets/monitoring) -> coo/artifacts-archive/
|
||||
# Usage: sudo bash root-archive-tmp-artifacts.sh [--dry-run]
|
||||
set -euo pipefail
|
||||
|
||||
DRY=0; [ "${1:-}" = "--dry-run" ] && DRY=1
|
||||
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
|
||||
|
||||
SRC=/tmp
|
||||
META_ROOT=/home/_crossfeed/tooling/agent-stack/artifacts-archive
|
||||
COO_ROOT=/home/_crossfeed/coo/artifacts-archive
|
||||
STAMP=$(date +%Y%m%d-%H%M%S)
|
||||
META_DEST="$META_ROOT/$STAMP"
|
||||
COO_DEST="$COO_ROOT/$STAMP"
|
||||
|
||||
# Never touch: system entries + .crush (live TSGCOO crush session data)
|
||||
is_system() {
|
||||
case "$1" in
|
||||
systemd-private-*|.X11-unix|.ICE-unix|.font-unix|.Test-unix|.XIM-unix|.IMAP-unix|\
|
||||
sddm*|xauth_*|ssh-*|scoped_dir*|.crush) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Meta-scoped: stack launchers, coordinate/crossfeed setup, prompts, boot captures, standdown kits
|
||||
is_meta() {
|
||||
case "$1" in
|
||||
launch-all-tsg-stacks.sh|launch-tsgcoo-stack.sh|launch-coordinate.sh|\
|
||||
setup-tsg-stack.sh|setup-tsgcoo-coordinate.sh|\
|
||||
tsg-pmo-autorelaunch.sh|tsg-stack-relaunch.sh.deprecated-20260828|tsg-crossfeed-setup.sh|\
|
||||
tsg-agent-stacks.service|tsgcoo-bootstrap.sh|tsgcoo-cleanup.sh|tsgcoo-crush-prompt.md|\
|
||||
tsgcoo-crush-prompt.oneline|tsgcoo-paste.sh|tsgcoo-pmo2.sh|tsgcoo-preflight.sh|\
|
||||
tsgcoo-screen-setup.sh|tsgcoo-stack-fix.sh|tsgcoo-stack-fix2.sh|\
|
||||
mk-coordinate.sh|rm-coordinate.sh|local-pmo.sh|pmo-check.sh|pmo-prompt.md|pmo-prompt.oneline|\
|
||||
prompt-knel-basis.md|prompt-legacy-fold.md|legacy-fold.log|\
|
||||
seed-intros.sh|run-seed.sh|run-setup.sh|\
|
||||
LIFTSHIFT-MANIFEST.md|liftshift-copy.sh|liftshift-copy2.sh|liftshift-manifest|\
|
||||
tsgcap1.txt|localpmo2.txt|localpmo3.txt|localpmo-after.txt|localpmo-boot.txt|\
|
||||
crushboot.txt|crushafter.txt|crushafter2.txt|hb1.txt|hb2.txt|yolotest.log|\
|
||||
tsgpmo2.txt|tsgpmo3.txt|tsgpmo-after.txt|tsgpmo-boot.txt|\
|
||||
pmo-before.txt|pmo-after.txt|pmo-yolo.txt|\
|
||||
work-before.txt|work-after.txt|work-screen.txt|work-screen2.txt|work-yolo.txt|\
|
||||
bodw2.txt|cco-pmo.txt|cco-work2.txt|\
|
||||
TSGBOD-pmo.txt|TSGBOD-work.txt|TSGCCO-pmo.txt|TSGCTO-pmo.txt|TSGCTO-work.txt|\
|
||||
CTOHandoff.md|COO-PMO-RESUME.md|human-feed-hud.txt|\
|
||||
offstage-kit|cos-pkg|cos-pkg-v2|\
|
||||
w.sh|w1.sh|tl.sh|logs.sh|fix.sh|verify.sh|restart.sh|provision.sh|dbg.sh|\
|
||||
inspect.sh|inspect-body.sh|inspect-wrap.sh|\
|
||||
install-unit.sh|install-body.sh|final2.sh|final-check.sh|final-topology.sh|\
|
||||
ssh-diag.sh|wrapper-extract.sh|w-extract.sh) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for d in /home/_crossfeed/coo "$META_ROOT" "$COO_ROOT"; do
|
||||
[ -d "$(dirname "$d")" ] || { echo "ERROR: missing $d"; exit 1; }
|
||||
done
|
||||
|
||||
if [ "$DRY" -eq 0 ]; then
|
||||
mkdir -p "$META_DEST" "$COO_DEST"
|
||||
fi
|
||||
META_MANIFEST="${META_DEST:-/tmp}/MANIFEST.txt"; COO_MANIFEST="${COO_DEST:-/tmp}/MANIFEST.txt"
|
||||
[ "$DRY" -eq 1 ] && { META_MANIFEST=/dev/null; COO_MANIFEST=/dev/null; }
|
||||
|
||||
meta_n=0; coo_n=0; sys_n=0
|
||||
for f in "$SRC"/* "$SRC"/.[!.]*; do
|
||||
[ -e "$f" ] || continue
|
||||
name=$(basename "$f")
|
||||
if is_system "$name"; then
|
||||
echo "SKIP(system) $name"; sys_n=$((sys_n+1)); continue
|
||||
fi
|
||||
meta=$(stat -c '%U:%G:%s:%y' "$f")
|
||||
if is_meta "$name"; then
|
||||
printf '%s|%s\n' "$name" "$meta" >> "$META_MANIFEST"
|
||||
echo "META $name ($meta)"
|
||||
[ "$DRY" -eq 0 ] && mv -- "$f" "$META_DEST/"
|
||||
meta_n=$((meta_n+1))
|
||||
else
|
||||
printf '%s|%s\n' "$name" "$meta" >> "$COO_MANIFEST"
|
||||
echo "COO $name ($meta)"
|
||||
[ "$DRY" -eq 0 ] && mv -- "$f" "$COO_DEST/"
|
||||
coo_n=$((coo_n+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$DRY" -eq 0 ]; then
|
||||
chgrp -R users "$META_ROOT" "$COO_ROOT"
|
||||
chmod -R g+rX "$META_ROOT" "$COO_ROOT"
|
||||
echo "DONE: meta=$meta_n -> $META_DEST | coo=$coo_n -> $COO_DEST | skipped(system)=$sys_n"
|
||||
else
|
||||
echo "DRY RUN: meta=$meta_n coo=$coo_n skipped=$sys_n — nothing moved"
|
||||
fi
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# root-fix-tooling-perms.sh — make _crossfeed/tooling uniformly group-writable
|
||||
# (collaboration across agent accounts) while keeping credentials/ locked down.
|
||||
# Usage: sudo bash ~/.coordinate/scripts/root-fix-tooling-perms.sh
|
||||
set -euo pipefail
|
||||
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
|
||||
|
||||
T=/home/_crossfeed/tooling
|
||||
[ -d "$T" ] || { echo "ERROR: $T missing"; exit 1; }
|
||||
|
||||
chgrp -R users "$T"
|
||||
# group rw, but NEVER widen credentials/ (skip symlinks; dangling links OK)
|
||||
find "$T" -path "$T/credentials" -prune -o ! -type l -print0 | xargs -0 -r chmod g+rwX || true
|
||||
find "$T" -path "$T/credentials" -prune -o -type d -print0 | xargs -0 -r chmod g+s || true
|
||||
# lock credentials to owner only
|
||||
chgrp -R reachableceo "$T/credentials" 2>/dev/null || true
|
||||
chmod -R o-rwx,g-rwx "$T/credentials" || true
|
||||
|
||||
echo "OK: tooling group-writable (setgid dirs), credentials owner-only"
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# root-migrate-coordinate.sh — move ~/.coordinate into _crossfeed/state/rceo (symlink back).
|
||||
# Run as root, when RCEO agents are idle (no work-turn screens, PMO/Work between turns):
|
||||
# sudo bash ~/.coordinate/scripts/root-migrate-coordinate.sh
|
||||
set -euo pipefail
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; }
|
||||
|
||||
SRC=/home/reachableceo/.coordinate
|
||||
STATE_ROOT=/home/_crossfeed/state
|
||||
DEST=$STATE_ROOT/rceo
|
||||
LINK=/home/reachableceo/.coordinate
|
||||
|
||||
if screen -ls 2>/dev/null | grep -q work-turn; then
|
||||
echo "ERROR: work-turn screens still running — let them finish first (or --force)"
|
||||
[ "${1:-}" = "--force" ] || exit 1
|
||||
fi
|
||||
|
||||
if [ -L "$LINK" ]; then
|
||||
echo "Already migrated: $LINK -> $(readlink "$LINK")"; exit 0
|
||||
fi
|
||||
[ -d "$SRC" ] || { echo "ERROR: $SRC missing"; exit 1; }
|
||||
[ -e "$DEST" ] && { echo "ERROR: $DEST already exists — resolve manually"; exit 1; }
|
||||
|
||||
mkdir -p "$STATE_ROOT"
|
||||
mv "$SRC" "$DEST"
|
||||
ln -s "$DEST" "$LINK"
|
||||
|
||||
chgrp -R users "$DEST"
|
||||
chmod -R g+rX "$DEST"
|
||||
chgrp users "$STATE_ROOT"; chmod g+rx "$STATE_ROOT"
|
||||
|
||||
echo "OK: state now at $DEST (owner reachableceo, group users read-only)"
|
||||
echo "NOTE: ~/projects symlink + all scripts keep working via $LINK"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# root-rename-inbox.sh — rename /home/_crossfeed/inbox/founder -> inbox/rceo
|
||||
# (Charles dislikes "founder" as a name for himself.) Idempotent. Root runs:
|
||||
# sudo bash ~/.coordinate/scripts/root-rename-inbox.sh
|
||||
set -euo pipefail
|
||||
[[ $EUID -eq 0 ]] || { echo "run as root: sudo bash $0"; exit 1; }
|
||||
src=/home/_crossfeed/inbox/founder
|
||||
dst=/home/_crossfeed/inbox/rceo
|
||||
if [[ -d $src && ! -e $dst ]]; then
|
||||
mv "$src" "$dst"
|
||||
elif [[ -d $dst ]]; then
|
||||
echo "already renamed"
|
||||
else
|
||||
echo "source missing: $src"; ls -la /home/_crossfeed/inbox/; exit 1
|
||||
fi
|
||||
chown reachableceo:users "$dst"
|
||||
chmod 3775 "$dst"
|
||||
ls -ld "$dst"
|
||||
echo "OK. Remind PMO to update PROTOCOL.md path references."
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# source: /home/_crossfeed/tooling/agent-stack @ HEAD
|
||||
# semaphore.sh — org-wide agent-turn concurrency gate (z.ai plan: 3
|
||||
# simultaneous API calls; we approximate by limiting concurrent ACTIVE
|
||||
# agent processes: headless work-turns + freshly-dispatched turns).
|
||||
#
|
||||
# Usage:
|
||||
# semaphore.sh status -> print active count / max
|
||||
# semaphore.sh try <tag> <secs> -> exit 0 + hold slot if free, else exit 1
|
||||
# semaphore.sh release <tag> -> release slot held by <tag>
|
||||
# semaphore.sh clean -> drop stale holds (>MaxAge, default 30m)
|
||||
#
|
||||
# Slot dir: /home/_crossfeed/metrics/active/ (one file per active turn,
|
||||
# filename = <tag>.pid; content = ISO start). Not a hard API-call gate —
|
||||
# it bounds AGENT TURN concurrency, the controllable unit.
|
||||
set -uo pipefail
|
||||
|
||||
METRICS=${METRICS:-/home/_crossfeed/metrics}
|
||||
ACTIVE="$METRICS/active"
|
||||
MAX=${MAX_CONCURRENT:-2} # headroom of 1 below z.ai cap for interactive PMO use
|
||||
STALE_SECS=${STALE_SECS:-1800}
|
||||
|
||||
mkdir -p "$ACTIVE" 2>/dev/null || true
|
||||
# graceful degrade: no writable metrics dir -> gate unavailable (exit 99)
|
||||
[ -w "$ACTIVE" ] || { [ "${1:-}" = status ] && echo "gate=unavailable"; exit 99; }
|
||||
|
||||
active_count() {
|
||||
local n=0 f
|
||||
for f in "$ACTIVE"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
if [ "$STALE_SECS" -gt 0 ]; then
|
||||
local age=$(( $(date +%s) - $(stat -c %Y "$f") ))
|
||||
[ "$age" -gt "$STALE_SECS" ] && continue
|
||||
fi
|
||||
n=$((n+1))
|
||||
done
|
||||
echo "$n"
|
||||
}
|
||||
|
||||
cmd=${1:-status}
|
||||
case "$cmd" in
|
||||
status)
|
||||
echo "active=$(active_count)/$MAX"
|
||||
[ "$(active_count)" -lt "$MAX" ] && exit 0 || exit 1
|
||||
;;
|
||||
try)
|
||||
tag=${2:?tag}; [ "$#" -ge 3 ] || { echo "usage: try <tag> <max-wait-secs>" >&2; exit 2; }
|
||||
wait_secs=$3; waited=0
|
||||
while true; do
|
||||
if [ "$(active_count)" -lt "$MAX" ]; then
|
||||
date -Is > "$ACTIVE/$tag.pid" 2>/dev/null && { echo "acquired: $tag"; exit 0; }
|
||||
echo "ERROR: cannot write $ACTIVE (perms?)" >&2; exit 2
|
||||
fi
|
||||
[ "$waited" -ge "$wait_secs" ] && { echo "timeout: no slot after ${wait_secs}s (active=$(active_count)/$MAX)" >&2; exit 1; }
|
||||
sleep 10; waited=$((waited+10))
|
||||
done
|
||||
;;
|
||||
release)
|
||||
tag=${2:?tag}
|
||||
rm -f "$ACTIVE/$tag.pid" && echo "released: $tag"
|
||||
;;
|
||||
clean)
|
||||
for f in "$ACTIVE"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
age=$(( $(date +%s) - $(stat -c %Y "$f") ))
|
||||
[ "$age" -gt "$STALE_SECS" ] && { rm -f "$f"; echo "dropped stale: $(basename "$f")"; }
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "usage: semaphore.sh {status|try <tag> <secs>|release <tag>|clean}" >&2; exit 2
|
||||
;;
|
||||
esac
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# work-chain.sh <chain-name> <task-file>... — sequential overnight work driver.
|
||||
# Runs dispatch-turn.sh per TASK in order. Skips tasks already REPORTed.
|
||||
# Stops on first failure; status -> ~/.coordinate/logs/<chain-name>.status
|
||||
set -uo pipefail
|
||||
CHAIN=${1:?chain name}; shift
|
||||
LOGDIR="$HOME/.coordinate/logs"
|
||||
STATUS="$LOGDIR/$CHAIN.status"
|
||||
inbox="$HOME/.coordinate/inbox-pmo"
|
||||
|
||||
echo "$(date -Is) chain start: $*" > "$STATUS"
|
||||
for task in "$@"; do
|
||||
slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//')
|
||||
if ls "$inbox"/REPORT-*"$slug"* >/dev/null 2>&1; then
|
||||
echo "$(date -Is) SKIP $slug (reported)" >> "$STATUS"; continue
|
||||
fi
|
||||
if [ ! -f "$task" ]; then
|
||||
echo "$(date -Is) FAIL $slug (task file missing) — chain halted" >> "$STATUS"
|
||||
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED missing task $slug.\r"
|
||||
exit 2
|
||||
fi
|
||||
echo "$(date -Is) RUN $slug" >> "$STATUS"
|
||||
bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "$CHAIN-$slug" "$task" \
|
||||
>> "$LOGDIR/$CHAIN-$slug.log" 2>&1
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "$(date -Is) FAIL $slug rc=$rc — chain halted" >> "$STATUS"
|
||||
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED at $slug - read $STATUS.\r"
|
||||
exit $rc
|
||||
fi
|
||||
echo "$(date -Is) OK $slug" >> "$STATUS"
|
||||
done
|
||||
echo "$(date -Is) chain COMPLETE" >> "$STATUS"
|
||||
screen -S RCEO-PMO -X stuff "CHAIN $CHAIN COMPLETE - read status + inbox-pmo.\r"
|
||||
Reference in New Issue
Block a user