From 79d93b6c4951fce112ae87509956e21798393b7f Mon Sep 17 00:00:00 2001 From: reachableceo Date: Sat, 29 Aug 2026 05:22:43 -0500 Subject: [PATCH] ops(scripts): dispatch, queue gating, heartbeat, quota probe dispatch-turn.sh (headless turn launcher with TURN_MODEL routing), queue-after.sh (serial gate chain + quota-wall guard), pmo-heartbeat.sh (PMO wake loop), quota-probe-then-chain.sh (post-wall relaunch). --- scripts/agent-metrics.sh | 76 +++++++++++++++++++ scripts/dispatch-turn.sh | 47 ++++++++++++ scripts/founder-crossfeed-inboxes.sh | 60 +++++++++++++++ scripts/founder-full-reset.sh | 105 ++++++++++++++++++++++++++ scripts/founder-tooling-and-yolo.sh | 50 ++++++++++++ scripts/founder-wire-tooling.sh | 46 +++++++++++ scripts/pmo-heartbeat.sh | 20 +++++ scripts/pmo-loop.sh | 41 ++++++++++ scripts/queue-after.sh | 23 ++++++ scripts/queue-next.sh | 27 +++++++ scripts/quota-probe-then-chain.sh | 36 +++++++++ scripts/relaunch-screen.sh | 22 ++++++ scripts/root-archive-tmp-artifacts.sh | 94 +++++++++++++++++++++++ scripts/root-fix-tooling-perms.sh | 19 +++++ scripts/root-migrate-coordinate.sh | 34 +++++++++ scripts/root-rename-inbox.sh | 19 +++++ scripts/semaphore.sh | 72 ++++++++++++++++++ scripts/work-chain.sh | 34 +++++++++ 18 files changed, 825 insertions(+) create mode 100755 scripts/agent-metrics.sh create mode 100755 scripts/dispatch-turn.sh create mode 100755 scripts/founder-crossfeed-inboxes.sh create mode 100755 scripts/founder-full-reset.sh create mode 100755 scripts/founder-tooling-and-yolo.sh create mode 100755 scripts/founder-wire-tooling.sh create mode 100755 scripts/pmo-heartbeat.sh create mode 100755 scripts/pmo-loop.sh create mode 100755 scripts/queue-after.sh create mode 100755 scripts/queue-next.sh create mode 100755 scripts/quota-probe-then-chain.sh create mode 100755 scripts/relaunch-screen.sh create mode 100755 scripts/root-archive-tmp-artifacts.sh create mode 100755 scripts/root-fix-tooling-perms.sh create mode 100755 scripts/root-migrate-coordinate.sh create mode 100755 scripts/root-rename-inbox.sh create mode 100755 scripts/semaphore.sh create mode 100755 scripts/work-chain.sh diff --git a/scripts/agent-metrics.sh b/scripts/agent-metrics.sh new file mode 100755 index 0000000..7dd42cd --- /dev/null +++ b/scripts/agent-metrics.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# source: /home/_crossfeed/tooling/agent-stack @ HEAD +# agent-metrics.sh — harvest per-vertical crush usage into the org ledger. +# Appends one JSONL line per session seen, deduped by session id via +# ledger/seen index. Idempotent; safe to run every timer tick. +# +# Ledger: /home/_crossfeed/metrics/ledger.jsonl +# Event: {"ts","vertical","role","session","model","prompt_tokens", +# "completion_tokens","total_tokens","cost"} +# Role heuristic: title contains 'PMO intro' -> pmo; 'INTRO-WORK' -> work; +# 'headless' -> workturn; else 'other'. +set -uo pipefail + +METRICS=${METRICS:-/home/_crossfeed/metrics} +LEDGER="$METRICS/ledger.jsonl" +SEEN="$METRICS/.seen-ids" +VERTICALS=(TSGBOD TSGCTO TSGCCO TSGCOO reachableceo) +RUNAS=${RUNAS:-sudo} # agent-metrics runs from the supervisor service (root) + +mkdir -p "$METRICS"; touch "$LEDGER" "$SEEN" + +emit() { printf '%s\n' "$1" >> "$LEDGER"; } + +for user in "${VERTICALS[@]}"; do + home=$(getent passwd "$user" | cut -d: -f6) + [ -n "$home" ] || continue + # shellcheck disable=SC2016 + json=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \ + bash -c "cd '$home' && crush session list --json 2>/dev/null" || true) + [ -n "$json" ] || continue + # one session id per line; token/cost fields only exist in `session show` + while IFS= read -r sid; do + [ -n "$sid" ] || continue + grep -q "$sid" "$SEEN" && continue + obj=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \ + bash -c "cd '$home' && crush session show '$sid' --json 2>/dev/null | head -c 4000" || true) + [ -n "$obj" ] || continue + title=$(printf '%s' "$obj" | grep -o '"title":"[^"]*"' | head -1 | cut -d'"' -f4) + role=other + case "$title" in + *PMO*intro*) role=pmo ;; + *INTRO-WORK*) role=work ;; + *headless*|*TSG*-Work*executing*) role=workturn ;; + esac + model=$(printf '%s' "$obj" | grep -o '"model":"[^"]*"' | head -1 | cut -d'"' -f4) + pt=$(printf '%s' "$obj" | grep -o '"prompt_tokens":[0-9]*' | head -1 | cut -d: -f2) + ct=$(printf '%s' "$obj" | grep -o '"completion_tokens":[0-9]*' | head -1 | cut -d: -f2) + tt=$(printf '%s' "$obj" | grep -o '"total_tokens":[0-9]*' | head -1 | cut -d: -f2) + cost=$(printf '%s' "$obj" | grep -o '"cost":[0-9.e-]*' | head -1 | cut -d: -f2) + ts=$(date -Is) + emit "{\"ts\":\"$ts\",\"vertical\":\"$user\",\"role\":\"$role\",\"session\":\"$sid\",\"model\":\"${model:-?}\",\"prompt_tokens\":${pt:-0},\"completion_tokens\":${ct:-0},\"total_tokens\":${tt:-0},\"cost\":${cost:-0}}" + echo "$sid" >> "$SEEN" + done < <(printf '%s' "$json" | grep -o '"id":"[0-9a-f]*"' | cut -d'"' -f4) +done + +# rolling summary +python3 - "$LEDGER" <<'PYEOF' > "$METRICS/SUMMARY.md" 2>/dev/null || true +import json, sys, collections, datetime +tot = collections.Counter(); cost = collections.Counter(); n = collections.Counter() +day = collections.Counter() +now = datetime.datetime.now().astimezone() +for line in open(sys.argv[1]): + try: e = json.loads(line) + except Exception: continue + v = e.get("vertical","?"); t = e.get("total_tokens",0) + tot[v] += t; cost[v] += e.get("cost",0.0); n[v] += 1 + try: + ts = datetime.datetime.fromisoformat(e["ts"]) + if (now-ts).total_seconds() < 86400: day[v] += t + except Exception: pass +print(f"# Agent usage summary — {now.isoformat()}") +print(f"- sessions ledgered total: {sum(n.values())}") +print("- all-time by vertical:") +for v in sorted(tot): + print(f" {v:<13} n={n[v]:<4} tokens={tot[v]:<10} cost={cost[v]:.4f} (24h tokens={day[v]})") +PYEOF diff --git a/scripts/dispatch-turn.sh b/scripts/dispatch-turn.sh new file mode 100755 index 0000000..0ee2287 --- /dev/null +++ b/scripts/dispatch-turn.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# source: /home/_crossfeed/tooling/agent-stack @ HEAD +# dispatch-turn.sh — headless work-turn launcher. +# Semaphore-gated (org concurrency cap), FRESH session by default (small +# context = quota-friendly; the turn self-bootstraps from the prompt file), +# success/fail doorbell to the vertical PMO screen. PMO usage: +# screen -dmS work-turn- \ +# ~/.coordinate/scripts/dispatch-turn.sh +set -uo pipefail + +SLUG=${1:?slug}; PROMPT=${2:?prompt-file} +TOOLING=${TOOLING:-/home/_crossfeed/tooling} +SEMI="$TOOLING/agent-stack/semaphore.sh" +LOG="$HOME/.coordinate/logs/$SLUG.log" +TAG="$USER-$SLUG" +WAIT=${WAIT_SECS:-300} + +[ -f "$PROMPT" ] || { echo "no prompt file: $PROMPT" >&2; exit 1; } + +semirc=0 +bash "$SEMI" try "$TAG" "$WAIT" >/dev/null 2>&1 || semirc=$? +if [ "$semirc" = 99 ]; then + echo "$(date -Is) semaphore gate unavailable - proceeding UNGATED (install metrics dirs)" >> "$LOG" +elif [ "$semirc" != 0 ]; then + echo "$(date -Is) semaphore rc=$semirc after ${WAIT}s budget - not launched" >> "$LOG" + screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn DEFERRED (concurrency cap) - rerun later: %s.\r' "$SLUG")" || true + exit 3 +fi +GATED=1; [ "$semirc" = 0 ] || GATED=0 + +cd "$HOME" +rc=0 +# 2026-08-29 ~06:30: Charles upgraded to MAX plan, all buckets zero. +# Quality-first: default back to glm-5.2; TURN_MODEL=glm-4.7-flash for +# grind/admin turns when wanted. +TURN_MODEL=${TURN_MODEL:-litellm/glm-5.2} +env HOME="$HOME" TERM=xterm-256color \ + crush run --quiet --model "$TURN_MODEL" "$(cat "$PROMPT")" >> "$LOG" 2>&1 || rc=$? + +bash "$SEMI" release "$TAG" >/dev/null 2>&1 || true + +if [ "$rc" = 0 ]; then + screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn done OK - read inbox-pmo + logs/%s.\r' "$SLUG")" || true +else + screen -S "${USER}-PMO" -X stuff "$(printf 'Work turn FAILED (rc=%s) - read logs/%s.\r' "$rc" "$SLUG")" || true +fi +exit "$rc" diff --git a/scripts/founder-crossfeed-inboxes.sh b/scripts/founder-crossfeed-inboxes.sh new file mode 100755 index 0000000..22abf6a --- /dev/null +++ b/scripts/founder-crossfeed-inboxes.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# founder-crossfeed-inboxes.sh — FOUNDER-RUN as root. +# Cross-vertical inbox interconnect under /home/_crossfeed/inbox/. +# Semantics per dir: owner=, group=users, mode 3775 +# (setgid+sticky): any vertical may CREATE files in any inbox; only the +# file's author or the inbox OWNER may move/delete. Homes stay 700. +set -uo pipefail +[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; } + +declare -A VMAP=( [bod]=TSGBOD [cco]=TSGCCO [cto]=TSGCTO [coo]=TSGCOO [founder]=reachableceo ) + +echo "== preflight ==" +for v in "${!VMAP[@]}"; do + acct=${VMAP[$v]}; id "$acct" >/dev/null 2>&1 || { echo "missing acct $acct" >&2; exit 1; } +done +echo "ok: all five accounts exist" + +echo "== create inbox tree ==" +for v in bod cco cto coo founder; do + acct=${VMAP[$v]} + d=/home/_crossfeed/inbox/$v + mkdir -p "$d" + chown "$acct":users "$d" + chmod 3775 "$d" + echo " $d -> $acct:users 3775" +done + +echo "== update crossfeed README ==" +grep -q 'inbox/' /home/_crossfeed/README.md || cat >> /home/_crossfeed/README.md <<'EOF' + +## inbox/ — cross-vertical direct messaging (2026-08-28) +/home/_crossfeed/inbox// — mode 3775, owner is the +receiving vertical, group users. Any vertical may DROP a file in any inbox +(name: MSG---.md). Only the author or the receiving +vertical consumes/moves/deletes. This is for lightweight cross-vertical +coordination when a Redmine/Discourse round trip is overkill; requests +with teeth still go via the founder or systems of record, never MSG files. +Receiving PMO scans own inbox at turn start (alongside peers' STATUS). +EOF +echo " README updated" + +echo "== drop NOTICE into each vertical's inbox-pmo ==" +for v in bod cco cto coo; do + acct=${VMAP[$v]} + home=$(getent passwd "$acct" | cut -d: -f6) + [ -d "$home/.coordinate/inbox-pmo" ] || continue + cat > "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md" <<'EOF' +Founder directive: /home/_crossfeed/inbox// is live. +Cross-vertical direct messaging: any PMO may drop +MSG---.md in another vertical's inbox (dirs 3775: anyone +creates, only author or inbox owner removes). Scan YOUR inbox at turn +start. Lightweight coordination only — requests with teeth go via the +founder or systems of record. Archive this notice after reading. +EOF + chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-crossfeed-inbox.md" + echo " NOTICE -> $acct" +done + +echo "== verify ==" +ls -la /home/_crossfeed/inbox/ diff --git a/scripts/founder-full-reset.sh b/scripts/founder-full-reset.sh new file mode 100755 index 0000000..0659058 --- /dev/null +++ b/scripts/founder-full-reset.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# founder-full-reset.sh — FOUNDER-RUN as reachableceo (sudo required). +# Org-wide clean reset of all FIVE verticals: TSGBOD TSGCCO TSGCTO TSGCOO RCEO. +# 1) kill every vertical screen (old 07:02 + new 11:13 generations alike) +# 2) clean stale /run/screen sockets +# 3) relaunch the four TSG stacks via /tmp/tsg-stack-relaunch.sh --yes +# (no live screens -> no duplicate names -> multiuser+acladd succeeds) +# 4) give RCEO the same wrapper pattern: pmo-loop.sh + pinned brains +# 5) verify + attach cheat sheet +# Does NOT touch AZHost/AgentZero or any other screens. RCEO-PMO/RCEO-Work +# (agent screens) die mid-run and return pinned (conversations preserved). +# Blank-slate lever (NOT default): before running, founder may +# touch ~/.coordinate/RELAUNCH-FRESH ~/.coordinate/RELAUNCH-FRESH-WORK +# but fresh sessions get NO intro injected by the wrapper — skip unless +# you plan to hand-feed the INTRO prompts after. +set -uo pipefail + +[ "$(id -un)" = "reachableceo" ] || { echo "run as reachableceo"; exit 1; } +sudo -v || { echo "sudo required"; exit 1; } + +USERS=(TSGBOD TSGCCO TSGCTO TSGCOO) + +echo "== 1/5 kill all vertical screens (both generations) ==" +pat='SCREEN -dmS (TSGBOD-|TSGCCO-|TSGCTO-|TSGCOO-|RCEO-)' +pids=$(pgrep -f "$pat" || true) +if [ -n "$pids" ]; then sudo kill $pids 2>/dev/null; fi +sleep 2 +pids=$(pgrep -f "$pat" || true) +if [ -n "$pids" ]; then sudo kill -9 $pids 2>/dev/null; fi +for u in "${USERS[@]}" reachableceo; do + sudo pkill -u "$u" -f pmo-loop.sh 2>/dev/null + sudo pkill -u "$u" -f 'crush --' 2>/dev/null + sudo pkill -u "$u" -fx 'crush' 2>/dev/null +done +sleep 1 + +echo "== 2/5 clean stale sockets ==" +for u in "${USERS[@]}" reachableceo; do + sudo rm -f "/run/screen/S-$u"/* 2>/dev/null +done + +echo "== 3/5 relaunch TSG stacks (wrappers + existing pins) ==" +if [ -f /tmp/tsg-stack-relaunch.sh ]; then + bash /tmp/tsg-stack-relaunch.sh --yes || echo "WARN: relaunch script failed — inspect its output" +else + echo "WARN: /tmp/tsg-stack-relaunch.sh missing — TSG stacks NOT relaunched" +fi + +echo "== 4/5 relaunch RCEO stack (wrapper parity) ==" +mkdir -p "$HOME/.coordinate/scripts" "$HOME/.coordinate/logs" +cat > "$HOME/.coordinate/scripts/pmo-loop.sh" <<'WRAP' +#!/usr/bin/env bash +# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen. +ROLE="${1:-pmo}" +case "$ROLE" in + pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;; + work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;; + *) echo "usage: $0 [pmo|work]" >&2; exit 1 ;; +esac +LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log" +cd "$HOME" +export TERM="${TERM:-xterm-256color}" +export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH" +while true; do + started=$(date +%s) + if [ -f "$FRESH" ]; then + rm -f "$FRESH" + echo "$(date -Is) relaunch FRESH" >> "$LOG" + crush + elif [ -s "$PIN" ]; then + sid=$(cat "$PIN") + echo "$(date -Is) relaunch PINNED $sid" >> "$LOG" + crush --session "$sid" + else + echo "$(date -Is) relaunch CONTINUE" >> "$LOG" + crush --continue + fi + rt=$(( $(date +%s) - started )) + if [ "$rt" -lt 60 ]; then + echo "$(date -Is) short run (${rt}s) - backoff 60s" >> "$LOG" + sleep 60 + else + sleep 2 + fi +done +WRAP +chmod 755 "$HOME/.coordinate/scripts/pmo-loop.sh" +echo 78cf63b8a72f4282 > "$HOME/.coordinate/PMO-SESSION-ID" +echo e1079fafb8c37d60 > "$HOME/.coordinate/WORK-SESSION-ID" +env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-PMO \ + /home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo +env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-Work \ + /home/reachableceo/.coordinate/scripts/pmo-loop.sh work + +echo "== 5/5 verify ==" +sleep 3 +screen -ls +for u in "${USERS[@]}"; do + echo "-- $u:" + sudo -u "$u" env TERM=dumb screen -ls 2>/dev/null | grep -E 'PMO|Work' || echo " NONE (check relaunch output)" +done +echo +echo "Attach: TSGxx-PMO / TSGxx-work aliases; RCEO: RCEO-PMO | RCEO-work" +echo "Reboot bring-up units (enabled): reachableceo-agent-stack.service," +echo "tsg-agent-stacks.service — safe to reboot-test after this." diff --git a/scripts/founder-tooling-and-yolo.sh b/scripts/founder-tooling-and-yolo.sh new file mode 100755 index 0000000..3002108 --- /dev/null +++ b/scripts/founder-tooling-and-yolo.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# founder-tooling-and-yolo.sh — FOUNDER-RUN: sudo bash ~/.coordinate/scripts/founder-tooling-and-yolo.sh +# 1) publish /home/_crossfeed/tooling (3 group-shared git repos) from PMO draft +# 2) fix org-wide --yolo regression: install canonical wrapper into the four +# TSG verticals + restart their screens (8 agents currently non-yolo) +# 3) deprecate /tmp/tsg-stack-relaunch.sh (v1, source of both regressions) +# 4) restart RCEO-PMO with canonical wrapper (PMO still non-yolo), then +# doorbell the PMO to finish live-repo verification + publish STATUS +set -uo pipefail +DRAFT=/home/reachableceo/.coordinate/tooling-draft +TOOLING=/home/_crossfeed/tooling +FOUNDER=reachableceo + +[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; } + +echo "== 1/4 publish tooling repos ==" +mkdir -p "$TOOLING" +cp -a "$DRAFT"/. "$TOOLING"/ +chown -R $FOUNDER:users "$TOOLING" +chmod 2775 "$TOOLING" +find "$TOOLING" -type d -exec chmod g+s {} + +find "$TOOLING" -type f -exec chmod g+rw {} + +chmod -R o-rwx "$TOOLING" +ls -la "$TOOLING" + +echo "== 2/4 TSG stacks: canonical wrapper (--yolo) + restart ==" +bash "$TOOLING/agent-stack/tsg-stack-relaunch-v2.sh" + +echo "== 3/4 deprecate /tmp v1 relaunch ==" +for f in /tmp/tsg-stack-relaunch.sh; do + [ -f "$f" ] && mv "$f" "$f.deprecated-20260828" && echo "moved: $f -> $f.deprecated-20260828" +done + +echo "== 4/4 RCEO-PMO restart with canonical wrapper + doorbell ==" +runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \ + screen -S RCEO-PMO -X quit 2>/dev/null || true +runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \ + screen -dmS RCEO-PMO /home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo +echo "waiting 20s for PMO crush to settle..." +sleep 20 +runuser -u $FOUNDER -- env HOME=/home/reachableceo TERM=xterm-256color \ + screen -S RCEO-PMO -X stuff $'Tooling live + yolo fix applied. Next turn: verify /home/_crossfeed/tooling, log + publish STATUS.\r' \ + && echo "doorbelled RCEO-PMO" || echo "doorbell failed - ping PMO manually" + +echo "== VERIFY ==" +for u in TSGBOD TSGCCO TSGCTO TSGCOO $FOUNDER; do + echo " $u sockets: $(ls /run/screen/S-$u 2>/dev/null | tr '\n' ' ')" +done +echo "yolo agents: $(pgrep -af 'crush --yolo' | wc -l) / expected 10" +echo "Attach: RCEO-PMO | RCEO-work; TSGxx-PMO | TSGxx-work aliases." diff --git a/scripts/founder-wire-tooling.sh b/scripts/founder-wire-tooling.sh new file mode 100755 index 0000000..0665e56 --- /dev/null +++ b/scripts/founder-wire-tooling.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# founder-wire-tooling.sh — FOUNDER-RUN as root. +# Wire the tooling repo (agent-stack @ current HEAD) into all four TSG +# verticals: canonical pmo-loop.sh (metrics-ledgered), dispatch-turn.sh, +# semaphore.sh, agent-metrics.sh into ~/.coordinate/scripts/. RCEO already +# wired itself. Supervisor units (tsg-supervisor.*) already org-wide. +# Stamped copies per provenance rule; fix forward in the repo, reinstall. +set -uo pipefail +[ "$(id -un)" = root ] || { echo "run as: sudo bash $0" >&2; exit 1; } + +TOOLING=/home/_crossfeed/tooling +SRC=$TOOLING/agent-stack +SHA=$(git -C "$SRC" rev-parse --short HEAD) +FILES="pmo-loop.sh dispatch-turn.sh semaphore.sh agent-metrics.sh" + +echo "== preflight ==" +for f in $FILES; do [ -f "$SRC/$f" ] || { echo "missing $SRC/$f" >&2; exit 1; }; done +echo "ok: source files present @ $SHA" + +for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do + home=$(getent passwd "$acct" | cut -d: -f6) + echo "== $acct ==" + mkdir -p "$home/.coordinate/scripts" "$home/.coordinate/logs" + for f in $FILES; do + install -o "$acct" -g "$acct" -m755 "$SRC/$f" "$home/.coordinate/scripts/$f" + done + sed -i "1a # installed: agent-stack @ $SHA ($(date -I))" "$home/.coordinate/scripts/pmo-loop.sh" + cat > "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md" < ~/.coordinate/scripts/dispatch-turn.sh +(semaphore-gated, fresh session, doorbell back to your PMO screen; see +GOVERNANCE.md in the repo: capture free, execution metered; founder gate +on every TASK still applies). Wrapper now logs launches to metrics. Fix +bugs IN THE REPO (commit as your uid via tooling/commit.sh), then rerun +this installer. Archive this notice after reading. +EOF + chown "$acct:$acct" "$home/.coordinate/inbox-pmo/NOTICE-tooling-wiring.md" + echo " wired + NOTICE" +done + +echo "== verify ==" +for acct in TSGBOD TSGCCO TSGCTO TSGCOO; do + home=$(getent passwd "$acct" | cut -d: -f6) + echo " $acct: $(ls "$home/.coordinate/scripts/" 2>/dev/null | tr '\n' ' ')" +done diff --git a/scripts/pmo-heartbeat.sh b/scripts/pmo-heartbeat.sh new file mode 100755 index 0000000..7931ead --- /dev/null +++ b/scripts/pmo-heartbeat.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# pmo-heartbeat.sh — wake the PMO session while the overnight chain runs. +# Self-terminates when inbox-work has no TASKs AND no semaphore slots are held. + +INBOX=/home/reachableceo/.coordinate/inbox-work +SLOTS=/home/_crossfeed/metrics/active +LOG=/home/reachableceo/.coordinate/logs/heartbeat.log +INTERVAL=3600 + +while true; do + pending=$(ls -1 "$INBOX"/TASK-*.md 2>/dev/null | wc -l) + active=$(ls -1 "$SLOTS"/*.pid 2>/dev/null | wc -l) + if [ "$pending" -eq 0 ] && [ "$active" -eq 0 ]; then + echo "$(date -Is) chain drained — heartbeat exiting" >> "$LOG" + exit 0 + fi + screen -S RCEO-PMO -X stuff "hb: chain check — verify reports, archive, mirror redmine, redispatch flakes, dispatch next$(printf '\r')" + echo "$(date -Is) wake sent (pending=$pending active=$active)" >> "$LOG" + sleep "$INTERVAL" +done diff --git a/scripts/pmo-loop.sh b/scripts/pmo-loop.sh new file mode 100755 index 0000000..c69ea8e --- /dev/null +++ b/scripts/pmo-loop.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen. +ROLE="${1:-pmo}" +case "$ROLE" in + pmo) FRESH="$HOME/.coordinate/RELAUNCH-FRESH"; PIN="$HOME/.coordinate/PMO-SESSION-ID" ;; + work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;; + *) echo "usage: $0 [pmo|work]" >&2; exit 1 ;; +esac +LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log" +LEDGER=/home/_crossfeed/metrics/launches.jsonl +ledger() { printf '%s\n' "$1" >> "$LEDGER" 2>/dev/null || true; } +cd "$HOME" +export TERM="${TERM:-xterm-256color}" +export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH" +while true; do + started=$(date +%s) + mode=continue + if [ -f "$FRESH" ]; then + rm -f "$FRESH" + mode=fresh + echo "$(date -Is) relaunch FRESH" >> "$LOG" + crush --yolo + elif [ -s "$PIN" ]; then + sid=$(cat "$PIN") + mode=pinned:$sid + echo "$(date -Is) relaunch PINNED $sid" >> "$LOG" + crush --yolo --session "$sid" + else + echo "$(date -Is) relaunch CONTINUE" >> "$LOG" + crush --yolo --continue + fi + rc=$? + rt=$(( $(date +%s) - started )) + ledger "{\"ts\":\"$(date -Is)\",\"user\":\"$USER\",\"role\":\"$ROLE\",\"mode\":\"$mode\",\"rc\":$rc,\"runtime_s\":$rt}" + if [ "$rt" -lt 60 ]; then + echo "$(date -Is) short run (${rt}s rc=$rc) - backoff 60s" >> "$LOG" + sleep 60 + else + sleep 2 + fi +done diff --git a/scripts/queue-after.sh b/scripts/queue-after.sh new file mode 100755 index 0000000..c5c3415 --- /dev/null +++ b/scripts/queue-after.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# queue-after.sh +# Wait until no longer exists, then dispatch the task turn. +# Keeps repo-touching turns serialized when two queue screens would overlap. +set -uo pipefail +GATE="$1"; SLUG="$2"; TASK="$3" +LOGDIR="$HOME/.coordinate/logs" +while screen -ls 2>/dev/null | grep -q "\.${GATE}[[:space:]]"; do + sleep 60 +done +# QUOTA WALL GUARD: if any recent turn log shows the z.ai usage wall, +# halt the chain instead of burning the next turn into the same wall. +WALLLOG=$(ls -t "$LOGDIR"/q-*.log 2>/dev/null | head -1) +if [ -n "$WALLLOG" ] && grep -q "Usage limit reached" "$WALLLOG" 2>/dev/null; then + echo "$(date -Is) QUOTA WALL detected in $WALLLOG — chain halted before $SLUG" >> "$LOGDIR/queue.status" + screen -S RCEO-PMO -X stuff "QUOTA WALL - chain halted before $SLUG. Do not dispatch until quota verified.$(printf '\r')" || true + exit 0 +fi +echo "$(date -Is) GATE $GATE cleared — RUN $SLUG" >> "$LOGDIR/queue.status" +bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$SLUG" "$TASK" \ + >> "$LOGDIR/queue-$SLUG.log" 2>&1 +rc=$? +echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo "FAIL(rc=$rc)") $SLUG" >> "$LOGDIR/queue.status" diff --git a/scripts/queue-next.sh b/scripts/queue-next.sh new file mode 100755 index 0000000..29e8fca --- /dev/null +++ b/scripts/queue-next.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# queue-next.sh — bash-only slot watcher: run queued TASK turns as slots free. +# rc=3 from dispatch = DEFERRED (cap) -> keep waiting on same task, don't exit. +set -uo pipefail +LOGDIR="$HOME/.coordinate/logs" +i=1 +while [ $i -le $# ]; do + task=${!i} + i=$((i+1)) + slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//') + while true; do + n=$(ls /home/_crossfeed/metrics/active/*.pid 2>/dev/null | wc -l) + [ "$n" -lt 2 ] && break + sleep 60 + done + echo "$(date -Is) RUN $slug" >> "$LOGDIR/queue.status" + bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "q-$slug" "$task" \ + >> "$LOGDIR/queue-$slug.log" 2>&1 + rc=$? + if [ "$rc" = 3 ]; then + echo "$(date -Is) DEFER $slug (cap) - re-queueing" >> "$LOGDIR/queue.status" + i=$((i-1)); sleep 30; continue + fi + echo "$(date -Is) $([ $rc -eq 0 ] && echo OK || echo FAIL) $slug rc=$rc" >> "$LOGDIR/queue.status" + [ $rc -ne 0 ] && exit $rc +done +echo "$(date -Is) queue complete" >> "$LOGDIR/queue.status" diff --git a/scripts/quota-probe-then-chain.sh b/scripts/quota-probe-then-chain.sh new file mode 100755 index 0000000..03720eb --- /dev/null +++ b/scripts/quota-probe-then-chain.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# quota-probe-then-chain.sh — wait for z.ai quota window to reset, then +# launch the serial build chain. Probe: 8-token glm-4.7-flash ping via the +# harness vkey. On first 200 -> fire work-q1..q4 chain and exit. +set -uo pipefail +VKEY_FILE="$HOME/.coordinate/secrets/mopac-harness-vkey.env" +LOG="$HOME/.coordinate/logs/queue.status" +PROBE_LOG="$HOME/.coordinate/logs/quota-probe.log" + +while true; do + ok=$(bash -c "set -a; source '$VKEY_FILE'; set +a; python3 - </dev/null) + echo "$(date -Is) probe: $ok" >> "$PROBE_LOG" + if [ "$ok" = "OK" ]; then + echo "$(date -Is) QUOTA BACK — launching build chain" >> "$LOG" + W="$HOME/.coordinate/scripts" + T="$HOME/.coordinate/inbox-work" + screen -dmS work-q1 bash "$W/queue-after.sh" NONE-GATE docs-standards "$T/TASK-20260828-2100-docs-standards.md" + screen -dmS work-q2 bash "$W/queue-after.sh" work-q1 events-receiver "$T/TASK-20260828-1975-events-receiver.md" + screen -dmS work-q3 bash "$W/queue-after.sh" work-q2 keyproxy-v0 "$T/TASK-20260828-2130-keyproxy-v0.md" + screen -dmS work-q4 bash "$W/queue-after.sh" work-q3 selfhost-core "$T/TASK-20260828-2131-selfhost-core.md" + echo "$(date -Is) chain armed: docs -> events -> keyproxy-v0 -> selfhost-core" >> "$LOG" + exit 0 + fi + sleep 1200 +done diff --git a/scripts/relaunch-screen.sh b/scripts/relaunch-screen.sh new file mode 100755 index 0000000..dc5731e --- /dev/null +++ b/scripts/relaunch-screen.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# relaunch-screen.sh [fuse] — restart an RCEO screen so it +# re-reads ~/.screenrc (termcapinfo scroll fix). Fuse >0 schedules via helper +# screen (used to restart RCEO-PMO from inside itself). Owner-run; no root. +set -euo pipefail +name="${1:?screen name}"; role="${2:?pmo|work}"; fuse="${3:-0}" +SELF="$HOME/.coordinate/scripts/relaunch-screen.sh" +if [ "$fuse" -gt 0 ]; then + screen -S RCEO-Relaunch -X quit 2>/dev/null || true + screen -dmS RCEO-Relaunch bash -c "sleep $fuse; $SELF $name $role 0" + echo "scheduled: $name restart in ${fuse}s" + exit 0 +fi +screen -S "$name" -X quit 2>/dev/null || true +sleep 2 +screen -dmS "$name" "$HOME/.coordinate/scripts/pmo-loop.sh" "$role" +sleep 5 +if screen -ls 2>/dev/null | grep -q "\.$name"; then + echo "OK: $name relaunched" +else + echo "FAILED: $name not present after relaunch" >&2; exit 1 +fi diff --git a/scripts/root-archive-tmp-artifacts.sh b/scripts/root-archive-tmp-artifacts.sh new file mode 100755 index 0000000..82579ba --- /dev/null +++ b/scripts/root-archive-tmp-artifacts.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# root-archive-tmp-artifacts.sh — archive /tmp agent artifacts into _crossfeed. +# meta (stack launchers/coordinate/crossfeed tooling) -> tooling/agent-stack/artifacts-archive/ +# everything else (ops/tickets/monitoring) -> coo/artifacts-archive/ +# Usage: sudo bash root-archive-tmp-artifacts.sh [--dry-run] +set -euo pipefail + +DRY=0; [ "${1:-}" = "--dry-run" ] && DRY=1 +[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; } + +SRC=/tmp +META_ROOT=/home/_crossfeed/tooling/agent-stack/artifacts-archive +COO_ROOT=/home/_crossfeed/coo/artifacts-archive +STAMP=$(date +%Y%m%d-%H%M%S) +META_DEST="$META_ROOT/$STAMP" +COO_DEST="$COO_ROOT/$STAMP" + +# Never touch: system entries + .crush (live TSGCOO crush session data) +is_system() { + case "$1" in + systemd-private-*|.X11-unix|.ICE-unix|.font-unix|.Test-unix|.XIM-unix|.IMAP-unix|\ + sddm*|xauth_*|ssh-*|scoped_dir*|.crush) return 0 ;; + *) return 1 ;; + esac +} + +# Meta-scoped: stack launchers, coordinate/crossfeed setup, prompts, boot captures, standdown kits +is_meta() { + case "$1" in + launch-all-tsg-stacks.sh|launch-tsgcoo-stack.sh|launch-coordinate.sh|\ + setup-tsg-stack.sh|setup-tsgcoo-coordinate.sh|\ + tsg-pmo-autorelaunch.sh|tsg-stack-relaunch.sh.deprecated-20260828|tsg-crossfeed-setup.sh|\ + tsg-agent-stacks.service|tsgcoo-bootstrap.sh|tsgcoo-cleanup.sh|tsgcoo-crush-prompt.md|\ + tsgcoo-crush-prompt.oneline|tsgcoo-paste.sh|tsgcoo-pmo2.sh|tsgcoo-preflight.sh|\ + tsgcoo-screen-setup.sh|tsgcoo-stack-fix.sh|tsgcoo-stack-fix2.sh|\ + mk-coordinate.sh|rm-coordinate.sh|local-pmo.sh|pmo-check.sh|pmo-prompt.md|pmo-prompt.oneline|\ + prompt-knel-basis.md|prompt-legacy-fold.md|legacy-fold.log|\ + seed-intros.sh|run-seed.sh|run-setup.sh|\ + LIFTSHIFT-MANIFEST.md|liftshift-copy.sh|liftshift-copy2.sh|liftshift-manifest|\ + tsgcap1.txt|localpmo2.txt|localpmo3.txt|localpmo-after.txt|localpmo-boot.txt|\ + crushboot.txt|crushafter.txt|crushafter2.txt|hb1.txt|hb2.txt|yolotest.log|\ + tsgpmo2.txt|tsgpmo3.txt|tsgpmo-after.txt|tsgpmo-boot.txt|\ + pmo-before.txt|pmo-after.txt|pmo-yolo.txt|\ + work-before.txt|work-after.txt|work-screen.txt|work-screen2.txt|work-yolo.txt|\ + bodw2.txt|cco-pmo.txt|cco-work2.txt|\ + TSGBOD-pmo.txt|TSGBOD-work.txt|TSGCCO-pmo.txt|TSGCTO-pmo.txt|TSGCTO-work.txt|\ + CTOHandoff.md|COO-PMO-RESUME.md|human-feed-hud.txt|\ + offstage-kit|cos-pkg|cos-pkg-v2|\ + w.sh|w1.sh|tl.sh|logs.sh|fix.sh|verify.sh|restart.sh|provision.sh|dbg.sh|\ + inspect.sh|inspect-body.sh|inspect-wrap.sh|\ + install-unit.sh|install-body.sh|final2.sh|final-check.sh|final-topology.sh|\ + ssh-diag.sh|wrapper-extract.sh|w-extract.sh) return 0 ;; + *) return 1 ;; + esac +} + +for d in /home/_crossfeed/coo "$META_ROOT" "$COO_ROOT"; do + [ -d "$(dirname "$d")" ] || { echo "ERROR: missing $d"; exit 1; } +done + +if [ "$DRY" -eq 0 ]; then + mkdir -p "$META_DEST" "$COO_DEST" +fi +META_MANIFEST="${META_DEST:-/tmp}/MANIFEST.txt"; COO_MANIFEST="${COO_DEST:-/tmp}/MANIFEST.txt" +[ "$DRY" -eq 1 ] && { META_MANIFEST=/dev/null; COO_MANIFEST=/dev/null; } + +meta_n=0; coo_n=0; sys_n=0 +for f in "$SRC"/* "$SRC"/.[!.]*; do + [ -e "$f" ] || continue + name=$(basename "$f") + if is_system "$name"; then + echo "SKIP(system) $name"; sys_n=$((sys_n+1)); continue + fi + meta=$(stat -c '%U:%G:%s:%y' "$f") + if is_meta "$name"; then + printf '%s|%s\n' "$name" "$meta" >> "$META_MANIFEST" + echo "META $name ($meta)" + [ "$DRY" -eq 0 ] && mv -- "$f" "$META_DEST/" + meta_n=$((meta_n+1)) + else + printf '%s|%s\n' "$name" "$meta" >> "$COO_MANIFEST" + echo "COO $name ($meta)" + [ "$DRY" -eq 0 ] && mv -- "$f" "$COO_DEST/" + coo_n=$((coo_n+1)) + fi +done + +if [ "$DRY" -eq 0 ]; then + chgrp -R users "$META_ROOT" "$COO_ROOT" + chmod -R g+rX "$META_ROOT" "$COO_ROOT" + echo "DONE: meta=$meta_n -> $META_DEST | coo=$coo_n -> $COO_DEST | skipped(system)=$sys_n" +else + echo "DRY RUN: meta=$meta_n coo=$coo_n skipped=$sys_n — nothing moved" +fi diff --git a/scripts/root-fix-tooling-perms.sh b/scripts/root-fix-tooling-perms.sh new file mode 100755 index 0000000..9a67870 --- /dev/null +++ b/scripts/root-fix-tooling-perms.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# root-fix-tooling-perms.sh — make _crossfeed/tooling uniformly group-writable +# (collaboration across agent accounts) while keeping credentials/ locked down. +# Usage: sudo bash ~/.coordinate/scripts/root-fix-tooling-perms.sh +set -euo pipefail +[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; } + +T=/home/_crossfeed/tooling +[ -d "$T" ] || { echo "ERROR: $T missing"; exit 1; } + +chgrp -R users "$T" +# group rw, but NEVER widen credentials/ (skip symlinks; dangling links OK) +find "$T" -path "$T/credentials" -prune -o ! -type l -print0 | xargs -0 -r chmod g+rwX || true +find "$T" -path "$T/credentials" -prune -o -type d -print0 | xargs -0 -r chmod g+s || true +# lock credentials to owner only +chgrp -R reachableceo "$T/credentials" 2>/dev/null || true +chmod -R o-rwx,g-rwx "$T/credentials" || true + +echo "OK: tooling group-writable (setgid dirs), credentials owner-only" diff --git a/scripts/root-migrate-coordinate.sh b/scripts/root-migrate-coordinate.sh new file mode 100755 index 0000000..158f049 --- /dev/null +++ b/scripts/root-migrate-coordinate.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# root-migrate-coordinate.sh — move ~/.coordinate into _crossfeed/state/rceo (symlink back). +# Run as root, when RCEO agents are idle (no work-turn screens, PMO/Work between turns): +# sudo bash ~/.coordinate/scripts/root-migrate-coordinate.sh +set -euo pipefail + +[ "$(id -u)" -eq 0 ] || { echo "ERROR: run with sudo"; exit 1; } + +SRC=/home/reachableceo/.coordinate +STATE_ROOT=/home/_crossfeed/state +DEST=$STATE_ROOT/rceo +LINK=/home/reachableceo/.coordinate + +if screen -ls 2>/dev/null | grep -q work-turn; then + echo "ERROR: work-turn screens still running — let them finish first (or --force)" + [ "${1:-}" = "--force" ] || exit 1 +fi + +if [ -L "$LINK" ]; then + echo "Already migrated: $LINK -> $(readlink "$LINK")"; exit 0 +fi +[ -d "$SRC" ] || { echo "ERROR: $SRC missing"; exit 1; } +[ -e "$DEST" ] && { echo "ERROR: $DEST already exists — resolve manually"; exit 1; } + +mkdir -p "$STATE_ROOT" +mv "$SRC" "$DEST" +ln -s "$DEST" "$LINK" + +chgrp -R users "$DEST" +chmod -R g+rX "$DEST" +chgrp users "$STATE_ROOT"; chmod g+rx "$STATE_ROOT" + +echo "OK: state now at $DEST (owner reachableceo, group users read-only)" +echo "NOTE: ~/projects symlink + all scripts keep working via $LINK" diff --git a/scripts/root-rename-inbox.sh b/scripts/root-rename-inbox.sh new file mode 100755 index 0000000..5a5f7d6 --- /dev/null +++ b/scripts/root-rename-inbox.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# root-rename-inbox.sh — rename /home/_crossfeed/inbox/founder -> inbox/rceo +# (Charles dislikes "founder" as a name for himself.) Idempotent. Root runs: +# sudo bash ~/.coordinate/scripts/root-rename-inbox.sh +set -euo pipefail +[[ $EUID -eq 0 ]] || { echo "run as root: sudo bash $0"; exit 1; } +src=/home/_crossfeed/inbox/founder +dst=/home/_crossfeed/inbox/rceo +if [[ -d $src && ! -e $dst ]]; then + mv "$src" "$dst" +elif [[ -d $dst ]]; then + echo "already renamed" +else + echo "source missing: $src"; ls -la /home/_crossfeed/inbox/; exit 1 +fi +chown reachableceo:users "$dst" +chmod 3775 "$dst" +ls -ld "$dst" +echo "OK. Remind PMO to update PROTOCOL.md path references." diff --git a/scripts/semaphore.sh b/scripts/semaphore.sh new file mode 100755 index 0000000..993ab25 --- /dev/null +++ b/scripts/semaphore.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# source: /home/_crossfeed/tooling/agent-stack @ HEAD +# semaphore.sh — org-wide agent-turn concurrency gate (z.ai plan: 3 +# simultaneous API calls; we approximate by limiting concurrent ACTIVE +# agent processes: headless work-turns + freshly-dispatched turns). +# +# Usage: +# semaphore.sh status -> print active count / max +# semaphore.sh try -> exit 0 + hold slot if free, else exit 1 +# semaphore.sh release -> release slot held by +# semaphore.sh clean -> drop stale holds (>MaxAge, default 30m) +# +# Slot dir: /home/_crossfeed/metrics/active/ (one file per active turn, +# filename = .pid; content = ISO start). Not a hard API-call gate — +# it bounds AGENT TURN concurrency, the controllable unit. +set -uo pipefail + +METRICS=${METRICS:-/home/_crossfeed/metrics} +ACTIVE="$METRICS/active" +MAX=${MAX_CONCURRENT:-2} # headroom of 1 below z.ai cap for interactive PMO use +STALE_SECS=${STALE_SECS:-1800} + +mkdir -p "$ACTIVE" 2>/dev/null || true +# graceful degrade: no writable metrics dir -> gate unavailable (exit 99) +[ -w "$ACTIVE" ] || { [ "${1:-}" = status ] && echo "gate=unavailable"; exit 99; } + +active_count() { + local n=0 f + for f in "$ACTIVE"/*; do + [ -f "$f" ] || continue + if [ "$STALE_SECS" -gt 0 ]; then + local age=$(( $(date +%s) - $(stat -c %Y "$f") )) + [ "$age" -gt "$STALE_SECS" ] && continue + fi + n=$((n+1)) + done + echo "$n" +} + +cmd=${1:-status} +case "$cmd" in + status) + echo "active=$(active_count)/$MAX" + [ "$(active_count)" -lt "$MAX" ] && exit 0 || exit 1 + ;; + try) + tag=${2:?tag}; [ "$#" -ge 3 ] || { echo "usage: try " >&2; exit 2; } + wait_secs=$3; waited=0 + while true; do + if [ "$(active_count)" -lt "$MAX" ]; then + date -Is > "$ACTIVE/$tag.pid" 2>/dev/null && { echo "acquired: $tag"; exit 0; } + echo "ERROR: cannot write $ACTIVE (perms?)" >&2; exit 2 + fi + [ "$waited" -ge "$wait_secs" ] && { echo "timeout: no slot after ${wait_secs}s (active=$(active_count)/$MAX)" >&2; exit 1; } + sleep 10; waited=$((waited+10)) + done + ;; + release) + tag=${2:?tag} + rm -f "$ACTIVE/$tag.pid" && echo "released: $tag" + ;; + clean) + for f in "$ACTIVE"/*; do + [ -f "$f" ] || continue + age=$(( $(date +%s) - $(stat -c %Y "$f") )) + [ "$age" -gt "$STALE_SECS" ] && { rm -f "$f"; echo "dropped stale: $(basename "$f")"; } + done + ;; + *) + echo "usage: semaphore.sh {status|try |release |clean}" >&2; exit 2 + ;; +esac diff --git a/scripts/work-chain.sh b/scripts/work-chain.sh new file mode 100755 index 0000000..02922bc --- /dev/null +++ b/scripts/work-chain.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# work-chain.sh ... — sequential overnight work driver. +# Runs dispatch-turn.sh per TASK in order. Skips tasks already REPORTed. +# Stops on first failure; status -> ~/.coordinate/logs/.status +set -uo pipefail +CHAIN=${1:?chain name}; shift +LOGDIR="$HOME/.coordinate/logs" +STATUS="$LOGDIR/$CHAIN.status" +inbox="$HOME/.coordinate/inbox-pmo" + +echo "$(date -Is) chain start: $*" > "$STATUS" +for task in "$@"; do + slug=$(basename "$task" .md | sed 's/^TASK-[0-9]*-[0-9]*-//') + if ls "$inbox"/REPORT-*"$slug"* >/dev/null 2>&1; then + echo "$(date -Is) SKIP $slug (reported)" >> "$STATUS"; continue + fi + if [ ! -f "$task" ]; then + echo "$(date -Is) FAIL $slug (task file missing) — chain halted" >> "$STATUS" + screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED missing task $slug.\r" + exit 2 + fi + echo "$(date -Is) RUN $slug" >> "$STATUS" + bash "$HOME/.coordinate/scripts/dispatch-turn.sh" "$CHAIN-$slug" "$task" \ + >> "$LOGDIR/$CHAIN-$slug.log" 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + echo "$(date -Is) FAIL $slug rc=$rc — chain halted" >> "$STATUS" + screen -S RCEO-PMO -X stuff "CHAIN $CHAIN HALTED at $slug - read $STATUS.\r" + exit $rc + fi + echo "$(date -Is) OK $slug" >> "$STATUS" +done +echo "$(date -Is) chain COMPLETE" >> "$STATUS" +screen -S RCEO-PMO -X stuff "CHAIN $CHAIN COMPLETE - read status + inbox-pmo.\r"