ops(scripts): dispatch, queue gating, heartbeat, quota probe

dispatch-turn.sh (headless turn launcher with TURN_MODEL routing), queue-after.sh (serial gate chain + quota-wall guard), pmo-heartbeat.sh (PMO wake loop), quota-probe-then-chain.sh (post-wall relaunch).
2026-08-29 05:22:43 -05:00
parent 458d92831e
commit 79d93b6c49
18 changed files with 825 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# founder-full-reset.sh — FOUNDER-RUN as reachableceo (sudo required).
+# Org-wide clean reset of all FIVE verticals: TSGBOD TSGCCO TSGCTO TSGCOO RCEO.
+#   1) kill every vertical screen (old 07:02 + new 11:13 generations alike)
+#   2) clean stale /run/screen sockets
+#   3) relaunch the four TSG stacks via /tmp/tsg-stack-relaunch.sh --yes
+#      (no live screens -> no duplicate names -> multiuser+acladd succeeds)
+#   4) give RCEO the same wrapper pattern: pmo-loop.sh + pinned brains
+#   5) verify + attach cheat sheet
+# Does NOT touch AZHost/AgentZero or any other screens. RCEO-PMO/RCEO-Work
+# (agent screens) die mid-run and return pinned (conversations preserved).
+# Blank-slate lever (NOT default): before running, founder may
+#   touch ~/.coordinate/RELAUNCH-FRESH ~/.coordinate/RELAUNCH-FRESH-WORK
+# but fresh sessions get NO intro injected by the wrapper — skip unless
+# you plan to hand-feed the INTRO prompts after.
+set -uo pipefail
+
+[ "$(id -un)" = "reachableceo" ] || { echo "run as reachableceo"; exit 1; }
+sudo -v || { echo "sudo required"; exit 1; }
+
+USERS=(TSGBOD TSGCCO TSGCTO TSGCOO)
+
+echo "== 1/5 kill all vertical screens (both generations) =="
+pat='SCREEN -dmS (TSGBOD-|TSGCCO-|TSGCTO-|TSGCOO-|RCEO-)'
+pids=$(pgrep -f "$pat" || true)
+if [ -n "$pids" ]; then sudo kill $pids 2>/dev/null; fi
+sleep 2
+pids=$(pgrep -f "$pat" || true)
+if [ -n "$pids" ]; then sudo kill -9 $pids 2>/dev/null; fi
+for u in "${USERS[@]}" reachableceo; do
+  sudo pkill -u "$u" -f pmo-loop.sh 2>/dev/null
+  sudo pkill -u "$u" -f 'crush --' 2>/dev/null
+  sudo pkill -u "$u" -fx 'crush' 2>/dev/null
+done
+sleep 1
+
+echo "== 2/5 clean stale sockets =="
+for u in "${USERS[@]}" reachableceo; do
+  sudo rm -f "/run/screen/S-$u"/* 2>/dev/null
+done
+
+echo "== 3/5 relaunch TSG stacks (wrappers + existing pins) =="
+if [ -f /tmp/tsg-stack-relaunch.sh ]; then
+  bash /tmp/tsg-stack-relaunch.sh --yes || echo "WARN: relaunch script failed — inspect its output"
+else
+  echo "WARN: /tmp/tsg-stack-relaunch.sh missing — TSG stacks NOT relaunched"
+fi
+
+echo "== 4/5 relaunch RCEO stack (wrapper parity) =="
+mkdir -p "$HOME/.coordinate/scripts" "$HOME/.coordinate/logs"
+cat > "$HOME/.coordinate/scripts/pmo-loop.sh" <<'WRAP'
+#!/usr/bin/env bash
+# pmo-loop.sh [pmo|work] — keep a crush session alive inside its screen.
+ROLE="${1:-pmo}"
+case "$ROLE" in
+  pmo)  FRESH="$HOME/.coordinate/RELAUNCH-FRESH";      PIN="$HOME/.coordinate/PMO-SESSION-ID" ;;
+  work) FRESH="$HOME/.coordinate/RELAUNCH-FRESH-WORK"; PIN="$HOME/.coordinate/WORK-SESSION-ID" ;;
+  *)    echo "usage: $0 [pmo|work]" >&2; exit 1 ;;
+esac
+LOG="$HOME/.coordinate/logs/${ROLE}-relaunch.log"
+cd "$HOME"
+export TERM="${TERM:-xterm-256color}"
+export PATH="$HOME/.local/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
+while true; do
+  started=$(date +%s)
+  if [ -f "$FRESH" ]; then
+    rm -f "$FRESH"
+    echo "$(date -Is) relaunch FRESH" >> "$LOG"
+    crush
+  elif [ -s "$PIN" ]; then
+    sid=$(cat "$PIN")
+    echo "$(date -Is) relaunch PINNED $sid" >> "$LOG"
+    crush --session "$sid"
+  else
+    echo "$(date -Is) relaunch CONTINUE" >> "$LOG"
+    crush --continue
+  fi
+  rt=$(( $(date +%s) - started ))
+  if [ "$rt" -lt 60 ]; then
+    echo "$(date -Is) short run (${rt}s) - backoff 60s" >> "$LOG"
+    sleep 60
+  else
+    sleep 2
+  fi
+done
+WRAP
+chmod 755 "$HOME/.coordinate/scripts/pmo-loop.sh"
+echo 78cf63b8a72f4282 > "$HOME/.coordinate/PMO-SESSION-ID"
+echo e1079fafb8c37d60 > "$HOME/.coordinate/WORK-SESSION-ID"
+env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-PMO \
+  /home/reachableceo/.coordinate/scripts/pmo-loop.sh pmo
+env HOME=/home/reachableceo TERM=xterm-256color screen -dmS RCEO-Work \
+  /home/reachableceo/.coordinate/scripts/pmo-loop.sh work
+
+echo "== 5/5 verify =="
+sleep 3
+screen -ls
+for u in "${USERS[@]}"; do
+  echo "-- $u:"
+  sudo -u "$u" env TERM=dumb screen -ls 2>/dev/null | grep -E 'PMO|Work' || echo "   NONE (check relaunch output)"
+done
+echo
+echo "Attach: TSGxx-PMO / TSGxx-work aliases; RCEO: RCEO-PMO | RCEO-work"
+echo "Reboot bring-up units (enabled): reachableceo-agent-stack.service,"
+echo "tsg-agent-stacks.service — safe to reboot-test after this."