ops(scripts): dispatch, queue gating, heartbeat, quota probe

dispatch-turn.sh (headless turn launcher with TURN_MODEL routing), queue-after.sh (serial gate chain + quota-wall guard), pmo-heartbeat.sh (PMO wake loop), quota-probe-then-chain.sh (post-wall relaunch).
2026-08-29 05:22:43 -05:00
parent 458d92831e
commit 79d93b6c49
18 changed files with 825 additions and 0 deletions
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# source: /home/_crossfeed/tooling/agent-stack @ HEAD
+# agent-metrics.sh — harvest per-vertical crush usage into the org ledger.
+# Appends one JSONL line per session seen, deduped by session id via
+# ledger/seen index. Idempotent; safe to run every timer tick.
+#
+# Ledger: /home/_crossfeed/metrics/ledger.jsonl
+# Event:  {"ts","vertical","role","session","model","prompt_tokens",
+#          "completion_tokens","total_tokens","cost"}
+# Role heuristic: title contains 'PMO intro' -> pmo; 'INTRO-WORK' -> work;
+# 'headless' -> workturn; else 'other'.
+set -uo pipefail
+
+METRICS=${METRICS:-/home/_crossfeed/metrics}
+LEDGER="$METRICS/ledger.jsonl"
+SEEN="$METRICS/.seen-ids"
+VERTICALS=(TSGBOD TSGCTO TSGCCO TSGCOO reachableceo)
+RUNAS=${RUNAS:-sudo}   # agent-metrics runs from the supervisor service (root)
+
+mkdir -p "$METRICS"; touch "$LEDGER" "$SEEN"
+
+emit() { printf '%s\n' "$1" >> "$LEDGER"; }
+
+for user in "${VERTICALS[@]}"; do
+  home=$(getent passwd "$user" | cut -d: -f6)
+  [ -n "$home" ] || continue
+  # shellcheck disable=SC2016
+  json=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
+    bash -c "cd '$home' && crush session list --json 2>/dev/null" || true)
+  [ -n "$json" ] || continue
+  # one session id per line; token/cost fields only exist in `session show`
+  while IFS= read -r sid; do
+    [ -n "$sid" ] || continue
+    grep -q "$sid" "$SEEN" && continue
+    obj=$($RUNAS runuser -u "$user" -- env HOME="$home" TERM=xterm-256color \
+      bash -c "cd '$home' && crush session show '$sid' --json 2>/dev/null | head -c 4000" || true)
+    [ -n "$obj" ] || continue
+    title=$(printf '%s' "$obj" | grep -o '"title":"[^"]*"' | head -1 | cut -d'"' -f4)
+    role=other
+    case "$title" in
+      *PMO*intro*) role=pmo ;;
+      *INTRO-WORK*) role=work ;;
+      *headless*|*TSG*-Work*executing*) role=workturn ;;
+    esac
+    model=$(printf '%s' "$obj" | grep -o '"model":"[^"]*"' | head -1 | cut -d'"' -f4)
+    pt=$(printf '%s' "$obj" | grep -o '"prompt_tokens":[0-9]*' | head -1 | cut -d: -f2)
+    ct=$(printf '%s' "$obj" | grep -o '"completion_tokens":[0-9]*' | head -1 | cut -d: -f2)
+    tt=$(printf '%s' "$obj" | grep -o '"total_tokens":[0-9]*' | head -1 | cut -d: -f2)
+    cost=$(printf '%s' "$obj" | grep -o '"cost":[0-9.e-]*' | head -1 | cut -d: -f2)
+    ts=$(date -Is)
+    emit "{\"ts\":\"$ts\",\"vertical\":\"$user\",\"role\":\"$role\",\"session\":\"$sid\",\"model\":\"${model:-?}\",\"prompt_tokens\":${pt:-0},\"completion_tokens\":${ct:-0},\"total_tokens\":${tt:-0},\"cost\":${cost:-0}}"
+    echo "$sid" >> "$SEEN"
+  done < <(printf '%s' "$json" | grep -o '"id":"[0-9a-f]*"' | cut -d'"' -f4)
+done
+
+# rolling summary
+python3 - "$LEDGER" <<'PYEOF' > "$METRICS/SUMMARY.md" 2>/dev/null || true
+import json, sys, collections, datetime
+tot = collections.Counter(); cost = collections.Counter(); n = collections.Counter()
+day = collections.Counter()
+now = datetime.datetime.now().astimezone()
+for line in open(sys.argv[1]):
+    try: e = json.loads(line)
+    except Exception: continue
+    v = e.get("vertical","?"); t = e.get("total_tokens",0)
+    tot[v] += t; cost[v] += e.get("cost",0.0); n[v] += 1
+    try:
+        ts = datetime.datetime.fromisoformat(e["ts"])
+        if (now-ts).total_seconds() < 86400: day[v] += t
+    except Exception: pass
+print(f"# Agent usage summary — {now.isoformat()}")
+print(f"- sessions ledgered total: {sum(n.values())}")
+print("- all-time by vertical:")
+for v in sorted(tot):
+    print(f"  {v:<13} n={n[v]:<4} tokens={tot[v]:<10} cost={cost[v]:.4f} (24h tokens={day[v]})")
+PYEOF