The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
166 lines
5.5 KiB
Go
166 lines
5.5 KiB
Go
package loop
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"ukrrs.com/mopac/harness/internal/config"
|
|
"ukrrs.com/mopac/harness/internal/quota"
|
|
"ukrrs.com/mopac/harness/internal/task"
|
|
)
|
|
|
|
// buildGates constructs the quota gate and resource config for the
|
|
// conductor. Quota/resources are opt-in ([quota] enabled / [resources]
|
|
// enabled); failures to build (bad schedule, unreachable redis) are fatal
|
|
// at startup because a silently missing gate is a silently burning plan.
|
|
func buildGates(cfg *config.Config, keys *config.KeyResolver) (*quota.Gate, error) {
|
|
if !cfg.Quota.Enabled && !cfg.Resources.Enabled {
|
|
return nil, nil
|
|
}
|
|
var shared *quota.SharedState
|
|
if cfg.Quota.RedisURL != "" {
|
|
var err error
|
|
shared, err = quota.NewSharedState(cfg.Quota.RedisURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("quota redis: %w", err)
|
|
}
|
|
}
|
|
if !cfg.Quota.Enabled {
|
|
return nil, nil
|
|
}
|
|
gate, err := quota.NewGate(cfg.Quota, keys, shared)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return gate, nil
|
|
}
|
|
|
|
// gateConsult is the pre-dispatch back-pressure check: quota snapshot +
|
|
// schedule + system resources. A defer never marks the task processed, so
|
|
// the next scan reconsiders it (defer is throttle, not rejection); the
|
|
// (task, updated_on, reason) tuple is logged once per process to keep the
|
|
// JSONL free of per-scan spam.
|
|
func (c *Conductor) gateConsult(ctx context.Context, state *loopState, t task.Task) bool {
|
|
if c.gate == nil && !c.resEnabled() {
|
|
return true
|
|
}
|
|
if reasons := c.resourceReasons(); len(reasons) > 0 {
|
|
c.logDefer(state, t, "resources busy: "+strings.Join(reasons, "; "))
|
|
return false
|
|
}
|
|
if c.gate == nil {
|
|
return true
|
|
}
|
|
d := c.gate.Decide(ctx, t.Class)
|
|
if d.Action == quota.ActionDefer {
|
|
c.logDefer(state, t, d.Reason)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// resourceReasons samples the host and returns threshold violations ([] =
|
|
// not busy, or the monitor is off/unreadable — read failures never defer).
|
|
func (c *Conductor) resourceReasons() []string {
|
|
rc := c.cfg.Resources
|
|
if !rc.Enabled {
|
|
return nil
|
|
}
|
|
st := quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot)
|
|
return quota.BusyCheck{
|
|
MaxLoadAvg: rc.MaxLoadAvg,
|
|
MinMemAvailableMB: rc.MinMemAvailableMB,
|
|
MinDiskFreeMB: rc.MinDiskFreeMB,
|
|
MaxIODelayPct: rc.MaxIODelayPct,
|
|
}.Evaluate(st)
|
|
}
|
|
|
|
func (c *Conductor) resEnabled() bool { return c.cfg.Resources.Enabled }
|
|
|
|
// logDefer records one defer event (deduped per task+reason in-process).
|
|
func (c *Conductor) logDefer(state *loopState, t task.Task, reason string) {
|
|
key := t.ID + "|" + t.UpdatedOn + "|" + reason
|
|
if c.deferLogged == nil {
|
|
c.deferLogged = map[string]bool{}
|
|
}
|
|
if c.deferLogged[key] {
|
|
return
|
|
}
|
|
c.deferLogged[key] = true
|
|
_ = state.log(loopEvent{Type: evDefer, TaskID: t.ID, UpdatedOn: t.UpdatedOn, Subject: t.Subject, Detail: reason})
|
|
fmt.Fprintf(c.out, "harness: loop: defer %s: %s (reconsidered next scan; never marked processed)\n", t.ID, reason)
|
|
}
|
|
|
|
// gateStatusLine is the startup/`quota status` one-liner.
|
|
func (c *Conductor) gateStatusLine(ctx context.Context) string {
|
|
if c.gate == nil && !c.resEnabled() {
|
|
return "off"
|
|
}
|
|
parts := []string{}
|
|
if c.gate != nil {
|
|
snap := c.gate.Snapshot(ctx)
|
|
parts = append(parts, c.gate.StatusLine(snap))
|
|
}
|
|
if c.resEnabled() {
|
|
st := quota.ReadResourceStats(c.cfg.Resources.ProcRoot, c.cfg.Resources.SysRoot, c.cfg.WorkRoot)
|
|
parts = append(parts, fmt.Sprintf("load=%.2f memAvail=%.0fMB diskFree=%.0fMB", st.LoadAvg1m, st.MemAvailable, st.DiskFree))
|
|
}
|
|
return strings.Join(parts, " | ")
|
|
}
|
|
|
|
// usageRow is one line of the per-class usage accounting (feeds the
|
|
// per-instance Discourse usage reports).
|
|
type usageRow struct {
|
|
Class string `json:"class"`
|
|
Turns int `json:"turns"`
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
Credits float64 `json:"credits"`
|
|
}
|
|
|
|
// UsageReport renders the per-class token+credit accounting table from a
|
|
// loop state dir (read-only; `harness quota status`).
|
|
func UsageReport(stateDir string) (string, error) {
|
|
s, err := openLoopStateReadOnly(stateDir)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer s.Close()
|
|
return usageReport(s.usageRows()), nil
|
|
}
|
|
|
|
// GateStatusLine is the exported one-liner for `harness quota` output.
|
|
func (c *Conductor) GateStatusLine(ctx context.Context) string {
|
|
return c.gateStatusLine(ctx)
|
|
}
|
|
|
|
// ResourceSample reads one host sample for the CLI (nil-safe when off).
|
|
func (c *Conductor) ResourceSample() (quota.ResourceStats, bool) {
|
|
if !c.resEnabled() {
|
|
return quota.ResourceStats{}, false
|
|
}
|
|
rc := c.cfg.Resources
|
|
return quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot), true
|
|
}
|
|
|
|
// usageReport renders the accounting table from the loop state.
|
|
func usageReport(rows []usageRow) string {
|
|
sort.Slice(rows, func(i, j int) bool { return rows[i].Class < rows[j].Class })
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "%-14s %6s %12s %12s %12s %10s\n", "CLASS", "TURNS", "PROMPT", "COMPLETION", "TOTAL", "CREDITS")
|
|
var tot usageRow
|
|
for _, r := range rows {
|
|
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", r.Class, r.Turns, r.PromptTokens, r.CompletionTokens, r.TotalTokens, r.Credits)
|
|
tot.Turns += r.Turns
|
|
tot.PromptTokens += r.PromptTokens
|
|
tot.CompletionTokens += r.CompletionTokens
|
|
tot.TotalTokens += r.TotalTokens
|
|
tot.Credits += r.Credits
|
|
}
|
|
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", "TOTAL", tot.Turns, tot.PromptTokens, tot.CompletionTokens, tot.TotalTokens, tot.Credits)
|
|
return b.String()
|
|
}
|