quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)

The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
This commit is contained in:
2026-08-29 05:37:15 -05:00
parent 9dbb20489c
commit fc518c475e
21 changed files with 2904 additions and 20 deletions
+26 -1
View File
@@ -15,6 +15,7 @@ import (
"time"
"ukrrs.com/mopac/harness/internal/models"
"ukrrs.com/mopac/harness/internal/quota"
"ukrrs.com/mopac/harness/internal/task"
"ukrrs.com/mopac/harness/internal/writeback"
)
@@ -69,6 +70,9 @@ func (c *Conductor) RunLoop(ctx context.Context, opts LoopOpts) error {
fmt.Fprintf(c.out, "harness: loop: vertical=%s poll=%s state=%s redmine=%s (SIGINT to stop)\n",
c.cfg.Vertical, interval, c.cfg.Loop.StateDir, c.redmineHost())
if status := c.gateStatusLine(ctx); status != "off" {
fmt.Fprintf(c.out, "harness: loop: gate: %s\n", status)
}
if gitea != nil {
fmt.Fprintf(c.out, "harness: loop: gitea report commit on (%s/%s)\n", c.cfg.Gitea.Owner, c.cfg.Gitea.Repo)
}
@@ -121,6 +125,13 @@ func (c *Conductor) scanOnce(ctx context.Context, state *loopState, writer *writ
fmt.Fprintf(c.out, "harness: loop: would dispatch %s (updated %s): %q\n", t.ID, t.UpdatedOn, t.Subject)
continue
}
// Back-pressure: quota buckets + peak schedule + host resources. A
// defer skips the turn WITHOUT consuming the task — the next scan
// reconsiders it (the 19:00-wall class of failure becomes a logged
// throttle instead of dead turns).
if !c.gateConsult(ctx, state, t) {
continue
}
c.dispatchTask(ctx, state, writer, gitea, t)
}
return nil
@@ -136,9 +147,23 @@ func (c *Conductor) dispatchTask(ctx context.Context, state *loopState, writer *
run, err := c.runTask(ctx, t)
if run != nil && run.turn != nil && run.reportPath != "" {
// Usage accounting: tokens + estimated z.ai credits, per class,
// into the JSONL (feeds the Discourse usage reports) and the
// shared quota state (all instances' estimates).
credits := 0.0
if c.gate != nil {
credits = c.gate.RecordTurn(quota.EstimateTurnInput{
Model: run.decision.Model,
PromptTokens: run.turn.PromptTokens,
CompletionTokens: run.turn.CompletionTokens,
Peak: c.gate.InPeak(),
})
}
_ = state.log(loopEvent{
Type: evReport, TaskID: t.ID, Model: run.decision.Model,
Type: evReport, TaskID: t.ID, Model: run.decision.Model, Class: t.Class,
ReportPath: run.reportPath, StopReason: run.turn.StopReason,
PromptTokens: run.turn.PromptTokens, CompletionTokens: run.turn.CompletionTokens,
TotalTokens: run.turn.TotalTokens, Credits: credits,
})
}
if err != nil {
+15 -3
View File
@@ -27,6 +27,7 @@ type fakeRedmine struct {
status string
updatedOn string
notes []string
class string
}
statusIDs map[string]int
noted []string // issue ids that got a note, in order
@@ -41,6 +42,7 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
status string
updatedOn string
notes []string
class string
}{},
statusIDs: map[string]int{"New": 1, "In Progress": 2, "Done": 3},
}
@@ -56,8 +58,12 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
b.WriteString(",")
}
first = false
fmt.Fprintf(&b, `{"id":%s,"subject":%q,"description":"do the thing","updated_on":%q,"status":{"name":%q},"custom_fields":[{"name":"Class","value":"primary"}]}`,
id, is.subject, is.updatedOn, is.status)
class := is.class
if class == "" {
class = "primary"
}
fmt.Fprintf(&b, `{"id":%s,"subject":%q,"description":"do the thing","updated_on":%q,"status":{"name":%q},"custom_fields":[{"name":"Class","value":%q}]}`,
id, is.subject, is.updatedOn, is.status, class)
}
b.WriteString(`]}`)
w.Header().Set("Content-Type", "application/json")
@@ -131,6 +137,11 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
}
func (f *fakeRedmine) addIssue(id, subject, status, updatedOn string) {
f.addIssueClass(id, subject, status, updatedOn, "primary")
}
// addIssueClass registers an issue with an explicit Class custom field.
func (f *fakeRedmine) addIssueClass(id, subject, status, updatedOn, class string) {
f.mu.Lock()
defer f.mu.Unlock()
f.issues[id] = struct {
@@ -138,7 +149,8 @@ func (f *fakeRedmine) addIssue(id, subject, status, updatedOn string) {
status string
updatedOn string
notes []string
}{subject: subject, status: status, updatedOn: updatedOn}
class string
}{subject: subject, status: status, updatedOn: updatedOn, class: class}
}
func (f *fakeRedmine) setUpdatedOn(id, updatedOn string) {
+165
View File
@@ -0,0 +1,165 @@
package loop
import (
"context"
"fmt"
"sort"
"strings"
"ukrrs.com/mopac/harness/internal/config"
"ukrrs.com/mopac/harness/internal/quota"
"ukrrs.com/mopac/harness/internal/task"
)
// buildGates constructs the quota gate and resource config for the
// conductor. Quota/resources are opt-in ([quota] enabled / [resources]
// enabled); failures to build (bad schedule, unreachable redis) are fatal
// at startup because a silently missing gate is a silently burning plan.
func buildGates(cfg *config.Config, keys *config.KeyResolver) (*quota.Gate, error) {
if !cfg.Quota.Enabled && !cfg.Resources.Enabled {
return nil, nil
}
var shared *quota.SharedState
if cfg.Quota.RedisURL != "" {
var err error
shared, err = quota.NewSharedState(cfg.Quota.RedisURL)
if err != nil {
return nil, fmt.Errorf("quota redis: %w", err)
}
}
if !cfg.Quota.Enabled {
return nil, nil
}
gate, err := quota.NewGate(cfg.Quota, keys, shared)
if err != nil {
return nil, err
}
return gate, nil
}
// gateConsult is the pre-dispatch back-pressure check: quota snapshot +
// schedule + system resources. A defer never marks the task processed, so
// the next scan reconsiders it (defer is throttle, not rejection); the
// (task, updated_on, reason) tuple is logged once per process to keep the
// JSONL free of per-scan spam.
func (c *Conductor) gateConsult(ctx context.Context, state *loopState, t task.Task) bool {
if c.gate == nil && !c.resEnabled() {
return true
}
if reasons := c.resourceReasons(); len(reasons) > 0 {
c.logDefer(state, t, "resources busy: "+strings.Join(reasons, "; "))
return false
}
if c.gate == nil {
return true
}
d := c.gate.Decide(ctx, t.Class)
if d.Action == quota.ActionDefer {
c.logDefer(state, t, d.Reason)
return false
}
return true
}
// resourceReasons samples the host and returns threshold violations ([] =
// not busy, or the monitor is off/unreadable — read failures never defer).
func (c *Conductor) resourceReasons() []string {
rc := c.cfg.Resources
if !rc.Enabled {
return nil
}
st := quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot)
return quota.BusyCheck{
MaxLoadAvg: rc.MaxLoadAvg,
MinMemAvailableMB: rc.MinMemAvailableMB,
MinDiskFreeMB: rc.MinDiskFreeMB,
MaxIODelayPct: rc.MaxIODelayPct,
}.Evaluate(st)
}
func (c *Conductor) resEnabled() bool { return c.cfg.Resources.Enabled }
// logDefer records one defer event (deduped per task+reason in-process).
func (c *Conductor) logDefer(state *loopState, t task.Task, reason string) {
key := t.ID + "|" + t.UpdatedOn + "|" + reason
if c.deferLogged == nil {
c.deferLogged = map[string]bool{}
}
if c.deferLogged[key] {
return
}
c.deferLogged[key] = true
_ = state.log(loopEvent{Type: evDefer, TaskID: t.ID, UpdatedOn: t.UpdatedOn, Subject: t.Subject, Detail: reason})
fmt.Fprintf(c.out, "harness: loop: defer %s: %s (reconsidered next scan; never marked processed)\n", t.ID, reason)
}
// gateStatusLine is the startup/`quota status` one-liner.
func (c *Conductor) gateStatusLine(ctx context.Context) string {
if c.gate == nil && !c.resEnabled() {
return "off"
}
parts := []string{}
if c.gate != nil {
snap := c.gate.Snapshot(ctx)
parts = append(parts, c.gate.StatusLine(snap))
}
if c.resEnabled() {
st := quota.ReadResourceStats(c.cfg.Resources.ProcRoot, c.cfg.Resources.SysRoot, c.cfg.WorkRoot)
parts = append(parts, fmt.Sprintf("load=%.2f memAvail=%.0fMB diskFree=%.0fMB", st.LoadAvg1m, st.MemAvailable, st.DiskFree))
}
return strings.Join(parts, " | ")
}
// usageRow is one line of the per-class usage accounting (feeds the
// per-instance Discourse usage reports).
type usageRow struct {
Class string `json:"class"`
Turns int `json:"turns"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
Credits float64 `json:"credits"`
}
// UsageReport renders the per-class token+credit accounting table from a
// loop state dir (read-only; `harness quota status`).
func UsageReport(stateDir string) (string, error) {
s, err := openLoopStateReadOnly(stateDir)
if err != nil {
return "", err
}
defer s.Close()
return usageReport(s.usageRows()), nil
}
// GateStatusLine is the exported one-liner for `harness quota` output.
func (c *Conductor) GateStatusLine(ctx context.Context) string {
return c.gateStatusLine(ctx)
}
// ResourceSample reads one host sample for the CLI (nil-safe when off).
func (c *Conductor) ResourceSample() (quota.ResourceStats, bool) {
if !c.resEnabled() {
return quota.ResourceStats{}, false
}
rc := c.cfg.Resources
return quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot), true
}
// usageReport renders the accounting table from the loop state.
func usageReport(rows []usageRow) string {
sort.Slice(rows, func(i, j int) bool { return rows[i].Class < rows[j].Class })
var b strings.Builder
fmt.Fprintf(&b, "%-14s %6s %12s %12s %12s %10s\n", "CLASS", "TURNS", "PROMPT", "COMPLETION", "TOTAL", "CREDITS")
var tot usageRow
for _, r := range rows {
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", r.Class, r.Turns, r.PromptTokens, r.CompletionTokens, r.TotalTokens, r.Credits)
tot.Turns += r.Turns
tot.PromptTokens += r.PromptTokens
tot.CompletionTokens += r.CompletionTokens
tot.TotalTokens += r.TotalTokens
tot.Credits += r.Credits
}
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", "TOTAL", tot.Turns, tot.PromptTokens, tot.CompletionTokens, tot.TotalTokens, tot.Credits)
return b.String()
}
+293
View File
@@ -0,0 +1,293 @@
package loop
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"ukrrs.com/mopac/harness/internal/config"
)
// fakeUsage is a mutable z.ai usage endpoint: the documented-shape JSON,
// with the weekly bucket flippable between "wall" (97%) and healthy (10%).
type fakeUsage struct {
wall atomic.Bool
srv *httptest.Server
}
func newFakeUsage(t *testing.T) *fakeUsage {
t.Helper()
f := &fakeUsage{}
mux := http.NewServeMux()
mux.HandleFunc("/usage", func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") == "" {
http.Error(w, "no bearer", http.StatusUnauthorized)
return
}
wk := 14000.0 // 10%
five := 2800.0
if f.wall.Load() {
wk = 135800.0 // 97%: the 2026-08-28 19:00 wall
five = 12000.0
}
fmt.Fprintf(w, `{"usage":{"five_hour":{"used_credits":%g,"total_credits":28000,"reset_at":"2026-08-29T22:00:00Z"},"weekly":{"used_credits":%g,"total_credits":140000,"reset_at":"2026-09-02T00:00:00Z"}}}`, five, wk)
})
f.srv = httptest.NewServer(mux)
t.Cleanup(f.srv.Close)
return f
}
func quotaTestConfig(t *testing.T, redmineURL, llmURL, usageURL string) *config.Config {
t.Helper()
cfg := loopTestConfig(t, redmineURL, llmURL)
cfg.Quota = quotaCfg(usageURL)
return cfg
}
func resCfg(procRoot string) config.ResourcesConfig {
return config.ResourcesConfig{
Enabled: true,
MaxLoadAvg: 6.0,
MinMemAvailableMB: 2048,
MinDiskFreeMB: 5120,
MaxIODelayPct: 90,
ProcRoot: procRoot,
SysRoot: procRoot,
}
}
func quotaCfg(usageURL string) config.QuotaConfig {
c := config.QuotaConfig{
Enabled: true, Account: "zai-1",
Plan5hCredits: 28000, PlanWeeklyCredits: 140000,
UsageURL: usageURL, KeyRef: "literal:zai-key",
PollIntervalSecs: 300, DeferAtPct: 85, BlockAtPct: 95,
PeakStart: "01:00", PeakEnd: "05:00", Timezone: "America/Chicago",
PeakWeekdaysOnly: true, PeakClasses: []string{"study", "read"},
}
if usageURL == "" {
c.KeyRef = ""
}
return c
}
// TestLoopQuotaWallDefersAndRecovers replays 2026-08-28 ~19:00 CST: the
// weekly bucket hit 97% and dispatched turns died at the provider. With the
// gate: the scan DEFERS the turn with a surfaced reason, the loop exits
// clean, the issue is NOT consumed (no note, not marked processed) — and
// once quota recovers the very next scan dispatches it.
func TestLoopQuotaWallDefersAndRecovers(t *testing.T) {
rm := newFakeRedmine(t)
rm.addIssue("490", "Big build task", "In Progress", "2026-08-28T21:30:00Z")
f := newFakeLLM(t, textMsg("the turn ran"))
usage := newFakeUsage(t)
usage.wall.Store(true) // the wall is up
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, usage.srv.URL+"/usage")
out := runOneScan(t, cfg, false)
if f.requestCount() != 0 {
t.Fatalf("LLM calls during the wall = %d, want 0 (turns must defer, not die)", f.requestCount())
}
for _, want := range []string{
"defer 490",
">= block 95%",
"weekly",
} {
if !strings.Contains(out, want) {
t.Errorf("output missing %q:\n%s", want, out)
}
}
if rm.noteCount("490") != 0 {
t.Errorf("deferred task must not be noted back")
}
// The defer is in the JSONL with the reason (json.Marshal HTML-escapes
// ">=", so match on the words); the task was NOT consumed.
statePath := filepath.Join(cfg.Loop.StateDir, "loop.jsonl")
data, err := os.ReadFile(statePath)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), `"type":"defer"`) || !strings.Contains(string(data), "block 95") {
t.Errorf("JSONL defer event missing:\n%s", data)
}
// Quota recovers: next scan dispatches (task was never marked processed).
usage.wall.Store(false)
out2 := runOneScan(t, cfg, false)
if f.requestCount() != 1 {
t.Fatalf("LLM calls after recovery = %d, want 1 (defer must not consume the task)", f.requestCount())
}
if !strings.Contains(out2, "dispatch 490") || !strings.Contains(out2, "noted REPORT on #490") {
t.Errorf("recovery scan output wrong:\n%s", out2)
}
}
// TestLoopPeakWindowDefersHeavyClass: inside 01:00-05:00 CST the heavy
// class defers with a peak reason while the flash-tier class runs.
func TestLoopPeakWindowDefersHeavyClass(t *testing.T) {
rm := newFakeRedmine(t)
rm.addIssue("491", "Heavy flagship work", "In Progress", "2026-08-28T21:30:00Z")
f := newFakeLLM(t, textMsg("flash reply"))
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, "") // estimates only
// Friday 2026-08-28 03:00 CST = inside the peak window.
peakAt := time.Date(2026, 8, 28, 3, 0, 0, 0, cst(t))
var out strings.Builder
cond, err := New(cfg, &out)
if err != nil {
t.Fatal(err)
}
cond.gate.SetClock(func() time.Time { return peakAt })
if err := cond.RunLoop(context.Background(), LoopOpts{Once: true}); err != nil {
t.Fatal(err)
}
if f.requestCount() != 0 {
t.Fatalf("heavy class dispatched in peak window (%d calls)", f.requestCount())
}
if !strings.Contains(out.String(), "peak window") {
t.Errorf("defer reason must name the peak window:\n%s", out.String())
}
// A flash-tier class in the same peak window runs.
rm.addIssueClass("492", "Light study work", "In Progress", "2026-08-28T21:31:00Z", "study")
out2 := runScanAt(t, cfg, peakAt)
if f.requestCount() != 1 {
t.Fatalf("flash class must dispatch in peak: %d calls", f.requestCount())
}
if !strings.Contains(out2, "dispatch 492") {
t.Errorf("flash dispatch missing:\n%s", out2)
}
// Off-peak (Friday 12:00 CST): the heavy task now dispatches.
offPeak := time.Date(2026, 8, 28, 12, 0, 0, 0, cst(t))
_ = runScanAt(t, cfg, offPeak)
if f.requestCount() != 2 {
t.Fatalf("heavy class must dispatch off-peak: %d calls", f.requestCount())
}
}
// TestLoopResourceBusyDefers: a pinned-high load fixture defers dispatch;
// the task stays unconsumed.
func TestLoopResourceBusyDefers(t *testing.T) {
rm := newFakeRedmine(t)
rm.addIssue("492", "Task during a build storm", "In Progress", "2026-08-28T21:30:00Z")
f := newFakeLLM(t, textMsg("never"))
cfg := loopTestConfig(t, rm.srv.URL, f.srv.URL)
proc := filepath.Join(t.TempDir(), "proc")
os.MkdirAll(proc, 0o755)
os.WriteFile(filepath.Join(proc, "loadavg"), []byte("41.20 30.00 15.00 5/900 1\n"), 0o644)
cfg.Resources = resCfg(proc)
out := runOneScan(t, cfg, false)
if f.requestCount() != 0 {
t.Fatalf("busy host must defer dispatch (%d calls)", f.requestCount())
}
if !strings.Contains(out, "resources busy") || !strings.Contains(out, "load 41.20") {
t.Errorf("defer reason must surface the resource violation:\n%s", out)
}
if rm.noteCount("492") != 0 {
t.Errorf("deferred task must not be noted")
}
}
// TestLoopUsageAccounting: after a dispatched turn, the JSONL report event
// carries class + tokens + estimated credits, and the usage table aggregates.
func TestLoopUsageAccounting(t *testing.T) {
rm := newFakeRedmine(t)
rm.addIssue("493", "Accounted task", "In Progress", "2026-08-28T21:30:00Z")
f := newFakeLLM(t, textMsg("reply with 20/40 tokens"))
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, "")
runScanAt(t, cfg, time.Date(2026, 8, 28, 12, 0, 0, 0, cst(t))) // off-peak Friday
data, err := os.ReadFile(filepath.Join(cfg.Loop.StateDir, "loop.jsonl"))
if err != nil {
t.Fatal(err)
}
var reportEv map[string]any
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
var ev map[string]any
if err := json.Unmarshal([]byte(line), &ev); err != nil {
t.Fatalf("bad JSONL line %q: %v", line, err)
}
if ev["type"] == "report" {
reportEv = ev
}
}
if reportEv == nil {
t.Fatal("no report event in JSONL")
}
if reportEv["class"] != "primary" {
t.Errorf("report event class = %v, want primary", reportEv["class"])
}
if reportEv["prompt_tokens"] != float64(20) || reportEv["completion_tokens"] != float64(40) {
t.Errorf("report tokens = %v", reportEv)
}
// This turn ran glm-5.3 off-peak: (20*6.9 + 40*24)/10000 / 2 = 0.0549
credits, _ := reportEv["credits"].(float64)
if credits < 0.0548 || credits > 0.0550 {
t.Errorf("credits = %v, want ~0.0549", credits)
}
report, err := UsageReport(cfg.Loop.StateDir)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(report, "primary") || !strings.Contains(report, "TOTAL") || !strings.Contains(report, "1") {
t.Errorf("usage table wrong:\n%s", report)
}
}
// TestLoopGateOffByDefault: with [quota] disabled the loop dispatches
// exactly as before (no behavior change for existing configs).
func TestLoopGateOffByDefault(t *testing.T) {
rm := newFakeRedmine(t)
rm.addIssue("494", "Plain task", "In Progress", "2026-08-28T21:30:00Z")
f := newFakeLLM(t, textMsg("plain reply"))
cfg := loopTestConfig(t, rm.srv.URL, f.srv.URL)
out := runOneScan(t, cfg, false)
if f.requestCount() != 1 {
t.Fatalf("LLM calls = %d, want 1", f.requestCount())
}
if strings.Contains(out, "defer") || strings.Contains(out, "gate:") {
t.Errorf("disabled gate must be silent:\n%s", out)
}
}
func cst(t *testing.T) *time.Location {
t.Helper()
loc, err := time.LoadLocation("America/Chicago")
if err != nil {
t.Fatal(err)
}
return loc
}
// runScanAt runs a --once scan with the quota gate clock pinned.
func runScanAt(t *testing.T, cfg *config.Config, at time.Time) string {
t.Helper()
var out strings.Builder
cond, err := New(cfg, &out)
if err != nil {
t.Fatal(err)
}
if cond.gate != nil {
cond.gate.SetClock(func() time.Time { return at })
}
if err := cond.RunLoop(context.Background(), LoopOpts{Once: true}); err != nil {
t.Fatalf("RunLoop: %v (output:\n%s)", err, out.String())
}
return out.String()
}
+13 -1
View File
@@ -17,6 +17,7 @@ import (
"ukrrs.com/mopac/harness/internal/intake"
"ukrrs.com/mopac/harness/internal/llm"
"ukrrs.com/mopac/harness/internal/models"
"ukrrs.com/mopac/harness/internal/quota"
"ukrrs.com/mopac/harness/internal/task"
"ukrrs.com/mopac/harness/internal/tools"
"ukrrs.com/mopac/harness/internal/writeback"
@@ -35,6 +36,9 @@ type Conductor struct {
bash *tools.BashTool
keys *config.KeyResolver
out io.Writer
gate *quota.Gate
// deferLogged dedups defer log lines per (task, updated_on, reason).
deferLogged map[string]bool
}
// New validates routing config and prepares the tool palette.
@@ -57,9 +61,17 @@ func New(cfg *config.Config, out io.Writer) (*Conductor, error) {
return nil, err
}
}
return &Conductor{cfg: cfg, router: router, bash: bash, keys: config.NewKeyResolver(cfg), out: out}, nil
keys := config.NewKeyResolver(cfg)
gate, err := buildGates(cfg, keys)
if err != nil {
return nil, err
}
return &Conductor{cfg: cfg, router: router, bash: bash, keys: keys, out: out, gate: gate, deferLogged: map[string]bool{}}, nil
}
// Gate exposes the quota gate (nil when off) for `harness quota status`.
func (c *Conductor) Gate() *quota.Gate { return c.gate }
// OnceOpts controls a single iteration.
type OnceOpts struct {
DryRun bool // intake + plan only, no LLM call
+59 -3
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
@@ -19,6 +20,7 @@ const (
evCommit = "commit" // REPORT committed to gitea (optional step)
evRefresh = "refresh" // dedup marker advanced to post-writeback updated_on
evError = "error" // a step failed; loop continues
evDefer = "defer" // dispatch deferred by the quota/resource gate (throttle, not rejection)
)
// loopEvent is one line of the append-only loop log. It doubles as the
@@ -31,11 +33,18 @@ type loopEvent struct {
UpdatedOn string `json:"updated_on,omitempty"`
Subject string `json:"subject,omitempty"`
Model string `json:"model,omitempty"`
Class string `json:"class,omitempty"` // usage accounting key
ReportPath string `json:"report_path,omitempty"`
StopReason string `json:"stop_reason,omitempty"`
StatusFrom string `json:"status_from,omitempty"`
StatusTo string `json:"status_to,omitempty"`
Detail string `json:"detail,omitempty"` // human context; never secrets
// Per-turn usage accounting (report events): tokens + estimated z.ai
// credits. Feeds the per-instance usage tables / Discourse reports.
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Credits float64 `json:"credits,omitempty"`
Detail string `json:"detail,omitempty"` // human context; never secrets
}
// loopState is the append-only loop log + dedup index: stateDir/loop.jsonl,
@@ -47,7 +56,18 @@ type loopState struct {
f *os.File
path string
seen map[string]string
readOnly bool // dry-run: dedup checks work, writes are refused
usage map[string]usageAcc // class -> per-class totals (report events)
readOnly bool // dry-run: dedup checks work, writes are refused
}
// usageAcc accumulates per-class turn accounting; it is rebuilt from the
// JSONL at open so usage reporting survives restarts.
type usageAcc struct {
Turns int `json:"turns"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
Credits float64 `json:"credits"`
}
func openLoopState(stateDir string) (*loopState, error) {
@@ -59,7 +79,7 @@ func openLoopState(stateDir string) (*loopState, error) {
if err != nil {
return nil, fmt.Errorf("open loop log: %w", err)
}
s := &loopState{path: path, f: f, seen: make(map[string]string)}
s := &loopState{path: path, f: f, seen: make(map[string]string), usage: make(map[string]usageAcc)}
if err := s.loadIndex(); err != nil {
f.Close()
return nil, err
@@ -73,6 +93,7 @@ func openLoopStateReadOnly(stateDir string) (*loopState, error) {
s := &loopState{
path: filepath.Join(stateDir, "loop.jsonl"),
seen: make(map[string]string),
usage: make(map[string]usageAcc),
readOnly: true,
}
if err := s.loadIndex(); err != nil {
@@ -100,6 +121,9 @@ func (s *loopState) loadIndex() error {
if ev.Type == evDispatch || ev.Type == evRefresh {
s.seen[ev.TaskID] = ev.UpdatedOn
}
if ev.Type == evReport {
s.addUsage(ev.Class, ev)
}
}
return sc.Err()
}
@@ -131,9 +155,41 @@ func (s *loopState) log(ev loopEvent) error {
if ev.Type == evDispatch || ev.Type == evRefresh {
s.seen[ev.TaskID] = ev.UpdatedOn
}
if ev.Type == evReport {
s.addUsage(ev.Class, ev)
}
return nil
}
func (s *loopState) addUsage(class string, ev loopEvent) {
if class == "" {
class = "(unknown)"
}
acc := s.usage[class]
acc.Turns++
acc.PromptTokens += ev.PromptTokens
acc.CompletionTokens += ev.CompletionTokens
acc.TotalTokens += ev.TotalTokens
acc.Credits += ev.Credits
s.usage[class] = acc
}
// usageRows snapshots the per-class accounting (sorted by class).
func (s *loopState) usageRows() []usageRow {
s.mu.Lock()
defer s.mu.Unlock()
rows := make([]usageRow, 0, len(s.usage))
for class, acc := range s.usage {
rows = append(rows, usageRow{
Class: class, Turns: acc.Turns,
PromptTokens: acc.PromptTokens, CompletionTokens: acc.CompletionTokens,
TotalTokens: acc.TotalTokens, Credits: acc.Credits,
})
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Class < rows[j].Class })
return rows
}
func (s *loopState) Close() error {
s.mu.Lock()
defer s.mu.Unlock()