quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)
The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
+26
-1
@@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"ukrrs.com/mopac/harness/internal/models"
|
||||
"ukrrs.com/mopac/harness/internal/quota"
|
||||
"ukrrs.com/mopac/harness/internal/task"
|
||||
"ukrrs.com/mopac/harness/internal/writeback"
|
||||
)
|
||||
@@ -69,6 +70,9 @@ func (c *Conductor) RunLoop(ctx context.Context, opts LoopOpts) error {
|
||||
|
||||
fmt.Fprintf(c.out, "harness: loop: vertical=%s poll=%s state=%s redmine=%s (SIGINT to stop)\n",
|
||||
c.cfg.Vertical, interval, c.cfg.Loop.StateDir, c.redmineHost())
|
||||
if status := c.gateStatusLine(ctx); status != "off" {
|
||||
fmt.Fprintf(c.out, "harness: loop: gate: %s\n", status)
|
||||
}
|
||||
if gitea != nil {
|
||||
fmt.Fprintf(c.out, "harness: loop: gitea report commit on (%s/%s)\n", c.cfg.Gitea.Owner, c.cfg.Gitea.Repo)
|
||||
}
|
||||
@@ -121,6 +125,13 @@ func (c *Conductor) scanOnce(ctx context.Context, state *loopState, writer *writ
|
||||
fmt.Fprintf(c.out, "harness: loop: would dispatch %s (updated %s): %q\n", t.ID, t.UpdatedOn, t.Subject)
|
||||
continue
|
||||
}
|
||||
// Back-pressure: quota buckets + peak schedule + host resources. A
|
||||
// defer skips the turn WITHOUT consuming the task — the next scan
|
||||
// reconsiders it (the 19:00-wall class of failure becomes a logged
|
||||
// throttle instead of dead turns).
|
||||
if !c.gateConsult(ctx, state, t) {
|
||||
continue
|
||||
}
|
||||
c.dispatchTask(ctx, state, writer, gitea, t)
|
||||
}
|
||||
return nil
|
||||
@@ -136,9 +147,23 @@ func (c *Conductor) dispatchTask(ctx context.Context, state *loopState, writer *
|
||||
|
||||
run, err := c.runTask(ctx, t)
|
||||
if run != nil && run.turn != nil && run.reportPath != "" {
|
||||
// Usage accounting: tokens + estimated z.ai credits, per class,
|
||||
// into the JSONL (feeds the Discourse usage reports) and the
|
||||
// shared quota state (all instances' estimates).
|
||||
credits := 0.0
|
||||
if c.gate != nil {
|
||||
credits = c.gate.RecordTurn(quota.EstimateTurnInput{
|
||||
Model: run.decision.Model,
|
||||
PromptTokens: run.turn.PromptTokens,
|
||||
CompletionTokens: run.turn.CompletionTokens,
|
||||
Peak: c.gate.InPeak(),
|
||||
})
|
||||
}
|
||||
_ = state.log(loopEvent{
|
||||
Type: evReport, TaskID: t.ID, Model: run.decision.Model,
|
||||
Type: evReport, TaskID: t.ID, Model: run.decision.Model, Class: t.Class,
|
||||
ReportPath: run.reportPath, StopReason: run.turn.StopReason,
|
||||
PromptTokens: run.turn.PromptTokens, CompletionTokens: run.turn.CompletionTokens,
|
||||
TotalTokens: run.turn.TotalTokens, Credits: credits,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -27,6 +27,7 @@ type fakeRedmine struct {
|
||||
status string
|
||||
updatedOn string
|
||||
notes []string
|
||||
class string
|
||||
}
|
||||
statusIDs map[string]int
|
||||
noted []string // issue ids that got a note, in order
|
||||
@@ -41,6 +42,7 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
|
||||
status string
|
||||
updatedOn string
|
||||
notes []string
|
||||
class string
|
||||
}{},
|
||||
statusIDs: map[string]int{"New": 1, "In Progress": 2, "Done": 3},
|
||||
}
|
||||
@@ -56,8 +58,12 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
|
||||
b.WriteString(",")
|
||||
}
|
||||
first = false
|
||||
fmt.Fprintf(&b, `{"id":%s,"subject":%q,"description":"do the thing","updated_on":%q,"status":{"name":%q},"custom_fields":[{"name":"Class","value":"primary"}]}`,
|
||||
id, is.subject, is.updatedOn, is.status)
|
||||
class := is.class
|
||||
if class == "" {
|
||||
class = "primary"
|
||||
}
|
||||
fmt.Fprintf(&b, `{"id":%s,"subject":%q,"description":"do the thing","updated_on":%q,"status":{"name":%q},"custom_fields":[{"name":"Class","value":%q}]}`,
|
||||
id, is.subject, is.updatedOn, is.status, class)
|
||||
}
|
||||
b.WriteString(`]}`)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@@ -131,6 +137,11 @@ func newFakeRedmine(t *testing.T) *fakeRedmine {
|
||||
}
|
||||
|
||||
func (f *fakeRedmine) addIssue(id, subject, status, updatedOn string) {
|
||||
f.addIssueClass(id, subject, status, updatedOn, "primary")
|
||||
}
|
||||
|
||||
// addIssueClass registers an issue with an explicit Class custom field.
|
||||
func (f *fakeRedmine) addIssueClass(id, subject, status, updatedOn, class string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.issues[id] = struct {
|
||||
@@ -138,7 +149,8 @@ func (f *fakeRedmine) addIssue(id, subject, status, updatedOn string) {
|
||||
status string
|
||||
updatedOn string
|
||||
notes []string
|
||||
}{subject: subject, status: status, updatedOn: updatedOn}
|
||||
class string
|
||||
}{subject: subject, status: status, updatedOn: updatedOn, class: class}
|
||||
}
|
||||
|
||||
func (f *fakeRedmine) setUpdatedOn(id, updatedOn string) {
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"ukrrs.com/mopac/harness/internal/config"
|
||||
"ukrrs.com/mopac/harness/internal/quota"
|
||||
"ukrrs.com/mopac/harness/internal/task"
|
||||
)
|
||||
|
||||
// buildGates constructs the quota gate and resource config for the
|
||||
// conductor. Quota/resources are opt-in ([quota] enabled / [resources]
|
||||
// enabled); failures to build (bad schedule, unreachable redis) are fatal
|
||||
// at startup because a silently missing gate is a silently burning plan.
|
||||
func buildGates(cfg *config.Config, keys *config.KeyResolver) (*quota.Gate, error) {
|
||||
if !cfg.Quota.Enabled && !cfg.Resources.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
var shared *quota.SharedState
|
||||
if cfg.Quota.RedisURL != "" {
|
||||
var err error
|
||||
shared, err = quota.NewSharedState(cfg.Quota.RedisURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("quota redis: %w", err)
|
||||
}
|
||||
}
|
||||
if !cfg.Quota.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
gate, err := quota.NewGate(cfg.Quota, keys, shared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gate, nil
|
||||
}
|
||||
|
||||
// gateConsult is the pre-dispatch back-pressure check: quota snapshot +
|
||||
// schedule + system resources. A defer never marks the task processed, so
|
||||
// the next scan reconsiders it (defer is throttle, not rejection); the
|
||||
// (task, updated_on, reason) tuple is logged once per process to keep the
|
||||
// JSONL free of per-scan spam.
|
||||
func (c *Conductor) gateConsult(ctx context.Context, state *loopState, t task.Task) bool {
|
||||
if c.gate == nil && !c.resEnabled() {
|
||||
return true
|
||||
}
|
||||
if reasons := c.resourceReasons(); len(reasons) > 0 {
|
||||
c.logDefer(state, t, "resources busy: "+strings.Join(reasons, "; "))
|
||||
return false
|
||||
}
|
||||
if c.gate == nil {
|
||||
return true
|
||||
}
|
||||
d := c.gate.Decide(ctx, t.Class)
|
||||
if d.Action == quota.ActionDefer {
|
||||
c.logDefer(state, t, d.Reason)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// resourceReasons samples the host and returns threshold violations ([] =
|
||||
// not busy, or the monitor is off/unreadable — read failures never defer).
|
||||
func (c *Conductor) resourceReasons() []string {
|
||||
rc := c.cfg.Resources
|
||||
if !rc.Enabled {
|
||||
return nil
|
||||
}
|
||||
st := quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot)
|
||||
return quota.BusyCheck{
|
||||
MaxLoadAvg: rc.MaxLoadAvg,
|
||||
MinMemAvailableMB: rc.MinMemAvailableMB,
|
||||
MinDiskFreeMB: rc.MinDiskFreeMB,
|
||||
MaxIODelayPct: rc.MaxIODelayPct,
|
||||
}.Evaluate(st)
|
||||
}
|
||||
|
||||
func (c *Conductor) resEnabled() bool { return c.cfg.Resources.Enabled }
|
||||
|
||||
// logDefer records one defer event (deduped per task+reason in-process).
|
||||
func (c *Conductor) logDefer(state *loopState, t task.Task, reason string) {
|
||||
key := t.ID + "|" + t.UpdatedOn + "|" + reason
|
||||
if c.deferLogged == nil {
|
||||
c.deferLogged = map[string]bool{}
|
||||
}
|
||||
if c.deferLogged[key] {
|
||||
return
|
||||
}
|
||||
c.deferLogged[key] = true
|
||||
_ = state.log(loopEvent{Type: evDefer, TaskID: t.ID, UpdatedOn: t.UpdatedOn, Subject: t.Subject, Detail: reason})
|
||||
fmt.Fprintf(c.out, "harness: loop: defer %s: %s (reconsidered next scan; never marked processed)\n", t.ID, reason)
|
||||
}
|
||||
|
||||
// gateStatusLine is the startup/`quota status` one-liner.
|
||||
func (c *Conductor) gateStatusLine(ctx context.Context) string {
|
||||
if c.gate == nil && !c.resEnabled() {
|
||||
return "off"
|
||||
}
|
||||
parts := []string{}
|
||||
if c.gate != nil {
|
||||
snap := c.gate.Snapshot(ctx)
|
||||
parts = append(parts, c.gate.StatusLine(snap))
|
||||
}
|
||||
if c.resEnabled() {
|
||||
st := quota.ReadResourceStats(c.cfg.Resources.ProcRoot, c.cfg.Resources.SysRoot, c.cfg.WorkRoot)
|
||||
parts = append(parts, fmt.Sprintf("load=%.2f memAvail=%.0fMB diskFree=%.0fMB", st.LoadAvg1m, st.MemAvailable, st.DiskFree))
|
||||
}
|
||||
return strings.Join(parts, " | ")
|
||||
}
|
||||
|
||||
// usageRow is one line of the per-class usage accounting (feeds the
|
||||
// per-instance Discourse usage reports).
|
||||
type usageRow struct {
|
||||
Class string `json:"class"`
|
||||
Turns int `json:"turns"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
Credits float64 `json:"credits"`
|
||||
}
|
||||
|
||||
// UsageReport renders the per-class token+credit accounting table from a
|
||||
// loop state dir (read-only; `harness quota status`).
|
||||
func UsageReport(stateDir string) (string, error) {
|
||||
s, err := openLoopStateReadOnly(stateDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer s.Close()
|
||||
return usageReport(s.usageRows()), nil
|
||||
}
|
||||
|
||||
// GateStatusLine is the exported one-liner for `harness quota` output.
|
||||
func (c *Conductor) GateStatusLine(ctx context.Context) string {
|
||||
return c.gateStatusLine(ctx)
|
||||
}
|
||||
|
||||
// ResourceSample reads one host sample for the CLI (nil-safe when off).
|
||||
func (c *Conductor) ResourceSample() (quota.ResourceStats, bool) {
|
||||
if !c.resEnabled() {
|
||||
return quota.ResourceStats{}, false
|
||||
}
|
||||
rc := c.cfg.Resources
|
||||
return quota.ReadResourceStats(rc.ProcRoot, rc.SysRoot, c.cfg.WorkRoot), true
|
||||
}
|
||||
|
||||
// usageReport renders the accounting table from the loop state.
|
||||
func usageReport(rows []usageRow) string {
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Class < rows[j].Class })
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%-14s %6s %12s %12s %12s %10s\n", "CLASS", "TURNS", "PROMPT", "COMPLETION", "TOTAL", "CREDITS")
|
||||
var tot usageRow
|
||||
for _, r := range rows {
|
||||
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", r.Class, r.Turns, r.PromptTokens, r.CompletionTokens, r.TotalTokens, r.Credits)
|
||||
tot.Turns += r.Turns
|
||||
tot.PromptTokens += r.PromptTokens
|
||||
tot.CompletionTokens += r.CompletionTokens
|
||||
tot.TotalTokens += r.TotalTokens
|
||||
tot.Credits += r.Credits
|
||||
}
|
||||
fmt.Fprintf(&b, "%-14s %6d %12d %12d %12d %10.2f\n", "TOTAL", tot.Turns, tot.PromptTokens, tot.CompletionTokens, tot.TotalTokens, tot.Credits)
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"ukrrs.com/mopac/harness/internal/config"
|
||||
)
|
||||
|
||||
// fakeUsage is a mutable z.ai usage endpoint: the documented-shape JSON,
|
||||
// with the weekly bucket flippable between "wall" (97%) and healthy (10%).
|
||||
type fakeUsage struct {
|
||||
wall atomic.Bool
|
||||
srv *httptest.Server
|
||||
}
|
||||
|
||||
func newFakeUsage(t *testing.T) *fakeUsage {
|
||||
t.Helper()
|
||||
f := &fakeUsage{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/usage", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") == "" {
|
||||
http.Error(w, "no bearer", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
wk := 14000.0 // 10%
|
||||
five := 2800.0
|
||||
if f.wall.Load() {
|
||||
wk = 135800.0 // 97%: the 2026-08-28 19:00 wall
|
||||
five = 12000.0
|
||||
}
|
||||
fmt.Fprintf(w, `{"usage":{"five_hour":{"used_credits":%g,"total_credits":28000,"reset_at":"2026-08-29T22:00:00Z"},"weekly":{"used_credits":%g,"total_credits":140000,"reset_at":"2026-09-02T00:00:00Z"}}}`, five, wk)
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func quotaTestConfig(t *testing.T, redmineURL, llmURL, usageURL string) *config.Config {
|
||||
t.Helper()
|
||||
cfg := loopTestConfig(t, redmineURL, llmURL)
|
||||
cfg.Quota = quotaCfg(usageURL)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func resCfg(procRoot string) config.ResourcesConfig {
|
||||
return config.ResourcesConfig{
|
||||
Enabled: true,
|
||||
MaxLoadAvg: 6.0,
|
||||
MinMemAvailableMB: 2048,
|
||||
MinDiskFreeMB: 5120,
|
||||
MaxIODelayPct: 90,
|
||||
ProcRoot: procRoot,
|
||||
SysRoot: procRoot,
|
||||
}
|
||||
}
|
||||
|
||||
func quotaCfg(usageURL string) config.QuotaConfig {
|
||||
c := config.QuotaConfig{
|
||||
Enabled: true, Account: "zai-1",
|
||||
Plan5hCredits: 28000, PlanWeeklyCredits: 140000,
|
||||
UsageURL: usageURL, KeyRef: "literal:zai-key",
|
||||
PollIntervalSecs: 300, DeferAtPct: 85, BlockAtPct: 95,
|
||||
PeakStart: "01:00", PeakEnd: "05:00", Timezone: "America/Chicago",
|
||||
PeakWeekdaysOnly: true, PeakClasses: []string{"study", "read"},
|
||||
}
|
||||
if usageURL == "" {
|
||||
c.KeyRef = ""
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// TestLoopQuotaWallDefersAndRecovers replays 2026-08-28 ~19:00 CST: the
|
||||
// weekly bucket hit 97% and dispatched turns died at the provider. With the
|
||||
// gate: the scan DEFERS the turn with a surfaced reason, the loop exits
|
||||
// clean, the issue is NOT consumed (no note, not marked processed) — and
|
||||
// once quota recovers the very next scan dispatches it.
|
||||
func TestLoopQuotaWallDefersAndRecovers(t *testing.T) {
|
||||
rm := newFakeRedmine(t)
|
||||
rm.addIssue("490", "Big build task", "In Progress", "2026-08-28T21:30:00Z")
|
||||
f := newFakeLLM(t, textMsg("the turn ran"))
|
||||
usage := newFakeUsage(t)
|
||||
usage.wall.Store(true) // the wall is up
|
||||
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, usage.srv.URL+"/usage")
|
||||
|
||||
out := runOneScan(t, cfg, false)
|
||||
|
||||
if f.requestCount() != 0 {
|
||||
t.Fatalf("LLM calls during the wall = %d, want 0 (turns must defer, not die)", f.requestCount())
|
||||
}
|
||||
for _, want := range []string{
|
||||
"defer 490",
|
||||
">= block 95%",
|
||||
"weekly",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if rm.noteCount("490") != 0 {
|
||||
t.Errorf("deferred task must not be noted back")
|
||||
}
|
||||
|
||||
// The defer is in the JSONL with the reason (json.Marshal HTML-escapes
|
||||
// ">=", so match on the words); the task was NOT consumed.
|
||||
statePath := filepath.Join(cfg.Loop.StateDir, "loop.jsonl")
|
||||
data, err := os.ReadFile(statePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"type":"defer"`) || !strings.Contains(string(data), "block 95") {
|
||||
t.Errorf("JSONL defer event missing:\n%s", data)
|
||||
}
|
||||
|
||||
// Quota recovers: next scan dispatches (task was never marked processed).
|
||||
usage.wall.Store(false)
|
||||
out2 := runOneScan(t, cfg, false)
|
||||
if f.requestCount() != 1 {
|
||||
t.Fatalf("LLM calls after recovery = %d, want 1 (defer must not consume the task)", f.requestCount())
|
||||
}
|
||||
if !strings.Contains(out2, "dispatch 490") || !strings.Contains(out2, "noted REPORT on #490") {
|
||||
t.Errorf("recovery scan output wrong:\n%s", out2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopPeakWindowDefersHeavyClass: inside 01:00-05:00 CST the heavy
|
||||
// class defers with a peak reason while the flash-tier class runs.
|
||||
func TestLoopPeakWindowDefersHeavyClass(t *testing.T) {
|
||||
rm := newFakeRedmine(t)
|
||||
rm.addIssue("491", "Heavy flagship work", "In Progress", "2026-08-28T21:30:00Z")
|
||||
f := newFakeLLM(t, textMsg("flash reply"))
|
||||
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, "") // estimates only
|
||||
|
||||
// Friday 2026-08-28 03:00 CST = inside the peak window.
|
||||
peakAt := time.Date(2026, 8, 28, 3, 0, 0, 0, cst(t))
|
||||
var out strings.Builder
|
||||
cond, err := New(cfg, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cond.gate.SetClock(func() time.Time { return peakAt })
|
||||
if err := cond.RunLoop(context.Background(), LoopOpts{Once: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.requestCount() != 0 {
|
||||
t.Fatalf("heavy class dispatched in peak window (%d calls)", f.requestCount())
|
||||
}
|
||||
if !strings.Contains(out.String(), "peak window") {
|
||||
t.Errorf("defer reason must name the peak window:\n%s", out.String())
|
||||
}
|
||||
|
||||
// A flash-tier class in the same peak window runs.
|
||||
rm.addIssueClass("492", "Light study work", "In Progress", "2026-08-28T21:31:00Z", "study")
|
||||
out2 := runScanAt(t, cfg, peakAt)
|
||||
if f.requestCount() != 1 {
|
||||
t.Fatalf("flash class must dispatch in peak: %d calls", f.requestCount())
|
||||
}
|
||||
if !strings.Contains(out2, "dispatch 492") {
|
||||
t.Errorf("flash dispatch missing:\n%s", out2)
|
||||
}
|
||||
|
||||
// Off-peak (Friday 12:00 CST): the heavy task now dispatches.
|
||||
offPeak := time.Date(2026, 8, 28, 12, 0, 0, 0, cst(t))
|
||||
_ = runScanAt(t, cfg, offPeak)
|
||||
if f.requestCount() != 2 {
|
||||
t.Fatalf("heavy class must dispatch off-peak: %d calls", f.requestCount())
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopResourceBusyDefers: a pinned-high load fixture defers dispatch;
|
||||
// the task stays unconsumed.
|
||||
func TestLoopResourceBusyDefers(t *testing.T) {
|
||||
rm := newFakeRedmine(t)
|
||||
rm.addIssue("492", "Task during a build storm", "In Progress", "2026-08-28T21:30:00Z")
|
||||
f := newFakeLLM(t, textMsg("never"))
|
||||
cfg := loopTestConfig(t, rm.srv.URL, f.srv.URL)
|
||||
|
||||
proc := filepath.Join(t.TempDir(), "proc")
|
||||
os.MkdirAll(proc, 0o755)
|
||||
os.WriteFile(filepath.Join(proc, "loadavg"), []byte("41.20 30.00 15.00 5/900 1\n"), 0o644)
|
||||
cfg.Resources = resCfg(proc)
|
||||
|
||||
out := runOneScan(t, cfg, false)
|
||||
if f.requestCount() != 0 {
|
||||
t.Fatalf("busy host must defer dispatch (%d calls)", f.requestCount())
|
||||
}
|
||||
if !strings.Contains(out, "resources busy") || !strings.Contains(out, "load 41.20") {
|
||||
t.Errorf("defer reason must surface the resource violation:\n%s", out)
|
||||
}
|
||||
if rm.noteCount("492") != 0 {
|
||||
t.Errorf("deferred task must not be noted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopUsageAccounting: after a dispatched turn, the JSONL report event
|
||||
// carries class + tokens + estimated credits, and the usage table aggregates.
|
||||
func TestLoopUsageAccounting(t *testing.T) {
|
||||
rm := newFakeRedmine(t)
|
||||
rm.addIssue("493", "Accounted task", "In Progress", "2026-08-28T21:30:00Z")
|
||||
f := newFakeLLM(t, textMsg("reply with 20/40 tokens"))
|
||||
cfg := quotaTestConfig(t, rm.srv.URL, f.srv.URL, "")
|
||||
|
||||
runScanAt(t, cfg, time.Date(2026, 8, 28, 12, 0, 0, 0, cst(t))) // off-peak Friday
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(cfg.Loop.StateDir, "loop.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reportEv map[string]any
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
|
||||
var ev map[string]any
|
||||
if err := json.Unmarshal([]byte(line), &ev); err != nil {
|
||||
t.Fatalf("bad JSONL line %q: %v", line, err)
|
||||
}
|
||||
if ev["type"] == "report" {
|
||||
reportEv = ev
|
||||
}
|
||||
}
|
||||
if reportEv == nil {
|
||||
t.Fatal("no report event in JSONL")
|
||||
}
|
||||
if reportEv["class"] != "primary" {
|
||||
t.Errorf("report event class = %v, want primary", reportEv["class"])
|
||||
}
|
||||
if reportEv["prompt_tokens"] != float64(20) || reportEv["completion_tokens"] != float64(40) {
|
||||
t.Errorf("report tokens = %v", reportEv)
|
||||
}
|
||||
// This turn ran glm-5.3 off-peak: (20*6.9 + 40*24)/10000 / 2 = 0.0549
|
||||
credits, _ := reportEv["credits"].(float64)
|
||||
if credits < 0.0548 || credits > 0.0550 {
|
||||
t.Errorf("credits = %v, want ~0.0549", credits)
|
||||
}
|
||||
|
||||
report, err := UsageReport(cfg.Loop.StateDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(report, "primary") || !strings.Contains(report, "TOTAL") || !strings.Contains(report, "1") {
|
||||
t.Errorf("usage table wrong:\n%s", report)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopGateOffByDefault: with [quota] disabled the loop dispatches
|
||||
// exactly as before (no behavior change for existing configs).
|
||||
func TestLoopGateOffByDefault(t *testing.T) {
|
||||
rm := newFakeRedmine(t)
|
||||
rm.addIssue("494", "Plain task", "In Progress", "2026-08-28T21:30:00Z")
|
||||
f := newFakeLLM(t, textMsg("plain reply"))
|
||||
cfg := loopTestConfig(t, rm.srv.URL, f.srv.URL)
|
||||
|
||||
out := runOneScan(t, cfg, false)
|
||||
if f.requestCount() != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", f.requestCount())
|
||||
}
|
||||
if strings.Contains(out, "defer") || strings.Contains(out, "gate:") {
|
||||
t.Errorf("disabled gate must be silent:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func cst(t *testing.T) *time.Location {
|
||||
t.Helper()
|
||||
loc, err := time.LoadLocation("America/Chicago")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
// runScanAt runs a --once scan with the quota gate clock pinned.
|
||||
func runScanAt(t *testing.T, cfg *config.Config, at time.Time) string {
|
||||
t.Helper()
|
||||
var out strings.Builder
|
||||
cond, err := New(cfg, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cond.gate != nil {
|
||||
cond.gate.SetClock(func() time.Time { return at })
|
||||
}
|
||||
if err := cond.RunLoop(context.Background(), LoopOpts{Once: true}); err != nil {
|
||||
t.Fatalf("RunLoop: %v (output:\n%s)", err, out.String())
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
+13
-1
@@ -17,6 +17,7 @@ import (
|
||||
"ukrrs.com/mopac/harness/internal/intake"
|
||||
"ukrrs.com/mopac/harness/internal/llm"
|
||||
"ukrrs.com/mopac/harness/internal/models"
|
||||
"ukrrs.com/mopac/harness/internal/quota"
|
||||
"ukrrs.com/mopac/harness/internal/task"
|
||||
"ukrrs.com/mopac/harness/internal/tools"
|
||||
"ukrrs.com/mopac/harness/internal/writeback"
|
||||
@@ -35,6 +36,9 @@ type Conductor struct {
|
||||
bash *tools.BashTool
|
||||
keys *config.KeyResolver
|
||||
out io.Writer
|
||||
gate *quota.Gate
|
||||
// deferLogged dedups defer log lines per (task, updated_on, reason).
|
||||
deferLogged map[string]bool
|
||||
}
|
||||
|
||||
// New validates routing config and prepares the tool palette.
|
||||
@@ -57,9 +61,17 @@ func New(cfg *config.Config, out io.Writer) (*Conductor, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Conductor{cfg: cfg, router: router, bash: bash, keys: config.NewKeyResolver(cfg), out: out}, nil
|
||||
keys := config.NewKeyResolver(cfg)
|
||||
gate, err := buildGates(cfg, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Conductor{cfg: cfg, router: router, bash: bash, keys: keys, out: out, gate: gate, deferLogged: map[string]bool{}}, nil
|
||||
}
|
||||
|
||||
// Gate exposes the quota gate (nil when off) for `harness quota status`.
|
||||
func (c *Conductor) Gate() *quota.Gate { return c.gate }
|
||||
|
||||
// OnceOpts controls a single iteration.
|
||||
type OnceOpts struct {
|
||||
DryRun bool // intake + plan only, no LLM call
|
||||
|
||||
+59
-3
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -19,6 +20,7 @@ const (
|
||||
evCommit = "commit" // REPORT committed to gitea (optional step)
|
||||
evRefresh = "refresh" // dedup marker advanced to post-writeback updated_on
|
||||
evError = "error" // a step failed; loop continues
|
||||
evDefer = "defer" // dispatch deferred by the quota/resource gate (throttle, not rejection)
|
||||
)
|
||||
|
||||
// loopEvent is one line of the append-only loop log. It doubles as the
|
||||
@@ -31,11 +33,18 @@ type loopEvent struct {
|
||||
UpdatedOn string `json:"updated_on,omitempty"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Class string `json:"class,omitempty"` // usage accounting key
|
||||
ReportPath string `json:"report_path,omitempty"`
|
||||
StopReason string `json:"stop_reason,omitempty"`
|
||||
StatusFrom string `json:"status_from,omitempty"`
|
||||
StatusTo string `json:"status_to,omitempty"`
|
||||
Detail string `json:"detail,omitempty"` // human context; never secrets
|
||||
// Per-turn usage accounting (report events): tokens + estimated z.ai
|
||||
// credits. Feeds the per-instance usage tables / Discourse reports.
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Credits float64 `json:"credits,omitempty"`
|
||||
Detail string `json:"detail,omitempty"` // human context; never secrets
|
||||
}
|
||||
|
||||
// loopState is the append-only loop log + dedup index: stateDir/loop.jsonl,
|
||||
@@ -47,7 +56,18 @@ type loopState struct {
|
||||
f *os.File
|
||||
path string
|
||||
seen map[string]string
|
||||
readOnly bool // dry-run: dedup checks work, writes are refused
|
||||
usage map[string]usageAcc // class -> per-class totals (report events)
|
||||
readOnly bool // dry-run: dedup checks work, writes are refused
|
||||
}
|
||||
|
||||
// usageAcc accumulates per-class turn accounting; it is rebuilt from the
|
||||
// JSONL at open so usage reporting survives restarts.
|
||||
type usageAcc struct {
|
||||
Turns int `json:"turns"`
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
Credits float64 `json:"credits"`
|
||||
}
|
||||
|
||||
func openLoopState(stateDir string) (*loopState, error) {
|
||||
@@ -59,7 +79,7 @@ func openLoopState(stateDir string) (*loopState, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open loop log: %w", err)
|
||||
}
|
||||
s := &loopState{path: path, f: f, seen: make(map[string]string)}
|
||||
s := &loopState{path: path, f: f, seen: make(map[string]string), usage: make(map[string]usageAcc)}
|
||||
if err := s.loadIndex(); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
@@ -73,6 +93,7 @@ func openLoopStateReadOnly(stateDir string) (*loopState, error) {
|
||||
s := &loopState{
|
||||
path: filepath.Join(stateDir, "loop.jsonl"),
|
||||
seen: make(map[string]string),
|
||||
usage: make(map[string]usageAcc),
|
||||
readOnly: true,
|
||||
}
|
||||
if err := s.loadIndex(); err != nil {
|
||||
@@ -100,6 +121,9 @@ func (s *loopState) loadIndex() error {
|
||||
if ev.Type == evDispatch || ev.Type == evRefresh {
|
||||
s.seen[ev.TaskID] = ev.UpdatedOn
|
||||
}
|
||||
if ev.Type == evReport {
|
||||
s.addUsage(ev.Class, ev)
|
||||
}
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
@@ -131,9 +155,41 @@ func (s *loopState) log(ev loopEvent) error {
|
||||
if ev.Type == evDispatch || ev.Type == evRefresh {
|
||||
s.seen[ev.TaskID] = ev.UpdatedOn
|
||||
}
|
||||
if ev.Type == evReport {
|
||||
s.addUsage(ev.Class, ev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *loopState) addUsage(class string, ev loopEvent) {
|
||||
if class == "" {
|
||||
class = "(unknown)"
|
||||
}
|
||||
acc := s.usage[class]
|
||||
acc.Turns++
|
||||
acc.PromptTokens += ev.PromptTokens
|
||||
acc.CompletionTokens += ev.CompletionTokens
|
||||
acc.TotalTokens += ev.TotalTokens
|
||||
acc.Credits += ev.Credits
|
||||
s.usage[class] = acc
|
||||
}
|
||||
|
||||
// usageRows snapshots the per-class accounting (sorted by class).
|
||||
func (s *loopState) usageRows() []usageRow {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
rows := make([]usageRow, 0, len(s.usage))
|
||||
for class, acc := range s.usage {
|
||||
rows = append(rows, usageRow{
|
||||
Class: class, Turns: acc.Turns,
|
||||
PromptTokens: acc.PromptTokens, CompletionTokens: acc.CompletionTokens,
|
||||
TotalTokens: acc.TotalTokens, Credits: acc.Credits,
|
||||
})
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Class < rows[j].Class })
|
||||
return rows
|
||||
}
|
||||
|
||||
func (s *loopState) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
Reference in New Issue
Block a user