quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)
The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
+118
-1
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"ukrrs.com/mopac/harness/internal/config"
|
||||
"ukrrs.com/mopac/harness/internal/events"
|
||||
"ukrrs.com/mopac/harness/internal/loop"
|
||||
"ukrrs.com/mopac/harness/internal/quota"
|
||||
"ukrrs.com/mopac/harness/internal/serve"
|
||||
)
|
||||
|
||||
@@ -27,6 +29,7 @@ Usage:
|
||||
harness loop [-config PATH] [-interval DUR] [-once] [-dry-run]
|
||||
harness events [-config PATH] [-listen ADDR]
|
||||
harness serve [-config PATH] [-listen ADDR]
|
||||
harness quota <status|probe|gate> [-config PATH]
|
||||
|
||||
once runs ONE conductor iteration and exits (chain by re-invoking; no daemon):
|
||||
intake (Redmine scope, or the [demo] issue) -> plan/model routing ->
|
||||
@@ -52,6 +55,15 @@ serve runs the OpenAI-compatible front door until SIGINT/SIGTERM (the
|
||||
non-streaming v0; tools off. Own port - coexists with events.
|
||||
Routes: POST /v1/chat/completions, GET /v1/models, GET /healthz.
|
||||
|
||||
quota is the Redmine 490/491 gate surface:
|
||||
status one-shot: quota snapshot (polled or estimated), peak window,
|
||||
host resources, and the per-class token+credit usage table from
|
||||
the loop state (the Discourse usage-report feed)
|
||||
probe poll [quota] usage_url once and print the parsed buckets (or the
|
||||
raw error; bearer key never printed)
|
||||
gate evaluate the back-pressure gates NOW: the decision for every
|
||||
[models.classes] class + resource thresholds
|
||||
|
||||
Flags:
|
||||
-config PATH config file (default $HARNESS_CONFIG or ./harness.toml)
|
||||
-dry-run once/loop: intake + plan only; no LLM call, no REPORT,
|
||||
@@ -61,7 +73,6 @@ Flags:
|
||||
-interval DUR loop: poll interval (overrides [loop] poll_interval_secs)
|
||||
-once loop: single scan then exit (cron-able)
|
||||
-listen ADDR events/serve: bind address (overrides [events]/[serve] listen)
|
||||
|
||||
Exit codes:
|
||||
0 ok (including "no tasks in scope"; loop: clean SIGINT stop)
|
||||
1 usage / config / routing / writeback error
|
||||
@@ -90,6 +101,8 @@ func run(args []string) int {
|
||||
return runEvents(args[1:])
|
||||
case "serve":
|
||||
return runServe(args[1:])
|
||||
case "quota":
|
||||
return runQuota(args[1:])
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "harness: unknown command %q\n\n%s", args[0], usage)
|
||||
return 1
|
||||
@@ -199,6 +212,110 @@ func runLoop(args []string) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func runQuota(args []string) int {
|
||||
sub := "status"
|
||||
var rest []string
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub = args[0]
|
||||
rest = args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "status", "probe", "gate":
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "harness: quota: unknown subcommand %q (want status|probe|gate)\n", sub)
|
||||
return 1
|
||||
}
|
||||
fs := flag.NewFlagSet("quota "+sub, flag.ContinueOnError)
|
||||
cfgPath := fs.String("config", "", "config file path")
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return 1
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprintf(os.Stderr, "harness: unexpected argument %q\n", fs.Arg(0))
|
||||
return 1
|
||||
}
|
||||
if *cfgPath == "" {
|
||||
*cfgPath = os.Getenv("HARNESS_CONFIG")
|
||||
}
|
||||
if *cfgPath == "" {
|
||||
*cfgPath = "harness.toml"
|
||||
}
|
||||
cfg, err := config.Load(*cfgPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
conductor, err := loop.New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
gate := conductor.Gate()
|
||||
|
||||
switch sub {
|
||||
case "probe":
|
||||
if gate == nil || cfg.Quota.UsageURL == "" {
|
||||
fmt.Println("harness: quota: no [quota] usage_url configured (running on estimates)")
|
||||
return 0
|
||||
}
|
||||
snap := gate.Snapshot(context.Background())
|
||||
fmt.Printf("harness: quota: account=%s source=%s fetched=%s\n", snap.Account, snap.Source, snap.FetchedAt.Format(time.RFC3339))
|
||||
for _, b := range snap.Buckets {
|
||||
reset := "-"
|
||||
if !b.WindowReset.IsZero() {
|
||||
reset = b.WindowReset.Format(time.RFC3339)
|
||||
}
|
||||
fmt.Printf(" %-7s used=%.0f limit=%.0f (%.1f%%) resets=%s\n", b.ID, b.Used, b.Limit, b.UsedPct(), reset)
|
||||
}
|
||||
return 0
|
||||
|
||||
case "gate":
|
||||
if gate == nil {
|
||||
fmt.Println("harness: quota gate off ([quota] enabled = false)")
|
||||
return 0
|
||||
}
|
||||
fmt.Printf("harness: quota gate: %s\n", conductor.GateStatusLine(context.Background()))
|
||||
for _, class := range sortedClasses(cfg) {
|
||||
d := gate.Decide(context.Background(), class)
|
||||
tag := "ALLOW"
|
||||
if d.Action == quota.ActionDefer {
|
||||
tag = "DEFER"
|
||||
}
|
||||
fmt.Printf(" %-6s class=%-12s %s\n", tag, class, d.Reason)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// status
|
||||
if gate == nil && !cfg.Resources.Enabled {
|
||||
fmt.Println("harness: quota gate off ([quota]/[resources] enabled = false)")
|
||||
} else if gate != nil {
|
||||
fmt.Printf("harness: quota: %s\n", conductor.GateStatusLine(context.Background()))
|
||||
fmt.Printf(" peak window: %s-%s %s weekdays_only=%v currently=%v\n",
|
||||
cfg.Quota.PeakStart, cfg.Quota.PeakEnd, cfg.Quota.Timezone, cfg.Quota.PeakWeekdaysOnly, gate.InPeak())
|
||||
}
|
||||
if st, ok := conductor.ResourceSample(); ok {
|
||||
fmt.Printf("harness: resources: load=%.2f memAvail=%.0fMB diskFree=%.0fMB ioDelay=%.1f%%(psi=%v)\n",
|
||||
st.LoadAvg1m, st.MemAvailable, st.DiskFree, st.IODelayPct, st.HasPSI)
|
||||
}
|
||||
report, err := loop.UsageReport(cfg.Loop.StateDir)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: usage report: %v\n", err)
|
||||
return 0
|
||||
}
|
||||
fmt.Printf("harness: usage by class (%s):\n%s", cfg.Loop.StateDir, report)
|
||||
return 0
|
||||
}
|
||||
|
||||
func sortedClasses(cfg *config.Config) []string {
|
||||
out := make([]string, 0, len(cfg.Models.Classes))
|
||||
for class := range cfg.Models.Classes {
|
||||
out = append(out, class)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func runEvents(args []string) int {
|
||||
fs := flag.NewFlagSet("events", flag.ContinueOnError)
|
||||
cfgPath := fs.String("config", "", "config file path")
|
||||
|
||||
Reference in New Issue
Block a user