quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)

The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
This commit is contained in:
2026-08-29 05:37:15 -05:00
parent 9dbb20489c
commit fc518c475e
21 changed files with 2904 additions and 20 deletions
+220
View File
@@ -2,7 +2,9 @@ package config
import (
"fmt"
"strconv"
"strings"
"time"
)
// Config is the full harness.toml surface. Defaults live in Default();
@@ -21,6 +23,8 @@ type Config struct {
Serve ServeConfig
KeyProxy KeyProxyConfig
Gitea GiteaConfig
Quota QuotaConfig
Resources ResourcesConfig
}
// LoopConfig is both the per-turn bound and the `harness loop` daemon
@@ -105,6 +109,65 @@ type GiteaConfig struct {
CommitReports bool
}
// QuotaConfig is the z.ai coding-plan quota gate: bucket limits, polling,
// back-pressure thresholds, the peak-hour schedule, and the optional
// shared-state hop (redis container) for multi-instance coordination.
// The quota gate NEVER hard-fails the loop; when it cannot see quota state
// it degrades to permissive (allow) with a logged caveat.
type QuotaConfig struct {
Enabled bool
// Account labels this instance's z.ai plan in the shared state so the
// nine harness instances across two hosts do not cross-contaminate.
Account string
// Plan credit buckets (docs: Lite 2k/10k, Pro 12k/60k, Max 28k/140k).
Plan5hCredits float64
PlanWeeklyCredits float64
// UsageURL + KeyRef: the z.ai usage/limits endpoint (bearer auth; the
// key is a ref, resolved at call time, never logged). Empty = no
// upstream polling; the gate runs on locally estimated consumption.
UsageURL string
KeyRef string
PollIntervalSecs int
// Back-pressure thresholds, percent of bucket used. defer_at: heavy
// classes defer, flash-tier continue. block_at: everything defers.
DeferAtPct float64
BlockAtPct float64
// Peak window (z.ai charges 3x-ish during peak; off-peak is 50% off).
// Defaults verified against the z.ai docs: peak = Mon-Fri 14:00-18:00
// Singapore (UTC+8) == 01:00-05:00 America/Chicago in winter.
PeakStart string // "HH:MM" in Timezone
PeakEnd string
Timezone string // IANA name; default America/Chicago (CST)
PeakWeekdaysOnly bool
// PeakClasses: task classes allowed to run inside the peak window
// (the LLM-lite / flash tier); every other class defers with a reason.
PeakClasses []string
// RedisURL: shared quota state for all instances of this account
// (docker container; no host packages). Empty = local-only estimate.
RedisURL string
}
// ResourcesConfig is the read-only system resource gate (Redmine 491):
// load average, memory available, work-root disk free, IO pressure. The
// loop defers dispatch while the host is busy; enforcement via cgroups is
// a deploy-time concern (see the runbook in README).
type ResourcesConfig struct {
Enabled bool
MaxLoadAvg float64 // 1m load average
MinMemAvailableMB float64
MinDiskFreeMB float64 // work_root filesystem
MaxIODelayPct float64 // /proc/pressure/io "some avg60" percent
// ProcRoot/SysRoot are seams for tests; default /proc and /sys.
ProcRoot string
SysRoot string
}
// EventsConfig is the `harness events` webhook receiver surface.
type EventsConfig struct {
Listen string // bind address (publish via docker -p)
@@ -157,6 +220,28 @@ func Default() *Config {
},
Serve: ServeConfig{Listen: ":8090"},
KeyProxy: KeyProxyConfig{CacheTTLSecs: 60},
Quota: QuotaConfig{
Enabled: false, // opt-in per instance until verified live
Plan5hCredits: 28000, // Max plan
PlanWeeklyCredits: 140000,
PollIntervalSecs: 300,
DeferAtPct: 85,
BlockAtPct: 95,
PeakStart: "01:00",
PeakEnd: "05:00",
Timezone: "America/Chicago",
PeakWeekdaysOnly: true,
PeakClasses: []string{"study", "read"},
},
Resources: ResourcesConfig{
Enabled: false,
MaxLoadAvg: 6.0,
MinMemAvailableMB: 2048,
MinDiskFreeMB: 5120,
MaxIODelayPct: 90.0,
ProcRoot: "/proc",
SysRoot: "/sys",
},
Demo: DemoConfig{
ID: "demo-1",
Subject: "MVP demo: GLM self-description",
@@ -343,6 +428,76 @@ func (c *Config) apply(doc TOMLDoc) error {
if v, ok := sv.StringList("enabled_models"); ok {
c.Serve.EnabledModels = v
}
qt := doc.Table("quota")
if v, ok := qt.Bool("enabled"); ok {
c.Quota.Enabled = v
}
if v, ok := qt.String("account"); ok {
c.Quota.Account = v
}
if v, ok := qt.Float("plan_5h_credits"); ok {
c.Quota.Plan5hCredits = v
}
if v, ok := qt.Float("plan_weekly_credits"); ok {
c.Quota.PlanWeeklyCredits = v
}
if v, ok := qt.String("usage_url"); ok {
c.Quota.UsageURL = v
}
if v, ok := qt.String("key_ref"); ok {
c.Quota.KeyRef = v
}
if v, ok := qt.Int("poll_interval_secs"); ok {
c.Quota.PollIntervalSecs = int(v)
}
if v, ok := qt.Float("defer_at_pct"); ok {
c.Quota.DeferAtPct = v
}
if v, ok := qt.Float("block_at_pct"); ok {
c.Quota.BlockAtPct = v
}
if v, ok := qt.String("peak_start"); ok {
c.Quota.PeakStart = v
}
if v, ok := qt.String("peak_end"); ok {
c.Quota.PeakEnd = v
}
if v, ok := qt.String("timezone"); ok {
c.Quota.Timezone = v
}
if v, ok := qt.Bool("peak_weekdays_only"); ok {
c.Quota.PeakWeekdaysOnly = v
}
if v, ok := qt.StringList("peak_classes"); ok {
c.Quota.PeakClasses = v
}
if v, ok := qt.String("redis_url"); ok {
c.Quota.RedisURL = v
}
rs := doc.Table("resources")
if v, ok := rs.Bool("enabled"); ok {
c.Resources.Enabled = v
}
if v, ok := rs.Float("max_load_avg"); ok {
c.Resources.MaxLoadAvg = v
}
if v, ok := rs.Float("min_mem_available_mb"); ok {
c.Resources.MinMemAvailableMB = v
}
if v, ok := rs.Float("min_disk_free_mb"); ok {
c.Resources.MinDiskFreeMB = v
}
if v, ok := rs.Float("max_io_delay_pct"); ok {
c.Resources.MaxIODelayPct = v
}
if v, ok := rs.String("proc_root"); ok {
c.Resources.ProcRoot = v
}
if v, ok := rs.String("sys_root"); ok {
c.Resources.SysRoot = v
}
return nil
}
@@ -484,5 +639,70 @@ func (c *Config) Validate() error {
return fmt.Errorf("[gitea]: %w", err)
}
}
// The quota gate is optional; when on, buckets, thresholds and the peak
// window must be coherent so the gate never divides by zero or wraps
// nonsense windows. usage_url without key_ref is an error (bearer-only).
if c.Quota.Enabled {
if c.Quota.Plan5hCredits <= 0 || c.Quota.PlanWeeklyCredits <= 0 {
return fmt.Errorf("[quota]: plan_5h_credits and plan_weekly_credits must be > 0")
}
if c.Quota.PollIntervalSecs < 1 {
return fmt.Errorf("[quota]: poll_interval_secs must be >= 1")
}
if !(c.Quota.DeferAtPct > 0 && c.Quota.DeferAtPct < 100) || !(c.Quota.BlockAtPct > 0 && c.Quota.BlockAtPct <= 100) {
return fmt.Errorf("[quota]: defer_at_pct and block_at_pct must be in (0,100]")
}
if c.Quota.DeferAtPct > c.Quota.BlockAtPct {
return fmt.Errorf("[quota]: defer_at_pct (%.0f) must be <= block_at_pct (%.0f)", c.Quota.DeferAtPct, c.Quota.BlockAtPct)
}
if _, err := ParseHHMM(c.Quota.PeakStart); err != nil {
return fmt.Errorf("[quota]: peak_start: %w", err)
}
if _, err := ParseHHMM(c.Quota.PeakEnd); err != nil {
return fmt.Errorf("[quota]: peak_end: %w", err)
}
if _, err := time.LoadLocation(c.Quota.Timezone); err != nil {
return fmt.Errorf("[quota]: unknown timezone %q", c.Quota.Timezone)
}
if c.Quota.Account == "" {
return fmt.Errorf("[quota]: account is required when the quota gate is enabled (shared-state bucket label)")
}
if c.Quota.UsageURL != "" && c.Quota.KeyRef == "" {
return fmt.Errorf("[quota]: key_ref is required when usage_url is set (bearer auth)")
}
if c.Quota.KeyRef != "" {
if err := CheckKeyRef(c.Quota.KeyRef); err != nil {
return fmt.Errorf("[quota]: %w", err)
}
}
}
// The resource gate is optional; when on, thresholds must be sane.
if c.Resources.Enabled {
if c.Resources.MaxLoadAvg <= 0 || c.Resources.MinMemAvailableMB <= 0 || c.Resources.MinDiskFreeMB <= 0 {
return fmt.Errorf("[resources]: thresholds must be > 0")
}
if !(c.Resources.MaxIODelayPct > 0 && c.Resources.MaxIODelayPct <= 100) {
return fmt.Errorf("[resources]: max_io_delay_pct must be in (0,100]")
}
}
return nil
}
// ParseHHMM parses a wall-clock time of day, "HH:MM" (24h).
func ParseHHMM(s string) (time.Duration, error) {
parts := strings.Split(s, ":")
if len(parts) != 2 || len(parts[0]) != 2 || len(parts[1]) != 2 {
return 0, fmt.Errorf("want HH:MM, got %q", s)
}
h, err := strconv.Atoi(parts[0])
if err != nil || h < 0 || h > 23 {
return 0, fmt.Errorf("bad hour in %q", s)
}
m, err := strconv.Atoi(parts[1])
if err != nil || m < 0 || m > 59 {
return 0, fmt.Errorf("bad minute in %q", s)
}
return time.Duration(h)*time.Hour + time.Duration(m)*time.Minute, nil
}