quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)
The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -2,7 +2,9 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config is the full harness.toml surface. Defaults live in Default();
|
||||
@@ -21,6 +23,8 @@ type Config struct {
|
||||
Serve ServeConfig
|
||||
KeyProxy KeyProxyConfig
|
||||
Gitea GiteaConfig
|
||||
Quota QuotaConfig
|
||||
Resources ResourcesConfig
|
||||
}
|
||||
|
||||
// LoopConfig is both the per-turn bound and the `harness loop` daemon
|
||||
@@ -105,6 +109,65 @@ type GiteaConfig struct {
|
||||
CommitReports bool
|
||||
}
|
||||
|
||||
// QuotaConfig is the z.ai coding-plan quota gate: bucket limits, polling,
|
||||
// back-pressure thresholds, the peak-hour schedule, and the optional
|
||||
// shared-state hop (redis container) for multi-instance coordination.
|
||||
// The quota gate NEVER hard-fails the loop; when it cannot see quota state
|
||||
// it degrades to permissive (allow) with a logged caveat.
|
||||
type QuotaConfig struct {
|
||||
Enabled bool
|
||||
|
||||
// Account labels this instance's z.ai plan in the shared state so the
|
||||
// nine harness instances across two hosts do not cross-contaminate.
|
||||
Account string
|
||||
|
||||
// Plan credit buckets (docs: Lite 2k/10k, Pro 12k/60k, Max 28k/140k).
|
||||
Plan5hCredits float64
|
||||
PlanWeeklyCredits float64
|
||||
|
||||
// UsageURL + KeyRef: the z.ai usage/limits endpoint (bearer auth; the
|
||||
// key is a ref, resolved at call time, never logged). Empty = no
|
||||
// upstream polling; the gate runs on locally estimated consumption.
|
||||
UsageURL string
|
||||
KeyRef string
|
||||
PollIntervalSecs int
|
||||
|
||||
// Back-pressure thresholds, percent of bucket used. defer_at: heavy
|
||||
// classes defer, flash-tier continue. block_at: everything defers.
|
||||
DeferAtPct float64
|
||||
BlockAtPct float64
|
||||
|
||||
// Peak window (z.ai charges 3x-ish during peak; off-peak is 50% off).
|
||||
// Defaults verified against the z.ai docs: peak = Mon-Fri 14:00-18:00
|
||||
// Singapore (UTC+8) == 01:00-05:00 America/Chicago in winter.
|
||||
PeakStart string // "HH:MM" in Timezone
|
||||
PeakEnd string
|
||||
Timezone string // IANA name; default America/Chicago (CST)
|
||||
PeakWeekdaysOnly bool
|
||||
// PeakClasses: task classes allowed to run inside the peak window
|
||||
// (the LLM-lite / flash tier); every other class defers with a reason.
|
||||
PeakClasses []string
|
||||
|
||||
// RedisURL: shared quota state for all instances of this account
|
||||
// (docker container; no host packages). Empty = local-only estimate.
|
||||
RedisURL string
|
||||
}
|
||||
|
||||
// ResourcesConfig is the read-only system resource gate (Redmine 491):
|
||||
// load average, memory available, work-root disk free, IO pressure. The
|
||||
// loop defers dispatch while the host is busy; enforcement via cgroups is
|
||||
// a deploy-time concern (see the runbook in README).
|
||||
type ResourcesConfig struct {
|
||||
Enabled bool
|
||||
MaxLoadAvg float64 // 1m load average
|
||||
MinMemAvailableMB float64
|
||||
MinDiskFreeMB float64 // work_root filesystem
|
||||
MaxIODelayPct float64 // /proc/pressure/io "some avg60" percent
|
||||
// ProcRoot/SysRoot are seams for tests; default /proc and /sys.
|
||||
ProcRoot string
|
||||
SysRoot string
|
||||
}
|
||||
|
||||
// EventsConfig is the `harness events` webhook receiver surface.
|
||||
type EventsConfig struct {
|
||||
Listen string // bind address (publish via docker -p)
|
||||
@@ -157,6 +220,28 @@ func Default() *Config {
|
||||
},
|
||||
Serve: ServeConfig{Listen: ":8090"},
|
||||
KeyProxy: KeyProxyConfig{CacheTTLSecs: 60},
|
||||
Quota: QuotaConfig{
|
||||
Enabled: false, // opt-in per instance until verified live
|
||||
Plan5hCredits: 28000, // Max plan
|
||||
PlanWeeklyCredits: 140000,
|
||||
PollIntervalSecs: 300,
|
||||
DeferAtPct: 85,
|
||||
BlockAtPct: 95,
|
||||
PeakStart: "01:00",
|
||||
PeakEnd: "05:00",
|
||||
Timezone: "America/Chicago",
|
||||
PeakWeekdaysOnly: true,
|
||||
PeakClasses: []string{"study", "read"},
|
||||
},
|
||||
Resources: ResourcesConfig{
|
||||
Enabled: false,
|
||||
MaxLoadAvg: 6.0,
|
||||
MinMemAvailableMB: 2048,
|
||||
MinDiskFreeMB: 5120,
|
||||
MaxIODelayPct: 90.0,
|
||||
ProcRoot: "/proc",
|
||||
SysRoot: "/sys",
|
||||
},
|
||||
Demo: DemoConfig{
|
||||
ID: "demo-1",
|
||||
Subject: "MVP demo: GLM self-description",
|
||||
@@ -343,6 +428,76 @@ func (c *Config) apply(doc TOMLDoc) error {
|
||||
if v, ok := sv.StringList("enabled_models"); ok {
|
||||
c.Serve.EnabledModels = v
|
||||
}
|
||||
|
||||
qt := doc.Table("quota")
|
||||
if v, ok := qt.Bool("enabled"); ok {
|
||||
c.Quota.Enabled = v
|
||||
}
|
||||
if v, ok := qt.String("account"); ok {
|
||||
c.Quota.Account = v
|
||||
}
|
||||
if v, ok := qt.Float("plan_5h_credits"); ok {
|
||||
c.Quota.Plan5hCredits = v
|
||||
}
|
||||
if v, ok := qt.Float("plan_weekly_credits"); ok {
|
||||
c.Quota.PlanWeeklyCredits = v
|
||||
}
|
||||
if v, ok := qt.String("usage_url"); ok {
|
||||
c.Quota.UsageURL = v
|
||||
}
|
||||
if v, ok := qt.String("key_ref"); ok {
|
||||
c.Quota.KeyRef = v
|
||||
}
|
||||
if v, ok := qt.Int("poll_interval_secs"); ok {
|
||||
c.Quota.PollIntervalSecs = int(v)
|
||||
}
|
||||
if v, ok := qt.Float("defer_at_pct"); ok {
|
||||
c.Quota.DeferAtPct = v
|
||||
}
|
||||
if v, ok := qt.Float("block_at_pct"); ok {
|
||||
c.Quota.BlockAtPct = v
|
||||
}
|
||||
if v, ok := qt.String("peak_start"); ok {
|
||||
c.Quota.PeakStart = v
|
||||
}
|
||||
if v, ok := qt.String("peak_end"); ok {
|
||||
c.Quota.PeakEnd = v
|
||||
}
|
||||
if v, ok := qt.String("timezone"); ok {
|
||||
c.Quota.Timezone = v
|
||||
}
|
||||
if v, ok := qt.Bool("peak_weekdays_only"); ok {
|
||||
c.Quota.PeakWeekdaysOnly = v
|
||||
}
|
||||
if v, ok := qt.StringList("peak_classes"); ok {
|
||||
c.Quota.PeakClasses = v
|
||||
}
|
||||
if v, ok := qt.String("redis_url"); ok {
|
||||
c.Quota.RedisURL = v
|
||||
}
|
||||
|
||||
rs := doc.Table("resources")
|
||||
if v, ok := rs.Bool("enabled"); ok {
|
||||
c.Resources.Enabled = v
|
||||
}
|
||||
if v, ok := rs.Float("max_load_avg"); ok {
|
||||
c.Resources.MaxLoadAvg = v
|
||||
}
|
||||
if v, ok := rs.Float("min_mem_available_mb"); ok {
|
||||
c.Resources.MinMemAvailableMB = v
|
||||
}
|
||||
if v, ok := rs.Float("min_disk_free_mb"); ok {
|
||||
c.Resources.MinDiskFreeMB = v
|
||||
}
|
||||
if v, ok := rs.Float("max_io_delay_pct"); ok {
|
||||
c.Resources.MaxIODelayPct = v
|
||||
}
|
||||
if v, ok := rs.String("proc_root"); ok {
|
||||
c.Resources.ProcRoot = v
|
||||
}
|
||||
if v, ok := rs.String("sys_root"); ok {
|
||||
c.Resources.SysRoot = v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -484,5 +639,70 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("[gitea]: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The quota gate is optional; when on, buckets, thresholds and the peak
|
||||
// window must be coherent so the gate never divides by zero or wraps
|
||||
// nonsense windows. usage_url without key_ref is an error (bearer-only).
|
||||
if c.Quota.Enabled {
|
||||
if c.Quota.Plan5hCredits <= 0 || c.Quota.PlanWeeklyCredits <= 0 {
|
||||
return fmt.Errorf("[quota]: plan_5h_credits and plan_weekly_credits must be > 0")
|
||||
}
|
||||
if c.Quota.PollIntervalSecs < 1 {
|
||||
return fmt.Errorf("[quota]: poll_interval_secs must be >= 1")
|
||||
}
|
||||
if !(c.Quota.DeferAtPct > 0 && c.Quota.DeferAtPct < 100) || !(c.Quota.BlockAtPct > 0 && c.Quota.BlockAtPct <= 100) {
|
||||
return fmt.Errorf("[quota]: defer_at_pct and block_at_pct must be in (0,100]")
|
||||
}
|
||||
if c.Quota.DeferAtPct > c.Quota.BlockAtPct {
|
||||
return fmt.Errorf("[quota]: defer_at_pct (%.0f) must be <= block_at_pct (%.0f)", c.Quota.DeferAtPct, c.Quota.BlockAtPct)
|
||||
}
|
||||
if _, err := ParseHHMM(c.Quota.PeakStart); err != nil {
|
||||
return fmt.Errorf("[quota]: peak_start: %w", err)
|
||||
}
|
||||
if _, err := ParseHHMM(c.Quota.PeakEnd); err != nil {
|
||||
return fmt.Errorf("[quota]: peak_end: %w", err)
|
||||
}
|
||||
if _, err := time.LoadLocation(c.Quota.Timezone); err != nil {
|
||||
return fmt.Errorf("[quota]: unknown timezone %q", c.Quota.Timezone)
|
||||
}
|
||||
if c.Quota.Account == "" {
|
||||
return fmt.Errorf("[quota]: account is required when the quota gate is enabled (shared-state bucket label)")
|
||||
}
|
||||
if c.Quota.UsageURL != "" && c.Quota.KeyRef == "" {
|
||||
return fmt.Errorf("[quota]: key_ref is required when usage_url is set (bearer auth)")
|
||||
}
|
||||
if c.Quota.KeyRef != "" {
|
||||
if err := CheckKeyRef(c.Quota.KeyRef); err != nil {
|
||||
return fmt.Errorf("[quota]: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The resource gate is optional; when on, thresholds must be sane.
|
||||
if c.Resources.Enabled {
|
||||
if c.Resources.MaxLoadAvg <= 0 || c.Resources.MinMemAvailableMB <= 0 || c.Resources.MinDiskFreeMB <= 0 {
|
||||
return fmt.Errorf("[resources]: thresholds must be > 0")
|
||||
}
|
||||
if !(c.Resources.MaxIODelayPct > 0 && c.Resources.MaxIODelayPct <= 100) {
|
||||
return fmt.Errorf("[resources]: max_io_delay_pct must be in (0,100]")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseHHMM parses a wall-clock time of day, "HH:MM" (24h).
|
||||
func ParseHHMM(s string) (time.Duration, error) {
|
||||
parts := strings.Split(s, ":")
|
||||
if len(parts) != 2 || len(parts[0]) != 2 || len(parts[1]) != 2 {
|
||||
return 0, fmt.Errorf("want HH:MM, got %q", s)
|
||||
}
|
||||
h, err := strconv.Atoi(parts[0])
|
||||
if err != nil || h < 0 || h > 23 {
|
||||
return 0, fmt.Errorf("bad hour in %q", s)
|
||||
}
|
||||
m, err := strconv.Atoi(parts[1])
|
||||
if err != nil || m < 0 || m > 59 {
|
||||
return 0, fmt.Errorf("bad minute in %q", s)
|
||||
}
|
||||
return time.Duration(h)*time.Hour + time.Duration(m)*time.Minute, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQuotaAndResourcesLoad(t *testing.T) {
|
||||
src := `
|
||||
vertical = "t"
|
||||
[loop]
|
||||
[redmine]
|
||||
url = "https://rm"
|
||||
key_ref = "env:K"
|
||||
scope_query = "p=1"
|
||||
[litellm]
|
||||
base_url = "http://l"
|
||||
key_ref = "env:K"
|
||||
[models]
|
||||
a = "m"
|
||||
default_tier = "a"
|
||||
[models.classes]
|
||||
x = "a"
|
||||
[quota]
|
||||
enabled = true
|
||||
account = "zai-max-1"
|
||||
plan_5h_credits = 28000
|
||||
plan_weekly_credits = 140000
|
||||
usage_url = "https://api.z.ai/api/coding/paas/v4/usage"
|
||||
key_ref = "env:HARNESS_ZAI_KEY"
|
||||
poll_interval_secs = 120
|
||||
defer_at_pct = 80
|
||||
block_at_pct = 92.5
|
||||
peak_start = "22:30"
|
||||
peak_end = "04:15"
|
||||
timezone = "America/Chicago"
|
||||
peak_weekdays_only = false
|
||||
peak_classes = ["study", "read", "summarize"]
|
||||
redis_url = "redis://192.168.3.78:6390/2"
|
||||
[resources]
|
||||
enabled = true
|
||||
max_load_avg = 5.5
|
||||
min_mem_available_mb = 1024
|
||||
min_disk_free_mb = 2048
|
||||
max_io_delay_pct = 75
|
||||
`
|
||||
path := filepath.Join(t.TempDir(), "harness.toml")
|
||||
if err := os.WriteFile(path, []byte(src), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
q := cfg.Quota
|
||||
if !q.Enabled || q.Account != "zai-max-1" || q.Plan5hCredits != 28000 || q.PlanWeeklyCredits != 140000 {
|
||||
t.Errorf("quota basics: %+v", q)
|
||||
}
|
||||
if q.UsageURL == "" || q.KeyRef != "env:HARNESS_ZAI_KEY" || q.PollIntervalSecs != 120 {
|
||||
t.Errorf("poller config: %+v", q)
|
||||
}
|
||||
if q.DeferAtPct != 80 || q.BlockAtPct != 92.5 {
|
||||
t.Errorf("thresholds: %+v", q)
|
||||
}
|
||||
if q.PeakStart != "22:30" || q.PeakEnd != "04:15" || q.Timezone != "America/Chicago" || q.PeakWeekdaysOnly {
|
||||
t.Errorf("peak window: %+v", q)
|
||||
}
|
||||
if len(q.PeakClasses) != 3 || q.PeakClasses[0] != "study" {
|
||||
t.Errorf("peak classes: %v", q.PeakClasses)
|
||||
}
|
||||
if q.RedisURL != "redis://192.168.3.78:6390/2" {
|
||||
t.Errorf("redis url: %q", q.RedisURL)
|
||||
}
|
||||
r := cfg.Resources
|
||||
if !r.Enabled || r.MaxLoadAvg != 5.5 || r.MinMemAvailableMB != 1024 || r.MinDiskFreeMB != 2048 || r.MaxIODelayPct != 75 {
|
||||
t.Errorf("resources: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotaValidateErrors(t *testing.T) {
|
||||
base := `
|
||||
vertical = "t"
|
||||
[litellm]
|
||||
base_url = "http://l"
|
||||
key_ref = "env:K"
|
||||
[models]
|
||||
a = "m"
|
||||
default_tier = "a"
|
||||
`
|
||||
cases := []struct {
|
||||
name, extra, want string
|
||||
}{
|
||||
{"no account", "[quota]\nenabled = true\n", "account is required"},
|
||||
{"bad bucket", "[quota]\nenabled = true\naccount = \"a\"\nplan_5h_credits = 0\n", "plan_5h_credits"},
|
||||
{"defer > block", "[quota]\nenabled = true\naccount = \"a\"\ndefer_at_pct = 99\nblock_at_pct = 95\n", "defer_at_pct"},
|
||||
{"bad window", "[quota]\nenabled = true\naccount = \"a\"\npeak_start = \"99:00\"\n", "peak_start"},
|
||||
{"bad tz", "[quota]\nenabled = true\naccount = \"a\"\ntimezone = \"Mars/Olympus\"\n", "timezone"},
|
||||
{"usage url without key", "[quota]\nenabled = true\naccount = \"a\"\nusage_url = \"https://x\"\n", "key_ref is required"},
|
||||
{"bad threshold", "[resources]\nenabled = true\nmax_load_avg = -1\n", "thresholds must be > 0"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "harness.toml")
|
||||
if err := os.WriteFile(path, []byte(base+c.extra), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := Load(path)
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Fatalf("err = %v, want containing %q", err, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFloatTomLValues(t *testing.T) {
|
||||
doc, err := ParseTOML("a = 6\nb = 6.25\nc = 1_000.5\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, ok := doc.Float("a"); !ok || v != 6 {
|
||||
t.Errorf("int as float = %v %v", v, ok)
|
||||
}
|
||||
if v, ok := doc.Float("b"); !ok || v != 6.25 {
|
||||
t.Errorf("float = %v %v", v, ok)
|
||||
}
|
||||
if v, ok := doc.Float("c"); !ok || v != 1000.5 {
|
||||
t.Errorf("underscore float = %v %v", v, ok)
|
||||
}
|
||||
}
|
||||
+16
-3
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -48,6 +49,15 @@ func (d TOMLDoc) Bool(key string) (bool, bool) {
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func (d TOMLDoc) Float(key string) (float64, bool) {
|
||||
if v, ok := d[key].(float64); ok {
|
||||
return v, true
|
||||
}
|
||||
// Bare integers read as floats too (max_load_avg = 6 parses as int64).
|
||||
v, ok := d[key].(int64)
|
||||
return float64(v), ok
|
||||
}
|
||||
|
||||
func (d TOMLDoc) StringList(key string) ([]string, bool) {
|
||||
raw, ok := d[key].([]any)
|
||||
if !ok {
|
||||
@@ -280,10 +290,13 @@ func parseScalar(v string, no int) (any, error) {
|
||||
return false, nil
|
||||
}
|
||||
n, err := parseTOMLInt(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness.toml:%d: unsupported value %q (want string, int, bool, or array)", no, v)
|
||||
if err == nil {
|
||||
return n, nil
|
||||
}
|
||||
return n, nil
|
||||
if f, ferr := strconv.ParseFloat(strings.ReplaceAll(v, "_", ""), 64); ferr == nil {
|
||||
return f, nil
|
||||
}
|
||||
return nil, fmt.Errorf("harness.toml:%d: unsupported value %q (want string, int, float, bool, or array)", no, v)
|
||||
}
|
||||
|
||||
func parseTOMLInt(v string) (int64, error) {
|
||||
|
||||
Reference in New Issue
Block a user