quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)

The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
This commit is contained in:
2026-08-29 05:37:15 -05:00
parent 9dbb20489c
commit fc518c475e
21 changed files with 2904 additions and 20 deletions
+143
View File
@@ -0,0 +1,143 @@
package quota
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
)
// ResourceStats is one read-only sample of host load: the numbers the
// resource gate compares against its thresholds. Zero fields mean "could
// not read" and never trigger a busy verdict on their own.
type ResourceStats struct {
LoadAvg1m float64
MemAvailable float64 // MB
DiskFree float64 // MB, at path
IODelayPct float64 // /proc/pressure/io "some avg60" percent; 0 if PSI absent
HasPSI bool
}
// ReadResourceStats samples /proc/loadavg, /proc/meminfo, /proc/pressure/io
// and statfs(path) — all read-only, no host packages. procRoot/sysRoot are
// seams (tests point them at fixture trees; production passes /proc, /sys).
func ReadResourceStats(procRoot, sysRoot, path string) ResourceStats {
var st ResourceStats
if la, ok := readLoadAvg(filepath.Join(procRoot, "loadavg")); ok {
st.LoadAvg1m = la
}
if mb, ok := readMemAvailable(filepath.Join(procRoot, "meminfo")); ok {
st.MemAvailable = mb
}
// PSI lives under /proc/pressure (io); older kernels expose none — the
// gate then skips the IO check rather than failing.
if p, ok := readIODelay(filepath.Join(procRoot, "pressure", "io")); ok {
st.IODelayPct, st.HasPSI = p, true
}
_ = sysRoot // reserved: /sys/class/... sources when PSI is absent
if df, ok := readDiskFree(path); ok {
st.DiskFree = df
}
return st
}
func readLoadAvg(path string) (float64, bool) {
data, err := os.ReadFile(path)
if err != nil {
return 0, false
}
fields := strings.Fields(string(data))
if len(fields) < 1 {
return 0, false
}
f, err := strconv.ParseFloat(fields[0], 64)
if err != nil {
return 0, false
}
return f, true
}
func readMemAvailable(path string) (float64, bool) {
f, err := os.Open(path)
if err != nil {
return 0, false
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
line := sc.Text()
if strings.HasPrefix(line, "MemAvailable:") {
fields := strings.Fields(line)
if len(fields) < 2 {
return 0, false
}
kb, err := strconv.ParseFloat(fields[1], 64)
if err != nil {
return 0, false
}
return kb / 1024, true
}
}
return 0, false
}
// readIODelay parses /proc/pressure/io, e.g.
// "some avg10=0.00 avg60=0.12 avg300=0.05 total=123456789" — avg60 is the
// steady-state signal (a build spiking IO shows here within a minute).
func readIODelay(path string) (float64, bool) {
data, err := os.ReadFile(path)
if err != nil {
return 0, false
}
for _, line := range strings.Split(string(data), "\n") {
if !strings.HasPrefix(line, "some ") {
continue
}
for _, field := range strings.Fields(line)[1:] {
if strings.HasPrefix(field, "avg60=") {
if v, err := strconv.ParseFloat(strings.TrimPrefix(field, "avg60="), 64); err == nil {
return v, true
}
}
}
}
return 0, false
}
func readDiskFree(path string) (float64, bool) {
var fs syscall.Statfs_t
if err := syscall.Statfs(path, &fs); err != nil {
return 0, false
}
return float64(fs.Bavail) * float64(fs.Bsize) / (1024 * 1024), true
}
// BusyCheck compares a sample against the resource thresholds. Violations
// are collected (all reported, not just the first) — the loop defers while
// any threshold trips, with every reason surfaced in the defer log line.
type BusyCheck struct {
MaxLoadAvg float64
MinMemAvailableMB float64
MinDiskFreeMB float64
MaxIODelayPct float64
}
func (c BusyCheck) Evaluate(st ResourceStats) []string {
var reasons []string
if c.MaxLoadAvg > 0 && st.LoadAvg1m > c.MaxLoadAvg {
reasons = append(reasons, fmt.Sprintf("load %.2f > %.2f", st.LoadAvg1m, c.MaxLoadAvg))
}
if c.MinMemAvailableMB > 0 && st.MemAvailable > 0 && st.MemAvailable < c.MinMemAvailableMB {
reasons = append(reasons, fmt.Sprintf("mem available %.0fMB < %.0fMB", st.MemAvailable, c.MinMemAvailableMB))
}
if c.MinDiskFreeMB > 0 && st.DiskFree > 0 && st.DiskFree < c.MinDiskFreeMB {
reasons = append(reasons, fmt.Sprintf("disk free %.0fMB < %.0fMB", st.DiskFree, c.MinDiskFreeMB))
}
if c.MaxIODelayPct > 0 && st.HasPSI && st.IODelayPct > c.MaxIODelayPct {
reasons = append(reasons, fmt.Sprintf("io delay %.1f%% > %.1f%%", st.IODelayPct, c.MaxIODelayPct))
}
return reasons
}