The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
144 lines
4.2 KiB
Go
144 lines
4.2 KiB
Go
package quota
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
)
|
|
|
|
// ResourceStats is one read-only sample of host load: the numbers the
|
|
// resource gate compares against its thresholds. Zero fields mean "could
|
|
// not read" and never trigger a busy verdict on their own.
|
|
type ResourceStats struct {
|
|
LoadAvg1m float64
|
|
MemAvailable float64 // MB
|
|
DiskFree float64 // MB, at path
|
|
IODelayPct float64 // /proc/pressure/io "some avg60" percent; 0 if PSI absent
|
|
HasPSI bool
|
|
}
|
|
|
|
// ReadResourceStats samples /proc/loadavg, /proc/meminfo, /proc/pressure/io
|
|
// and statfs(path) — all read-only, no host packages. procRoot/sysRoot are
|
|
// seams (tests point them at fixture trees; production passes /proc, /sys).
|
|
func ReadResourceStats(procRoot, sysRoot, path string) ResourceStats {
|
|
var st ResourceStats
|
|
if la, ok := readLoadAvg(filepath.Join(procRoot, "loadavg")); ok {
|
|
st.LoadAvg1m = la
|
|
}
|
|
if mb, ok := readMemAvailable(filepath.Join(procRoot, "meminfo")); ok {
|
|
st.MemAvailable = mb
|
|
}
|
|
// PSI lives under /proc/pressure (io); older kernels expose none — the
|
|
// gate then skips the IO check rather than failing.
|
|
if p, ok := readIODelay(filepath.Join(procRoot, "pressure", "io")); ok {
|
|
st.IODelayPct, st.HasPSI = p, true
|
|
}
|
|
_ = sysRoot // reserved: /sys/class/... sources when PSI is absent
|
|
if df, ok := readDiskFree(path); ok {
|
|
st.DiskFree = df
|
|
}
|
|
return st
|
|
}
|
|
|
|
func readLoadAvg(path string) (float64, bool) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
fields := strings.Fields(string(data))
|
|
if len(fields) < 1 {
|
|
return 0, false
|
|
}
|
|
f, err := strconv.ParseFloat(fields[0], 64)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return f, true
|
|
}
|
|
|
|
func readMemAvailable(path string) (float64, bool) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
line := sc.Text()
|
|
if strings.HasPrefix(line, "MemAvailable:") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
return 0, false
|
|
}
|
|
kb, err := strconv.ParseFloat(fields[1], 64)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return kb / 1024, true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// readIODelay parses /proc/pressure/io, e.g.
|
|
// "some avg10=0.00 avg60=0.12 avg300=0.05 total=123456789" — avg60 is the
|
|
// steady-state signal (a build spiking IO shows here within a minute).
|
|
func readIODelay(path string) (float64, bool) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
if !strings.HasPrefix(line, "some ") {
|
|
continue
|
|
}
|
|
for _, field := range strings.Fields(line)[1:] {
|
|
if strings.HasPrefix(field, "avg60=") {
|
|
if v, err := strconv.ParseFloat(strings.TrimPrefix(field, "avg60="), 64); err == nil {
|
|
return v, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func readDiskFree(path string) (float64, bool) {
|
|
var fs syscall.Statfs_t
|
|
if err := syscall.Statfs(path, &fs); err != nil {
|
|
return 0, false
|
|
}
|
|
return float64(fs.Bavail) * float64(fs.Bsize) / (1024 * 1024), true
|
|
}
|
|
|
|
// BusyCheck compares a sample against the resource thresholds. Violations
|
|
// are collected (all reported, not just the first) — the loop defers while
|
|
// any threshold trips, with every reason surfaced in the defer log line.
|
|
type BusyCheck struct {
|
|
MaxLoadAvg float64
|
|
MinMemAvailableMB float64
|
|
MinDiskFreeMB float64
|
|
MaxIODelayPct float64
|
|
}
|
|
|
|
func (c BusyCheck) Evaluate(st ResourceStats) []string {
|
|
var reasons []string
|
|
if c.MaxLoadAvg > 0 && st.LoadAvg1m > c.MaxLoadAvg {
|
|
reasons = append(reasons, fmt.Sprintf("load %.2f > %.2f", st.LoadAvg1m, c.MaxLoadAvg))
|
|
}
|
|
if c.MinMemAvailableMB > 0 && st.MemAvailable > 0 && st.MemAvailable < c.MinMemAvailableMB {
|
|
reasons = append(reasons, fmt.Sprintf("mem available %.0fMB < %.0fMB", st.MemAvailable, c.MinMemAvailableMB))
|
|
}
|
|
if c.MinDiskFreeMB > 0 && st.DiskFree > 0 && st.DiskFree < c.MinDiskFreeMB {
|
|
reasons = append(reasons, fmt.Sprintf("disk free %.0fMB < %.0fMB", st.DiskFree, c.MinDiskFreeMB))
|
|
}
|
|
if c.MaxIODelayPct > 0 && st.HasPSI && st.IODelayPct > c.MaxIODelayPct {
|
|
reasons = append(reasons, fmt.Sprintf("io delay %.1f%% > %.1f%%", st.IODelayPct, c.MaxIODelayPct))
|
|
}
|
|
return reasons
|
|
}
|