Files
MOPAC/internal/quota/state.go
T
mrcharles fc518c475e quota: z.ai credit-bucket back-pressure + resource gate + usage accounting (Redmine 490+491)
The loop now consults quota and host state before every dispatch and
DEFERS gated work with a logged reason instead of letting turns die at
the provider (the 2026-08-28 19:00 quota-wall failure mode, replayed as
a test). Adds internal/quota: 5h/weekly credit buckets (provider poll
when z.ai ships an endpoint - fake-server tested - else locally
estimated from the documented credit formula), TZ-aware peak window
(default 01:00-05:00 America/Chicago weekdays, matching the documented
z.ai peak Mon-Fri 14:00-18:00 Singapore), block/defer thresholds, a
read-only load/mem/disk/IO-PSI monitor, an optional redis shared-state
hop (stdlib RESP2 mini-client) so all instances of an account
coordinate, per-class token+credit accounting in loop.jsonl, and
`harness quota status|probe|gate`. Config: [quota] + [resources]
sections; README runbook covers the redis container and deploy-time
cgroup enforcement.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
2026-08-29 05:37:15 -05:00

234 lines
6.1 KiB
Go

package quota
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"net"
"net/url"
"strconv"
"strings"
"sync"
"time"
)
// SharedState is the cross-instance quota state bus: one redis container on
// the LAN (DECISION 2026-08-29, see REPORT-20260829-0500-quota) so the nine
// harness instances across two hosts read/write the same quota snapshot and
// credit estimates. It is a deliberately tiny stdlib-only RESP2 client
// (SET/GET/INCRBYFLOAT/EXPIRE — the whole surface this package needs), no
// host packages and no external Go deps. Every op fails soft: with redis
// down the gate degrades to this instance's local estimate and the loop
// keeps running.
type SharedState struct {
addr string
db int
mu sync.Mutex
conn net.Conn
rw *bufio.ReadWriter
dialTO time.Duration
}
// NewSharedState parses redis://host:port[/db]; empty URL returns nil (the
// nil state is valid and fully functional as a no-op).
func NewSharedState(redisURL string) (*SharedState, error) {
if redisURL == "" {
return nil, nil
}
u, err := url.Parse(redisURL)
if err != nil {
return nil, fmt.Errorf("redis url: %w", err)
}
host := u.Host
if host == "" {
return nil, fmt.Errorf("redis url: missing host")
}
if !strings.Contains(host, ":") {
host += ":6379"
}
db := 0
if s := strings.TrimPrefix(u.Path, "/"); s != "" {
if db, err = strconv.Atoi(s); err != nil {
return nil, fmt.Errorf("redis url db: %w", err)
}
}
return &SharedState{addr: host, db: db, dialTO: 3 * time.Second}, nil
}
// Close releases the connection, if any.
func (s *SharedState) Close() error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.conn != nil {
err := s.conn.Close()
s.conn, s.rw = nil, nil
return err
}
return nil
}
// command runs one RESP command and returns the reply. Transport failures
// reset the connection so the next op redials once.
func (s *SharedState) command(args ...string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn == nil {
conn, err := net.DialTimeout("tcp", s.addr, s.dialTO)
if err != nil {
return "", fmt.Errorf("redis dial %s: %w", s.addr, err)
}
s.conn = conn
s.rw = bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
if s.db != 0 {
if _, err := s.execLocked("SELECT", strconv.Itoa(s.db)); err != nil {
s.resetLocked()
return "", err
}
}
}
reply, err := s.execLocked(args...)
if err != nil {
s.resetLocked()
}
return reply, err
}
// execLocked writes one command and reads one reply (mutex held).
func (s *SharedState) execLocked(args ...string) (string, error) {
var b strings.Builder
fmt.Fprintf(&b, "*%d\r\n", len(args))
for _, a := range args {
fmt.Fprintf(&b, "$%d\r\n%s\r\n", len(a), a)
}
if _, err := s.rw.WriteString(b.String()); err != nil {
return "", fmt.Errorf("redis write: %w", err)
}
if err := s.rw.Flush(); err != nil {
return "", fmt.Errorf("redis flush: %w", err)
}
return readReply(s.rw.Reader)
}
func (s *SharedState) resetLocked() {
if s.conn != nil {
_ = s.conn.Close()
}
s.conn, s.rw = nil, nil
}
// readReply parses one RESP2 reply: +simple / -error / :integer / $bulk.
func readReply(r *bufio.Reader) (string, error) {
line, err := r.ReadString('\n')
if err != nil {
return "", fmt.Errorf("redis read: %w", err)
}
line = strings.TrimRight(line, "\r\n")
if line == "" {
return "", errors.New("redis: empty reply")
}
switch line[0] {
case '+', ':':
return line[1:], nil
case '-':
return "", fmt.Errorf("redis: %s", line[1:])
case '$':
n, err := strconv.Atoi(line[1:])
if err != nil {
return "", fmt.Errorf("redis bulk len: %w", err)
}
if n < 0 {
return "", nil // nil bulk: missing key
}
buf := make([]byte, n+2)
if _, err := ioReadFull(r, buf); err != nil {
return "", fmt.Errorf("redis bulk read: %w", err)
}
return string(buf[:n]), nil
default:
return "", fmt.Errorf("redis: unexpected reply %q", line)
}
}
func ioReadFull(r *bufio.Reader, buf []byte) (int, error) {
total := 0
for total < len(buf) {
n, err := r.Read(buf[total:])
total += n
if err != nil {
return total, err
}
}
return total, nil
}
// Key layout (one account's plan is one shared universe):
//
// mopac:quota:<account>:snapshot — latest polled QuotaSnapshot (JSON)
// mopac:quota:<account>:est:5h — estimated credits in the rolling 5h window
// mopac:quota:<account>:est:weekly — estimated credits in the plan week
//
// Estimate keys carry the window id in the VALUE-side bookkeeping done by
// the caller (Gate): 5h keys expire after 5h+slack; weekly keys are keyed
// by ISO week via the caller and need no expiry.
func (s *SharedState) get(key string) (string, error) {
if s == nil {
return "", nil
}
return s.command("GET", key)
}
func (s *SharedState) setEx(key, val string, ttl time.Duration) error {
if s == nil {
return nil
}
_, err := s.command("SET", key, val, "EX", strconv.Itoa(int(ttl.Seconds())))
return err
}
// incrByFloat adds delta to key (creating at delta) and returns the new value.
func (s *SharedState) incrByFloat(key string, delta float64) (float64, error) {
if s == nil {
return 0, nil
}
reply, err := s.command("INCRBYFLOAT", key, strconv.FormatFloat(delta, 'f', 6, 64))
if err != nil {
return 0, err
}
return strconv.ParseFloat(strings.TrimSpace(reply), 64)
}
// PublishSnapshot stores the polled snapshot for all instances to read.
func (s *SharedState) PublishSnapshot(snap *QuotaSnapshot, ttl time.Duration) error {
if s == nil || snap == nil {
return nil
}
body, err := json.Marshal(snap)
if err != nil {
return err
}
return s.setEx(snapshotKey(snap.Account), string(body), ttl)
}
// LoadSnapshot returns the last published snapshot, or nil when absent.
func (s *SharedState) LoadSnapshot(account string) (*QuotaSnapshot, error) {
body, err := s.get(snapshotKey(account))
if err != nil {
return nil, err
}
if body == "" {
return nil, nil
}
var snap QuotaSnapshot
if err := json.Unmarshal([]byte(body), &snap); err != nil {
return nil, fmt.Errorf("shared snapshot decode: %w", err)
}
return &snap, nil
}
func snapshotKey(account string) string { return "mopac:quota:" + account + ":snapshot" }