From 591d345371ff6b5fc6e04e24d9f2c4a5674d703f Mon Sep 17 00:00:00 2001 From: reachableceo Date: Fri, 28 Aug 2026 19:20:51 -0500 Subject: [PATCH] harness: config layer + model routing v0 (tier map, no heuristics) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit harness.toml loading via a stdlib-only TOML subset parser (tables, bare keys, strings/ints/bools, multi-line arrays; anything richer fails loudly). Secrets are refs only (env:/file:/literal:, bw: reserved) and are redacted from every error path. Model routing v0: static class -> tier alias -> concrete model map per the DESIGN model-selection layer; requests carry the resolved concrete model and unknown classes fail hard so routing stays auditable. Table-driven tests cover the parser, validation, key refs, and routing decisions. 💘 Generated with Crush Assisted-by: Crush:glm-5.2 --- go.mod | 2 + internal/config/config.go | 265 ++++++++++++++++++++ internal/config/config_test.go | 202 ++++++++++++++++ internal/config/keys.go | 78 ++++++ internal/config/toml.go | 425 +++++++++++++++++++++++++++++++++ internal/config/toml_test.go | 139 +++++++++++ internal/models/models.go | 68 ++++++ internal/models/models_test.go | 78 ++++++ 8 files changed, 1257 insertions(+) create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/config/keys.go create mode 100644 internal/config/toml.go create mode 100644 internal/config/toml_test.go create mode 100644 internal/models/models.go create mode 100644 internal/models/models_test.go diff --git a/go.mod b/go.mod index 8e3bc02..28a9181 100644 --- a/go.mod +++ b/go.mod @@ -1 +1,3 @@ +module git.knownelement.com/reachableceo/MOPAC/harness + go 1.24 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..43edcc4 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,265 @@ +package config + +import ( + "fmt" +) + +// Config is the full harness.toml surface. Defaults live in Default(); +// apply() overlays the parsed document; Validate() enforces invariants. +type Config struct { + Vertical string + WorkRoot string + ReportDir string + Loop LoopConfig + Redmine RedmineConfig + LiteLLM LiteLLMConfig + Models ModelsConfig + Bash BashConfig + Demo DemoConfig +} + +type LoopConfig struct { + MaxRounds int +} + +type RedmineConfig struct { + URL string + KeyRef string + ScopeQuery string + ScopeQueryID int + ClassField string + DefaultClass string + Limit int +} + +type LiteLLMConfig struct { + BaseURL string + KeyRef string + TimeoutSecs int + MaxRetries int +} + +type ModelsConfig struct { + // Tiers maps tier aliases (mopac-study, ...) to concrete proxy models. + Tiers map[string]string + // Classes maps task classes (study, code, ...) to tier aliases. + Classes map[string]string + // DefaultTier is used when a task carries no class. + DefaultTier string +} + +type BashConfig struct { + Enabled bool + Allow []string + Deny []string + DefaultAllow bool + TimeoutSecs int + MaxOutputBytes int +} + +type DemoConfig struct { + ID string + Subject string + Prompt string + Class string +} + +// Default returns the built-in defaults for every field. +func Default() *Config { + return &Config{ + WorkRoot: ".", + ReportDir: "reports", + Loop: LoopConfig{MaxRounds: 8}, + Redmine: RedmineConfig{ + ClassField: "Class", + DefaultClass: "primary", + Limit: 50, + }, + LiteLLM: LiteLLMConfig{TimeoutSecs: 120, MaxRetries: 2}, + Models: ModelsConfig{ + Tiers: map[string]string{}, + Classes: map[string]string{}, + DefaultTier: "mopac-primary", + }, + Bash: BashConfig{Enabled: true, TimeoutSecs: 60, MaxOutputBytes: 100_000}, + Demo: DemoConfig{ + ID: "demo-1", + Subject: "MVP demo: GLM self-description", + Prompt: "tell me about yourself", + Class: "primary", + }, + } +} + +func (c *Config) apply(doc TOMLDoc) error { + if v, ok := doc.String("vertical"); ok { + c.Vertical = v + } + if v, ok := doc.String("work_root"); ok { + c.WorkRoot = v + } + if v, ok := doc.String("report_dir"); ok { + c.ReportDir = v + } + if v, ok := doc.Table("loop").Int("max_rounds"); ok { + c.Loop.MaxRounds = int(v) + } + + rm := doc.Table("redmine") + if v, ok := rm.String("url"); ok { + c.Redmine.URL = v + } + if v, ok := rm.String("key_ref"); ok { + c.Redmine.KeyRef = v + } + if v, ok := rm.String("scope_query"); ok { + c.Redmine.ScopeQuery = v + } + if v, ok := rm.Int("scope_query_id"); ok { + c.Redmine.ScopeQueryID = int(v) + } + if v, ok := rm.String("class_field"); ok { + c.Redmine.ClassField = v + } + if v, ok := rm.String("default_class"); ok { + c.Redmine.DefaultClass = v + } + if v, ok := rm.Int("limit"); ok { + c.Redmine.Limit = int(v) + } + + lt := doc.Table("litellm") + if v, ok := lt.String("base_url"); ok { + c.LiteLLM.BaseURL = v + } + if v, ok := lt.String("key_ref"); ok { + c.LiteLLM.KeyRef = v + } + if v, ok := lt.Int("timeout_secs"); ok { + c.LiteLLM.TimeoutSecs = int(v) + } + if v, ok := lt.Int("max_retries"); ok { + c.LiteLLM.MaxRetries = int(v) + } + + md := doc.Table("models") + for _, k := range md.Keys() { + if k == "default_tier" { + continue + } + if v, ok := md.String(k); ok { + c.Models.Tiers[k] = v + } + } + if v, ok := md.String("default_tier"); ok { + c.Models.DefaultTier = v + } + cls := doc.Table("models", "classes") + for _, k := range cls.Keys() { + if v, ok := cls.String(k); ok { + c.Models.Classes[k] = v + } + } + + bt := doc.Table("tools", "bash") + if v, ok := bt.Bool("enabled"); ok { + c.Bash.Enabled = v + } + if v, ok := bt.StringList("allow"); ok { + c.Bash.Allow = v + } + if v, ok := bt.StringList("deny"); ok { + c.Bash.Deny = v + } + if v, ok := bt.String("default"); ok { + switch v { + case "allow": + c.Bash.DefaultAllow = true + case "deny": + c.Bash.DefaultAllow = false + default: + return fmt.Errorf("[tools.bash]: default must be \"allow\" or \"deny\", got %q", v) + } + } + if v, ok := bt.Int("timeout_secs"); ok { + c.Bash.TimeoutSecs = int(v) + } + if v, ok := bt.Int("max_output_bytes"); ok { + c.Bash.MaxOutputBytes = int(v) + } + + dm := doc.Table("demo") + if v, ok := dm.String("id"); ok { + c.Demo.ID = v + } + if v, ok := dm.String("subject"); ok { + c.Demo.Subject = v + } + if v, ok := dm.String("prompt"); ok { + c.Demo.Prompt = v + } + if v, ok := dm.String("class"); ok { + c.Demo.Class = v + } + return nil +} + +// Validate enforces the invariants the conductor relies on. +func (c *Config) Validate() error { + if c.Vertical == "" { + return fmt.Errorf("[core]: vertical is required") + } + if c.Loop.MaxRounds < 1 { + return fmt.Errorf("[loop]: max_rounds must be >= 1") + } + if c.LiteLLM.BaseURL == "" { + return fmt.Errorf("[litellm]: base_url is required") + } + if c.LiteLLM.KeyRef == "" { + return fmt.Errorf("[litellm]: key_ref is required (env:NAME, file:PATH, or literal:VALUE)") + } + if err := CheckKeyRef(c.LiteLLM.KeyRef); err != nil { + return fmt.Errorf("[litellm]: %w", err) + } + if c.LiteLLM.TimeoutSecs < 1 || c.LiteLLM.MaxRetries < 0 { + return fmt.Errorf("[litellm]: bad timeout_secs/max_retries") + } + + // Redmine is optional (demo-only configs); if any piece is set, the rest + // of the minimum set must be too. + rc := c.Redmine + if rc.URL != "" || rc.KeyRef != "" || rc.ScopeQuery != "" || rc.ScopeQueryID != 0 { + if rc.URL == "" { + return fmt.Errorf("[redmine]: url is required when redmine is configured") + } + if rc.KeyRef == "" { + return fmt.Errorf("[redmine]: key_ref is required when redmine is configured") + } + if err := CheckKeyRef(rc.KeyRef); err != nil { + return fmt.Errorf("[redmine]: %w", err) + } + if rc.ScopeQuery == "" && rc.ScopeQueryID == 0 { + return fmt.Errorf("[redmine]: scope_query or scope_query_id is required") + } + } + + if len(c.Models.Tiers) == 0 { + return fmt.Errorf("[models]: at least one tier alias is required") + } + if _, ok := c.Models.Tiers[c.Models.DefaultTier]; !ok { + return fmt.Errorf("[models]: default_tier %q has no entry in [models]", c.Models.DefaultTier) + } + for class, tier := range c.Models.Classes { + if _, ok := c.Models.Tiers[tier]; !ok { + return fmt.Errorf("[models.classes]: class %q points at unknown tier %q", class, tier) + } + } + + if c.Bash.TimeoutSecs < 1 || c.Bash.MaxOutputBytes < 1 { + return fmt.Errorf("[tools.bash]: bad timeout_secs/max_output_bytes") + } + if c.Demo.Prompt == "" { + return fmt.Errorf("[demo]: prompt is required") + } + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..3b0be74 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,202 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const testCfg = ` +vertical = "teststack" +work_root = "/tmp/harness-test" +report_dir = "out" + +[redmine] +url = "https://rm.test" +key_ref = "env:HARNESS_TEST_RM_KEY" +scope_query = "project=x&status_id=open" + +[litellm] +base_url = "http://litellm.test:4000" +key_ref = "env:HARNESS_TEST_LLM_KEY" + +[models] +mopac-study = "glm-4.7-flash" +mopac-code = "glm-5.2" +mopac-review = "glm-5-turbo" +mopac-primary = "glm-5.3" +default_tier = "mopac-primary" + +[models.classes] +study = "mopac-study" +code = "mopac-code" +primary = "mopac-primary" + +[tools.bash] +allow = ["pwd", "ls *"] +deny = ["sudo *"] +` + +func writeTemp(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "harness.toml") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestLoadAppliesDocAndDefaults(t *testing.T) { + cfg, err := Load(writeTemp(t, testCfg)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Vertical != "teststack" || cfg.WorkRoot != "/tmp/harness-test" || cfg.ReportDir != "out" { + t.Errorf("core fields wrong: %+v", cfg) + } + if cfg.Loop.MaxRounds != 8 { + t.Errorf("default max_rounds = %d, want 8", cfg.Loop.MaxRounds) + } + if cfg.LiteLLM.BaseURL != "http://litellm.test:4000" || cfg.LiteLLM.MaxRetries != 2 { + t.Errorf("litellm wrong: %+v", cfg.LiteLLM) + } + wantTiers := map[string]string{ + "mopac-study": "glm-4.7-flash", + "mopac-code": "glm-5.2", + "mopac-review": "glm-5-turbo", + "mopac-primary": "glm-5.3", + } + for tier, model := range wantTiers { + if got, ok := cfg.Models.Tiers[tier]; !ok || got != model { + t.Errorf("tier %s = %q ok=%v, want %q", tier, got, ok, model) + } + } + if cfg.Models.Classes["code"] != "mopac-code" { + t.Errorf("class map wrong: %+v", cfg.Models.Classes) + } + if len(cfg.Bash.Allow) != 2 || len(cfg.Bash.Deny) != 1 { + t.Errorf("bash lists wrong: %v / %v", cfg.Bash.Allow, cfg.Bash.Deny) + } + if cfg.Bash.TimeoutSecs != 60 || cfg.Bash.MaxOutputBytes != 100_000 { + t.Errorf("bash defaults wrong: %+v", cfg.Bash) + } + if cfg.Demo.Prompt != "tell me about yourself" { + t.Errorf("demo default prompt = %q", cfg.Demo.Prompt) + } +} + +func TestLoadTrackedExampleFile(t *testing.T) { + // The example ships to users; it must always parse and validate as-is. + path := filepath.Join("..", "..", "harness.toml.example") + if _, err := os.Stat(path); err != nil { + t.Skipf("example not found: %v", err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load(harness.toml.example): %v", err) + } + if cfg.Vertical != "demo" || cfg.Models.Tiers["mopac-primary"] != "glm-5.3" { + t.Errorf("example parsed wrong: %+v", cfg.Models.Tiers) + } + if len(cfg.Models.Classes) != 9 { + t.Errorf("example classes = %d, want 9", len(cfg.Models.Classes)) + } +} + +func TestLoadValidationErrors(t *testing.T) { + cases := []struct { + name string + mut func(string) string + want string + }{ + { + name: "missing vertical", + mut: func(s string) string { return strings.Replace(s, `vertical = "teststack"`, "", 1) }, + want: "vertical is required", + }, + { + name: "missing litellm url", + mut: func(s string) string { return strings.Replace(s, `base_url = "http://litellm.test:4000"`, "", 1) }, + want: "base_url is required", + }, + { + name: "bare key value", + mut: func(s string) string { return strings.Replace(s, `key_ref = "env:HARNESS_TEST_LLM_KEY"`, `key_ref = "sk-or-whatever"`, 1) }, + want: "env:, file:, literal:, or bw:", + }, + { + name: "bw ref not implemented", + mut: func(s string) string { return strings.Replace(s, `key_ref = "env:HARNESS_TEST_LLM_KEY"`, `key_ref = "bw:item"`, 1) }, + want: "not implemented", + }, + { + name: "redmine half configured", + mut: func(s string) string { return strings.Replace(s, `scope_query = "project=x&status_id=open"`, "", 1) }, + want: "scope_query or scope_query_id is required", + }, + { + name: "class to unknown tier", + mut: func(s string) string { return strings.Replace(s, `code = "mopac-code"`, `code = "mopac-nope"`, 1) }, + want: `unknown tier "mopac-nope"`, + }, + { + name: "default tier missing", + mut: func(s string) string { return strings.Replace(s, `mopac-primary = "glm-5.3"`, "", 1) }, + want: `default_tier "mopac-primary" has no entry`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(writeTemp(t, tc.mut(testCfg))) + if err == nil { + t.Fatalf("expected error containing %q, got nil", tc.want) + } + if !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error %q does not contain %q", err, tc.want) + } + }) + } +} + +func TestDemoOnlyConfigIsValid(t *testing.T) { + // No [redmine] at all: --demo still works. + cfg, err := Load(writeTemp(t, ` +vertical = "demo" +[litellm] +base_url = "http://x:4001" +key_ref = "literal:test-key" +[models] +mopac-primary = "glm-5.3" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Redmine.URL != "" { + t.Errorf("redmine should be unset") + } +} + +func TestResolveKeyRef(t *testing.T) { + t.Setenv("HARNESS_TEST_KEY", "sekrit") + if v, err := ResolveKeyRef("env:HARNESS_TEST_KEY"); err != nil || v != "sekrit" { + t.Errorf("env ref = %q err=%v", v, err) + } + if _, err := ResolveKeyRef("env:HARNESS_TEST_UNSET"); err == nil { + t.Errorf("unset env should error") + } + + f := filepath.Join(t.TempDir(), "key") + os.WriteFile(f, []byte(" filekey\n"), 0o600) + if v, err := ResolveKeyRef("file:" + f); err != nil || v != "filekey" { + t.Errorf("file ref = %q err=%v", v, err) + } + if v, err := ResolveKeyRef("literal:abc"); err != nil || v != "abc" { + t.Errorf("literal ref = %q err=%v", v, err) + } + // Secrets must never leak through the redaction path used in errors. + _, err := ResolveKeyRef("super-secret-value") + if err == nil || strings.Contains(err.Error(), "super-secret-value") { + t.Errorf("bare ref error leaks value: %v", err) + } +} diff --git a/internal/config/keys.go b/internal/config/keys.go new file mode 100644 index 0000000..c084373 --- /dev/null +++ b/internal/config/keys.go @@ -0,0 +1,78 @@ +package config + +import ( + "fmt" + "os" + "strings" +) + +// Secret key references. Values are never logged and never stored by the +// harness; only their resolved bytes reach the outgoing HTTP headers. +// +// Formats: +// +// env:NAME environment variable (must be set and non-empty) +// file:PATH file whose trimmed contents are the key +// literal:VALUE inline key (last resort; still never logged) +// bw:REF bitwarden item (reserved; lands with the bw wrapper, phase 3) +func CheckKeyRef(ref string) error { + switch { + case ref == "": + return fmt.Errorf("empty key reference") + case strings.HasPrefix(ref, "env:"): + if len(ref) <= 4 { + return fmt.Errorf("key ref %q: env: needs a variable name", redact(ref)) + } + case strings.HasPrefix(ref, "file:"): + if len(ref) <= 5 { + return fmt.Errorf("key ref %q: file: needs a path", redact(ref)) + } + case strings.HasPrefix(ref, "literal:"): + if len(ref) <= 8 { + return fmt.Errorf("key ref: literal: value is empty") + } + case strings.HasPrefix(ref, "bw:"): + return fmt.Errorf("key ref bw: not implemented yet (bitwarden wrapper lands in build phase 3)") + default: + return fmt.Errorf("key ref must use env:, file:, literal:, or bw: prefixes (unrecognized ref redacted)") + } + return nil +} + +// ResolveKeyRef resolves a key reference to its value. Never log the result. +func ResolveKeyRef(ref string) (string, error) { + if err := CheckKeyRef(ref); err != nil { + return "", err + } + switch { + case strings.HasPrefix(ref, "env:"): + v := os.Getenv(ref[4:]) + if v == "" { + return "", fmt.Errorf("environment variable %s is not set", ref[4:]) + } + return v, nil + case strings.HasPrefix(ref, "file:"): + data, err := os.ReadFile(ref[5:]) + if err != nil { + return "", fmt.Errorf("read key file: %w", err) + } + v := strings.TrimSpace(string(data)) + if v == "" { + return "", fmt.Errorf("key file %s is empty", ref[5:]) + } + return v, nil + case strings.HasPrefix(ref, "literal:"): + return ref[8:], nil + } + return "", fmt.Errorf("unreachable key ref branch") +} + +// redact masks everything after a recognized prefix so malformed refs can +// be reported without echoing a possibly-pasted secret. Refs without a +// recognizable prefix are never echoed at all. +func redact(ref string) string { + if i := strings.IndexByte(ref, ':'); i >= 0 && len(ref) > i+1 { + return ref[:i+1] + "****" + } + return "" +} diff --git a/internal/config/toml.go b/internal/config/toml.go new file mode 100644 index 0000000..ec8250f --- /dev/null +++ b/internal/config/toml.go @@ -0,0 +1,425 @@ +// Package config implements the harness.toml surface for the MOPAC harness. +// +// The TOML parser in toml.go is a deliberate stdlib-only subset: tables, +// bare keys, basic strings, integers, booleans, and (single- or multi-line) +// string/int/bool arrays. Anything richer is a parse error so misconfigured +// files fail loudly instead of silently. +package config + +import ( + "fmt" + "os" + "sort" + "strings" +) + +// TOMLDoc is the parsed representation of harness.toml: nested tables are +// TOMLDoc values; leaves are string, int64, bool, or []any of those. +type TOMLDoc map[string]any + +// Table returns the nested table at the given key path, or an empty doc. +func (d TOMLDoc) Table(keys ...string) TOMLDoc { + cur := any(d) + for _, k := range keys { + m, ok := cur.(TOMLDoc) + if !ok { + return TOMLDoc{} + } + cur = m[k] + } + if m, ok := cur.(TOMLDoc); ok { + return m + } + return TOMLDoc{} +} + +func (d TOMLDoc) String(key string) (string, bool) { + v, ok := d[key].(string) + return v, ok +} + +func (d TOMLDoc) Int(key string) (int64, bool) { + v, ok := d[key].(int64) + return v, ok +} + +func (d TOMLDoc) Bool(key string) (bool, bool) { + v, ok := d[key].(bool) + return v, ok +} + +func (d TOMLDoc) StringList(key string) ([]string, bool) { + raw, ok := d[key].([]any) + if !ok { + return nil, false + } + out := make([]string, 0, len(raw)) + for _, e := range raw { + s, ok := e.(string) + if !ok { + return nil, false + } + out = append(out, s) + } + return out, true +} + +// Keys returns the table's key names in sorted order. +func (d TOMLDoc) Keys() []string { + out := make([]string, 0, len(d)) + for k := range d { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// ParseTOML parses the harness.toml subset. +func ParseTOML(src string) (TOMLDoc, error) { + lines, err := logicalLines(src) + if err != nil { + return nil, err + } + root := TOMLDoc{} + cur := root + for _, ln := range lines { + if strings.HasPrefix(ln.text, "[") { + path, err := parseTableName(ln.text, ln.no) + if err != nil { + return nil, err + } + cur, err = descend(root, path, ln.no) + if err != nil { + return nil, err + } + continue + } + if err := parseKeyValue(cur, ln.text, ln.no); err != nil { + return nil, err + } + } + return root, nil +} + +type logicalLine struct { + no int + text string +} + +// logicalLines joins physical lines into logical ones: comments are stripped +// (quote-aware), blank lines dropped, and unbalanced array brackets pull the +// next line in so multi-line arrays work. Strings may not span lines. +func logicalLines(src string) ([]logicalLine, error) { + var out []logicalLine + var buf strings.Builder + curNo := 0 + depth := 0 + for i, raw := range strings.Split(src, "\n") { + no := i + 1 + line := stripComment(strings.TrimSuffix(raw, "\r")) + if buf.Len() == 0 { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + curNo = no + buf.WriteString(trimmed) + } else { + buf.WriteString(" ") + buf.WriteString(strings.TrimSpace(line)) + } + inStr, esc := false, false + d := 0 + for _, r := range line { + switch { + case esc: + esc = false + case inStr && r == '\\': + esc = true + case r == '"': + inStr = !inStr + case !inStr && r == '[': + d++ + case !inStr && r == ']': + d-- + } + } + if inStr { + return nil, fmt.Errorf("harness.toml:%d: unterminated string", no) + } + if depth+d < 0 { + return nil, fmt.Errorf("harness.toml:%d: unbalanced ']'", no) + } + depth += d + if depth == 0 { + out = append(out, logicalLine{no: curNo, text: buf.String()}) + buf.Reset() + } + } + if depth != 0 { + return nil, fmt.Errorf("harness.toml:%d: unterminated array (missing ']')", curNo) + } + return out, nil +} + +// stripComment removes a trailing # comment, ignoring # inside strings. +func stripComment(line string) string { + inStr, esc := false, false + for i, r := range line { + switch { + case esc: + esc = false + case inStr && r == '\\': + esc = true + case r == '"': + inStr = !inStr + case r == '#' && !inStr: + return line[:i] + } + } + return line +} + +func parseTableName(text string, no int) ([]string, error) { + if !strings.HasSuffix(text, "]") { + return nil, fmt.Errorf("harness.toml:%d: malformed table header %q", no, text) + } + inner := strings.TrimSuffix(strings.TrimPrefix(text, "["), "]") + if inner == "" { + return nil, fmt.Errorf("harness.toml:%d: empty table name", no) + } + parts := strings.Split(inner, ".") + for _, p := range parts { + if !isBareKey(p) { + return nil, fmt.Errorf("harness.toml:%d: invalid table name segment %q", no, p) + } + } + return parts, nil +} + +func descend(root TOMLDoc, path []string, no int) (TOMLDoc, error) { + cur := root + for _, k := range path { + if existing, ok := cur[k]; ok { + sub, ok := existing.(TOMLDoc) + if !ok { + return nil, fmt.Errorf("harness.toml:%d: key %q already has a value, cannot become table", no, k) + } + cur = sub + continue + } + sub := TOMLDoc{} + cur[k] = sub + cur = sub + } + return cur, nil +} + +func parseKeyValue(m TOMLDoc, text string, no int) error { + eq := strings.IndexByte(text, '=') + if eq < 0 { + return fmt.Errorf("harness.toml:%d: expected key = value, got %q", no, text) + } + key := strings.TrimSpace(text[:eq]) + if !isBareKey(key) { + return fmt.Errorf("harness.toml:%d: invalid key %q", no, key) + } + valStr := strings.TrimSpace(text[eq+1:]) + if valStr == "" { + return fmt.Errorf("harness.toml:%d: missing value for key %q", no, key) + } + val, err := parseValue(valStr, no) + if err != nil { + return err + } + if _, exists := m[key]; exists { + return fmt.Errorf("harness.toml:%d: duplicate key %q", no, key) + } + m[key] = val + return nil +} + +func isBareKey(s string) bool { + if s == "" { + return false + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '-': + default: + return false + } + } + return true +} + +func parseValue(v string, no int) (any, error) { + switch { + case strings.HasPrefix(v, `"`): + return parseStringValue(v, no) + case strings.HasPrefix(v, "["): + return parseArrayValue(v, no) + default: + return parseScalar(v, no) + } +} + +func parseScalar(v string, no int) (any, error) { + switch v { + case "true": + return true, nil + case "false": + return false, nil + } + n, err := parseTOMLInt(v) + if err != nil { + return nil, fmt.Errorf("harness.toml:%d: unsupported value %q (want string, int, bool, or array)", no, v) + } + return n, nil +} + +func parseTOMLInt(v string) (int64, error) { + s := strings.ReplaceAll(v, "_", "") + return parseSignedInt(s) +} + +func parseSignedInt(s string) (int64, error) { + var n int64 + i := 0 + neg := false + if i < len(s) && (s[i] == '+' || s[i] == '-') { + neg = s[i] == '-' + i++ + } + if i >= len(s) { + return 0, fmt.Errorf("not an integer: %q", s) + } + for ; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return 0, fmt.Errorf("not an integer: %q", s) + } + n = n*10 + int64(s[i]-'0') + if n < 0 { + return 0, fmt.Errorf("integer overflow: %q", s) + } + } + if neg { + n = -n + } + return n, nil +} + +func parseStringValue(v string, no int) (string, error) { + if len(v) < 2 { + return "", fmt.Errorf("harness.toml:%d: unterminated string %q", no, v) + } + var b strings.Builder + i := 1 + for i < len(v) { + c := v[i] + if c == '"' { + if rest := strings.TrimSpace(v[i+1:]); rest != "" { + return "", fmt.Errorf("harness.toml:%d: unexpected trailing content after string: %q", no, rest) + } + return b.String(), nil + } + if c != '\\' { + b.WriteByte(c) + i++ + continue + } + if i+1 >= len(v) { + return "", fmt.Errorf("harness.toml:%d: dangling escape in %q", no, v) + } + switch v[i+1] { + case '"': + b.WriteByte('"') + case '\\': + b.WriteByte('\\') + case 'n': + b.WriteByte('\n') + case 't': + b.WriteByte('\t') + case 'r': + b.WriteByte('\r') + default: + return "", fmt.Errorf("harness.toml:%d: unsupported escape \\%c", no, v[i+1]) + } + i += 2 + } + return "", fmt.Errorf("harness.toml:%d: unterminated string", no) +} + +func parseArrayValue(v string, no int) ([]any, error) { + if !strings.HasSuffix(v, "]") { + return nil, fmt.Errorf("harness.toml:%d: unterminated array %q", no, v) + } + inner := strings.TrimSpace(v[1 : len(v)-1]) + if inner == "" { + return []any{}, nil + } + var out []any + elems := splitTopLevel(inner, ',') + if len(elems) > 0 && strings.TrimSpace(elems[len(elems)-1]) == "" { + elems = elems[:len(elems)-1] // tolerate a trailing comma + } + for _, elem := range elems { + elem = strings.TrimSpace(elem) + if elem == "" { + return nil, fmt.Errorf("harness.toml:%d: empty array element in %q", no, v) + } + val, err := parseValue(elem, no) + if err != nil { + return nil, err + } + out = append(out, val) + } + return out, nil +} + +// splitTopLevel splits on sep outside of double-quoted strings. +func splitTopLevel(s string, sep rune) []string { + var out []string + var cur strings.Builder + inStr, esc := false, false + for _, r := range s { + switch { + case esc: + esc = false + cur.WriteRune(r) + case inStr && r == '\\': + esc = true + cur.WriteRune(r) + case r == '"': + inStr = !inStr + cur.WriteRune(r) + case r == sep && !inStr: + out = append(out, cur.String()) + cur.Reset() + default: + cur.WriteRune(r) + } + } + out = append(out, cur.String()) + return out +} + +// Load reads, parses, and validates harness.toml at path. +func Load(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("config: read %s: %w", path, err) + } + doc, err := ParseTOML(string(data)) + if err != nil { + return nil, fmt.Errorf("config: %w", err) + } + cfg := Default() + if err := cfg.apply(doc); err != nil { + return nil, fmt.Errorf("config: %w", err) + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("config: %w", err) + } + return cfg, nil +} diff --git a/internal/config/toml_test.go b/internal/config/toml_test.go new file mode 100644 index 0000000..1ccb44a --- /dev/null +++ b/internal/config/toml_test.go @@ -0,0 +1,139 @@ +package config + +import ( + "strings" + "testing" +) + +func TestParseTOMLValid(t *testing.T) { + src := ` +# top comment +vertical = "demo" # trailing comment +work_root = "." + +[loop] +max_rounds = 4 + +[redmine] +url = "https://rm.example.org" +scope_query_id = 42 +class_field = "Class" + +[litellm] +base_url = "http://h:4001" +timeout_secs = 30 +max_retries = 3 + +[models] +mopac-study = "glm-4.7-flash" # hyphenated bare key +default_tier = "mopac-study" + +[models.classes] +study = "mopac-study" +code = "mopac-code" + +[tools.bash] +enabled = false +default = "deny" +allow = ["pwd", "ls *", "cat *"] +deny = [ + "sudo *", # comment inside array + "ssh *", +] + +[deep.nested.table] +flag = true +note = "quote \" and # inside string" +empty = [] +` + doc, err := ParseTOML(src) + if err != nil { + t.Fatalf("ParseTOML: %v", err) + } + tests := []struct { + name string + got any + want any + }{ + {"vertical", str(doc.String("vertical")), "demo"}, + {"work_root", str(doc.String("work_root")), "."}, + {"max_rounds", intg(doc.Table("loop").Int("max_rounds")), int64(4)}, + {"scope_query_id", intg(doc.Table("redmine").Int("scope_query_id")), int64(42)}, + {"class_field", str(doc.Table("redmine").String("class_field")), "Class"}, + {"timeout_secs", intg(doc.Table("litellm").Int("timeout_secs")), int64(30)}, + {"max_retries", intg(doc.Table("litellm").Int("max_retries")), int64(3)}, + {"tier", str(doc.Table("models").String("mopac-study")), "glm-4.7-flash"}, + {"default_tier", str(doc.Table("models").String("default_tier")), "mopac-study"}, + {"class map", str(doc.Table("models", "classes").String("study")), "mopac-study"}, + {"bash enabled", boolean(doc.Table("tools", "bash").Bool("enabled")), false}, + {"deep bool", boolean(doc.Table("deep", "nested", "table").Bool("flag")), true}, + {"escaped string", str(doc.Table("deep", "nested", "table").String("note")), `quote " and # inside string`}, + } + for _, tc := range tests { + if tc.got != tc.want { + t.Errorf("%s = %v, want %v", tc.name, tc.got, tc.want) + } + } + + allow, ok := doc.Table("tools", "bash").StringList("allow") + if !ok || len(allow) != 3 || allow[1] != "ls *" { + t.Errorf("allow list = %v ok=%v", allow, ok) + } + deny, ok := doc.Table("tools", "bash").StringList("deny") + if !ok || len(deny) != 2 || deny[0] != "sudo *" { + t.Errorf("deny list = %v ok=%v", deny, ok) + } + if empty, ok := doc.Table("deep", "nested", "table").StringList("empty"); !ok || len(empty) != 0 { + t.Errorf("empty array = %v ok=%v", empty, ok) + } +} + +func str(v string, ok bool) string { + if !ok { + panic("string key missing") + } + return v +} + +func intg(v int64, ok bool) int64 { + if !ok { + panic("int key missing") + } + return v +} + +func boolean(v bool, ok bool) bool { + if !ok { + panic("bool key missing") + } + return v +} + +func TestParseTOMLErrors(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + {"unterminated string", `a = "oops`, "unterminated string"}, + {"bare value", `a = sk-123`, "unsupported value"}, + {"missing value", `a =`, "missing value"}, + {"bad key", `a b = "c"`, "invalid key"}, + {"duplicate key", "a = \"1\"\na = \"2\"", "duplicate key"}, + {"scalar to table", "a = \"1\"\n[a.b]", "cannot become table"}, + {"unterminated array", "a = [\n\"x\",", "unterminated array"}, + {"bad table name", `[a.]`, "invalid table name"}, + {"no equals", `[t]\njustakey`, "expected key = value"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := ParseTOML(strings.ReplaceAll(tc.src, `\n`, "\n")) + if err == nil { + t.Fatalf("expected error containing %q, got nil", tc.want) + } + if !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error %q does not contain %q", err, tc.want) + } + }) + } +} diff --git a/internal/models/models.go b/internal/models/models.go new file mode 100644 index 0000000..70068e5 --- /dev/null +++ b/internal/models/models.go @@ -0,0 +1,68 @@ +// Package models implements MODEL ROUTING v0: a static, config-only map from +// task class -> tier alias -> concrete proxy model. There is deliberately no +// heuristic code: the tier map in harness.toml is the whole contract, and an +// unknown class is a hard error so routing stays auditable. +package models + +import ( + "fmt" + "sort" +) + +// Router resolves task classes to concrete model names. +type Router struct { + tiers map[string]string // tier alias -> concrete model + classes map[string]string // task class -> tier alias + defaultTier string +} + +// Decision records one routing outcome for the REPORT trail. +type Decision struct { + Class string + Tier string + Model string +} + +// NewRouter validates the config maps and returns a Router. +func NewRouter(tiers, classes map[string]string, defaultTier string) (*Router, error) { + if len(tiers) == 0 { + return nil, fmt.Errorf("models: tier map is empty") + } + if _, ok := tiers[defaultTier]; !ok { + return nil, fmt.Errorf("models: default tier %q has no model entry", defaultTier) + } + for class, tier := range classes { + if _, ok := tiers[tier]; !ok { + return nil, fmt.Errorf("models: class %q points at unknown tier %q", class, tier) + } + } + return &Router{tiers: tiers, classes: classes, defaultTier: defaultTier}, nil +} + +// Resolve maps a task class to its tier and concrete model. An empty class +// falls back to the default tier; an unknown class is an error. +func (r *Router) Resolve(class string) (Decision, error) { + tier := r.defaultTier + if class != "" { + t, ok := r.classes[class] + if !ok { + return Decision{}, fmt.Errorf("models: unknown task class %q (add it to [models.classes] in harness.toml)", class) + } + tier = t + } + model, ok := r.tiers[tier] + if !ok { + return Decision{}, fmt.Errorf("models: tier %q has no model entry", tier) + } + return Decision{Class: class, Tier: tier, Model: model}, nil +} + +// Tiers returns the sorted tier aliases known to the router. +func (r *Router) Tiers() []string { + out := make([]string, 0, len(r.tiers)) + for k := range r.tiers { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/models/models_test.go b/internal/models/models_test.go new file mode 100644 index 0000000..2517de8 --- /dev/null +++ b/internal/models/models_test.go @@ -0,0 +1,78 @@ +package models + +import ( + "strings" + "testing" +) + +func testRouter(t *testing.T) *Router { + t.Helper() + tiers := map[string]string{ + "mopac-study": "glm-4.7-flash", + "mopac-code": "glm-5.2", + "mopac-review": "glm-5-turbo", + "mopac-primary": "glm-5.3", + "mopac-vision": "glm-4.6v", + } + classes := map[string]string{ + "study": "mopac-study", + "read": "mopac-study", + "code": "mopac-code", + "review": "mopac-review", + "primary": "mopac-primary", + } + r, err := NewRouter(tiers, classes, "mopac-primary") + if err != nil { + t.Fatalf("NewRouter: %v", err) + } + return r +} + +func TestResolve(t *testing.T) { + cases := []struct { + class string + tier string + model string + }{ + {"study", "mopac-study", "glm-4.7-flash"}, + {"read", "mopac-study", "glm-4.7-flash"}, + {"code", "mopac-code", "glm-5.2"}, + {"review", "mopac-review", "glm-5-turbo"}, + {"primary", "mopac-primary", "glm-5.3"}, + {"", "mopac-primary", "glm-5.3"}, // no class -> default tier + } + r := testRouter(t) + for _, tc := range cases { + d, err := r.Resolve(tc.class) + if err != nil { + t.Fatalf("Resolve(%q): %v", tc.class, err) + } + if d.Tier != tc.tier || d.Model != tc.model { + t.Errorf("Resolve(%q) = %s/%s, want %s/%s", tc.class, d.Tier, d.Model, tc.tier, tc.model) + } + } +} + +func TestResolveUnknownClassIsError(t *testing.T) { + r := testRouter(t) + _, err := r.Resolve("urgent") + if err == nil { + t.Fatal("unknown class must error (routing stays auditable)") + } + if !strings.Contains(err.Error(), `[models.classes]`) { + t.Errorf("error should point at the config section: %v", err) + } +} + +func TestNewRouterValidation(t *testing.T) { + tiers := map[string]string{"mopac-primary": "glm-5.3"} + if _, err := NewRouter(tiers, nil, "mopac-missing"); err == nil || !strings.Contains(err.Error(), "default tier") { + t.Errorf("missing default tier: err=%v", err) + } + if _, err := NewRouter(tiers, map[string]string{"x": "mopac-nope"}, "mopac-primary"); err == nil || !strings.Contains(err.Error(), "unknown tier") { + t.Errorf("class to unknown tier: err=%v", err) + } + if _, err := NewRouter(nil, nil, "mopac-primary"); err == nil { + t.Errorf("empty tiers must error") + } +}