harness: config layer + model routing v0 (tier map, no heuristics)
harness.toml loading via a stdlib-only TOML subset parser (tables, bare
keys, strings/ints/bools, multi-line arrays; anything richer fails loudly).
Secrets are refs only (env:/file:/literal:, bw: reserved) and are redacted
from every error path. Model routing v0: static class -> tier alias ->
concrete model map per the DESIGN model-selection layer; requests carry the
resolved concrete model and unknown classes fail hard so routing stays
auditable. Table-driven tests cover the parser, validation, key refs, and
routing decisions.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const testCfg = `
|
||||
vertical = "teststack"
|
||||
work_root = "/tmp/harness-test"
|
||||
report_dir = "out"
|
||||
|
||||
[redmine]
|
||||
url = "https://rm.test"
|
||||
key_ref = "env:HARNESS_TEST_RM_KEY"
|
||||
scope_query = "project=x&status_id=open"
|
||||
|
||||
[litellm]
|
||||
base_url = "http://litellm.test:4000"
|
||||
key_ref = "env:HARNESS_TEST_LLM_KEY"
|
||||
|
||||
[models]
|
||||
mopac-study = "glm-4.7-flash"
|
||||
mopac-code = "glm-5.2"
|
||||
mopac-review = "glm-5-turbo"
|
||||
mopac-primary = "glm-5.3"
|
||||
default_tier = "mopac-primary"
|
||||
|
||||
[models.classes]
|
||||
study = "mopac-study"
|
||||
code = "mopac-code"
|
||||
primary = "mopac-primary"
|
||||
|
||||
[tools.bash]
|
||||
allow = ["pwd", "ls *"]
|
||||
deny = ["sudo *"]
|
||||
`
|
||||
|
||||
func writeTemp(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "harness.toml")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestLoadAppliesDocAndDefaults(t *testing.T) {
|
||||
cfg, err := Load(writeTemp(t, testCfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.Vertical != "teststack" || cfg.WorkRoot != "/tmp/harness-test" || cfg.ReportDir != "out" {
|
||||
t.Errorf("core fields wrong: %+v", cfg)
|
||||
}
|
||||
if cfg.Loop.MaxRounds != 8 {
|
||||
t.Errorf("default max_rounds = %d, want 8", cfg.Loop.MaxRounds)
|
||||
}
|
||||
if cfg.LiteLLM.BaseURL != "http://litellm.test:4000" || cfg.LiteLLM.MaxRetries != 2 {
|
||||
t.Errorf("litellm wrong: %+v", cfg.LiteLLM)
|
||||
}
|
||||
wantTiers := map[string]string{
|
||||
"mopac-study": "glm-4.7-flash",
|
||||
"mopac-code": "glm-5.2",
|
||||
"mopac-review": "glm-5-turbo",
|
||||
"mopac-primary": "glm-5.3",
|
||||
}
|
||||
for tier, model := range wantTiers {
|
||||
if got, ok := cfg.Models.Tiers[tier]; !ok || got != model {
|
||||
t.Errorf("tier %s = %q ok=%v, want %q", tier, got, ok, model)
|
||||
}
|
||||
}
|
||||
if cfg.Models.Classes["code"] != "mopac-code" {
|
||||
t.Errorf("class map wrong: %+v", cfg.Models.Classes)
|
||||
}
|
||||
if len(cfg.Bash.Allow) != 2 || len(cfg.Bash.Deny) != 1 {
|
||||
t.Errorf("bash lists wrong: %v / %v", cfg.Bash.Allow, cfg.Bash.Deny)
|
||||
}
|
||||
if cfg.Bash.TimeoutSecs != 60 || cfg.Bash.MaxOutputBytes != 100_000 {
|
||||
t.Errorf("bash defaults wrong: %+v", cfg.Bash)
|
||||
}
|
||||
if cfg.Demo.Prompt != "tell me about yourself" {
|
||||
t.Errorf("demo default prompt = %q", cfg.Demo.Prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadTrackedExampleFile(t *testing.T) {
|
||||
// The example ships to users; it must always parse and validate as-is.
|
||||
path := filepath.Join("..", "..", "harness.toml.example")
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Skipf("example not found: %v", err)
|
||||
}
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(harness.toml.example): %v", err)
|
||||
}
|
||||
if cfg.Vertical != "demo" || cfg.Models.Tiers["mopac-primary"] != "glm-5.3" {
|
||||
t.Errorf("example parsed wrong: %+v", cfg.Models.Tiers)
|
||||
}
|
||||
if len(cfg.Models.Classes) != 9 {
|
||||
t.Errorf("example classes = %d, want 9", len(cfg.Models.Classes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValidationErrors(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mut func(string) string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "missing vertical",
|
||||
mut: func(s string) string { return strings.Replace(s, `vertical = "teststack"`, "", 1) },
|
||||
want: "vertical is required",
|
||||
},
|
||||
{
|
||||
name: "missing litellm url",
|
||||
mut: func(s string) string { return strings.Replace(s, `base_url = "http://litellm.test:4000"`, "", 1) },
|
||||
want: "base_url is required",
|
||||
},
|
||||
{
|
||||
name: "bare key value",
|
||||
mut: func(s string) string { return strings.Replace(s, `key_ref = "env:HARNESS_TEST_LLM_KEY"`, `key_ref = "sk-or-whatever"`, 1) },
|
||||
want: "env:, file:, literal:, or bw:",
|
||||
},
|
||||
{
|
||||
name: "bw ref not implemented",
|
||||
mut: func(s string) string { return strings.Replace(s, `key_ref = "env:HARNESS_TEST_LLM_KEY"`, `key_ref = "bw:item"`, 1) },
|
||||
want: "not implemented",
|
||||
},
|
||||
{
|
||||
name: "redmine half configured",
|
||||
mut: func(s string) string { return strings.Replace(s, `scope_query = "project=x&status_id=open"`, "", 1) },
|
||||
want: "scope_query or scope_query_id is required",
|
||||
},
|
||||
{
|
||||
name: "class to unknown tier",
|
||||
mut: func(s string) string { return strings.Replace(s, `code = "mopac-code"`, `code = "mopac-nope"`, 1) },
|
||||
want: `unknown tier "mopac-nope"`,
|
||||
},
|
||||
{
|
||||
name: "default tier missing",
|
||||
mut: func(s string) string { return strings.Replace(s, `mopac-primary = "glm-5.3"`, "", 1) },
|
||||
want: `default_tier "mopac-primary" has no entry`,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := Load(writeTemp(t, tc.mut(testCfg)))
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q, got nil", tc.want)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("error %q does not contain %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDemoOnlyConfigIsValid(t *testing.T) {
|
||||
// No [redmine] at all: --demo still works.
|
||||
cfg, err := Load(writeTemp(t, `
|
||||
vertical = "demo"
|
||||
[litellm]
|
||||
base_url = "http://x:4001"
|
||||
key_ref = "literal:test-key"
|
||||
[models]
|
||||
mopac-primary = "glm-5.3"
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.Redmine.URL != "" {
|
||||
t.Errorf("redmine should be unset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveKeyRef(t *testing.T) {
|
||||
t.Setenv("HARNESS_TEST_KEY", "sekrit")
|
||||
if v, err := ResolveKeyRef("env:HARNESS_TEST_KEY"); err != nil || v != "sekrit" {
|
||||
t.Errorf("env ref = %q err=%v", v, err)
|
||||
}
|
||||
if _, err := ResolveKeyRef("env:HARNESS_TEST_UNSET"); err == nil {
|
||||
t.Errorf("unset env should error")
|
||||
}
|
||||
|
||||
f := filepath.Join(t.TempDir(), "key")
|
||||
os.WriteFile(f, []byte(" filekey\n"), 0o600)
|
||||
if v, err := ResolveKeyRef("file:" + f); err != nil || v != "filekey" {
|
||||
t.Errorf("file ref = %q err=%v", v, err)
|
||||
}
|
||||
if v, err := ResolveKeyRef("literal:abc"); err != nil || v != "abc" {
|
||||
t.Errorf("literal ref = %q err=%v", v, err)
|
||||
}
|
||||
// Secrets must never leak through the redaction path used in errors.
|
||||
_, err := ResolveKeyRef("super-secret-value")
|
||||
if err == nil || strings.Contains(err.Error(), "super-secret-value") {
|
||||
t.Errorf("bare ref error leaks value: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user