mrcharles
62d3781d71
`harness once` runs one bounded iteration (intake -> routing -> bounded turn
-> REPORT) then exits, so callers chain it without a daemon. The bounded
turn loops request -> tool calls -> results up to max_rounds; gate denials
feed back to the model and are counted instead of failing the turn.
--dry-run prints the plan (task, resolved model, tool bounds) and makes zero
LLM calls; --demo runs the [demo] issue ("tell me about yourself" through
LiteLLM -> GLM self-description as the REPORT), the MVP acceptance bar.
Distinct exit codes for config/usage, intake, and llm failures. Ships
harness.toml.example (scope query, tier map, allow-lists); real configs are
gitignored. Loop tests run end-to-end against a scripted fake OpenAI server:
demo turn, dry-run zero-call, tool round-trip, denial counting, round limit,
and error-class mapping.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
320 lines
9.4 KiB
Go
320 lines
9.4 KiB
Go
package loop
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
|
|
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
|
|
)
|
|
|
|
// fakeLLM is a scripted OpenAI-compatible server that records every request.
|
|
type fakeLLM struct {
|
|
mu sync.Mutex
|
|
requests []llm.ChatRequest
|
|
srv *httptest.Server
|
|
}
|
|
|
|
func toolCallMsg(id, name, args string) string {
|
|
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`,
|
|
id, name, args)
|
|
}
|
|
|
|
func textMsg(content string) string {
|
|
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content)
|
|
}
|
|
|
|
// newFakeLLM serves the scripted response bodies in order (repeating the
|
|
// last one if the turn asks for more).
|
|
func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM {
|
|
t.Helper()
|
|
f := &fakeLLM{}
|
|
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var req llm.ChatRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Errorf("decode request: %v", err)
|
|
}
|
|
f.mu.Lock()
|
|
f.requests = append(f.requests, req)
|
|
n := len(f.requests)
|
|
f.mu.Unlock()
|
|
body := bodies[len(bodies)-1]
|
|
if n <= len(bodies) {
|
|
body = bodies[n-1]
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(body))
|
|
}))
|
|
t.Cleanup(f.srv.Close)
|
|
return f
|
|
}
|
|
|
|
func (f *fakeLLM) requestCount() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.requests)
|
|
}
|
|
|
|
func (f *fakeLLM) request(i int) llm.ChatRequest {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.requests[i]
|
|
}
|
|
|
|
func testConfig(t *testing.T, llmURL string) *config.Config {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
cfg := config.Default()
|
|
cfg.Vertical = "teststack"
|
|
cfg.WorkRoot = dir
|
|
cfg.ReportDir = filepath.Join(dir, "reports")
|
|
cfg.LiteLLM.BaseURL = llmURL
|
|
cfg.LiteLLM.KeyRef = "literal:test-key"
|
|
cfg.LiteLLM.MaxRetries = 0
|
|
cfg.Models.Tiers = map[string]string{
|
|
"mopac-study": "glm-4.7-flash",
|
|
"mopac-code": "glm-5.2",
|
|
"mopac-review": "glm-5-turbo",
|
|
"mopac-primary": "glm-5.3",
|
|
}
|
|
cfg.Models.Classes = map[string]string{
|
|
"study": "mopac-study",
|
|
"code": "mopac-code",
|
|
"primary": "mopac-primary",
|
|
}
|
|
cfg.Bash.Allow = []string{"echo *", "pwd"}
|
|
cfg.Bash.Deny = []string{"sudo *"}
|
|
return cfg
|
|
}
|
|
|
|
func TestOnceDemoEndToEnd(t *testing.T) {
|
|
f := newFakeLLM(t, textMsg("I am GLM, a large language model."))
|
|
cfg := testConfig(t, f.srv.URL)
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if err != nil {
|
|
t.Fatalf("Once: %v", err)
|
|
}
|
|
// MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT.
|
|
if f.requestCount() != 1 {
|
|
t.Fatalf("LLM calls = %d, want 1", f.requestCount())
|
|
}
|
|
req := f.request(0)
|
|
if req.Model != "glm-5.3" {
|
|
t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model)
|
|
}
|
|
if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" {
|
|
t.Errorf("messages = %+v", req.Messages)
|
|
}
|
|
if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" {
|
|
t.Errorf("turn = %+v", res.Turn)
|
|
}
|
|
if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" {
|
|
t.Errorf("routed = %+v", res.Routed)
|
|
}
|
|
body, err := os.ReadFile(res.ReportPath)
|
|
if err != nil {
|
|
t.Fatalf("REPORT: %v", err)
|
|
}
|
|
s := string(body)
|
|
for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} {
|
|
if !strings.Contains(s, want) {
|
|
t.Errorf("REPORT missing %q", want)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil {
|
|
t.Errorf("REPORT-latest.md: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOnceDryRunMakesNoLLMCall(t *testing.T) {
|
|
f := newFakeLLM(t, textMsg("unused"))
|
|
cfg := testConfig(t, f.srv.URL)
|
|
var out strings.Builder
|
|
cond, err := New(cfg, &out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true})
|
|
if err != nil {
|
|
t.Fatalf("Once: %v", err)
|
|
}
|
|
if f.requestCount() != 0 {
|
|
t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount())
|
|
}
|
|
if res.ReportPath != "" {
|
|
t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath)
|
|
}
|
|
for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} {
|
|
if !strings.Contains(out.String(), want) {
|
|
t.Errorf("plan output missing %q:\n%s", want, out.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOnceToolRoundTrip(t *testing.T) {
|
|
f := newFakeLLM(t,
|
|
toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`),
|
|
textMsg("Tool ran fine."),
|
|
)
|
|
cfg := testConfig(t, f.srv.URL)
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if err != nil {
|
|
t.Fatalf("Once: %v", err)
|
|
}
|
|
if f.requestCount() != 2 {
|
|
t.Fatalf("LLM calls = %d, want 2", f.requestCount())
|
|
}
|
|
if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 {
|
|
t.Errorf("turn = %+v", res.Turn)
|
|
}
|
|
// Second request must carry the assistant tool_call and the tool result.
|
|
second := f.request(1)
|
|
roles := make([]string, len(second.Messages))
|
|
for i, m := range second.Messages {
|
|
roles[i] = m.Role
|
|
}
|
|
want := []string{"system", "user", "assistant", "tool"}
|
|
if fmt.Sprint(roles) != fmt.Sprint(want) {
|
|
t.Errorf("second request roles = %v, want %v", roles, want)
|
|
}
|
|
toolMsg := second.Messages[3]
|
|
if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") {
|
|
t.Errorf("tool message = %+v", toolMsg)
|
|
}
|
|
}
|
|
|
|
func TestOnceToolDeniedCounts(t *testing.T) {
|
|
f := newFakeLLM(t,
|
|
toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`),
|
|
textMsg("Understood, permission was denied."),
|
|
)
|
|
cfg := testConfig(t, f.srv.URL)
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if err != nil {
|
|
t.Fatalf("denied tool call must not fail the turn: %v", err)
|
|
}
|
|
if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 {
|
|
t.Errorf("turn = %+v", res.Turn)
|
|
}
|
|
toolMsg := f.request(1).Messages[3]
|
|
if !strings.Contains(toolMsg.Content, "permission denied") {
|
|
t.Errorf("model should see the denial, got %q", toolMsg.Content)
|
|
}
|
|
}
|
|
|
|
func TestOnceRoundLimitBounded(t *testing.T) {
|
|
f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`))
|
|
cfg := testConfig(t, f.srv.URL)
|
|
cfg.Loop.MaxRounds = 3
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if err != nil {
|
|
t.Fatalf("round limit is not an error: %v", err)
|
|
}
|
|
if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" {
|
|
t.Errorf("turn = %+v", res.Turn)
|
|
}
|
|
if f.requestCount() != 3 {
|
|
t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount())
|
|
}
|
|
if res.ReportPath == "" {
|
|
t.Errorf("round_limit turn should still write a REPORT")
|
|
}
|
|
}
|
|
|
|
func TestOnceLLMFailureMapsToErrLLM(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "no key for you", http.StatusUnauthorized)
|
|
}))
|
|
defer srv.Close()
|
|
cfg := testConfig(t, srv.URL)
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if !errors.Is(err, ErrLLM) {
|
|
t.Fatalf("err = %v, want ErrLLM", err)
|
|
}
|
|
}
|
|
|
|
func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "down", http.StatusInternalServerError)
|
|
}))
|
|
defer srv.Close()
|
|
cfg := testConfig(t, "http://unused")
|
|
cfg.Redmine.URL = srv.URL
|
|
cfg.Redmine.KeyRef = "literal:rm-key"
|
|
cfg.Redmine.ScopeQuery = "project=x"
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = cond.Once(context.Background(), OnceOpts{})
|
|
if !errors.Is(err, ErrIntake) {
|
|
t.Fatalf("err = %v, want ErrIntake", err)
|
|
}
|
|
}
|
|
|
|
func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) {
|
|
cfg := testConfig(t, "http://unused")
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = cond.Once(context.Background(), OnceOpts{})
|
|
if !errors.Is(err, ErrIntake) {
|
|
t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err)
|
|
}
|
|
}
|
|
|
|
func TestOnceTaskIDFilter(t *testing.T) {
|
|
cfg := testConfig(t, "http://unused")
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"})
|
|
if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") {
|
|
t.Fatalf("err = %v, want not-in-scope ErrIntake", err)
|
|
}
|
|
}
|
|
|
|
func TestOnceUnknownClassIsRoutingError(t *testing.T) {
|
|
cfg := testConfig(t, "http://unused")
|
|
cfg.Demo.Class = "definitely-not-a-class"
|
|
cond, err := New(cfg, os.Stdout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
|
|
if err == nil || !strings.Contains(err.Error(), "unknown task class") {
|
|
t.Fatalf("err = %v, want unknown-class error", err)
|
|
}
|
|
}
|