package loop import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "ukrrs.com/mopac/harness/internal/config" "ukrrs.com/mopac/harness/internal/llm" ) // fakeLLM is a scripted OpenAI-compatible server that records every request. type fakeLLM struct { mu sync.Mutex requests []llm.ChatRequest srv *httptest.Server } func toolCallMsg(id, name, args string) string { return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`, id, name, args) } func textMsg(content string) string { return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content) } // newFakeLLM serves the scripted response bodies in order (repeating the // last one if the turn asks for more). func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM { t.Helper() f := &fakeLLM{} f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req llm.ChatRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("decode request: %v", err) } f.mu.Lock() f.requests = append(f.requests, req) n := len(f.requests) f.mu.Unlock() body := bodies[len(bodies)-1] if n <= len(bodies) { body = bodies[n-1] } w.Header().Set("Content-Type", "application/json") w.Write([]byte(body)) })) t.Cleanup(f.srv.Close) return f } func (f *fakeLLM) requestCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.requests) } func (f *fakeLLM) request(i int) llm.ChatRequest { f.mu.Lock() defer f.mu.Unlock() return f.requests[i] } func testConfig(t *testing.T, llmURL string) *config.Config { t.Helper() dir := t.TempDir() cfg := config.Default() cfg.Vertical = "teststack" cfg.WorkRoot = dir cfg.ReportDir = filepath.Join(dir, "reports") cfg.LiteLLM.BaseURL = llmURL cfg.LiteLLM.KeyRef = "literal:test-key" cfg.LiteLLM.MaxRetries = 0 cfg.Models.Tiers = map[string]string{ "mopac-study": "glm-4.7-flash", "mopac-code": "glm-5.2", "mopac-review": "glm-5-turbo", "mopac-primary": "glm-5.3", } cfg.Models.Classes = map[string]string{ "study": "mopac-study", "code": "mopac-code", "primary": "mopac-primary", } cfg.Bash.Allow = []string{"echo *", "pwd"} cfg.Bash.Deny = []string{"sudo *"} return cfg } func TestOnceDemoEndToEnd(t *testing.T) { f := newFakeLLM(t, textMsg("I am GLM, a large language model.")) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("Once: %v", err) } // MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT. if f.requestCount() != 1 { t.Fatalf("LLM calls = %d, want 1", f.requestCount()) } req := f.request(0) if req.Model != "glm-5.3" { t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model) } if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" { t.Errorf("messages = %+v", req.Messages) } if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" { t.Errorf("turn = %+v", res.Turn) } if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" { t.Errorf("routed = %+v", res.Routed) } body, err := os.ReadFile(res.ReportPath) if err != nil { t.Fatalf("REPORT: %v", err) } s := string(body) for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} { if !strings.Contains(s, want) { t.Errorf("REPORT missing %q", want) } } if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil { t.Errorf("REPORT-latest.md: %v", err) } } func TestOnceDryRunMakesNoLLMCall(t *testing.T) { f := newFakeLLM(t, textMsg("unused")) cfg := testConfig(t, f.srv.URL) var out strings.Builder cond, err := New(cfg, &out) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true}) if err != nil { t.Fatalf("Once: %v", err) } if f.requestCount() != 0 { t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount()) } if res.ReportPath != "" { t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath) } for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} { if !strings.Contains(out.String(), want) { t.Errorf("plan output missing %q:\n%s", want, out.String()) } } } func TestOnceToolRoundTrip(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`), textMsg("Tool ran fine."), ) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("Once: %v", err) } if f.requestCount() != 2 { t.Fatalf("LLM calls = %d, want 2", f.requestCount()) } if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 { t.Errorf("turn = %+v", res.Turn) } // Second request must carry the assistant tool_call and the tool result. second := f.request(1) roles := make([]string, len(second.Messages)) for i, m := range second.Messages { roles[i] = m.Role } want := []string{"system", "user", "assistant", "tool"} if fmt.Sprint(roles) != fmt.Sprint(want) { t.Errorf("second request roles = %v, want %v", roles, want) } toolMsg := second.Messages[3] if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") { t.Errorf("tool message = %+v", toolMsg) } } func TestOnceToolDeniedCounts(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`), textMsg("Understood, permission was denied."), ) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("denied tool call must not fail the turn: %v", err) } if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 { t.Errorf("turn = %+v", res.Turn) } toolMsg := f.request(1).Messages[3] if !strings.Contains(toolMsg.Content, "permission denied") { t.Errorf("model should see the denial, got %q", toolMsg.Content) } } func TestOnceRoundLimitBounded(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`)) cfg := testConfig(t, f.srv.URL) cfg.Loop.MaxRounds = 3 cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("round limit is not an error: %v", err) } if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" { t.Errorf("turn = %+v", res.Turn) } if f.requestCount() != 3 { t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount()) } if res.ReportPath == "" { t.Errorf("round_limit turn should still write a REPORT") } } func TestOnceLLMFailureMapsToErrLLM(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no key for you", http.StatusUnauthorized) })) defer srv.Close() cfg := testConfig(t, srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true}) if !errors.Is(err, ErrLLM) { t.Fatalf("err = %v, want ErrLLM", err) } } func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "down", http.StatusInternalServerError) })) defer srv.Close() cfg := testConfig(t, "http://unused") cfg.Redmine.URL = srv.URL cfg.Redmine.KeyRef = "literal:rm-key" cfg.Redmine.ScopeQuery = "project=x" cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{}) if !errors.Is(err, ErrIntake) { t.Fatalf("err = %v, want ErrIntake", err) } } func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) { cfg := testConfig(t, "http://unused") cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{}) if !errors.Is(err, ErrIntake) { t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err) } } func TestOnceTaskIDFilter(t *testing.T) { cfg := testConfig(t, "http://unused") cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"}) if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") { t.Fatalf("err = %v, want not-in-scope ErrIntake", err) } } func TestOnceUnknownClassIsRoutingError(t *testing.T) { cfg := testConfig(t, "http://unused") cfg.Demo.Class = "definitely-not-a-class" cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true}) if err == nil || !strings.Contains(err.Error(), "unknown task class") { t.Fatalf("err = %v, want unknown-class error", err) } }