package loop import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "ukrrs.com/mopac/harness/internal/config" "ukrrs.com/mopac/harness/internal/llm" ) // fakeLLM is a scripted OpenAI-compatible server that records every request. type fakeLLM struct { mu sync.Mutex requests []llm.ChatRequest srv *httptest.Server } func toolCallMsg(id, name, args string) string { return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`, id, name, args) } func textMsg(content string) string { return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content) } // newFakeLLM serves the scripted response bodies in order (repeating the // last one if the turn asks for more). func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM { t.Helper() f := &fakeLLM{} f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req llm.ChatRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("decode request: %v", err) } f.mu.Lock() f.requests = append(f.requests, req) n := len(f.requests) f.mu.Unlock() body := bodies[len(bodies)-1] if n <= len(bodies) { body = bodies[n-1] } w.Header().Set("Content-Type", "application/json") w.Write([]byte(body)) })) t.Cleanup(f.srv.Close) return f } func (f *fakeLLM) requestCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.requests) } func (f *fakeLLM) request(i int) llm.ChatRequest { f.mu.Lock() defer f.mu.Unlock() return f.requests[i] } func testConfig(t *testing.T, llmURL string) *config.Config { t.Helper() dir := t.TempDir() cfg := config.Default() cfg.Vertical = "teststack" cfg.WorkRoot = dir cfg.ReportDir = filepath.Join(dir, "reports") cfg.LiteLLM.BaseURL = llmURL cfg.LiteLLM.KeyRef = "literal:test-key" cfg.LiteLLM.MaxRetries = 0 cfg.Models.Tiers = map[string]string{ "mopac-study": "glm-4.7-flash", "mopac-code": "glm-5.2", "mopac-review": "glm-5-turbo", "mopac-primary": "glm-5.3", } cfg.Models.Classes = map[string]string{ "study": "mopac-study", "code": "mopac-code", "primary": "mopac-primary", } cfg.Bash.Allow = []string{"echo *", "pwd"} cfg.Bash.Deny = []string{"sudo *"} return cfg } func TestOnceDemoEndToEnd(t *testing.T) { f := newFakeLLM(t, textMsg("I am GLM, a large language model.")) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("Once: %v", err) } // MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT. if f.requestCount() != 1 { t.Fatalf("LLM calls = %d, want 1", f.requestCount()) } req := f.request(0) if req.Model != "glm-5.3" { t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model) } if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" { t.Errorf("messages = %+v", req.Messages) } if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" { t.Errorf("turn = %+v", res.Turn) } if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" { t.Errorf("routed = %+v", res.Routed) } body, err := os.ReadFile(res.ReportPath) if err != nil { t.Fatalf("REPORT: %v", err) } s := string(body) for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} { if !strings.Contains(s, want) { t.Errorf("REPORT missing %q", want) } } if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil { t.Errorf("REPORT-latest.md: %v", err) } } func TestOnceDryRunMakesNoLLMCall(t *testing.T) { f := newFakeLLM(t, textMsg("unused")) cfg := testConfig(t, f.srv.URL) var out strings.Builder cond, err := New(cfg, &out) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true}) if err != nil { t.Fatalf("Once: %v", err) } if f.requestCount() != 0 { t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount()) } if res.ReportPath != "" { t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath) } for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} { if !strings.Contains(out.String(), want) { t.Errorf("plan output missing %q:\n%s", want, out.String()) } } } func TestOnceToolRoundTrip(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`), textMsg("Tool ran fine."), ) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("Once: %v", err) } if f.requestCount() != 2 { t.Fatalf("LLM calls = %d, want 2", f.requestCount()) } if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 { t.Errorf("turn = %+v", res.Turn) } // Second request must carry the assistant tool_call and the tool result. second := f.request(1) roles := make([]string, len(second.Messages)) for i, m := range second.Messages { roles[i] = m.Role } want := []string{"system", "user", "assistant", "tool"} if fmt.Sprint(roles) != fmt.Sprint(want) { t.Errorf("second request roles = %v, want %v", roles, want) } toolMsg := second.Messages[3] if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") { t.Errorf("tool message = %+v", toolMsg) } } func TestOnceToolDeniedCounts(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`), textMsg("Understood, permission was denied."), ) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("denied tool call must not fail the turn: %v", err) } if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 { t.Errorf("turn = %+v", res.Turn) } toolMsg := f.request(1).Messages[3] if !strings.Contains(toolMsg.Content, "permission denied") { t.Errorf("model should see the denial, got %q", toolMsg.Content) } } func TestOnceRoundLimitBounded(t *testing.T) { f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`)) cfg := testConfig(t, f.srv.URL) cfg.Loop.MaxRounds = 3 cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } res, err := cond.Once(context.Background(), OnceOpts{Demo: true}) if err != nil { t.Fatalf("round limit is not an error: %v", err) } if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" { t.Errorf("turn = %+v", res.Turn) } if f.requestCount() != 3 { t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount()) } if res.ReportPath == "" { t.Errorf("round_limit turn should still write a REPORT") } } func TestOnceLLMFailureMapsToErrLLM(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no key for you", http.StatusUnauthorized) })) defer srv.Close() cfg := testConfig(t, srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true}) if !errors.Is(err, ErrLLM) { t.Fatalf("err = %v, want ErrLLM", err) } } func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "down", http.StatusInternalServerError) })) defer srv.Close() cfg := testConfig(t, "http://unused") cfg.Redmine.URL = srv.URL cfg.Redmine.KeyRef = "literal:rm-key" cfg.Redmine.ScopeQuery = "project=x" cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{}) if !errors.Is(err, ErrIntake) { t.Fatalf("err = %v, want ErrIntake", err) } } func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) { cfg := testConfig(t, "http://unused") cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{}) if !errors.Is(err, ErrIntake) { t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err) } } func TestOnceTaskIDFilter(t *testing.T) { cfg := testConfig(t, "http://unused") cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"}) if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") { t.Fatalf("err = %v, want not-in-scope ErrIntake", err) } } func TestOnceUnknownClassIsRoutingError(t *testing.T) { cfg := testConfig(t, "http://unused") cfg.Demo.Class = "definitely-not-a-class" cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } _, err = cond.Once(context.Background(), OnceOpts{Demo: true}) if err == nil || !strings.Contains(err.Error(), "unknown task class") { t.Fatalf("err = %v, want unknown-class error", err) } } func TestServeTurnPrependsSystemPromptWhenMissing(t *testing.T) { f := newFakeLLM(t, textMsg("front door reply")) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } turn, err := cond.ServeTurn(context.Background(), []llm.Message{{Role: "user", Content: "hello there"}}, "glm-5.3", ServeTurnOpts{}) if err != nil { t.Fatalf("ServeTurn: %v", err) } if turn.Content != "front door reply" || turn.StopReason != "complete" { t.Errorf("turn = %+v", turn) } req := f.request(0) if req.Model != "glm-5.3" { t.Errorf("request model = %q, want glm-5.3", req.Model) } if len(req.Messages) != 2 || req.Messages[0].Role != "system" { t.Fatalf("messages = %+v, want [system, user]", req.Messages) } if !strings.Contains(req.Messages[0].Content, "teststack") || !strings.Contains(req.Messages[0].Content, "OpenWebUI") { t.Errorf("prepended system prompt = %q, want vertical identity", req.Messages[0].Content) } if req.Messages[1].Role != "user" || req.Messages[1].Content != "hello there" { t.Errorf("user message = %+v", req.Messages[1]) } } func TestServeTurnPreservesClientSystemMessage(t *testing.T) { f := newFakeLLM(t, textMsg("aye")) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } history := []llm.Message{ {Role: "system", Content: "You are a pirate."}, {Role: "user", Content: "greet me"}, {Role: "assistant", Content: "ahoy"}, {Role: "user", Content: "again"}, } if _, err := cond.ServeTurn(context.Background(), history, "glm-5.3", ServeTurnOpts{}); err != nil { t.Fatalf("ServeTurn: %v", err) } got := f.request(0).Messages if len(got) != len(history) { t.Fatalf("history length = %d, want %d (client system message wins, no prepend)", len(got), len(history)) } for i, m := range history { if got[i].Role != m.Role || got[i].Content != m.Content { t.Errorf("message[%d] = %+v, want %+v", i, got[i], m) } } } func TestServeTurnToolsOffNeverExecutes(t *testing.T) { // pwd IS on the allow list: if the palette leaked into the serve path // the command would run and its output would come back as the tool // result. Tools are off, so the model must instead see the disabled // refusal and the turn must count a denial. f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`), textMsg("No tools then, here is the answer."), ) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } turn, err := cond.ServeTurn(context.Background(), []llm.Message{{Role: "user", Content: "run pwd"}}, "glm-5.3", ServeTurnOpts{}) if err != nil { t.Fatalf("ServeTurn: %v", err) } if f.requestCount() != 2 { t.Fatalf("LLM calls = %d, want 2 (refused tool call fed back)", f.requestCount()) } if turn.ToolCalls != 1 || turn.Denied != 1 { t.Errorf("turn = %+v, want 1 refused tool call", turn) } toolMsg := f.request(1).Messages[len(f.request(1).Messages)-1] if toolMsg.Role != "tool" || !strings.Contains(toolMsg.Content, "tools are disabled") { t.Errorf("tool result = %+v, want the disabled refusal", toolMsg) } if turn.Content != "No tools then, here is the answer." { t.Errorf("turn content = %q", turn.Content) } } func TestServeTurnForwardsKnobs(t *testing.T) { f := newFakeLLM(t, textMsg("ok")) cfg := testConfig(t, f.srv.URL) cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } temp := 0.7 if _, err := cond.ServeTurn(context.Background(), []llm.Message{{Role: "user", Content: "hi"}}, "glm-5.3", ServeTurnOpts{MaxTokens: 512, Temperature: &temp}); err != nil { t.Fatalf("ServeTurn: %v", err) } req := f.request(0) if req.MaxTokens != 512 { t.Errorf("max_tokens = %d, want 512 forwarded", req.MaxTokens) } if req.Temperature == nil || *req.Temperature != 0.7 { t.Errorf("temperature = %v, want 0.7 forwarded", req.Temperature) } if len(req.Tools) != 0 { t.Errorf("tools = %+v, want none on the serve path", req.Tools) } } func TestRouterAccessor(t *testing.T) { cfg := testConfig(t, "http://unused") cond, err := New(cfg, os.Stdout) if err != nil { t.Fatal(err) } classes := cond.Router().Classes() want := []string{"code", "primary", "study"} if fmt.Sprint(classes) != fmt.Sprint(want) { t.Errorf("Router().Classes() = %v, want %v", classes, want) } }