MOPAC/internal/loop/loop_test.go

package loop

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"ukrrs.com/mopac/harness/internal/config"
	"ukrrs.com/mopac/harness/internal/llm"
)

// fakeLLM is a scripted OpenAI-compatible server that records every request.
type fakeLLM struct {
	mu       sync.Mutex
	requests []llm.ChatRequest
	srv      *httptest.Server
}

func toolCallMsg(id, name, args string) string {
	return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`,
		id, name, args)
}

func textMsg(content string) string {
	return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content)
}

// newFakeLLM serves the scripted response bodies in order (repeating the
// last one if the turn asks for more).
func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM {
	t.Helper()
	f := &fakeLLM{}
	f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var req llm.ChatRequest
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			t.Errorf("decode request: %v", err)
		}
		f.mu.Lock()
		f.requests = append(f.requests, req)
		n := len(f.requests)
		f.mu.Unlock()
		body := bodies[len(bodies)-1]
		if n <= len(bodies) {
			body = bodies[n-1]
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write([]byte(body))
	}))
	t.Cleanup(f.srv.Close)
	return f
}

func (f *fakeLLM) requestCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.requests)
}

func (f *fakeLLM) request(i int) llm.ChatRequest {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.requests[i]
}

func testConfig(t *testing.T, llmURL string) *config.Config {
	t.Helper()
	dir := t.TempDir()
	cfg := config.Default()
	cfg.Vertical = "teststack"
	cfg.WorkRoot = dir
	cfg.ReportDir = filepath.Join(dir, "reports")
	cfg.LiteLLM.BaseURL = llmURL
	cfg.LiteLLM.KeyRef = "literal:test-key"
	cfg.LiteLLM.MaxRetries = 0
	cfg.Models.Tiers = map[string]string{
		"mopac-study":   "glm-4.7-flash",
		"mopac-code":    "glm-5.2",
		"mopac-review":  "glm-5-turbo",
		"mopac-primary": "glm-5.3",
	}
	cfg.Models.Classes = map[string]string{
		"study":   "mopac-study",
		"code":    "mopac-code",
		"primary": "mopac-primary",
	}
	cfg.Bash.Allow = []string{"echo *", "pwd"}
	cfg.Bash.Deny = []string{"sudo *"}
	return cfg
}

func TestOnceDemoEndToEnd(t *testing.T) {
	f := newFakeLLM(t, textMsg("I am GLM, a large language model."))
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
	if err != nil {
		t.Fatalf("Once: %v", err)
	}
	// MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT.
	if f.requestCount() != 1 {
		t.Fatalf("LLM calls = %d, want 1", f.requestCount())
	}
	req := f.request(0)
	if req.Model != "glm-5.3" {
		t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model)
	}
	if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" {
		t.Errorf("messages = %+v", req.Messages)
	}
	if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" {
		t.Errorf("turn = %+v", res.Turn)
	}
	if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" {
		t.Errorf("routed = %+v", res.Routed)
	}
	body, err := os.ReadFile(res.ReportPath)
	if err != nil {
		t.Fatalf("REPORT: %v", err)
	}
	s := string(body)
	for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} {
		if !strings.Contains(s, want) {
			t.Errorf("REPORT missing %q", want)
		}
	}
	if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil {
		t.Errorf("REPORT-latest.md: %v", err)
	}
}

func TestOnceDryRunMakesNoLLMCall(t *testing.T) {
	f := newFakeLLM(t, textMsg("unused"))
	cfg := testConfig(t, f.srv.URL)
	var out strings.Builder
	cond, err := New(cfg, &out)
	if err != nil {
		t.Fatal(err)
	}
	res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true})
	if err != nil {
		t.Fatalf("Once: %v", err)
	}
	if f.requestCount() != 0 {
		t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount())
	}
	if res.ReportPath != "" {
		t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath)
	}
	for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} {
		if !strings.Contains(out.String(), want) {
			t.Errorf("plan output missing %q:\n%s", want, out.String())
		}
	}
}

func TestOnceToolRoundTrip(t *testing.T) {
	f := newFakeLLM(t,
		toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`),
		textMsg("Tool ran fine."),
	)
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
	if err != nil {
		t.Fatalf("Once: %v", err)
	}
	if f.requestCount() != 2 {
		t.Fatalf("LLM calls = %d, want 2", f.requestCount())
	}
	if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 {
		t.Errorf("turn = %+v", res.Turn)
	}
	// Second request must carry the assistant tool_call and the tool result.
	second := f.request(1)
	roles := make([]string, len(second.Messages))
	for i, m := range second.Messages {
		roles[i] = m.Role
	}
	want := []string{"system", "user", "assistant", "tool"}
	if fmt.Sprint(roles) != fmt.Sprint(want) {
		t.Errorf("second request roles = %v, want %v", roles, want)
	}
	toolMsg := second.Messages[3]
	if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") {
		t.Errorf("tool message = %+v", toolMsg)
	}
}

func TestOnceToolDeniedCounts(t *testing.T) {
	f := newFakeLLM(t,
		toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`),
		textMsg("Understood, permission was denied."),
	)
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
	if err != nil {
		t.Fatalf("denied tool call must not fail the turn: %v", err)
	}
	if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 {
		t.Errorf("turn = %+v", res.Turn)
	}
	toolMsg := f.request(1).Messages[3]
	if !strings.Contains(toolMsg.Content, "permission denied") {
		t.Errorf("model should see the denial, got %q", toolMsg.Content)
	}
}

func TestOnceRoundLimitBounded(t *testing.T) {
	f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`))
	cfg := testConfig(t, f.srv.URL)
	cfg.Loop.MaxRounds = 3
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
	if err != nil {
		t.Fatalf("round limit is not an error: %v", err)
	}
	if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" {
		t.Errorf("turn = %+v", res.Turn)
	}
	if f.requestCount() != 3 {
		t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount())
	}
	if res.ReportPath == "" {
		t.Errorf("round_limit turn should still write a REPORT")
	}
}

func TestOnceLLMFailureMapsToErrLLM(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "no key for you", http.StatusUnauthorized)
	}))
	defer srv.Close()
	cfg := testConfig(t, srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
	if !errors.Is(err, ErrLLM) {
		t.Fatalf("err = %v, want ErrLLM", err)
	}
}

func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "down", http.StatusInternalServerError)
	}))
	defer srv.Close()
	cfg := testConfig(t, "http://unused")
	cfg.Redmine.URL = srv.URL
	cfg.Redmine.KeyRef = "literal:rm-key"
	cfg.Redmine.ScopeQuery = "project=x"
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	_, err = cond.Once(context.Background(), OnceOpts{})
	if !errors.Is(err, ErrIntake) {
		t.Fatalf("err = %v, want ErrIntake", err)
	}
}

func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) {
	cfg := testConfig(t, "http://unused")
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	_, err = cond.Once(context.Background(), OnceOpts{})
	if !errors.Is(err, ErrIntake) {
		t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err)
	}
}

func TestOnceTaskIDFilter(t *testing.T) {
	cfg := testConfig(t, "http://unused")
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	_, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"})
	if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") {
		t.Fatalf("err = %v, want not-in-scope ErrIntake", err)
	}
}

func TestOnceUnknownClassIsRoutingError(t *testing.T) {
	cfg := testConfig(t, "http://unused")
	cfg.Demo.Class = "definitely-not-a-class"
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
	if err == nil || !strings.Contains(err.Error(), "unknown task class") {
		t.Fatalf("err = %v, want unknown-class error", err)
	}
}

func TestServeTurnPrependsSystemPromptWhenMissing(t *testing.T) {
	f := newFakeLLM(t, textMsg("front door reply"))
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	turn, err := cond.ServeTurn(context.Background(),
		[]llm.Message{{Role: "user", Content: "hello there"}}, "glm-5.3", ServeTurnOpts{})
	if err != nil {
		t.Fatalf("ServeTurn: %v", err)
	}
	if turn.Content != "front door reply" || turn.StopReason != "complete" {
		t.Errorf("turn = %+v", turn)
	}
	req := f.request(0)
	if req.Model != "glm-5.3" {
		t.Errorf("request model = %q, want glm-5.3", req.Model)
	}
	if len(req.Messages) != 2 || req.Messages[0].Role != "system" {
		t.Fatalf("messages = %+v, want [system, user]", req.Messages)
	}
	if !strings.Contains(req.Messages[0].Content, "teststack") || !strings.Contains(req.Messages[0].Content, "OpenWebUI") {
		t.Errorf("prepended system prompt = %q, want vertical identity", req.Messages[0].Content)
	}
	if req.Messages[1].Role != "user" || req.Messages[1].Content != "hello there" {
		t.Errorf("user message = %+v", req.Messages[1])
	}
}

func TestServeTurnPreservesClientSystemMessage(t *testing.T) {
	f := newFakeLLM(t, textMsg("aye"))
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	history := []llm.Message{
		{Role: "system", Content: "You are a pirate."},
		{Role: "user", Content: "greet me"},
		{Role: "assistant", Content: "ahoy"},
		{Role: "user", Content: "again"},
	}
	if _, err := cond.ServeTurn(context.Background(), history, "glm-5.3", ServeTurnOpts{}); err != nil {
		t.Fatalf("ServeTurn: %v", err)
	}
	got := f.request(0).Messages
	if len(got) != len(history) {
		t.Fatalf("history length = %d, want %d (client system message wins, no prepend)", len(got), len(history))
	}
	for i, m := range history {
		if got[i].Role != m.Role || got[i].Content != m.Content {
			t.Errorf("message[%d] = %+v, want %+v", i, got[i], m)
		}
	}
}

func TestServeTurnToolsOffNeverExecutes(t *testing.T) {
	// pwd IS on the allow list: if the palette leaked into the serve path
	// the command would run and its output would come back as the tool
	// result. Tools are off, so the model must instead see the disabled
	// refusal and the turn must count a denial.
	f := newFakeLLM(t,
		toolCallMsg("call-1", "bash", `{"command":"pwd"}`),
		textMsg("No tools then, here is the answer."),
	)
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	turn, err := cond.ServeTurn(context.Background(),
		[]llm.Message{{Role: "user", Content: "run pwd"}}, "glm-5.3", ServeTurnOpts{})
	if err != nil {
		t.Fatalf("ServeTurn: %v", err)
	}
	if f.requestCount() != 2 {
		t.Fatalf("LLM calls = %d, want 2 (refused tool call fed back)", f.requestCount())
	}
	if turn.ToolCalls != 1 || turn.Denied != 1 {
		t.Errorf("turn = %+v, want 1 refused tool call", turn)
	}
	toolMsg := f.request(1).Messages[len(f.request(1).Messages)-1]
	if toolMsg.Role != "tool" || !strings.Contains(toolMsg.Content, "tools are disabled") {
		t.Errorf("tool result = %+v, want the disabled refusal", toolMsg)
	}
	if turn.Content != "No tools then, here is the answer." {
		t.Errorf("turn content = %q", turn.Content)
	}
}

func TestServeTurnForwardsKnobs(t *testing.T) {
	f := newFakeLLM(t, textMsg("ok"))
	cfg := testConfig(t, f.srv.URL)
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	temp := 0.7
	if _, err := cond.ServeTurn(context.Background(),
		[]llm.Message{{Role: "user", Content: "hi"}}, "glm-5.3",
		ServeTurnOpts{MaxTokens: 512, Temperature: &temp}); err != nil {
		t.Fatalf("ServeTurn: %v", err)
	}
	req := f.request(0)
	if req.MaxTokens != 512 {
		t.Errorf("max_tokens = %d, want 512 forwarded", req.MaxTokens)
	}
	if req.Temperature == nil || *req.Temperature != 0.7 {
		t.Errorf("temperature = %v, want 0.7 forwarded", req.Temperature)
	}
	if len(req.Tools) != 0 {
		t.Errorf("tools = %+v, want none on the serve path", req.Tools)
	}
}

func TestRouterAccessor(t *testing.T) {
	cfg := testConfig(t, "http://unused")
	cond, err := New(cfg, os.Stdout)
	if err != nil {
		t.Fatal(err)
	}
	classes := cond.Router().Classes()
	want := []string{"code", "primary", "study"}
	if fmt.Sprint(classes) != fmt.Sprint(want) {
		t.Errorf("Router().Classes() = %v, want %v", classes, want)
	}
}