package serve import ( "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "ukrrs.com/mopac/harness/internal/config" "ukrrs.com/mopac/harness/internal/llm" "ukrrs.com/mopac/harness/internal/loop" ) const testVKey = "owui-test-vkey" // fakeUpstream is a scripted OpenAI-compatible upstream that records every // request (the "fake OpenAI client against the real server" pattern). type fakeUpstream struct { mu sync.Mutex requests []llm.ChatRequest srv *httptest.Server } func (f *fakeUpstream) handler(t *testing.T, bodies ...string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req llm.ChatRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("decode upstream request: %v", err) } f.mu.Lock() f.requests = append(f.requests, req) n := len(f.requests) f.mu.Unlock() body := bodies[len(bodies)-1] if n <= len(bodies) { body = bodies[n-1] } w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, body) }) } func textBody(content string, prompt, completion int) string { return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%q},"finish_reason":"stop"}],"usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}}`, content, prompt, completion, prompt+completion) } func (f *fakeUpstream) request(i int) llm.ChatRequest { f.mu.Lock() defer f.mu.Unlock() return f.requests[i] } func (f *fakeUpstream) count() int { f.mu.Lock() defer f.mu.Unlock() return len(f.requests) } // newStack wires fake upstream + conductor + serve server, the same way // `harness serve` does in main. func newStack(t *testing.T, bodies ...string) (*Server, *fakeUpstream, *httptest.Server) { t.Helper() up := &fakeUpstream{} up.srv = httptest.NewServer(up.handler(t, bodies...)) t.Cleanup(up.srv.Close) dir := t.TempDir() cfg := config.Default() cfg.Vertical = "teststack" cfg.WorkRoot = dir cfg.ReportDir = filepath.Join(dir, "reports") cfg.LiteLLM.BaseURL = up.srv.URL cfg.LiteLLM.KeyRef = "literal:upstream-key" cfg.LiteLLM.MaxRetries = 0 cfg.Models.Tiers = map[string]string{ "mopac-study": "glm-4.7-flash", "mopac-code": "glm-5.2", "mopac-review": "glm-5-turbo", "mopac-primary": "glm-5.3", } cfg.Models.Classes = map[string]string{ "study": "mopac-study", "read": "mopac-study", "code": "mopac-code", "primary": "mopac-primary", } cfg.Serve.VKeyRef = "literal:" + testVKey cond, err := loop.New(cfg, io.Discard) if err != nil { t.Fatalf("loop.New: %v", err) } srv, err := NewServer(cfg.Serve, cond.Router(), cond, io.Discard) if err != nil { t.Fatalf("NewServer: %v", err) } ts := httptest.NewServer(srv.Handler()) t.Cleanup(ts.Close) return srv, up, ts } func do(t *testing.T, method, url, auth string, body string) (int, string) { t.Helper() var rd io.Reader if body != "" { rd = strings.NewReader(body) } req, err := http.NewRequest(method, url, rd) if err != nil { t.Fatal(err) } if auth != "" { req.Header.Set("Authorization", auth) } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) return resp.StatusCode, string(b) } func bearer(key string) string { return "Bearer " + key } func TestAuth(t *testing.T) { _, _, ts := newStack(t, textBody("hi", 5, 5)) chatURL := ts.URL + "/v1/chat/completions" body := `{"model":"mopac-code","messages":[{"role":"user","content":"hi"}]}` modelsURL := ts.URL + "/v1/models" cases := []struct { name string url string method string auth string body string wantCode int }{ {"chat no auth header", chatURL, http.MethodPost, "", body, 401}, {"chat malformed scheme", chatURL, http.MethodPost, "Basic " + testVKey, body, 401}, {"chat wrong key", chatURL, http.MethodPost, bearer("wrong-key"), body, 401}, {"chat right key", chatURL, http.MethodPost, bearer(testVKey), body, 200}, {"models no auth", modelsURL, http.MethodGet, "", "", 401}, {"models wrong key", modelsURL, http.MethodGet, bearer("nope"), "", 401}, {"models right key", modelsURL, http.MethodGet, bearer(testVKey), "", 200}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { code, respBody := do(t, tc.method, tc.url, tc.auth, tc.body) if code != tc.wantCode { t.Errorf("code = %d, want %d (body %s)", code, tc.wantCode, respBody) } if code == 401 { if !strings.Contains(respBody, "invalid api key") { t.Errorf("401 body = %q, want the generic message", respBody) } if strings.Contains(respBody, testVKey) { t.Errorf("401 body leaks the vkey: %s", respBody) } } }) } } func TestModelsCatalog(t *testing.T) { srv, _, ts := newStack(t) code, body := do(t, http.MethodGet, ts.URL+"/v1/models", bearer(testVKey), "") if code != 200 { t.Fatalf("code = %d body %s", code, body) } var list modelList if err := json.Unmarshal([]byte(body), &list); err != nil { t.Fatalf("decode: %v", err) } var ids []string for _, m := range list.Data { ids = append(ids, m.ID) if m.Object != "model" || m.OwnedBy != "mopac" || m.Created == 0 { t.Errorf("entry shape wrong: %+v", m) } } want := "mopac-code mopac-primary mopac-read mopac-study" if strings.Join(ids, " ") != want { t.Errorf("catalog = %v, want %v", ids, want) } if srv.ModelNames()[0] != "mopac-code" { t.Errorf("ModelNames() = %v", srv.ModelNames()) } } func TestModelsCatalogEnabledSubset(t *testing.T) { // Rebuild the stack with enabled_models narrowed to two classes. up := &fakeUpstream{} up.srv = httptest.NewServer(up.handler(t, textBody("x", 1, 1))) defer up.srv.Close() cfg := config.Default() cfg.Vertical = "teststack" cfg.LiteLLM.BaseURL = up.srv.URL cfg.LiteLLM.KeyRef = "literal:k" cfg.Models.Tiers = map[string]string{"mopac-study": "glm-4.7-flash", "mopac-code": "glm-5.2", "mopac-primary": "glm-5.3"} cfg.Models.Classes = map[string]string{"study": "mopac-study", "code": "mopac-code", "primary": "mopac-primary"} cfg.Serve.VKeyRef = "literal:" + testVKey cfg.Serve.EnabledModels = []string{"mopac-code"} cond, err := loop.New(cfg, io.Discard) if err != nil { t.Fatal(err) } srv, err := NewServer(cfg.Serve, cond.Router(), cond, io.Discard) if err != nil { t.Fatal(err) } ts := httptest.NewServer(srv.Handler()) defer ts.Close() code, body := do(t, http.MethodGet, ts.URL+"/v1/models", bearer(testVKey), "") if code != 200 { t.Fatalf("code = %d body %s", code, body) } var list modelList if err := json.Unmarshal([]byte(body), &list); err != nil { t.Fatal(err) } if len(list.Data) != 1 || list.Data[0].ID != "mopac-code" { t.Errorf("catalog = %+v, want only mopac-code", list.Data) } // A disabled model must now be unknown. code, body = do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-study","messages":[{"role":"user","content":"x"}]}`) if code != 400 || !strings.Contains(body, "unknown model") { t.Errorf("disabled model: code=%d body=%s, want 400 unknown model", code, body) } } func TestChatEndToEnd(t *testing.T) { _, up, ts := newStack(t, textBody("the serve reply", 20, 40)) code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-code","messages":[{"role":"user","content":"write a function"}]}`) if code != 200 { t.Fatalf("code = %d body %s", code, body) } var out chatCompletion if err := json.Unmarshal([]byte(body), &out); err != nil { t.Fatalf("decode: %v (%s)", err, body) } if !strings.HasPrefix(out.ID, "chatcmpl-") || out.Object != "chat.completion" { t.Errorf("id/object wrong: %q %q", out.ID, out.Object) } if out.Model != "mopac-code" { t.Errorf("response model = %q, want the requested name echoed", out.Model) } if len(out.Choices) != 1 || out.Choices[0].Message.Role != "assistant" || out.Choices[0].Message.Content != "the serve reply" || out.Choices[0].FinishReason != "stop" { t.Errorf("choices wrong: %+v", out.Choices) } if out.Usage.PromptTokens != 20 || out.Usage.CompletionTokens != 40 || out.Usage.TotalTokens != 60 { t.Errorf("usage wrong: %+v", out.Usage) } // Routing: the request the upstream saw must carry the CONCRETE model // resolved through the tier map, and no tools. req := up.request(0) if req.Model != "glm-5.2" { t.Errorf("upstream model = %q, want glm-5.2 (mopac-code tier)", req.Model) } if len(req.Tools) != 0 { t.Errorf("upstream tools = %+v, want none (v0 serve turns are pure chat)", req.Tools) } } func TestChatHistoryAssembly(t *testing.T) { // OWUI sends the whole conversation each call. The system message is // the client's own (no harness prepend) and the history order is // preserved verbatim; only when no system message leads does the // conductor inject the vertical identity prompt. _, up, ts := newStack(t, textBody("reply", 1, 1)) payload := `{"model":"mopac-primary","messages":[ {"role":"system","content":"You are the test persona."}, {"role":"user","content":"first question"}, {"role":"assistant","content":"first answer"}, {"role":"user","content":"second question"} ]}` if code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), payload); code != 200 { t.Fatalf("code = %d body %s", code, body) } got := up.request(0).Messages if len(got) != 4 { t.Fatalf("upstream history length = %d, want 4 (client system wins)", len(got)) } wantRoles := "system user assistant user" var roles []string for _, m := range got { roles = append(roles, m.Role) } if strings.Join(roles, " ") != wantRoles { t.Errorf("roles = %v, want %v", roles, wantRoles) } if got[0].Content != "You are the test persona." || got[3].Content != "second question" { t.Errorf("history contents wrong: %+v", got) } } func TestChatPrependsSystemWhenMissing(t *testing.T) { _, up, ts := newStack(t, textBody("r", 1, 1)) if code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-study","messages":[{"role":"user","content":"hi"}]}`); code != 200 { t.Fatalf("code = %d body %s", code, body) } got := up.request(0).Messages if len(got) != 2 || got[0].Role != "system" || got[1].Role != "user" { t.Fatalf("messages = %+v, want [system, user]", got) } if !strings.Contains(got[0].Content, "teststack") { t.Errorf("prepended system prompt lacks vertical identity: %q", got[0].Content) } // flash tier routes through the same class map if up.request(0).Model != "glm-4.7-flash" { t.Errorf("upstream model = %q, want glm-4.7-flash (mopac-study tier)", up.request(0).Model) } } func TestChatUsageAccountingAcrossRounds(t *testing.T) { // A model that hallucinates a tool call first (tools are off) then // answers: usage must be the SUM over both rounds. toolCall := `{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"bash","arguments":"{\"command\":\"pwd\"}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}` _, up, ts := newStack(t, toolCall, textBody("final text", 30, 25)) code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-primary","messages":[{"role":"user","content":"go"}]}`) if code != 200 { t.Fatalf("code = %d body %s", code, body) } if up.count() != 2 { t.Fatalf("upstream calls = %d, want 2", up.count()) } // The refused tool call must come back as a tool result, never executed. second := up.request(1).Messages last := second[len(second)-1] if last.Role != "tool" || !strings.Contains(last.Content, "tools are disabled") { t.Errorf("tool result = %+v, want the disabled refusal", last) } var out chatCompletion if err := json.Unmarshal([]byte(body), &out); err != nil { t.Fatal(err) } if out.Usage.PromptTokens != 40 || out.Usage.CompletionTokens != 30 || out.Usage.TotalTokens != 70 { t.Errorf("usage = %+v, want sums over both rounds (40/30/70)", out.Usage) } if out.Choices[0].Message.Content != "final text" { t.Errorf("content = %q", out.Choices[0].Message.Content) } } func TestChatForwardsKnobs(t *testing.T) { _, up, ts := newStack(t, textBody("r", 1, 1)) if code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-code","messages":[{"role":"user","content":"hi"}],"temperature":0.3,"max_tokens":256}`); code != 200 { t.Fatalf("code = %d body %s", code, body) } req := up.request(0) if req.Temperature == nil || *req.Temperature != 0.3 { t.Errorf("temperature = %v, want 0.3 forwarded", req.Temperature) } if req.MaxTokens != 256 { t.Errorf("max_tokens = %d, want 256 forwarded", req.MaxTokens) } } func TestChatBadRequestTable(t *testing.T) { _, _, ts := newStack(t, textBody("unused", 1, 1)) url := ts.URL + "/v1/chat/completions" cases := []struct { name string body string wantCode int wantBody string }{ { "unknown model names valid ones", `{"model":"gpt-4","messages":[{"role":"user","content":"x"}]}`, 400, `valid models: mopac-code, mopac-primary, mopac-read, mopac-study`, }, {"stream requested", `{"model":"mopac-code","stream":true,"messages":[{"role":"user","content":"x"}]}`, 400, "streaming is not supported"}, {"empty messages", `{"model":"mopac-code","messages":[]}`, 400, "messages must not be empty"}, {"bad role", `{"model":"mopac-code","messages":[{"role":"tool","content":"x"}]}`, 400, "unsupported role"}, {"malformed json", `{not json`, 400, "malformed request body"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { code, body := do(t, http.MethodPost, url, bearer(testVKey), tc.body) if code != tc.wantCode { t.Errorf("code = %d, want %d (body %s)", code, tc.wantCode, body) } if !strings.Contains(body, tc.wantBody) { t.Errorf("body %q does not contain %q", body, tc.wantBody) } }) } } func TestChatUpstreamErrorIs502(t *testing.T) { up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "upstream exploded", http.StatusInternalServerError) })) defer up.Close() cfg := config.Default() cfg.Vertical = "teststack" cfg.LiteLLM.BaseURL = up.URL cfg.LiteLLM.KeyRef = "literal:k" cfg.LiteLLM.MaxRetries = 0 cfg.Models.Tiers = map[string]string{"mopac-primary": "glm-5.3"} cfg.Models.Classes = map[string]string{"primary": "mopac-primary"} cfg.Serve.VKeyRef = "literal:" + testVKey cond, err := loop.New(cfg, io.Discard) if err != nil { t.Fatal(err) } srv, err := NewServer(cfg.Serve, cond.Router(), cond, io.Discard) if err != nil { t.Fatal(err) } ts := httptest.NewServer(srv.Handler()) defer ts.Close() code, body := do(t, http.MethodPost, ts.URL+"/v1/chat/completions", bearer(testVKey), `{"model":"mopac-primary","messages":[{"role":"user","content":"x"}]}`) if code != http.StatusBadGateway { t.Fatalf("code = %d, want 502 (body %s)", code, body) } if strings.Contains(body, "exploded") { t.Errorf("502 body must not echo upstream details: %s", body) } } func TestNewServerFailFast(t *testing.T) { cfg := config.Default() cfg.Vertical = "teststack" cfg.LiteLLM.BaseURL = "http://unused" cfg.LiteLLM.KeyRef = "literal:k" cfg.Models.Tiers = map[string]string{"mopac-primary": "glm-5.3"} cfg.Models.Classes = map[string]string{"primary": "mopac-primary"} cond, err := loop.New(cfg, os.Stdout) if err != nil { t.Fatal(err) } if _, err := NewServer(cfg.Serve, cond.Router(), cond, io.Discard); err == nil || !strings.Contains(err.Error(), "vkey_ref is required") { t.Errorf("missing vkey_ref: err = %v", err) } cfg.Serve.VKeyRef = "literal:" + testVKey cfg.Serve.EnabledModels = []string{"mopac-nope"} if _, err := NewServer(cfg.Serve, cond.Router(), cond, io.Discard); err == nil || !strings.Contains(err.Error(), "not servable") { t.Errorf("bad enabled_models: err = %v", err) } } func TestHealthz(t *testing.T) { _, _, ts := newStack(t) // No auth on healthz (liveness probe, same as the events receiver). code, body := do(t, http.MethodGet, ts.URL+"/healthz", "", "") if code != 200 || !strings.Contains(body, `"status":"ok"`) { t.Errorf("healthz: code=%d body=%s", code, body) } } // Compile-time check that the conductor satisfies the Turner contract. var _ Turner = (*loop.Conductor)(nil)