harness: single-shot conductor + CLI with MVP demo path

`harness once` runs one bounded iteration (intake -> routing -> bounded turn
-> REPORT) then exits, so callers chain it without a daemon. The bounded
turn loops request -> tool calls -> results up to max_rounds; gate denials
feed back to the model and are counted instead of failing the turn.
--dry-run prints the plan (task, resolved model, tool bounds) and makes zero
LLM calls; --demo runs the [demo] issue ("tell me about yourself" through
LiteLLM -> GLM self-description as the REPORT), the MVP acceptance bar.
Distinct exit codes for config/usage, intake, and llm failures. Ships
harness.toml.example (scope query, tier map, allow-lists); real configs are
gitignored. Loop tests run end-to-end against a scripted fake OpenAI server:
demo turn, dry-run zero-call, tool round-trip, denial counting, round limit,
and error-class mapping.

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
This commit is contained in:
2026-08-28 19:21:04 -05:00
parent aefa73aa90
commit 62d3781d71
6 changed files with 862 additions and 1 deletions
+4
View File
@@ -0,0 +1,4 @@
harness.toml
reports/
bin/
*.test
+13 -1
View File
@@ -1,4 +1,16 @@
# MOPAC harness
Headless multi-vertical agent harness in Go. AGPLv3. SoR: Redmine project MOPAC; docs: Discourse; code: Gitea reachableceo/MOPAC.
Spec: DESIGN.md. Status: see Redmine MOPAC project issues.
Spec: DESIGN.md. Status: see Redmine MOPAC project issues and REPORT.md.
## Build & run (v0 skeleton)
go build ./... && go vet ./... && go test ./...
cp harness.toml.example harness.toml # then set the key refs
./bin/harness once --dry-run --demo # intake + plan, no LLM call
HARNESS_LITELLM_KEY=sk-... ./bin/harness once --demo # MVP demo turn
`harness once` runs ONE conductor iteration (intake -> model routing ->
bounded turn via LiteLLM -> REPORT file) and exits; chain iterations by
re-invoking. There is no daemon. Exit codes: 0 ok, 1 config/usage,
2 intake, 4 llm/turn. See `harness help`.
+111
View File
@@ -0,0 +1,111 @@
// Command harness is the MOPAC harness CLI. v0 surface: `harness once`.
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/loop"
)
const usage = `MOPAC harness (v0)
Usage:
harness once [-config PATH] [-dry-run] [-demo] [-task-id ID]
once runs ONE conductor iteration and exits (chain by re-invoking; no daemon):
intake (Redmine scope, or the [demo] issue) -> plan/model routing ->
bounded turn via LiteLLM -> REPORT file writeback.
Flags:
-config PATH config file (default $HARNESS_CONFIG or ./harness.toml)
-dry-run intake + plan only; no LLM call, no REPORT
-demo run the [demo] issue instead of Redmine intake
-task-id ID run only the task/issue with this id
Exit codes:
0 ok (including "no tasks in scope")
1 usage / config / routing / writeback error
2 intake error
4 llm / turn error
`
func main() {
os.Exit(run(os.Args[1:]))
}
func run(args []string) int {
if len(args) == 0 {
fmt.Fprint(os.Stderr, usage)
return 1
}
switch args[0] {
case "help", "-h", "--help":
fmt.Print(usage)
return 0
case "once":
return runOnce(args[1:])
default:
fmt.Fprintf(os.Stderr, "harness: unknown command %q\n\n%s", args[0], usage)
return 1
}
}
func runOnce(args []string) int {
fs := flag.NewFlagSet("once", flag.ContinueOnError)
cfgPath := fs.String("config", "", "config file path")
dryRun := fs.Bool("dry-run", false, "intake + plan only, no LLM call")
demo := fs.Bool("demo", false, "use the [demo] issue instead of Redmine")
taskID := fs.String("task-id", "", "run only this task id")
if err := fs.Parse(args); err != nil {
return 1
}
if fs.NArg() > 0 {
fmt.Fprintf(os.Stderr, "harness: unexpected argument %q\n", fs.Arg(0))
return 1
}
if *cfgPath == "" {
*cfgPath = os.Getenv("HARNESS_CONFIG")
}
if *cfgPath == "" {
*cfgPath = "harness.toml"
}
cfg, err := config.Load(*cfgPath)
if err != nil {
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
return 1
}
conductor, err := loop.New(cfg, os.Stdout)
if err != nil {
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
return 1
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
_, err = conductor.Once(ctx, loop.OnceOpts{
DryRun: *dryRun,
Demo: *demo,
TaskID: *taskID,
})
if err != nil {
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
switch {
case errors.Is(err, loop.ErrIntake):
return 2
case errors.Is(err, loop.ErrLLM):
return 4
default:
return 1
}
}
return 0
}
+104
View File
@@ -0,0 +1,104 @@
# MOPAC harness configuration, v0 (2026-08-28).
#
# Copy to harness.toml and adjust. harness.toml is gitignored: secrets never
# live in this file, only refs (env:NAME | file:PATH | literal:VALUE; the
# bw: bitwarden ref lands with the key wrapper in build phase 3).
# Vertical / stack identity for this harness instance.
vertical = "demo"
# Where the bash tool executes and REPORTs land (relative to the CWD the
# harness is started from).
work_root = "."
report_dir = "reports"
[loop]
# Bounded turn: max LLM round trips per task (tool calls included).
max_rounds = 8
[redmine]
url = "https://rm.example.org"
key_ref = "env:HARNESS_REDMINE_KEY"
# Released scope: raw /issues.json filter params...
scope_query = "project=mopac&status_id=released&limit=25"
# ...or a saved custom query id (scope_query wins when both are set).
# scope_query_id = 42
# Custom field carrying the TASK class; issues without it get default_class.
class_field = "Class"
default_class = "primary"
[litellm]
base_url = "http://192.168.3.78:4001"
key_ref = "env:HARNESS_LITELLM_KEY"
timeout_secs = 120
max_retries = 2
# MODEL ROUTING v0 (static, config-only, no heuristics): [models] is the
# tier map (tier alias -> concrete proxy model); [models.classes] maps task
# classes to tiers. Requests go out with the CONCRETE model name resolved
# here. Swapping models (e.g. glm-4.7-flash -> glm-5.3-flash once it is
# configured on the proxy) is a one-line edit in this file.
[models]
mopac-study = "glm-4.7-flash" # flash tier
mopac-code = "glm-5.2" # flagship
mopac-review = "glm-5-turbo" # mid
mopac-primary = "glm-5.3" # default / flagship+
mopac-vision = "glm-4.6v" # vision when needed
default_tier = "mopac-primary"
[models.classes]
study = "mopac-study"
read = "mopac-study"
code = "mopac-code"
architecture = "mopac-code"
review = "mopac-review"
summarize = "mopac-review"
writeback = "mopac-review"
vision = "mopac-vision"
primary = "mopac-primary"
# Exec tool: allow-listed bash. Org preset: deny-first, read-leaning allow
# list, no sudo/ssh/network exfil. Compound commands are checked segment by
# segment; command substitution and subshells are always denied.
[tools.bash]
enabled = true
timeout_secs = 60
max_output_bytes = 100000
default = "deny"
deny = [
"sudo *",
"ssh *",
"scp *",
"nc *",
"curl *",
"wget *",
"rm -rf *",
]
allow = [
"pwd",
"ls *",
"cat *",
"head *",
"tail *",
"grep *",
"find *",
"wc *",
"echo *",
"env",
"git status",
"git diff *",
"git log *",
"git show *",
"go version",
"go build *",
"go vet *",
"go test *",
]
# MVP demo (Charles, 2026-08-28 ~19:00): `harness once --demo` runs this
# issue through LiteLLM; the GLM self-description lands as the REPORT.
[demo]
id = "demo-1"
subject = "MVP demo: GLM self-description"
prompt = "tell me about yourself"
class = "primary"
+311
View File
@@ -0,0 +1,311 @@
// Package loop is the conductor: one bounded iteration per call
// (intake -> gate -> bounded turn -> REPORT writeback). Callers chain
// iterations by re-invoking; there is no daemon.
package loop
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"time"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/intake"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/models"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/task"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/tools"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/writeback"
)
// Sentinel error classes; the CLI maps these to distinct exit codes.
var (
ErrIntake = errors.New("intake error")
ErrLLM = errors.New("llm error")
)
// Conductor wires config, routing, tools, and clients into single-shot runs.
type Conductor struct {
cfg *config.Config
router *models.Router
bash *tools.BashTool
out io.Writer
}
// New validates routing config and prepares the tool palette.
func New(cfg *config.Config, out io.Writer) (*Conductor, error) {
router, err := models.NewRouter(cfg.Models.Tiers, cfg.Models.Classes, cfg.Models.DefaultTier)
if err != nil {
return nil, fmt.Errorf("model routing: %w", err)
}
var bash *tools.BashTool
if cfg.Bash.Enabled {
bash, err = tools.NewBashTool(tools.BashOptions{
WorkRoot: cfg.WorkRoot,
Allow: cfg.Bash.Allow,
Deny: cfg.Bash.Deny,
DefaultAllow: cfg.Bash.DefaultAllow,
Timeout: time.Duration(cfg.Bash.TimeoutSecs) * time.Second,
MaxOutputBytes: cfg.Bash.MaxOutputBytes,
})
if err != nil {
return nil, err
}
}
return &Conductor{cfg: cfg, router: router, bash: bash, out: out}, nil
}
// OnceOpts controls a single iteration.
type OnceOpts struct {
DryRun bool // intake + plan only, no LLM call
Demo bool // use the [demo] issue instead of Redmine intake
TaskID string // run only the task with this id (chaining aid)
}
// OnceResult reports what one iteration did.
type OnceResult struct {
TasksSeen int
Task task.Task
Routed models.Decision
ReportPath string
Turn *TurnResult
}
// Once runs ONE conductor iteration then returns. Zero tasks in scope is a
// success (exit 0) so chained invocations stay cheap.
func (c *Conductor) Once(ctx context.Context, opts OnceOpts) (*OnceResult, error) {
tasks, err := c.intake(ctx, opts)
if err != nil {
return nil, err
}
res := &OnceResult{TasksSeen: len(tasks)}
if opts.TaskID != "" {
var kept []task.Task
for _, t := range tasks {
if t.ID == opts.TaskID {
kept = append(kept, t)
}
}
if len(kept) == 0 {
return res, fmt.Errorf("%w: task id %q not in scope", ErrIntake, opts.TaskID)
}
tasks = kept
}
if len(tasks) == 0 {
fmt.Fprintf(c.out, "harness: no tasks in scope; nothing to do\n")
return res, nil
}
t := tasks[0]
res.Task = t
decision, err := c.router.Resolve(t.Class)
if err != nil {
return res, err
}
res.Routed = decision
fmt.Fprintf(c.out, "harness: task %s (%s): %q class=%s -> %s -> %s\n",
t.ID, t.Source, t.Subject, t.Class, decision.Tier, decision.Model)
if opts.DryRun {
c.printPlan(t, decision)
fmt.Fprintf(c.out, "harness: dry-run complete, no LLM call made\n")
return res, nil
}
client, err := c.llmClient()
if err != nil {
return res, err
}
start := time.Now()
turn, err := c.turn(ctx, client, t, decision.Model)
if turn != nil {
res.Turn = turn
}
if err != nil {
// Keep whatever content the turn produced before failing.
if turn != nil && turn.Content != "" {
_, _ = c.writeReport(t, decision, turn, start, err)
}
return res, fmt.Errorf("%w: %v", ErrLLM, err)
}
path, err := c.writeReport(t, decision, turn, start, nil)
if err != nil {
return res, err
}
res.ReportPath = path
return res, nil
}
// TurnResult is the bounded-turn outcome recorded in the REPORT.
type TurnResult struct {
Content string
Rounds int
ToolCalls int
Denied int
PromptTokens int
CompletionTokens int
TotalTokens int
StopReason string
}
// turn runs the bounded LLM loop: request, execute tool calls, repeat until
// the model replies with plain content or the round bound is hit.
func (c *Conductor) turn(ctx context.Context, client *llm.Client, t task.Task, model string) (*TurnResult, error) {
msgs := []llm.Message{
{Role: "system", Content: c.systemPrompt()},
{Role: "user", Content: t.Prompt},
}
var palette []llm.Tool
if c.bash != nil {
palette = append(palette, c.bash.Definition())
}
res := &TurnResult{}
for round := 1; round <= c.cfg.Loop.MaxRounds; round++ {
resp, err := client.Chat(ctx, llm.ChatRequest{Model: model, Messages: msgs, Tools: palette})
if err != nil {
res.StopReason = "llm_error"
return res, err
}
res.Rounds = round
res.PromptTokens += resp.Usage.PromptTokens
res.CompletionTokens += resp.Usage.CompletionTokens
res.TotalTokens += resp.Usage.TotalTokens
m := resp.Choices[0].Message
if m.Content != "" {
res.Content = m.Content
}
if len(m.ToolCalls) == 0 {
res.StopReason = "complete"
return res, nil
}
msgs = append(msgs, m)
for _, tc := range m.ToolCalls {
out, derr := c.execTool(ctx, tc)
res.ToolCalls++
if derr != nil {
res.Denied++
}
msgs = append(msgs, llm.Message{Role: "tool", ToolCallID: tc.ID, Content: out})
}
}
res.StopReason = "round_limit"
return res, nil
}
// execTool dispatches one tool call; the returned string is the model-facing
// result. The returned error is non-nil only for gate denials (counted).
func (c *Conductor) execTool(ctx context.Context, call llm.ToolCall) (string, error) {
if c.bash == nil || call.Function.Name != "bash" {
return fmt.Sprintf("error: unknown tool %q", call.Function.Name), nil
}
out, err := c.bash.Run(ctx, json.RawMessage(call.Function.Arguments))
if err != nil {
if errors.Is(err, tools.ErrDenied) {
return fmt.Sprintf("permission denied: %v", err), tools.ErrDenied
}
if out != "" {
return fmt.Sprintf("%s\nerror: %v", out, err), nil
}
return fmt.Sprintf("error: %v", err), nil
}
if out == "" {
out = "(no output)"
}
return out, nil
}
func (c *Conductor) intake(ctx context.Context, opts OnceOpts) ([]task.Task, error) {
if opts.Demo {
fmt.Fprintf(c.out, "harness: intake: demo issue from harness.toml [demo]\n")
return []task.Task{intake.DemoTask(c.cfg.Demo)}, nil
}
rc := c.cfg.Redmine
if rc.URL == "" {
return nil, fmt.Errorf("%w: no [redmine] url configured (use --demo for the demo issue)", ErrIntake)
}
key, err := config.ResolveKeyRef(rc.KeyRef)
if err != nil {
return nil, fmt.Errorf("%w: redmine key: %v", ErrIntake, err)
}
client := intake.NewRedmineClient(rc, key)
tasks, err := client.ListTasks(ctx)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrIntake, err)
}
fmt.Fprintf(c.out, "harness: intake: %d task(s) in redmine scope\n", len(tasks))
return tasks, nil
}
func (c *Conductor) llmClient() (*llm.Client, error) {
key, err := config.ResolveKeyRef(c.cfg.LiteLLM.KeyRef)
if err != nil {
return nil, fmt.Errorf("litellm key: %w", err)
}
return llm.NewClient(
c.cfg.LiteLLM.BaseURL,
key,
time.Duration(c.cfg.LiteLLM.TimeoutSecs)*time.Second,
c.cfg.LiteLLM.MaxRetries,
), nil
}
func (c *Conductor) writeReport(t task.Task, d models.Decision, turn *TurnResult, start time.Time, turnErr error) (string, error) {
r := writeback.Report{
Time: time.Now().UTC(),
Vertical: c.cfg.Vertical,
Task: t,
Class: t.Class,
Tier: d.Tier,
Model: d.Model,
PromptTokens: turn.PromptTokens,
CompletionTokens: turn.CompletionTokens,
TotalTokens: turn.TotalTokens,
Rounds: turn.Rounds,
ToolCalls: turn.ToolCalls,
Denied: turn.Denied,
Duration: time.Since(start),
StopReason: turn.StopReason,
Content: turn.Content,
}
if turnErr != nil {
r.StopReason = "error: " + turnErr.Error()
}
path, err := writeback.Write(c.cfg.ReportDir, r)
if err != nil {
return "", err
}
fmt.Fprintf(c.out, "harness: REPORT %s\n", path)
return path, nil
}
func (c *Conductor) printPlan(t task.Task, d models.Decision) {
fmt.Fprintf(c.out, "PLAN (dry-run)\n")
fmt.Fprintf(c.out, " vertical: %s\n", c.cfg.Vertical)
fmt.Fprintf(c.out, " task: [%s] %s (source %s)\n", t.ID, t.Subject, t.Source)
fmt.Fprintf(c.out, " routing: class %q -> tier %s -> model %s\n", t.Class, d.Tier, d.Model)
if c.bash != nil {
fmt.Fprintf(c.out, " tools: bash (allow=%d deny=%d default=%s timeout=%ds)\n",
len(c.cfg.Bash.Allow), len(c.cfg.Bash.Deny), denyAllow(c.cfg.Bash.DefaultAllow), c.cfg.Bash.TimeoutSecs)
} else {
fmt.Fprintf(c.out, " tools: none\n")
}
fmt.Fprintf(c.out, " bound: max %d rounds\n", c.cfg.Loop.MaxRounds)
}
func denyAllow(defaultAllow bool) string {
if defaultAllow {
return "allow"
}
return "deny"
}
func (c *Conductor) systemPrompt() string {
return fmt.Sprintf(`You are %s, an agent run by the MOPAC harness (headless, bounded turn).
Complete the assigned task. Your final reply is written verbatim to the turn
REPORT. Use the bash tool only when necessary; commands run in %s and must
match the configured allow-list (unmatched or denied commands fail).`,
c.cfg.Vertical, c.cfg.WorkRoot)
}
+319
View File
@@ -0,0 +1,319 @@
package loop
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
)
// fakeLLM is a scripted OpenAI-compatible server that records every request.
type fakeLLM struct {
mu sync.Mutex
requests []llm.ChatRequest
srv *httptest.Server
}
func toolCallMsg(id, name, args string) string {
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`,
id, name, args)
}
func textMsg(content string) string {
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content)
}
// newFakeLLM serves the scripted response bodies in order (repeating the
// last one if the turn asks for more).
func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM {
t.Helper()
f := &fakeLLM{}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req llm.ChatRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Errorf("decode request: %v", err)
}
f.mu.Lock()
f.requests = append(f.requests, req)
n := len(f.requests)
f.mu.Unlock()
body := bodies[len(bodies)-1]
if n <= len(bodies) {
body = bodies[n-1]
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(body))
}))
t.Cleanup(f.srv.Close)
return f
}
func (f *fakeLLM) requestCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.requests)
}
func (f *fakeLLM) request(i int) llm.ChatRequest {
f.mu.Lock()
defer f.mu.Unlock()
return f.requests[i]
}
func testConfig(t *testing.T, llmURL string) *config.Config {
t.Helper()
dir := t.TempDir()
cfg := config.Default()
cfg.Vertical = "teststack"
cfg.WorkRoot = dir
cfg.ReportDir = filepath.Join(dir, "reports")
cfg.LiteLLM.BaseURL = llmURL
cfg.LiteLLM.KeyRef = "literal:test-key"
cfg.LiteLLM.MaxRetries = 0
cfg.Models.Tiers = map[string]string{
"mopac-study": "glm-4.7-flash",
"mopac-code": "glm-5.2",
"mopac-review": "glm-5-turbo",
"mopac-primary": "glm-5.3",
}
cfg.Models.Classes = map[string]string{
"study": "mopac-study",
"code": "mopac-code",
"primary": "mopac-primary",
}
cfg.Bash.Allow = []string{"echo *", "pwd"}
cfg.Bash.Deny = []string{"sudo *"}
return cfg
}
func TestOnceDemoEndToEnd(t *testing.T) {
f := newFakeLLM(t, textMsg("I am GLM, a large language model."))
cfg := testConfig(t, f.srv.URL)
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
if err != nil {
t.Fatalf("Once: %v", err)
}
// MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT.
if f.requestCount() != 1 {
t.Fatalf("LLM calls = %d, want 1", f.requestCount())
}
req := f.request(0)
if req.Model != "glm-5.3" {
t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model)
}
if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" {
t.Errorf("messages = %+v", req.Messages)
}
if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" {
t.Errorf("turn = %+v", res.Turn)
}
if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" {
t.Errorf("routed = %+v", res.Routed)
}
body, err := os.ReadFile(res.ReportPath)
if err != nil {
t.Fatalf("REPORT: %v", err)
}
s := string(body)
for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} {
if !strings.Contains(s, want) {
t.Errorf("REPORT missing %q", want)
}
}
if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil {
t.Errorf("REPORT-latest.md: %v", err)
}
}
func TestOnceDryRunMakesNoLLMCall(t *testing.T) {
f := newFakeLLM(t, textMsg("unused"))
cfg := testConfig(t, f.srv.URL)
var out strings.Builder
cond, err := New(cfg, &out)
if err != nil {
t.Fatal(err)
}
res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true})
if err != nil {
t.Fatalf("Once: %v", err)
}
if f.requestCount() != 0 {
t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount())
}
if res.ReportPath != "" {
t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath)
}
for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} {
if !strings.Contains(out.String(), want) {
t.Errorf("plan output missing %q:\n%s", want, out.String())
}
}
}
func TestOnceToolRoundTrip(t *testing.T) {
f := newFakeLLM(t,
toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`),
textMsg("Tool ran fine."),
)
cfg := testConfig(t, f.srv.URL)
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
if err != nil {
t.Fatalf("Once: %v", err)
}
if f.requestCount() != 2 {
t.Fatalf("LLM calls = %d, want 2", f.requestCount())
}
if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 {
t.Errorf("turn = %+v", res.Turn)
}
// Second request must carry the assistant tool_call and the tool result.
second := f.request(1)
roles := make([]string, len(second.Messages))
for i, m := range second.Messages {
roles[i] = m.Role
}
want := []string{"system", "user", "assistant", "tool"}
if fmt.Sprint(roles) != fmt.Sprint(want) {
t.Errorf("second request roles = %v, want %v", roles, want)
}
toolMsg := second.Messages[3]
if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") {
t.Errorf("tool message = %+v", toolMsg)
}
}
func TestOnceToolDeniedCounts(t *testing.T) {
f := newFakeLLM(t,
toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`),
textMsg("Understood, permission was denied."),
)
cfg := testConfig(t, f.srv.URL)
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
if err != nil {
t.Fatalf("denied tool call must not fail the turn: %v", err)
}
if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 {
t.Errorf("turn = %+v", res.Turn)
}
toolMsg := f.request(1).Messages[3]
if !strings.Contains(toolMsg.Content, "permission denied") {
t.Errorf("model should see the denial, got %q", toolMsg.Content)
}
}
func TestOnceRoundLimitBounded(t *testing.T) {
f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`))
cfg := testConfig(t, f.srv.URL)
cfg.Loop.MaxRounds = 3
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
if err != nil {
t.Fatalf("round limit is not an error: %v", err)
}
if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" {
t.Errorf("turn = %+v", res.Turn)
}
if f.requestCount() != 3 {
t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount())
}
if res.ReportPath == "" {
t.Errorf("round_limit turn should still write a REPORT")
}
}
func TestOnceLLMFailureMapsToErrLLM(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "no key for you", http.StatusUnauthorized)
}))
defer srv.Close()
cfg := testConfig(t, srv.URL)
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
if !errors.Is(err, ErrLLM) {
t.Fatalf("err = %v, want ErrLLM", err)
}
}
func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "down", http.StatusInternalServerError)
}))
defer srv.Close()
cfg := testConfig(t, "http://unused")
cfg.Redmine.URL = srv.URL
cfg.Redmine.KeyRef = "literal:rm-key"
cfg.Redmine.ScopeQuery = "project=x"
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
_, err = cond.Once(context.Background(), OnceOpts{})
if !errors.Is(err, ErrIntake) {
t.Fatalf("err = %v, want ErrIntake", err)
}
}
func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) {
cfg := testConfig(t, "http://unused")
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
_, err = cond.Once(context.Background(), OnceOpts{})
if !errors.Is(err, ErrIntake) {
t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err)
}
}
func TestOnceTaskIDFilter(t *testing.T) {
cfg := testConfig(t, "http://unused")
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
_, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"})
if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") {
t.Fatalf("err = %v, want not-in-scope ErrIntake", err)
}
}
func TestOnceUnknownClassIsRoutingError(t *testing.T) {
cfg := testConfig(t, "http://unused")
cfg.Demo.Class = "definitely-not-a-class"
cond, err := New(cfg, os.Stdout)
if err != nil {
t.Fatal(err)
}
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
if err == nil || !strings.Contains(err.Error(), "unknown task class") {
t.Fatalf("err = %v, want unknown-class error", err)
}
}