harness: single-shot conductor + CLI with MVP demo path
`harness once` runs one bounded iteration (intake -> routing -> bounded turn
-> REPORT) then exits, so callers chain it without a daemon. The bounded
turn loops request -> tool calls -> results up to max_rounds; gate denials
feed back to the model and are counted instead of failing the turn.
--dry-run prints the plan (task, resolved model, tool bounds) and makes zero
LLM calls; --demo runs the [demo] issue ("tell me about yourself" through
LiteLLM -> GLM self-description as the REPORT), the MVP acceptance bar.
Distinct exit codes for config/usage, intake, and llm failures. Ships
harness.toml.example (scope query, tier map, allow-lists); real configs are
gitignored. Loop tests run end-to-end against a scripted fake OpenAI server:
demo turn, dry-run zero-call, tool round-trip, denial counting, round limit,
and error-class mapping.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
harness.toml
|
||||
reports/
|
||||
bin/
|
||||
*.test
|
||||
@@ -1,4 +1,16 @@
|
||||
# MOPAC harness
|
||||
|
||||
Headless multi-vertical agent harness in Go. AGPLv3. SoR: Redmine project MOPAC; docs: Discourse; code: Gitea reachableceo/MOPAC.
|
||||
Spec: DESIGN.md. Status: see Redmine MOPAC project issues.
|
||||
Spec: DESIGN.md. Status: see Redmine MOPAC project issues and REPORT.md.
|
||||
|
||||
## Build & run (v0 skeleton)
|
||||
|
||||
go build ./... && go vet ./... && go test ./...
|
||||
cp harness.toml.example harness.toml # then set the key refs
|
||||
./bin/harness once --dry-run --demo # intake + plan, no LLM call
|
||||
HARNESS_LITELLM_KEY=sk-... ./bin/harness once --demo # MVP demo turn
|
||||
|
||||
`harness once` runs ONE conductor iteration (intake -> model routing ->
|
||||
bounded turn via LiteLLM -> REPORT file) and exits; chain iterations by
|
||||
re-invoking. There is no daemon. Exit codes: 0 ok, 1 config/usage,
|
||||
2 intake, 4 llm/turn. See `harness help`.
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// Command harness is the MOPAC harness CLI. v0 surface: `harness once`.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/loop"
|
||||
)
|
||||
|
||||
const usage = `MOPAC harness (v0)
|
||||
|
||||
Usage:
|
||||
harness once [-config PATH] [-dry-run] [-demo] [-task-id ID]
|
||||
|
||||
once runs ONE conductor iteration and exits (chain by re-invoking; no daemon):
|
||||
intake (Redmine scope, or the [demo] issue) -> plan/model routing ->
|
||||
bounded turn via LiteLLM -> REPORT file writeback.
|
||||
|
||||
Flags:
|
||||
-config PATH config file (default $HARNESS_CONFIG or ./harness.toml)
|
||||
-dry-run intake + plan only; no LLM call, no REPORT
|
||||
-demo run the [demo] issue instead of Redmine intake
|
||||
-task-id ID run only the task/issue with this id
|
||||
|
||||
Exit codes:
|
||||
0 ok (including "no tasks in scope")
|
||||
1 usage / config / routing / writeback error
|
||||
2 intake error
|
||||
4 llm / turn error
|
||||
`
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
return 1
|
||||
}
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
fmt.Print(usage)
|
||||
return 0
|
||||
case "once":
|
||||
return runOnce(args[1:])
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "harness: unknown command %q\n\n%s", args[0], usage)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func runOnce(args []string) int {
|
||||
fs := flag.NewFlagSet("once", flag.ContinueOnError)
|
||||
cfgPath := fs.String("config", "", "config file path")
|
||||
dryRun := fs.Bool("dry-run", false, "intake + plan only, no LLM call")
|
||||
demo := fs.Bool("demo", false, "use the [demo] issue instead of Redmine")
|
||||
taskID := fs.String("task-id", "", "run only this task id")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprintf(os.Stderr, "harness: unexpected argument %q\n", fs.Arg(0))
|
||||
return 1
|
||||
}
|
||||
|
||||
if *cfgPath == "" {
|
||||
*cfgPath = os.Getenv("HARNESS_CONFIG")
|
||||
}
|
||||
if *cfgPath == "" {
|
||||
*cfgPath = "harness.toml"
|
||||
}
|
||||
cfg, err := config.Load(*cfgPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
conductor, err := loop.New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
_, err = conductor.Once(ctx, loop.OnceOpts{
|
||||
DryRun: *dryRun,
|
||||
Demo: *demo,
|
||||
TaskID: *taskID,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "harness: %v\n", err)
|
||||
switch {
|
||||
case errors.Is(err, loop.ErrIntake):
|
||||
return 2
|
||||
case errors.Is(err, loop.ErrLLM):
|
||||
return 4
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
# MOPAC harness configuration, v0 (2026-08-28).
|
||||
#
|
||||
# Copy to harness.toml and adjust. harness.toml is gitignored: secrets never
|
||||
# live in this file, only refs (env:NAME | file:PATH | literal:VALUE; the
|
||||
# bw: bitwarden ref lands with the key wrapper in build phase 3).
|
||||
|
||||
# Vertical / stack identity for this harness instance.
|
||||
vertical = "demo"
|
||||
|
||||
# Where the bash tool executes and REPORTs land (relative to the CWD the
|
||||
# harness is started from).
|
||||
work_root = "."
|
||||
report_dir = "reports"
|
||||
|
||||
[loop]
|
||||
# Bounded turn: max LLM round trips per task (tool calls included).
|
||||
max_rounds = 8
|
||||
|
||||
[redmine]
|
||||
url = "https://rm.example.org"
|
||||
key_ref = "env:HARNESS_REDMINE_KEY"
|
||||
# Released scope: raw /issues.json filter params...
|
||||
scope_query = "project=mopac&status_id=released&limit=25"
|
||||
# ...or a saved custom query id (scope_query wins when both are set).
|
||||
# scope_query_id = 42
|
||||
# Custom field carrying the TASK class; issues without it get default_class.
|
||||
class_field = "Class"
|
||||
default_class = "primary"
|
||||
|
||||
[litellm]
|
||||
base_url = "http://192.168.3.78:4001"
|
||||
key_ref = "env:HARNESS_LITELLM_KEY"
|
||||
timeout_secs = 120
|
||||
max_retries = 2
|
||||
|
||||
# MODEL ROUTING v0 (static, config-only, no heuristics): [models] is the
|
||||
# tier map (tier alias -> concrete proxy model); [models.classes] maps task
|
||||
# classes to tiers. Requests go out with the CONCRETE model name resolved
|
||||
# here. Swapping models (e.g. glm-4.7-flash -> glm-5.3-flash once it is
|
||||
# configured on the proxy) is a one-line edit in this file.
|
||||
[models]
|
||||
mopac-study = "glm-4.7-flash" # flash tier
|
||||
mopac-code = "glm-5.2" # flagship
|
||||
mopac-review = "glm-5-turbo" # mid
|
||||
mopac-primary = "glm-5.3" # default / flagship+
|
||||
mopac-vision = "glm-4.6v" # vision when needed
|
||||
default_tier = "mopac-primary"
|
||||
|
||||
[models.classes]
|
||||
study = "mopac-study"
|
||||
read = "mopac-study"
|
||||
code = "mopac-code"
|
||||
architecture = "mopac-code"
|
||||
review = "mopac-review"
|
||||
summarize = "mopac-review"
|
||||
writeback = "mopac-review"
|
||||
vision = "mopac-vision"
|
||||
primary = "mopac-primary"
|
||||
|
||||
# Exec tool: allow-listed bash. Org preset: deny-first, read-leaning allow
|
||||
# list, no sudo/ssh/network exfil. Compound commands are checked segment by
|
||||
# segment; command substitution and subshells are always denied.
|
||||
[tools.bash]
|
||||
enabled = true
|
||||
timeout_secs = 60
|
||||
max_output_bytes = 100000
|
||||
default = "deny"
|
||||
deny = [
|
||||
"sudo *",
|
||||
"ssh *",
|
||||
"scp *",
|
||||
"nc *",
|
||||
"curl *",
|
||||
"wget *",
|
||||
"rm -rf *",
|
||||
]
|
||||
allow = [
|
||||
"pwd",
|
||||
"ls *",
|
||||
"cat *",
|
||||
"head *",
|
||||
"tail *",
|
||||
"grep *",
|
||||
"find *",
|
||||
"wc *",
|
||||
"echo *",
|
||||
"env",
|
||||
"git status",
|
||||
"git diff *",
|
||||
"git log *",
|
||||
"git show *",
|
||||
"go version",
|
||||
"go build *",
|
||||
"go vet *",
|
||||
"go test *",
|
||||
]
|
||||
|
||||
# MVP demo (Charles, 2026-08-28 ~19:00): `harness once --demo` runs this
|
||||
# issue through LiteLLM; the GLM self-description lands as the REPORT.
|
||||
[demo]
|
||||
id = "demo-1"
|
||||
subject = "MVP demo: GLM self-description"
|
||||
prompt = "tell me about yourself"
|
||||
class = "primary"
|
||||
@@ -0,0 +1,311 @@
|
||||
// Package loop is the conductor: one bounded iteration per call
|
||||
// (intake -> gate -> bounded turn -> REPORT writeback). Callers chain
|
||||
// iterations by re-invoking; there is no daemon.
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/intake"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/models"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/task"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/tools"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/writeback"
|
||||
)
|
||||
|
||||
// Sentinel error classes; the CLI maps these to distinct exit codes.
|
||||
var (
|
||||
ErrIntake = errors.New("intake error")
|
||||
ErrLLM = errors.New("llm error")
|
||||
)
|
||||
|
||||
// Conductor wires config, routing, tools, and clients into single-shot runs.
|
||||
type Conductor struct {
|
||||
cfg *config.Config
|
||||
router *models.Router
|
||||
bash *tools.BashTool
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
// New validates routing config and prepares the tool palette.
|
||||
func New(cfg *config.Config, out io.Writer) (*Conductor, error) {
|
||||
router, err := models.NewRouter(cfg.Models.Tiers, cfg.Models.Classes, cfg.Models.DefaultTier)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("model routing: %w", err)
|
||||
}
|
||||
var bash *tools.BashTool
|
||||
if cfg.Bash.Enabled {
|
||||
bash, err = tools.NewBashTool(tools.BashOptions{
|
||||
WorkRoot: cfg.WorkRoot,
|
||||
Allow: cfg.Bash.Allow,
|
||||
Deny: cfg.Bash.Deny,
|
||||
DefaultAllow: cfg.Bash.DefaultAllow,
|
||||
Timeout: time.Duration(cfg.Bash.TimeoutSecs) * time.Second,
|
||||
MaxOutputBytes: cfg.Bash.MaxOutputBytes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Conductor{cfg: cfg, router: router, bash: bash, out: out}, nil
|
||||
}
|
||||
|
||||
// OnceOpts controls a single iteration.
|
||||
type OnceOpts struct {
|
||||
DryRun bool // intake + plan only, no LLM call
|
||||
Demo bool // use the [demo] issue instead of Redmine intake
|
||||
TaskID string // run only the task with this id (chaining aid)
|
||||
}
|
||||
|
||||
// OnceResult reports what one iteration did.
|
||||
type OnceResult struct {
|
||||
TasksSeen int
|
||||
Task task.Task
|
||||
Routed models.Decision
|
||||
ReportPath string
|
||||
Turn *TurnResult
|
||||
}
|
||||
|
||||
// Once runs ONE conductor iteration then returns. Zero tasks in scope is a
|
||||
// success (exit 0) so chained invocations stay cheap.
|
||||
func (c *Conductor) Once(ctx context.Context, opts OnceOpts) (*OnceResult, error) {
|
||||
tasks, err := c.intake(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &OnceResult{TasksSeen: len(tasks)}
|
||||
if opts.TaskID != "" {
|
||||
var kept []task.Task
|
||||
for _, t := range tasks {
|
||||
if t.ID == opts.TaskID {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return res, fmt.Errorf("%w: task id %q not in scope", ErrIntake, opts.TaskID)
|
||||
}
|
||||
tasks = kept
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
fmt.Fprintf(c.out, "harness: no tasks in scope; nothing to do\n")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
t := tasks[0]
|
||||
res.Task = t
|
||||
decision, err := c.router.Resolve(t.Class)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Routed = decision
|
||||
fmt.Fprintf(c.out, "harness: task %s (%s): %q class=%s -> %s -> %s\n",
|
||||
t.ID, t.Source, t.Subject, t.Class, decision.Tier, decision.Model)
|
||||
|
||||
if opts.DryRun {
|
||||
c.printPlan(t, decision)
|
||||
fmt.Fprintf(c.out, "harness: dry-run complete, no LLM call made\n")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
client, err := c.llmClient()
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
start := time.Now()
|
||||
turn, err := c.turn(ctx, client, t, decision.Model)
|
||||
if turn != nil {
|
||||
res.Turn = turn
|
||||
}
|
||||
if err != nil {
|
||||
// Keep whatever content the turn produced before failing.
|
||||
if turn != nil && turn.Content != "" {
|
||||
_, _ = c.writeReport(t, decision, turn, start, err)
|
||||
}
|
||||
return res, fmt.Errorf("%w: %v", ErrLLM, err)
|
||||
}
|
||||
path, err := c.writeReport(t, decision, turn, start, nil)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.ReportPath = path
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// TurnResult is the bounded-turn outcome recorded in the REPORT.
|
||||
type TurnResult struct {
|
||||
Content string
|
||||
Rounds int
|
||||
ToolCalls int
|
||||
Denied int
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
StopReason string
|
||||
}
|
||||
|
||||
// turn runs the bounded LLM loop: request, execute tool calls, repeat until
|
||||
// the model replies with plain content or the round bound is hit.
|
||||
func (c *Conductor) turn(ctx context.Context, client *llm.Client, t task.Task, model string) (*TurnResult, error) {
|
||||
msgs := []llm.Message{
|
||||
{Role: "system", Content: c.systemPrompt()},
|
||||
{Role: "user", Content: t.Prompt},
|
||||
}
|
||||
var palette []llm.Tool
|
||||
if c.bash != nil {
|
||||
palette = append(palette, c.bash.Definition())
|
||||
}
|
||||
res := &TurnResult{}
|
||||
for round := 1; round <= c.cfg.Loop.MaxRounds; round++ {
|
||||
resp, err := client.Chat(ctx, llm.ChatRequest{Model: model, Messages: msgs, Tools: palette})
|
||||
if err != nil {
|
||||
res.StopReason = "llm_error"
|
||||
return res, err
|
||||
}
|
||||
res.Rounds = round
|
||||
res.PromptTokens += resp.Usage.PromptTokens
|
||||
res.CompletionTokens += resp.Usage.CompletionTokens
|
||||
res.TotalTokens += resp.Usage.TotalTokens
|
||||
|
||||
m := resp.Choices[0].Message
|
||||
if m.Content != "" {
|
||||
res.Content = m.Content
|
||||
}
|
||||
if len(m.ToolCalls) == 0 {
|
||||
res.StopReason = "complete"
|
||||
return res, nil
|
||||
}
|
||||
msgs = append(msgs, m)
|
||||
for _, tc := range m.ToolCalls {
|
||||
out, derr := c.execTool(ctx, tc)
|
||||
res.ToolCalls++
|
||||
if derr != nil {
|
||||
res.Denied++
|
||||
}
|
||||
msgs = append(msgs, llm.Message{Role: "tool", ToolCallID: tc.ID, Content: out})
|
||||
}
|
||||
}
|
||||
res.StopReason = "round_limit"
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// execTool dispatches one tool call; the returned string is the model-facing
|
||||
// result. The returned error is non-nil only for gate denials (counted).
|
||||
func (c *Conductor) execTool(ctx context.Context, call llm.ToolCall) (string, error) {
|
||||
if c.bash == nil || call.Function.Name != "bash" {
|
||||
return fmt.Sprintf("error: unknown tool %q", call.Function.Name), nil
|
||||
}
|
||||
out, err := c.bash.Run(ctx, json.RawMessage(call.Function.Arguments))
|
||||
if err != nil {
|
||||
if errors.Is(err, tools.ErrDenied) {
|
||||
return fmt.Sprintf("permission denied: %v", err), tools.ErrDenied
|
||||
}
|
||||
if out != "" {
|
||||
return fmt.Sprintf("%s\nerror: %v", out, err), nil
|
||||
}
|
||||
return fmt.Sprintf("error: %v", err), nil
|
||||
}
|
||||
if out == "" {
|
||||
out = "(no output)"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) intake(ctx context.Context, opts OnceOpts) ([]task.Task, error) {
|
||||
if opts.Demo {
|
||||
fmt.Fprintf(c.out, "harness: intake: demo issue from harness.toml [demo]\n")
|
||||
return []task.Task{intake.DemoTask(c.cfg.Demo)}, nil
|
||||
}
|
||||
rc := c.cfg.Redmine
|
||||
if rc.URL == "" {
|
||||
return nil, fmt.Errorf("%w: no [redmine] url configured (use --demo for the demo issue)", ErrIntake)
|
||||
}
|
||||
key, err := config.ResolveKeyRef(rc.KeyRef)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: redmine key: %v", ErrIntake, err)
|
||||
}
|
||||
client := intake.NewRedmineClient(rc, key)
|
||||
tasks, err := client.ListTasks(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrIntake, err)
|
||||
}
|
||||
fmt.Fprintf(c.out, "harness: intake: %d task(s) in redmine scope\n", len(tasks))
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) llmClient() (*llm.Client, error) {
|
||||
key, err := config.ResolveKeyRef(c.cfg.LiteLLM.KeyRef)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("litellm key: %w", err)
|
||||
}
|
||||
return llm.NewClient(
|
||||
c.cfg.LiteLLM.BaseURL,
|
||||
key,
|
||||
time.Duration(c.cfg.LiteLLM.TimeoutSecs)*time.Second,
|
||||
c.cfg.LiteLLM.MaxRetries,
|
||||
), nil
|
||||
}
|
||||
|
||||
func (c *Conductor) writeReport(t task.Task, d models.Decision, turn *TurnResult, start time.Time, turnErr error) (string, error) {
|
||||
r := writeback.Report{
|
||||
Time: time.Now().UTC(),
|
||||
Vertical: c.cfg.Vertical,
|
||||
Task: t,
|
||||
Class: t.Class,
|
||||
Tier: d.Tier,
|
||||
Model: d.Model,
|
||||
PromptTokens: turn.PromptTokens,
|
||||
CompletionTokens: turn.CompletionTokens,
|
||||
TotalTokens: turn.TotalTokens,
|
||||
Rounds: turn.Rounds,
|
||||
ToolCalls: turn.ToolCalls,
|
||||
Denied: turn.Denied,
|
||||
Duration: time.Since(start),
|
||||
StopReason: turn.StopReason,
|
||||
Content: turn.Content,
|
||||
}
|
||||
if turnErr != nil {
|
||||
r.StopReason = "error: " + turnErr.Error()
|
||||
}
|
||||
path, err := writeback.Write(c.cfg.ReportDir, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(c.out, "harness: REPORT %s\n", path)
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) printPlan(t task.Task, d models.Decision) {
|
||||
fmt.Fprintf(c.out, "PLAN (dry-run)\n")
|
||||
fmt.Fprintf(c.out, " vertical: %s\n", c.cfg.Vertical)
|
||||
fmt.Fprintf(c.out, " task: [%s] %s (source %s)\n", t.ID, t.Subject, t.Source)
|
||||
fmt.Fprintf(c.out, " routing: class %q -> tier %s -> model %s\n", t.Class, d.Tier, d.Model)
|
||||
if c.bash != nil {
|
||||
fmt.Fprintf(c.out, " tools: bash (allow=%d deny=%d default=%s timeout=%ds)\n",
|
||||
len(c.cfg.Bash.Allow), len(c.cfg.Bash.Deny), denyAllow(c.cfg.Bash.DefaultAllow), c.cfg.Bash.TimeoutSecs)
|
||||
} else {
|
||||
fmt.Fprintf(c.out, " tools: none\n")
|
||||
}
|
||||
fmt.Fprintf(c.out, " bound: max %d rounds\n", c.cfg.Loop.MaxRounds)
|
||||
}
|
||||
|
||||
func denyAllow(defaultAllow bool) string {
|
||||
if defaultAllow {
|
||||
return "allow"
|
||||
}
|
||||
return "deny"
|
||||
}
|
||||
|
||||
func (c *Conductor) systemPrompt() string {
|
||||
return fmt.Sprintf(`You are %s, an agent run by the MOPAC harness (headless, bounded turn).
|
||||
Complete the assigned task. Your final reply is written verbatim to the turn
|
||||
REPORT. Use the bash tool only when necessary; commands run in %s and must
|
||||
match the configured allow-list (unmatched or denied commands fail).`,
|
||||
c.cfg.Vertical, c.cfg.WorkRoot)
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
|
||||
)
|
||||
|
||||
// fakeLLM is a scripted OpenAI-compatible server that records every request.
|
||||
type fakeLLM struct {
|
||||
mu sync.Mutex
|
||||
requests []llm.ChatRequest
|
||||
srv *httptest.Server
|
||||
}
|
||||
|
||||
func toolCallMsg(id, name, args string) string {
|
||||
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":%[1]q,"type":"function","function":{"name":%[2]q,"arguments":%[3]q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`,
|
||||
id, name, args)
|
||||
}
|
||||
|
||||
func textMsg(content string) string {
|
||||
return fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%[1]q},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":40,"total_tokens":60}}`, content)
|
||||
}
|
||||
|
||||
// newFakeLLM serves the scripted response bodies in order (repeating the
|
||||
// last one if the turn asks for more).
|
||||
func newFakeLLM(t *testing.T, bodies ...string) *fakeLLM {
|
||||
t.Helper()
|
||||
f := &fakeLLM{}
|
||||
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req llm.ChatRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Errorf("decode request: %v", err)
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.requests = append(f.requests, req)
|
||||
n := len(f.requests)
|
||||
f.mu.Unlock()
|
||||
body := bodies[len(bodies)-1]
|
||||
if n <= len(bodies) {
|
||||
body = bodies[n-1]
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeLLM) requestCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.requests)
|
||||
}
|
||||
|
||||
func (f *fakeLLM) request(i int) llm.ChatRequest {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.requests[i]
|
||||
}
|
||||
|
||||
func testConfig(t *testing.T, llmURL string) *config.Config {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
cfg := config.Default()
|
||||
cfg.Vertical = "teststack"
|
||||
cfg.WorkRoot = dir
|
||||
cfg.ReportDir = filepath.Join(dir, "reports")
|
||||
cfg.LiteLLM.BaseURL = llmURL
|
||||
cfg.LiteLLM.KeyRef = "literal:test-key"
|
||||
cfg.LiteLLM.MaxRetries = 0
|
||||
cfg.Models.Tiers = map[string]string{
|
||||
"mopac-study": "glm-4.7-flash",
|
||||
"mopac-code": "glm-5.2",
|
||||
"mopac-review": "glm-5-turbo",
|
||||
"mopac-primary": "glm-5.3",
|
||||
}
|
||||
cfg.Models.Classes = map[string]string{
|
||||
"study": "mopac-study",
|
||||
"code": "mopac-code",
|
||||
"primary": "mopac-primary",
|
||||
}
|
||||
cfg.Bash.Allow = []string{"echo *", "pwd"}
|
||||
cfg.Bash.Deny = []string{"sudo *"}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestOnceDemoEndToEnd(t *testing.T) {
|
||||
f := newFakeLLM(t, textMsg("I am GLM, a large language model."))
|
||||
cfg := testConfig(t, f.srv.URL)
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Once: %v", err)
|
||||
}
|
||||
// MVP demo bar: prompt "tell me about yourself" -> GLM reply -> REPORT.
|
||||
if f.requestCount() != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", f.requestCount())
|
||||
}
|
||||
req := f.request(0)
|
||||
if req.Model != "glm-5.3" {
|
||||
t.Errorf("request model = %q, want concrete glm-5.3 (class routing)", req.Model)
|
||||
}
|
||||
if len(req.Messages) < 2 || req.Messages[0].Role != "system" || req.Messages[1].Content != "tell me about yourself" {
|
||||
t.Errorf("messages = %+v", req.Messages)
|
||||
}
|
||||
if res.Turn == nil || res.Turn.Content != "I am GLM, a large language model." || res.Turn.StopReason != "complete" {
|
||||
t.Errorf("turn = %+v", res.Turn)
|
||||
}
|
||||
if res.Routed.Tier != "mopac-primary" || res.Routed.Model != "glm-5.3" {
|
||||
t.Errorf("routed = %+v", res.Routed)
|
||||
}
|
||||
body, err := os.ReadFile(res.ReportPath)
|
||||
if err != nil {
|
||||
t.Fatalf("REPORT: %v", err)
|
||||
}
|
||||
s := string(body)
|
||||
for _, want := range []string{"model: glm-5.3 (tier mopac-primary, class primary)", "## Result", "I am GLM, a large language model.", "total=60 tokens"} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("REPORT missing %q", want)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(cfg.ReportDir, "REPORT-latest.md")); err != nil {
|
||||
t.Errorf("REPORT-latest.md: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceDryRunMakesNoLLMCall(t *testing.T) {
|
||||
f := newFakeLLM(t, textMsg("unused"))
|
||||
cfg := testConfig(t, f.srv.URL)
|
||||
var out strings.Builder
|
||||
cond, err := New(cfg, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := cond.Once(context.Background(), OnceOpts{Demo: true, DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Once: %v", err)
|
||||
}
|
||||
if f.requestCount() != 0 {
|
||||
t.Errorf("dry-run made %d LLM calls, want 0", f.requestCount())
|
||||
}
|
||||
if res.ReportPath != "" {
|
||||
t.Errorf("dry-run must not write a REPORT, wrote %s", res.ReportPath)
|
||||
}
|
||||
for _, want := range []string{"dry-run", "glm-5.3", `class "primary" -> tier mopac-primary`} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("plan output missing %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceToolRoundTrip(t *testing.T) {
|
||||
f := newFakeLLM(t,
|
||||
toolCallMsg("call-1", "bash", `{"command":"echo hi from tool"}`),
|
||||
textMsg("Tool ran fine."),
|
||||
)
|
||||
cfg := testConfig(t, f.srv.URL)
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Once: %v", err)
|
||||
}
|
||||
if f.requestCount() != 2 {
|
||||
t.Fatalf("LLM calls = %d, want 2", f.requestCount())
|
||||
}
|
||||
if res.Turn.Rounds != 2 || res.Turn.ToolCalls != 1 || res.Turn.Denied != 0 {
|
||||
t.Errorf("turn = %+v", res.Turn)
|
||||
}
|
||||
// Second request must carry the assistant tool_call and the tool result.
|
||||
second := f.request(1)
|
||||
roles := make([]string, len(second.Messages))
|
||||
for i, m := range second.Messages {
|
||||
roles[i] = m.Role
|
||||
}
|
||||
want := []string{"system", "user", "assistant", "tool"}
|
||||
if fmt.Sprint(roles) != fmt.Sprint(want) {
|
||||
t.Errorf("second request roles = %v, want %v", roles, want)
|
||||
}
|
||||
toolMsg := second.Messages[3]
|
||||
if toolMsg.ToolCallID != "call-1" || !strings.Contains(toolMsg.Content, "hi from tool") {
|
||||
t.Errorf("tool message = %+v", toolMsg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceToolDeniedCounts(t *testing.T) {
|
||||
f := newFakeLLM(t,
|
||||
toolCallMsg("call-9", "bash", `{"command":"sudo rm -rf /"}`),
|
||||
textMsg("Understood, permission was denied."),
|
||||
)
|
||||
cfg := testConfig(t, f.srv.URL)
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if err != nil {
|
||||
t.Fatalf("denied tool call must not fail the turn: %v", err)
|
||||
}
|
||||
if res.Turn.Denied != 1 || res.Turn.ToolCalls != 1 {
|
||||
t.Errorf("turn = %+v", res.Turn)
|
||||
}
|
||||
toolMsg := f.request(1).Messages[3]
|
||||
if !strings.Contains(toolMsg.Content, "permission denied") {
|
||||
t.Errorf("model should see the denial, got %q", toolMsg.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceRoundLimitBounded(t *testing.T) {
|
||||
f := newFakeLLM(t, toolCallMsg("call-1", "bash", `{"command":"pwd"}`))
|
||||
cfg := testConfig(t, f.srv.URL)
|
||||
cfg.Loop.MaxRounds = 3
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if err != nil {
|
||||
t.Fatalf("round limit is not an error: %v", err)
|
||||
}
|
||||
if res.Turn.Rounds != 3 || res.Turn.StopReason != "round_limit" {
|
||||
t.Errorf("turn = %+v", res.Turn)
|
||||
}
|
||||
if f.requestCount() != 3 {
|
||||
t.Errorf("LLM calls = %d, want 3 (bound enforced)", f.requestCount())
|
||||
}
|
||||
if res.ReportPath == "" {
|
||||
t.Errorf("round_limit turn should still write a REPORT")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceLLMFailureMapsToErrLLM(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "no key for you", http.StatusUnauthorized)
|
||||
}))
|
||||
defer srv.Close()
|
||||
cfg := testConfig(t, srv.URL)
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if !errors.Is(err, ErrLLM) {
|
||||
t.Fatalf("err = %v, want ErrLLM", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceIntakeFailureMapsToErrIntake(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "down", http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
cfg := testConfig(t, "http://unused")
|
||||
cfg.Redmine.URL = srv.URL
|
||||
cfg.Redmine.KeyRef = "literal:rm-key"
|
||||
cfg.Redmine.ScopeQuery = "project=x"
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cond.Once(context.Background(), OnceOpts{})
|
||||
if !errors.Is(err, ErrIntake) {
|
||||
t.Fatalf("err = %v, want ErrIntake", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceNoRedmineWithoutDemoIsIntakeError(t *testing.T) {
|
||||
cfg := testConfig(t, "http://unused")
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cond.Once(context.Background(), OnceOpts{})
|
||||
if !errors.Is(err, ErrIntake) {
|
||||
t.Fatalf("err = %v, want ErrIntake (hint toward --demo)", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceTaskIDFilter(t *testing.T) {
|
||||
cfg := testConfig(t, "http://unused")
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cond.Once(context.Background(), OnceOpts{Demo: true, TaskID: "wrong-id"})
|
||||
if !errors.Is(err, ErrIntake) || !strings.Contains(err.Error(), "not in scope") {
|
||||
t.Fatalf("err = %v, want not-in-scope ErrIntake", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnceUnknownClassIsRoutingError(t *testing.T) {
|
||||
cfg := testConfig(t, "http://unused")
|
||||
cfg.Demo.Class = "definitely-not-a-class"
|
||||
cond, err := New(cfg, os.Stdout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cond.Once(context.Background(), OnceOpts{Demo: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown task class") {
|
||||
t.Fatalf("err = %v, want unknown-class error", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user