harness: single-shot conductor + CLI with MVP demo path
`harness once` runs one bounded iteration (intake -> routing -> bounded turn
-> REPORT) then exits, so callers chain it without a daemon. The bounded
turn loops request -> tool calls -> results up to max_rounds; gate denials
feed back to the model and are counted instead of failing the turn.
--dry-run prints the plan (task, resolved model, tool bounds) and makes zero
LLM calls; --demo runs the [demo] issue ("tell me about yourself" through
LiteLLM -> GLM self-description as the REPORT), the MVP acceptance bar.
Distinct exit codes for config/usage, intake, and llm failures. Ships
harness.toml.example (scope query, tier map, allow-lists); real configs are
gitignored. Loop tests run end-to-end against a scripted fake OpenAI server:
demo turn, dry-run zero-call, tool round-trip, denial counting, round limit,
and error-class mapping.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
// Package loop is the conductor: one bounded iteration per call
|
||||
// (intake -> gate -> bounded turn -> REPORT writeback). Callers chain
|
||||
// iterations by re-invoking; there is no daemon.
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/config"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/intake"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/llm"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/models"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/task"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/tools"
|
||||
"git.knownelement.com/reachableceo/MOPAC/harness/internal/writeback"
|
||||
)
|
||||
|
||||
// Sentinel error classes; the CLI maps these to distinct exit codes.
|
||||
var (
|
||||
ErrIntake = errors.New("intake error")
|
||||
ErrLLM = errors.New("llm error")
|
||||
)
|
||||
|
||||
// Conductor wires config, routing, tools, and clients into single-shot runs.
|
||||
type Conductor struct {
|
||||
cfg *config.Config
|
||||
router *models.Router
|
||||
bash *tools.BashTool
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
// New validates routing config and prepares the tool palette.
|
||||
func New(cfg *config.Config, out io.Writer) (*Conductor, error) {
|
||||
router, err := models.NewRouter(cfg.Models.Tiers, cfg.Models.Classes, cfg.Models.DefaultTier)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("model routing: %w", err)
|
||||
}
|
||||
var bash *tools.BashTool
|
||||
if cfg.Bash.Enabled {
|
||||
bash, err = tools.NewBashTool(tools.BashOptions{
|
||||
WorkRoot: cfg.WorkRoot,
|
||||
Allow: cfg.Bash.Allow,
|
||||
Deny: cfg.Bash.Deny,
|
||||
DefaultAllow: cfg.Bash.DefaultAllow,
|
||||
Timeout: time.Duration(cfg.Bash.TimeoutSecs) * time.Second,
|
||||
MaxOutputBytes: cfg.Bash.MaxOutputBytes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Conductor{cfg: cfg, router: router, bash: bash, out: out}, nil
|
||||
}
|
||||
|
||||
// OnceOpts controls a single iteration.
|
||||
type OnceOpts struct {
|
||||
DryRun bool // intake + plan only, no LLM call
|
||||
Demo bool // use the [demo] issue instead of Redmine intake
|
||||
TaskID string // run only the task with this id (chaining aid)
|
||||
}
|
||||
|
||||
// OnceResult reports what one iteration did.
|
||||
type OnceResult struct {
|
||||
TasksSeen int
|
||||
Task task.Task
|
||||
Routed models.Decision
|
||||
ReportPath string
|
||||
Turn *TurnResult
|
||||
}
|
||||
|
||||
// Once runs ONE conductor iteration then returns. Zero tasks in scope is a
|
||||
// success (exit 0) so chained invocations stay cheap.
|
||||
func (c *Conductor) Once(ctx context.Context, opts OnceOpts) (*OnceResult, error) {
|
||||
tasks, err := c.intake(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &OnceResult{TasksSeen: len(tasks)}
|
||||
if opts.TaskID != "" {
|
||||
var kept []task.Task
|
||||
for _, t := range tasks {
|
||||
if t.ID == opts.TaskID {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return res, fmt.Errorf("%w: task id %q not in scope", ErrIntake, opts.TaskID)
|
||||
}
|
||||
tasks = kept
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
fmt.Fprintf(c.out, "harness: no tasks in scope; nothing to do\n")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
t := tasks[0]
|
||||
res.Task = t
|
||||
decision, err := c.router.Resolve(t.Class)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Routed = decision
|
||||
fmt.Fprintf(c.out, "harness: task %s (%s): %q class=%s -> %s -> %s\n",
|
||||
t.ID, t.Source, t.Subject, t.Class, decision.Tier, decision.Model)
|
||||
|
||||
if opts.DryRun {
|
||||
c.printPlan(t, decision)
|
||||
fmt.Fprintf(c.out, "harness: dry-run complete, no LLM call made\n")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
client, err := c.llmClient()
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
start := time.Now()
|
||||
turn, err := c.turn(ctx, client, t, decision.Model)
|
||||
if turn != nil {
|
||||
res.Turn = turn
|
||||
}
|
||||
if err != nil {
|
||||
// Keep whatever content the turn produced before failing.
|
||||
if turn != nil && turn.Content != "" {
|
||||
_, _ = c.writeReport(t, decision, turn, start, err)
|
||||
}
|
||||
return res, fmt.Errorf("%w: %v", ErrLLM, err)
|
||||
}
|
||||
path, err := c.writeReport(t, decision, turn, start, nil)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.ReportPath = path
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// TurnResult is the bounded-turn outcome recorded in the REPORT.
|
||||
type TurnResult struct {
|
||||
Content string
|
||||
Rounds int
|
||||
ToolCalls int
|
||||
Denied int
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
StopReason string
|
||||
}
|
||||
|
||||
// turn runs the bounded LLM loop: request, execute tool calls, repeat until
|
||||
// the model replies with plain content or the round bound is hit.
|
||||
func (c *Conductor) turn(ctx context.Context, client *llm.Client, t task.Task, model string) (*TurnResult, error) {
|
||||
msgs := []llm.Message{
|
||||
{Role: "system", Content: c.systemPrompt()},
|
||||
{Role: "user", Content: t.Prompt},
|
||||
}
|
||||
var palette []llm.Tool
|
||||
if c.bash != nil {
|
||||
palette = append(palette, c.bash.Definition())
|
||||
}
|
||||
res := &TurnResult{}
|
||||
for round := 1; round <= c.cfg.Loop.MaxRounds; round++ {
|
||||
resp, err := client.Chat(ctx, llm.ChatRequest{Model: model, Messages: msgs, Tools: palette})
|
||||
if err != nil {
|
||||
res.StopReason = "llm_error"
|
||||
return res, err
|
||||
}
|
||||
res.Rounds = round
|
||||
res.PromptTokens += resp.Usage.PromptTokens
|
||||
res.CompletionTokens += resp.Usage.CompletionTokens
|
||||
res.TotalTokens += resp.Usage.TotalTokens
|
||||
|
||||
m := resp.Choices[0].Message
|
||||
if m.Content != "" {
|
||||
res.Content = m.Content
|
||||
}
|
||||
if len(m.ToolCalls) == 0 {
|
||||
res.StopReason = "complete"
|
||||
return res, nil
|
||||
}
|
||||
msgs = append(msgs, m)
|
||||
for _, tc := range m.ToolCalls {
|
||||
out, derr := c.execTool(ctx, tc)
|
||||
res.ToolCalls++
|
||||
if derr != nil {
|
||||
res.Denied++
|
||||
}
|
||||
msgs = append(msgs, llm.Message{Role: "tool", ToolCallID: tc.ID, Content: out})
|
||||
}
|
||||
}
|
||||
res.StopReason = "round_limit"
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// execTool dispatches one tool call; the returned string is the model-facing
|
||||
// result. The returned error is non-nil only for gate denials (counted).
|
||||
func (c *Conductor) execTool(ctx context.Context, call llm.ToolCall) (string, error) {
|
||||
if c.bash == nil || call.Function.Name != "bash" {
|
||||
return fmt.Sprintf("error: unknown tool %q", call.Function.Name), nil
|
||||
}
|
||||
out, err := c.bash.Run(ctx, json.RawMessage(call.Function.Arguments))
|
||||
if err != nil {
|
||||
if errors.Is(err, tools.ErrDenied) {
|
||||
return fmt.Sprintf("permission denied: %v", err), tools.ErrDenied
|
||||
}
|
||||
if out != "" {
|
||||
return fmt.Sprintf("%s\nerror: %v", out, err), nil
|
||||
}
|
||||
return fmt.Sprintf("error: %v", err), nil
|
||||
}
|
||||
if out == "" {
|
||||
out = "(no output)"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) intake(ctx context.Context, opts OnceOpts) ([]task.Task, error) {
|
||||
if opts.Demo {
|
||||
fmt.Fprintf(c.out, "harness: intake: demo issue from harness.toml [demo]\n")
|
||||
return []task.Task{intake.DemoTask(c.cfg.Demo)}, nil
|
||||
}
|
||||
rc := c.cfg.Redmine
|
||||
if rc.URL == "" {
|
||||
return nil, fmt.Errorf("%w: no [redmine] url configured (use --demo for the demo issue)", ErrIntake)
|
||||
}
|
||||
key, err := config.ResolveKeyRef(rc.KeyRef)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: redmine key: %v", ErrIntake, err)
|
||||
}
|
||||
client := intake.NewRedmineClient(rc, key)
|
||||
tasks, err := client.ListTasks(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrIntake, err)
|
||||
}
|
||||
fmt.Fprintf(c.out, "harness: intake: %d task(s) in redmine scope\n", len(tasks))
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) llmClient() (*llm.Client, error) {
|
||||
key, err := config.ResolveKeyRef(c.cfg.LiteLLM.KeyRef)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("litellm key: %w", err)
|
||||
}
|
||||
return llm.NewClient(
|
||||
c.cfg.LiteLLM.BaseURL,
|
||||
key,
|
||||
time.Duration(c.cfg.LiteLLM.TimeoutSecs)*time.Second,
|
||||
c.cfg.LiteLLM.MaxRetries,
|
||||
), nil
|
||||
}
|
||||
|
||||
func (c *Conductor) writeReport(t task.Task, d models.Decision, turn *TurnResult, start time.Time, turnErr error) (string, error) {
|
||||
r := writeback.Report{
|
||||
Time: time.Now().UTC(),
|
||||
Vertical: c.cfg.Vertical,
|
||||
Task: t,
|
||||
Class: t.Class,
|
||||
Tier: d.Tier,
|
||||
Model: d.Model,
|
||||
PromptTokens: turn.PromptTokens,
|
||||
CompletionTokens: turn.CompletionTokens,
|
||||
TotalTokens: turn.TotalTokens,
|
||||
Rounds: turn.Rounds,
|
||||
ToolCalls: turn.ToolCalls,
|
||||
Denied: turn.Denied,
|
||||
Duration: time.Since(start),
|
||||
StopReason: turn.StopReason,
|
||||
Content: turn.Content,
|
||||
}
|
||||
if turnErr != nil {
|
||||
r.StopReason = "error: " + turnErr.Error()
|
||||
}
|
||||
path, err := writeback.Write(c.cfg.ReportDir, r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(c.out, "harness: REPORT %s\n", path)
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (c *Conductor) printPlan(t task.Task, d models.Decision) {
|
||||
fmt.Fprintf(c.out, "PLAN (dry-run)\n")
|
||||
fmt.Fprintf(c.out, " vertical: %s\n", c.cfg.Vertical)
|
||||
fmt.Fprintf(c.out, " task: [%s] %s (source %s)\n", t.ID, t.Subject, t.Source)
|
||||
fmt.Fprintf(c.out, " routing: class %q -> tier %s -> model %s\n", t.Class, d.Tier, d.Model)
|
||||
if c.bash != nil {
|
||||
fmt.Fprintf(c.out, " tools: bash (allow=%d deny=%d default=%s timeout=%ds)\n",
|
||||
len(c.cfg.Bash.Allow), len(c.cfg.Bash.Deny), denyAllow(c.cfg.Bash.DefaultAllow), c.cfg.Bash.TimeoutSecs)
|
||||
} else {
|
||||
fmt.Fprintf(c.out, " tools: none\n")
|
||||
}
|
||||
fmt.Fprintf(c.out, " bound: max %d rounds\n", c.cfg.Loop.MaxRounds)
|
||||
}
|
||||
|
||||
func denyAllow(defaultAllow bool) string {
|
||||
if defaultAllow {
|
||||
return "allow"
|
||||
}
|
||||
return "deny"
|
||||
}
|
||||
|
||||
func (c *Conductor) systemPrompt() string {
|
||||
return fmt.Sprintf(`You are %s, an agent run by the MOPAC harness (headless, bounded turn).
|
||||
Complete the assigned task. Your final reply is written verbatim to the turn
|
||||
REPORT. Use the bash tool only when necessary; commands run in %s and must
|
||||
match the configured allow-list (unmatched or denied commands fail).`,
|
||||
c.cfg.Vertical, c.cfg.WorkRoot)
|
||||
}
|
||||
Reference in New Issue
Block a user