From b7799ea0de0d286e9e192c32489cc0715d556210 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Sat, 29 Aug 2026 05:50:41 -0500 Subject: [PATCH] deploy: 9-account/2-host packaging + static release build (Redmine 494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything needed for one ~10-minute Charles window on ultix-streaming and ultix-offstage, with zero root/systemd on the target accounts: - deploy/accounts.tsv: the fleet authority (account, host, vertical, Redmine project, quota group, ports). Port scheme events=4100+index, serve=8090+index with a global 0-8 index so per-account daemons on one host never collide; loop state is per-account under ~/.mopac. - deploy/install-account.sh : idempotent installer run AS the target user; renders harness.toml from deploy/harness.toml.in, writes a 0600 env-secrets template, refuses to overwrite existing config or secrets (re-run = the upgrade path), generates mopac-start/stop. - deploy/runbook.md: exact Charles sequence (build, stage, install, secrets bootstrap, verify, start via nohup or cron @reboot, rollback) with the account-port table, per-host time estimates and assumptions. - Makefile: release target (digest-pinned docker builder, CGO off, linux/amd64 static, stripped) plus check/deploy-test entrypoints. - deploy/tests.sh: 13 packaging tests (TSV scheme, template substitution for all 9 accounts, idempotence, refuse-to-overwrite, binary lookup, rendered config loads via a dry-run); README deploy section. πŸ’˜ Generated with Crush Assisted-by: Crush:glm-5.2 --- Makefile | 33 ++++++ README.md | 47 +++++++- deploy/accounts.tsv | 28 +++++ deploy/harness.toml.in | 177 ++++++++++++++++++++++++++++++ deploy/install-account.sh | 203 ++++++++++++++++++++++++++++++++++ deploy/runbook.md | 222 ++++++++++++++++++++++++++++++++++++++ deploy/tests.sh | 180 +++++++++++++++++++++++++++++++ 7 files changed, 889 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 deploy/accounts.tsv create mode 100644 deploy/harness.toml.in create mode 100755 deploy/install-account.sh create mode 100644 deploy/runbook.md create mode 100755 deploy/tests.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bef8df0 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +# MOPAC harness Makefile. Everything routes through the digest-pinned +# Docker builder (DESIGN "ALL dev work in Docker"); the host runs +# containers, never toolchains. dev.sh stays the dev entrypoint β€” the +# targets here are the deploy-facing surface. + +IMAGE := golang@sha256:e8c859f5632dcfde7b32d2012b4351728f6437930887c2f6a91ea242459e5514 +# = golang:1.26-bookworm (digest-pinned; alpine lacks bash for tests) + +.PHONY: release check deploy-test clean + +# Static linux/amd64 release binary for the 9-account deploy (Redmine +# 494): CGO off -> no libc dependency, runs in any Linux account with +# zero host packages. Output is gitignored (bin/). +release: + docker run --rm -v "$(CURDIR):/h" -w /h \ + -u $$(id -u):$$(id -g) -e HOME=/tmp -e GOFLAGS=-buildvcs=false \ + -e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 \ + $(IMAGE) go build -trimpath -ldflags='-s -w' \ + -o bin/harness-linux-amd64 ./cmd/harness + @file bin/harness-linux-amd64 2>/dev/null || ls -l bin/harness-linux-amd64 + +# Pre-commit gate: build + vet + test, all inside the builder. +check: + ./dev.sh check + +# Deploy packaging tests: accounts.tsv scheme, template substitution, +# installer idempotence + refuse-to-overwrite in a fake HOME. +# (Pure shell, no docker needed β€” it tests the shell deliverables.) +deploy-test: + ./deploy/tests.sh + +clean: + rm -rf bin/harness bin/harness-linux-amd64 diff --git a/README.md b/README.md index 6706b69..7849fbe 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ logged reason and is reconsidered next scan, never hard-failed; per-class token+credit accounting lands in the loop JSONL (`harness quota status`). `harness serve` (OWUI front door, LAN 8090), `harness loop` (fake-Redmine e2e test-asserted), the MVP demo path and `harness events` (port 4100) -all live. +all live. Multi-account deploy packaging (Redmine 494): `make release` + +`deploy/` β€” 9 accounts / 2 hosts, per-account ports + state, no root. ## Quickstart @@ -46,6 +47,11 @@ docker run --rm -v "$PWD:/h" -w /h \ (that digest = `golang:1.26-bookworm`; alpine lacks bash, which the exec tool's tests need.) +`make release` builds the deployable static binary +(`bin/harness-linux-amd64`, CGO off) through the same pinned builder; +`make deploy-test` tests the multi-account packaging β€” see +"Deploy: 9 accounts / 2 hosts" below. + Expected output (tail): ```text @@ -565,10 +571,49 @@ docker run ... --memory 4g --cpus 2 --pids-limit 512 \ or a systemd slice for non-container deploys (`CPUQuota=200%`, `MemoryMax=4G`, `IOWeight`). The gate catches what the limits don't. +## Deploy: 9 accounts / 2 hosts (Redmine 494) + +The full multi-account packaging lives in [`deploy/`](deploy/) β€” one +harness instance per Linux account (5 on ultix-streaming, 4 on +ultix-offstage), each entirely under that account's `~/.mopac/` (binary, +config, 0600 env secrets, state, reports). No root, no systemd, no host +packages: `sudo` is only used to switch identity when installing into the +other accounts. + +```sh +make release # static linux/amd64 binary: bin/harness-linux-amd64 + # (docker builder, CGO_ENABLED=0 β€” runs anywhere) +make deploy-test # packaging tests: port scheme, template substitution, + # installer idempotence in a fake HOME +``` + +- [`deploy/accounts.tsv`](deploy/accounts.tsv) β€” the fleet authority: + account, host, vertical, Redmine project, quota group, and the port + scheme `events = 4100 + index` / `serve = 8090 + index` (index 0-8, so + the daemons of any two accounts on one host never collide; loop state + is per-account under `~/.mopac/state/`). +- [`deploy/install-account.sh `](deploy/install-account.sh) β€” + idempotent installer, run AS the target user: creates + `~/.mopac/{bin,state,reports,work}`, installs the static binary, + renders `harness.toml` from [`deploy/harness.toml.in`](deploy/harness.toml.in) + + the TSV row, writes the 0600 `~/.mopac/env` secrets template and + `mopac-start`/`mopac-stop` helpers. Existing `harness.toml`/`env` are + NEVER overwritten (re-running = the upgrade path). +- [`deploy/runbook.md`](deploy/runbook.md) β€” the exact Charles sequence: + build once, stage per host, install per account, secrets bootstrap, + verify (`once --dry-run --demo`, healthz, first `loop --once --dry-run`), + start (nohup helpers + optional cron `@reboot`), rollback (stop + + `rm -rf ~/.mopac`), per-host time estimates. + +Generated configs keep `[quota]`/`[resources]` commented with values +pre-filled per account β€” flipping them on is a per-account one-liner +once the shared redis container (above) is up. + ## Docs and links - [DESIGN.md](DESIGN.md) β€” design spec (hard rules, build order, org model) - [REPORT.md](REPORT.md) β€” current build status +- [docs/SPEC-20260829-charles-brief.md](docs/SPEC-20260829-charles-brief.md) β€” Charles's end-to-end design brief (spec of record) - [docs/PORTING-NOTES-crush.md](docs/PORTING-NOTES-crush.md) β€” sessions/MCP/provider study of the crush agent - [docs/PORTING-NOTES-maki.md](docs/PORTING-NOTES-maki.md) β€” permission parsing and token-reduction study of maki - [docs/PORTING-NOTES-secrets.md](docs/PORTING-NOTES-secrets.md) β€” Bitwarden secrets study; feeds the bitwarden-go tool diff --git a/deploy/accounts.tsv b/deploy/accounts.tsv new file mode 100644 index 0000000..b84602a --- /dev/null +++ b/deploy/accounts.tsv @@ -0,0 +1,28 @@ +# MOPAC multi-account fleet β€” SPEC-20260829-charles-brief.md, Redmine 494. +# Tab-separated; lines starting with '#' are comments. This file is the +# authority for deploy/install-account.sh and the account-port table in +# deploy/runbook.md β€” edit here, never in generated configs. +# +# Port scheme (collision-free across BOTH hosts, index is global 0-8): +# events_port = 4100 + index (`harness events` webhook receiver) +# serve_port = 8090 + index (`harness serve` OWUI front door) +# +# quota_account groups instances sharing one z.ai coding plan (two Max +# plans assumed: one per host β€” one-line edit per row when the real +# key-to-host grouping is confirmed). +# +# redmine_project is the Redmine project identifier used in the generated +# scope_query (project=&status_id=released&limit=25). The projects must +# exist with these identifiers (runbook bootstrap step 0), or edit this +# column to the real identifiers and re-render. +# +# account host index vertical redmine_project quota_account events_port serve_port +reachableceo ultix-streaming 0 rceo mopac-rceo zai-max-1 4100 8090 +TSGBOD ultix-streaming 1 bod mopac-bod zai-max-1 4101 8091 +TSGCOO ultix-streaming 2 coo mopac-coo zai-max-1 4102 8092 +TSGCTO ultix-streaming 3 cto mopac-cto zai-max-1 4103 8093 +TSGCCO ultix-streaming 4 cco mopac-cco zai-max-1 4104 8094 +reachableceo-offstage ultix-offstage 5 rceo-offstage mopac-rceo-offstage zai-max-2 4105 8095 +COSRCEO-Personal ultix-offstage 6 cos-rceo-personal mopac-cos-rceo-personal zai-max-2 4106 8096 +COSRCEO-Biz ultix-offstage 7 cos-rceo-biz mopac-cos-rceo-biz zai-max-2 4107 8097 +COSWFO ultix-offstage 8 cos-wfo mopac-cos-wfo zai-max-2 4108 8098 diff --git a/deploy/harness.toml.in b/deploy/harness.toml.in new file mode 100644 index 0000000..b95028c --- /dev/null +++ b/deploy/harness.toml.in @@ -0,0 +1,177 @@ +# MOPAC harness configuration for @ACCOUNT@ (@HOST@, index @INDEX@) β€” +# generated by deploy/install-account.sh from deploy/accounts.tsv +# (Redmine 494). Hand-edits are ALLOWED and survive re-runs of the +# installer (it refuses to overwrite an existing harness.toml). +# +# No secrets in this file, ever β€” only env: refs resolved at runtime from +# ~/.mopac/env (0600, see deploy/runbook.md bootstrap step). + +# Vertical / stack identity for this harness instance. +vertical = "@VERTICAL@" + +# Absolute paths (substituted at install): the bash tool executes in +# work_root; REPORTs land in report_dir; daemons keep state under state/. +work_root = "@HOME@/.mopac/work" +report_dir = "@HOME@/.mopac/reports" + +[loop] +# Bounded turn: max LLM round trips per task (tool calls included). +max_rounds = 8 +# `harness loop` daemon: Redmine scan interval and state location +# (append-only loop.jsonl, dedup by issue id + updated_on). +poll_interval_secs = 120 +state_dir = "@HOME@/.mopac/state/loop" + +[redmine] +url = "https://projects.knownelement.com" +key_ref = "env:HARNESS_REDMINE_KEY" +# Released scope for this account's vertical (project from accounts.tsv): +scope_query = "project=@REDMINE_PROJECT@&status_id=released&limit=25" +# class_field: Redmine custom field carrying the TASK class; issues +# without it get default_class. +class_field = "Class" +default_class = "primary" + +# After a REPORT is noted back on an issue, an issue whose CURRENT status +# matches a key here moves to the value (names resolved via +# /issue_statuses.json). Set the pair that moves work OUT of the released +# scope for this project's workflow, e.g. (uncomment + adjust names): +# [redmine.status_map] +# "Released" = "Done" +# Empty/absent map = leave status alone (the loop still advances its own +# dedup marker past its writebacks β€” no redispatch storm). + +[litellm] +base_url = "http://192.168.3.78:4001" +key_ref = "env:HARNESS_LITELLM_KEY" +timeout_secs = 120 +max_retries = 2 + +# GITEA (optional, off by default): `harness loop` commits each REPORT +# file to this repo right after writing it. Per-account REPORT repos can +# be wired later by uncommenting (key goes into ~/.mopac/env). +# [gitea] +# url = "https://git.knownelement.com" +# key_ref = "env:HARNESS_GITEA_KEY" +# owner = "ukrrs" +# repo = "MOPAC-reports" +# branch = "main" +# commit_reports = false + +# MODEL ROUTING v0 (static, config-only): [models] is the tier map (tier +# alias -> concrete proxy model); [models.classes] maps task classes to +# tiers. Swapping models is a one-line edit here. +[models] +mopac-study = "glm-4.7-flash" # flash tier +mopac-code = "glm-5.2" # flagship +mopac-review = "glm-5-turbo" # mid +mopac-primary = "glm-5.3" # default / flagship+ +mopac-vision = "glm-4.6v" # vision when needed +default_tier = "mopac-primary" + +[models.classes] +study = "mopac-study" +read = "mopac-study" +code = "mopac-code" +architecture = "mopac-code" +review = "mopac-review" +summarize = "mopac-review" +writeback = "mopac-review" +vision = "mopac-vision" +primary = "mopac-primary" + +# Exec tool: allow-listed bash. Org preset: deny-first, read-leaning allow +# list, no sudo/ssh/network exfil. +[tools.bash] +enabled = true +timeout_secs = 60 +max_output_bytes = 100000 +default = "deny" +deny = [ + "sudo *", + "ssh *", + "scp *", + "nc *", + "curl *", + "wget *", + "rm -rf *", +] +allow = [ + "pwd", + "ls *", + "cat *", + "head *", + "tail *", + "grep *", + "find *", + "wc *", + "echo *", + "env", + "git status", + "git diff *", + "git log *", + "git show *", + "go version", + "go build *", + "go vet *", + "go test *", +] + +# Smoke-test issue for this install: `harness once --dry-run --demo` runs +# the plan path with NO secrets and NO LLM call β€” the runbook verify step. +[demo] +id = "demo-@VERTICAL@" +subject = "@ACCOUNT@ install smoke" +prompt = "introduce yourself and your vertical" +class = "primary" + +# EVENTS: the `harness events` webhook receiver. Port @EVENTS_PORT@ = +# 4100 + index from accounts.tsv β€” unique per account per host. +[events] +listen = ":@EVENTS_PORT@" +state_dir = "@HOME@/.mopac/state/events" + +[events.redmine] +secret_ref = "env:HARNESS_REDMINE_WEBHOOK_SECRET" + +[events.discourse] +secret_ref = "env:HARNESS_DISCOURSE_WEBHOOK_SECRET" + +[events.gitea] +secret_ref = "env:HARNESS_GITEA_WEBHOOK_SECRET" + +# SERVE: the OpenAI-compatible front door for OpenWebUI. Port +# @SERVE_PORT@ = 8090 + index from accounts.tsv. Bearer vkey auth β€” the +# same value goes into the OWUI connection config for this account. +[serve] +listen = ":@SERVE_PORT@" +vkey_ref = "env:HARNESS_SERVE_VKEY" + +# QUOTA GATE (Redmine 490): values pre-filled for this account's plan +# group (@QUOTA_ACCOUNT@); flip enabled = true once the shared redis +# container is up (README runbook) and the grouping is confirmed. +# [quota] +# enabled = false +# account = "@QUOTA_ACCOUNT@" +# plan_5h_credits = 28000 +# plan_weekly_credits = 140000 +# usage_url = "" +# key_ref = "env:HARNESS_ZAI_KEY" +# poll_interval_secs = 300 +# defer_at_pct = 85 +# block_at_pct = 95 +# peak_start = "01:00" +# peak_end = "05:00" +# timezone = "America/Chicago" +# peak_weekdays_only = true +# peak_classes = ["study", "read"] +# redis_url = "redis://192.168.3.78:6390/0" + +# RESOURCE GATE (Redmine 491): read-only host monitor; the loop defers +# dispatch while busy. Flip enabled = true per host capacity. +# [resources] +# enabled = false +# max_load_avg = 6.0 +# min_mem_available_mb = 2048 +# min_disk_free_mb = 5120 +# max_io_delay_pct = 90.0 diff --git a/deploy/install-account.sh b/deploy/install-account.sh new file mode 100755 index 0000000..f44e067 --- /dev/null +++ b/deploy/install-account.sh @@ -0,0 +1,203 @@ +#!/bin/sh +# install-account.sh β€” idempotent per-account MOPAC harness installer +# (Redmine 494, SPEC-20260829-charles-brief.md). +# +# Runs AS the target user on the target host (no root, no sudo): creates +# ~/.mopac/{bin,state,reports,work}, installs the static linux/amd64 +# binary (built once via `make release` in the Docker builder), renders +# harness.toml from deploy/harness.toml.in + the deploy/accounts.tsv row, +# and writes the 0600 env-secrets template plus start/stop helpers. +# +# Idempotence contract (deploy/tests.sh asserts all of this): +# - re-running is always safe: existing harness.toml and ~/.mopac/env +# are NEVER overwritten (delete them to re-render); +# - the binary and the generated start/stop helpers ARE refreshed +# (that is the upgrade path: make release, re-stage, re-run); +# - exit 0 on success (including "already installed, kept"), non-zero +# only on genuine errors (unknown account, missing binary/template). +# +# Usage: deploy/install-account.sh +# Env: MOPAC_HARNESS_BIN explicit binary path (overrides lookup; +# used by deploy/tests.sh with a stub binary) +# +# Lookup order for the binary: +# 1. $MOPAC_HARNESS_BIN +# 2. /../bin/harness-linux-amd64 (repo layout, staged tgz) +# 3. /bin/harness-linux-amd64 +# 4. /harness-linux-amd64 + +set -eu + +die() { echo "install-account.sh: $*" >&2; exit 1; } + +[ $# -eq 1 ] || die "usage: install-account.sh (accounts: see accounts.tsv)" +ACCOUNT=$1 + +DEPLOY_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TSV="$DEPLOY_DIR/accounts.tsv" +TEMPLATE="$DEPLOY_DIR/harness.toml.in" +[ -r "$TSV" ] || die "accounts.tsv not found next to this script ($DEPLOY_DIR)" +[ -r "$TEMPLATE" ] || die "harness.toml.in not found next to this script" + +# --- account row from the TSV (tab-separated, exact case-sensitive match) +ROW=$(awk -F'\t' -v a="$ACCOUNT" '$1 !~ /^#/ && $1 == a {print}' "$TSV") +[ -n "$ROW" ] || die "account '$ACCOUNT' not in $TSV" +[ "$(printf '%s\n' "$ROW" | wc -l)" -eq 1 ] || die "account '$ACCOUNT' has duplicate rows in accounts.tsv" + +HOST=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $2}') +INDEX=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $3}') +VERTICAL=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $4}') +REDMINE_PROJECT=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $5}') +QUOTA_ACCOUNT=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $6}') +EVENTS_PORT=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $7}') +SERVE_PORT=$(printf '%s\n' "$ROW" | awk -F'\t' '{print $8}') +[ -n "$HOST" ] && [ -n "$INDEX" ] && [ -n "$VERTICAL" ] && [ -n "$REDMINE_PROJECT" ] \ + && [ -n "$QUOTA_ACCOUNT" ] && [ -n "$EVENTS_PORT" ] && [ -n "$SERVE_PORT" ] \ + || die "malformed row for '$ACCOUNT' in accounts.tsv (need 8 columns)" + +# --- concurrency guard: ports must follow the fleet scheme (unique per +# account, so multiple daemons on one host never collide) +[ "$EVENTS_PORT" -eq $((4100 + INDEX)) ] || die "$ACCOUNT: events_port $EVENTS_PORT != 4100+$INDEX (accounts.tsv scheme broken)" +[ "$SERVE_PORT" -eq $((8090 + INDEX)) ] || die "$ACCOUNT: serve_port $SERVE_PORT != 8090+$INDEX (accounts.tsv scheme broken)" + +# --- locate the static binary +BIN="" +for cand in "${MOPAC_HARNESS_BIN:-}" "$DEPLOY_DIR/../bin/harness-linux-amd64" \ + "$DEPLOY_DIR/bin/harness-linux-amd64" "$DEPLOY_DIR/harness-linux-amd64"; do + [ -n "$cand" ] && [ -f "$cand" ] && { BIN=$cand; break; } +done +[ -n "$BIN" ] || die "static binary not found (run 'make release' and stage bin/harness-linux-amd64 next to deploy/, or set MOPAC_HARNESS_BIN)" + +HOME_DIR=$(CDPATH= cd && pwd) +MOPAC="$HOME_DIR/.mopac" + +# --- advisory: wrong host? (hostname check never blocks an install) +THIS_HOST=$(hostname -s 2>/dev/null || echo unknown) +if [ "$THIS_HOST" != "$HOST" ]; then + echo "install-account.sh: NOTE: account '$ACCOUNT' belongs on '$HOST' but this host is '$THIS_HOST' (continuing)" +fi + +# --- 1. directory skeleton +mkdir -p "$MOPAC/bin" "$MOPAC/state/loop" "$MOPAC/state/events" "$MOPAC/reports" "$MOPAC/work" +chmod 700 "$MOPAC" "$MOPAC/state" + +# --- 2. binary (always refreshed: the upgrade path) +cp "$BIN" "$MOPAC/bin/harness.new" +chmod 0755 "$MOPAC/bin/harness.new" +mv -f "$MOPAC/bin/harness.new" "$MOPAC/bin/harness" +echo "installed binary: $MOPAC/bin/harness ($(wc -c <"$MOPAC/bin/harness" | tr -d ' ') bytes)" + +# --- 3. harness.toml (rendered once; NEVER overwritten) +if [ -e "$MOPAC/harness.toml" ]; then + echo "kept existing: $MOPAC/harness.toml (delete it to re-render)" +else + sed -e "s|@ACCOUNT@|$ACCOUNT|g" \ + -e "s|@HOST@|$HOST|g" \ + -e "s|@INDEX@|$INDEX|g" \ + -e "s|@VERTICAL@|$VERTICAL|g" \ + -e "s|@REDMINE_PROJECT@|$REDMINE_PROJECT|g" \ + -e "s|@QUOTA_ACCOUNT@|$QUOTA_ACCOUNT|g" \ + -e "s|@EVENTS_PORT@|$EVENTS_PORT|g" \ + -e "s|@SERVE_PORT@|$SERVE_PORT|g" \ + -e "s|@HOME@|$HOME_DIR|g" \ + "$TEMPLATE" > "$MOPAC/harness.toml.new" + if grep -q '@[A-Z_][A-Z0-9_]*@' "$MOPAC/harness.toml.new"; then + rm -f "$MOPAC/harness.toml.new" + die "template substitution left placeholders for '$ACCOUNT' (template/template mismatch)" + fi + mv "$MOPAC/harness.toml.new" "$MOPAC/harness.toml" + echo "rendered: $MOPAC/harness.toml (vertical=$VERTICAL project=$REDMINE_PROJECT events=:$EVENTS_PORT serve=:$SERVE_PORT)" +fi + +# --- 4. secrets template (0600; NEVER overwritten) +if [ -e "$MOPAC/env" ]; then + echo "kept existing: $MOPAC/env (fill it if not yet filled)" +else + ENV_TMP="$MOPAC/env.new" + cat >"$ENV_TMP" <"$START.new" <<'EOF' +#!/bin/sh +# Generated by install-account.sh for @ACCOUNT@ β€” safe to regenerate. +# Starts the three daemons under this account (no root, no systemd): +# loop (Redmine poll -> bounded turns) +# events (webhook receiver, :@EVENTS_PORT@) +# serve (OWUI front door, :@SERVE_PORT@) +# Idempotent: daemons with a live pid are skipped. Logs + pids land in +# ~/.mopac/state/. Survives logout (nohup); for reboot persistence add +# the cron @reboot line from deploy/runbook.md. +set -eu +H="${HOME%/}/.mopac" +if [ -r "$H/env" ]; then . "$H/env"; fi +cd "$H" +start() { # name args... + name=$1; shift + if [ -s "$H/state/$name.pid" ] && kill -0 "$(cat "$H/state/$name.pid")" 2>/dev/null; then + echo "$name already running (pid $(cat "$H/state/$name.pid"))" + return 0 + fi + nohup "$H/bin/harness" "$@" >>"$H/state/$name.log" 2>&1 & + echo $! >"$H/state/$name.pid" + echo "started $name (pid $!)" +} +start loop -config "$H/harness.toml" +start events -config "$H/harness.toml" -listen ":@EVENTS_PORT@" +start serve -config "$H/harness.toml" -listen ":@SERVE_PORT@" +EOF +mkscript "$START" 0755 + +STOP="$MOPAC/bin/mopac-stop" +sed -e "s|@ACCOUNT@|$ACCOUNT|g" >"$STOP.new" <<'EOF' +#!/bin/sh +# Generated by install-account.sh for @ACCOUNT@ β€” safe to regenerate. +# Stops the daemons started by mopac-start (SIGTERM; the harness exits +# cleanly on SIGINT/SIGTERM). Stale pids are cleaned up. +H="${HOME%/}/.mopac" +for name in loop events serve; do + f="$H/state/$name.pid" + if [ -s "$f" ]; then + pid=$(cat "$f") + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" && echo "stopped $name (pid $pid)" + else + echo "$name not running (stale pid $pid)" + fi + rm -f "$f" + else + echo "$name not running (no pid file)" + fi +done +EOF +mkscript "$STOP" 0755 +echo "generated: $MOPAC/bin/mopac-start, $MOPAC/bin/mopac-stop" + +echo +echo "account '$ACCOUNT' on $HOST ready. Next (runbook):" +echo " 1. fill in ~/.mopac/env (Bitwarden/Vault values)" +echo " 2. smoke: ~/.mopac/bin/harness once --dry-run --demo -config ~/.mopac/harness.toml" +echo " 3. start: ~/.mopac/bin/mopac-start" diff --git a/deploy/runbook.md b/deploy/runbook.md new file mode 100644 index 0000000..a40ba80 --- /dev/null +++ b/deploy/runbook.md @@ -0,0 +1,222 @@ +# Runbook β€” MOPAC 9-account deploy (Redmine 494, 2 hosts) + +Everything below is one Charles window. The PMO runtime cannot ssh/sudo; +this runbook is the whole deployment. Mechanical steps only (no decisions). +Estimated total: **~10 min mechanical** (build + stage + install + start + +verify), plus **~1 min/account** for the secrets bootstrap if not prefilled. + +Fleet (authority: `deploy/accounts.tsv`): + +| # | account | host | vertical | redmine project | events | serve | +|---|---------|------|----------|-----------------|--------|-------| +| 0 | reachableceo | ultix-streaming | rceo | mopac-rceo | 4100 | 8090 | +| 1 | TSGBOD | ultix-streaming | bod | mopac-bod | 4101 | 8091 | +| 2 | TSGCOO | ultix-streaming | coo | mopac-coo | 4102 | 8092 | +| 3 | TSGCTO | ultix-streaming | cto | mopac-cto | 4103 | 8093 | +| 4 | TSGCCO | ultix-streaming | cco | mopac-cco | 4104 | 8094 | +| 5 | reachableceo-offstage | ultix-offstage | rceo-offstage | mopac-rceo-offstage | 4105 | 8095 | +| 6 | COSRCEO-Personal | ultix-offstage | cos-rceo-personal | mopac-cos-rceo-personal | 4106 | 8096 | +| 7 | COSRCEO-Biz | ultix-offstage | cos-rceo-biz | mopac-cos-rceo-biz | 4107 | 8097 | +| 8 | COSWFO | ultix-offstage | cos-wfo | mopac-cos-wfo | 4108 | 8098 | + +Port scheme: `events = 4100 + index`, `serve = 8090 + index`, index is +global 0-8 β€” so the three daemons of any account never collide with +another account's on the same host (loop has no port; state dirs are +per-account under `~/.mopac/state/`). Nothing listens on 0.0.0.0 roots, +no systemd units, no root anywhere. + +**No-root model**: each instance lives entirely in the target account's +`~/.mopac/` (binary, config, secrets, state, reports). The install script +runs AS the target user. On ultix-streaming Charles is `reachableceo` +with passwordless sudo; on ultix-offstage he ssh's as +`reachableceo-offstage` (also passwordless sudo) β€” sudo is used ONLY to +switch identity (`sudo -u `), never to install system-wide +anything. + +--- + +## Step 0 β€” one-time bootstrap (before the window, ~5 min) + +These are Redmine/Gitea side, not host side: + +1. **Redmine projects**: create (or rename) one project per vertical with + EXACTLY the identifiers above (`mopac-rceo`, `mopac-bod`, ...), or edit + the `redmine_project` column in `deploy/accounts.tsv` to the real + identifiers BEFORE step 2 (it feeds the generated `scope_query`). +2. **API keys** (per account, from the account's cloudron user in + Bitwarden): Redmine API key, LiteLLM virtual key, three webhook + secrets, serve vkey. These go into each account's `~/.mopac/env` in + step 3. +3. Optional, for `[quota]` later: the shared redis container per the + README "Runbook: shared quota state" section. The generated configs + ship with `[quota]`/`[resources]` commented out β€” flipping them on is + a per-account one-liner, no redeploy needed. + +## Step 1 β€” build once (~2-3 min, any docker host, e.g. your workstation) + +```sh +git clone https://git.knownelement.com/ukrrs/MOPAC.git # or git pull +cd MOPAC +make release # docker builder: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 +``` + +Output: `bin/harness-linux-amd64` (static, stripped). Sanity-check it: + +```sh +file bin/harness-linux-amd64 # must say: statically linked +./bin/harness-linux-amd64 help | head -5 +``` + +## Step 2 β€” stage the bundle per host (~1 min per host) + +```sh +tar czf /tmp/mopac-deploy.tgz deploy bin/harness-linux-amd64 +scp /tmp/mopac-deploy.tgz reachableceo@ultix-streaming:/tmp/ +scp /tmp/mopac-deploy.tgz reachableceo-offstage@ultix-offstage:/tmp/ +``` + +## Step 3 β€” install per account (~30s per account, idempotent) + +### ultix-streaming (5 accounts) + +```sh +ssh reachableceo@ultix-streaming +mkdir -p ~/mopac-deploy-bundle && tar xzf /tmp/mopac-deploy.tgz -C ~/mopac-deploy-bundle + +# yourself, no sudo: +~/mopac-deploy-bundle/deploy/install-account.sh reachableceo + +# the other four, as each user (sudo = identity switch only): +for acct in TSGBOD TSGCOO TSGCTO TSGCCO; do + sudo -u "$acct" -H sh -c \ + "mkdir -p ~/mopac-deploy-bundle && tar xzf /tmp/mopac-deploy.tgz -C ~/mopac-deploy-bundle && ~/mopac-deploy-bundle/deploy/install-account.sh $acct" +done +``` + +### ultix-offstage (4 accounts) + +```sh +ssh reachableceo-offstage@ultix-offstage +mkdir -p ~/mopac-deploy-bundle && tar xzf /tmp/mopac-deploy.tgz -C ~/mopac-deploy-bundle +~/mopac-deploy-bundle/deploy/install-account.sh reachableceo-offstage + +for acct in COSRCEO-Personal COSRCEO-Biz COSWFO; do + sudo -u "$acct" -H sh -c \ + "mkdir -p ~/mopac-deploy-bundle && tar xzf /tmp/mopac-deploy.tgz -C ~/mopac-deploy-bundle && ~/mopac-deploy-bundle/deploy/install-account.sh $acct" +done +``` + +(Variant: if direct ssh to each COS account is set up, `scp` + run the +installer per account without sudo β€” same result.) + +Installer contract: creates `~/.mopac/{bin,state/loop,state/events,reports,work}`, +installs the binary, renders `~/.mopac/harness.toml`, creates `~/.mopac/env` +(0600) and `~/.mopac/bin/mopac-{start,stop}`. **Refuses to overwrite an +existing `harness.toml` or `env`** β€” hand-edits survive; re-running is the +upgrade path (binary + helpers refresh, config kept). + +## Step 4 β€” secrets bootstrap (~1 min per account) + +As each account (same `sudo -u -H sh -c` pattern, or ssh), fill +`~/.mopac/env` from Bitwarden β€” the installer printed exactly this: + +```sh +vi ~/.mopac/env # 0600 already; values: HARNESS_REDMINE_KEY, + # HARNESS_LITELLM_KEY, HARNESS_{REDMINE,DISCOURSE,GITEA}_WEBHOOK_SECRET, + # HARNESS_SERVE_VKEY +``` + +`harness.toml` stays secret-free (env refs only). If an env value stays +empty: loop turns fail fast/clean on the missing key, `events` refuses to +start until at least one webhook secret resolves β€” visible in +`~/.mopac/state/events.log`. + +## Step 5 β€” verify BEFORE starting (~10s per account) + +Still as the target account: + +```sh +# config + routing smoke, NO secrets, NO LLM call β€” must exit 0 and print +# the PLAN (dry-run) block: +~/.mopac/bin/harness once --dry-run --demo -config ~/.mopac/harness.toml + +# first real scan of the account's Redmine scope (needs HARNESS_REDMINE_KEY +# in ~/.mopac/env); prints what would dispatch, writes nothing: +. ~/.mopac/env && ~/.mopac/bin/harness loop --once --dry-run -config ~/.mopac/harness.toml +``` + +## Step 6 β€” start (~15s per account) + +```sh +~/.mopac/bin/mopac-start # starts loop + events + serve (nohup, pid files) +``` + +Then verify the two listeners (healthz; curl or any GET client): + +```sh +. ~/.mopac/env +curl -fsS http://127.0.0.1:4100/healthz # events, this account's port from the table +curl -fsS http://127.0.0.1:8090/healthz # serve, this account's port from the table +curl -fsS -H "Authorization: Bearer $HARNESS_SERVE_VKEY" http://127.0.0.1:8090/v1/models +tail -n 5 ~/.mopac/state/loop.log +``` + +Expected: `{"status":"ok"}` from both healthz endpoints, a JSON model +list from `/v1/models` (the `mopac-*` catalog), loop log showing scan +lines with the account's scope. + +### Reboot persistence β€” pick ONE (no systemd: that needs root, by design) + +**Option A, cron `@reboot`** (recommended; per account): + +```sh +( crontab -l 2>/dev/null | grep -v mopac-start; \ + echo '@reboot $HOME/.mopac/bin/mopac-start >> $HOME/.mopac/state/cron.log 2>&1' ) | crontab - +``` + +**Option B, manual**: re-run `~/.mopac/bin/mopac-start` after a reboot +(idempotent β€” safe to run anytime; it skips live daemons). + +## Step 7 β€” rollback (per account, ~30s) + +Stop the daemons and remove the install; Redmine/Gitea data is untouched +(the harness only reads issues and writes notes/REPORTs you can see): + +```sh +~/.mopac/bin/mopac-stop +rm -rf ~/.mopac # full removal, or keep evidence: +# mv ~/.mopac ~/.mopac.disabled-$(date +%Y%m%d-%H%M%S) +crontab -l | grep -v mopac-start | crontab - # if option A was used +``` + +`mopac-stop` is SIGTERM (clean stop); a stuck daemon: `kill -9 $(cat +~/.mopac/state/.pid)`. State/logs/reports live only under +`~/.mopac/` β€” removing it removes every trace of the instance. + +--- + +## Time budget (the ~10-minute window) + +| Step | ultix-streaming (5 accts) | ultix-offstage (4 accts) | +|------|---------------------------|--------------------------| +| stage + install (2+3) | ~3 min | ~2.5 min | +| secrets bootstrap (4) | ~5 min (unless prefilled) | ~4 min | +| verify + start + healthz (5+6) | ~2 min | ~1.5 min | +| **total** | **~10 min** (mechanical: ~5) | **~8 min** (mechanical: ~4) | + +Build (step 1) and bootstrap (step 0) happen before the window. + +## Assumptions (one-line fixes, no redeploy) + +- **Redmine project identifiers** are as listed; wrong ones β†’ edit + `scope_query` in the account's `harness.toml` (installer never + overwrites it) or edit `accounts.tsv` + re-render. +- **Quota grouping** assumes one z.ai Max plan per host + (`zai-max-1`/`zai-max-2`); when the real key-to-account mapping is + confirmed, edit `[quota] account` per `harness.toml` before flipping + `enabled = true`. +- Host short names are `ultix-streaming`/`ultix-offstage` (the installer + prints an advisory note, never blocks, on a mismatch). +- `[redmine.status_map]` is intentionally empty in generated configs: + set the Releasedβ†’Done pair per project workflow when adopting the loop + for real work (see the comment block in the generated file). diff --git a/deploy/tests.sh b/deploy/tests.sh new file mode 100755 index 0000000..00451ee --- /dev/null +++ b/deploy/tests.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# deploy/tests.sh β€” packaging tests for the 9-account deploy (Redmine +# 494). Pure POSIX shell, no docker, no network, no root: +# +# T1 accounts.tsv: 9 rows, spec-exact account/host lists, 8 columns, +# unique accounts, port scheme events=4100+idx / serve=8090+idx, +# (host,port) uniqueness across the fleet +# T2 render: installer runs for ALL 9 accounts against a stub binary +# in fake HOMEs; harness.toml has no unsubstituted @PLACEHOLDER@s +# and carries the row's vertical/ports/project; env is 0600; +# start/stop helpers exist, executable, correct ports +# T3 idempotence: re-run exits 0, harness.toml + env byte-identical +# T4 refuse-to-overwrite: hand-edited harness.toml marker survives +# T5 unknown account -> non-zero exit +# T6 binary lookup order (../bin/harness-linux-amd64 staging layout) +# T7 generated TOML actually loads: `once --dry-run --demo` via the +# dev binary when bin/harness exists (skipped otherwise) +# +# Usage: ./deploy/tests.sh (or: make deploy-test) + +set -u + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +TSV="$ROOT/deploy/accounts.tsv" +INSTALL="$ROOT/deploy/install-account.sh" +TEMPLATE="$ROOT/deploy/harness.toml.in" +PASS=0 +FAIL=0 + +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +bad() { FAIL=$((FAIL + 1)); echo "FAIL $1"; } +check() { # desc condition-already-evaluated-by-caller + if [ "$2" = 0 ]; then ok "$1"; else bad "$1"; fi +} + +# ---------------------------------------------------------------- T1 +t1() { + # 9 data rows + n=$(awk -F'\t' '$1 !~ /^#/ && NF >= 8 {c++} END {print c + 0}' "$TSV") + check "T1 accounts.tsv has 9 account rows" "$([ "$n" -eq 9 ]; echo $?)" + + # exact account/host sets from the spec + spec_streaming="reachableceo TSGBOD TSGCOO TSGCTO TSGCCO" + spec_offstage="reachableceo-offstage COSRCEO-Personal COSRCEO-Biz COSWFO" + got_streaming=$(awk -F'\t' '$1 !~ /^#/ && $2 == "ultix-streaming" {print $1}' "$TSV" | sort | tr '\n' ' ') + got_offstage=$(awk -F'\t' '$1 !~ /^#/ && $2 == "ultix-offstage" {print $1}' "$TSV" | sort | tr '\n' ' ') + want_streaming=$(printf '%s\n' $spec_streaming | sort | tr '\n' ' ') + want_offstage=$(printf '%s\n' $spec_offstage | sort | tr '\n' ' ') + check "T1 ultix-streaming accounts match the spec" "$([ "$got_streaming" = "$want_streaming" ]; echo $?)" + check "T1 ultix-offstage accounts match the spec" "$([ "$got_offstage" = "$want_offstage" ]; echo $?)" + + # unique account names, unique host:index, unique (host,events/serve) ports + dup=$(awk -F'\t' '$1 !~ /^#/ {print $1}' "$TSV" | sort | uniq -d | wc -l | tr -d ' ') + check "T1 account names unique" "$([ "$dup" -eq 0 ]; echo $?)" + dup=$(awk -F'\t' '$1 !~ /^#/ {print $2" "$7}' "$TSV" | sort | uniq -d | wc -l | tr -d ' ') + check "T1 (host, events_port) unique" "$([ "$dup" -eq 0 ]; echo $?)" + dup=$(awk -F'\t' '$1 !~ /^#/ {print $2" "$8}' "$TSV" | sort | uniq -d | wc -l | tr -d ' ') + check "T1 (host, serve_port) unique" "$([ "$dup" -eq 0 ]; echo $?)" + + # port derivation + row sanity (also enforced by the installer itself) + badrows=$(awk -F'\t' '$1 !~ /^#/ { + if (NF != 8 || $3 !~ /^[0-9]+$/ || $7 != 4100 + $3 || $8 != 8090 + $3) bad++ + } END {print bad + 0}' "$TSV") + check "T1 every row: 8 cols, events=4100+index, serve=8090+index" "$([ "$badrows" -eq 0 ]; echo $?)" +} + +# ---------------------------------------------------------------- T2-T6 +t2_t6() { + TMP=$(mktemp -d /tmp/mopac-deploy-test.XXXXXX) || { bad "T2 mktemp"; return; } + trap 'rm -rf "$TMP"' EXIT INT TERM + + # stub binary (real one not needed to test the packaging) + printf '#!/bin/sh\necho stub-harness "$@"\n' >"$TMP/harness-linux-amd64" + chmod 755 "$TMP/harness-linux-amd64" + + rc_all=0 + for acct in $(awk -F'\t' '$1 !~ /^#/ {print $1}' "$TSV"); do + row=$(awk -F'\t' -v a="$acct" '$1 == a {print}' "$TSV") + idx=$(printf '%s\n' "$row" | awk -F'\t' '{print $3}') + vert=$(printf '%s\n' "$row" | awk -F'\t' '{print $4}') + proj=$(printf '%s\n' "$row" | awk -F'\t' '{print $5}') + ev=$((4100 + idx)); sv=$((8090 + idx)) + FAKEHOME="$TMP/home-$acct" + mkdir -p "$FAKEHOME" + + HOME="$FAKEHOME" MOPAC_HARNESS_BIN="$TMP/harness-linux-amd64" \ + sh "$INSTALL" "$acct" >"$TMP/out-$acct" 2>&1 + rc=$? + [ "$rc" -eq 0 ] || { bad "T2 install $acct exits 0 (got $rc)"; sed 's/^/ /' "$TMP/out-$acct"; rc_all=1; continue; } + + cfg="$FAKEHOME/.mopac/harness.toml" + for cond in \ + "[ -f \"\$cfg\" ]" \ + "[ -x \"\$FAKEHOME/.mopac/bin/harness\" ]" \ + "[ -x \"\$FAKEHOME/.mopac/bin/mopac-start\" ]" \ + "[ -x \"\$FAKEHOME/.mopac/bin/mopac-stop\" ]" \ + "[ -d \"\$FAKEHOME/.mopac/state/loop\" ]" \ + "[ -d \"\$FAKEHOME/.mopac/state/events\" ]" \ + "[ -d \"\$FAKEHOME/.mopac/reports\" ]" \ + "[ -d \"\$FAKEHOME/.mopac/work\" ]" \ + "! grep -q '@[A-Z_][A-Z0-9_]*@' \"\$cfg\"" \ + "grep -q \"vertical = \\\"\$vert\\\"\" \"\$cfg\"" \ + "grep -q 'listen = \":'\$ev'\"' \"\$cfg\"" \ + "grep -q 'listen = \":'\$sv'\"' \"\$cfg\"" \ + "grep -q \"project=\$proj&\" \"\$cfg\"" \ + "grep -q \"\$FAKEHOME/.mopac/state/loop\" \"\$cfg\"" \ + "grep -q \":\$ev\" \"\$FAKEHOME/.mopac/bin/mopac-start\"" \ + "[ \"\$(stat -c %a \"\$FAKEHOME/.mopac/env\")\" = 600 ]" + do + if ! eval "$cond"; then + bad "T2 $acct: $cond" + rc_all=1 + fi + done + done + [ "$rc_all" -eq 0 ] && ok "T2 render all 9 accounts: files, modes, substitutions, ports" + + # --- T3 idempotence (re-run one account: exit 0, config + env unchanged) + acct=TSGBOD + FAKEHOME="$TMP/home-$acct" + sum_cfg=$(cksum "$FAKEHOME/.mopac/harness.toml" | awk '{print $1, $2}') + sum_env=$(cksum "$FAKEHOME/.mopac/env" | awk '{print $1, $2}') + HOME="$FAKEHOME" MOPAC_HARNESS_BIN="$TMP/harness-linux-amd64" \ + sh "$INSTALL" "$acct" >"$TMP/out-rerun" 2>&1 + rc=$? + [ "$rc" -eq 0 ] && \ + [ "$sum_cfg" = "$(cksum "$FAKEHOME/.mopac/harness.toml" | awk '{print $1, $2}')" ] && \ + [ "$sum_env" = "$(cksum "$FAKEHOME/.mopac/env" | awk '{print $1, $2}')" ] \ + && ok "T3 idempotent re-run: exit 0, harness.toml + env byte-identical" \ + || bad "T3 idempotent re-run (rc=$rc)" + + # --- T4 hand-edited config survives (refuse to overwrite) + echo "# hand-edited marker" >>"$FAKEHOME/.mopac/harness.toml" + HOME="$FAKEHOME" MOPAC_HARNESS_BIN="$TMP/harness-linux-amd64" \ + sh "$INSTALL" "$acct" >"$TMP/out-keep" 2>&1 + rc=$? + grep -q "# hand-edited marker" "$FAKEHOME/.mopac/harness.toml" && [ "$rc" -eq 0 ] \ + && grep -q "kept existing" "$TMP/out-keep" \ + && ok "T4 existing harness.toml kept on re-run (exit 0, marker intact)" \ + || bad "T4 refuse-to-overwrite (rc=$rc)" + + # --- T5 unknown account + HOME="$TMP/home-nowhere" MOPAC_HARNESS_BIN="$TMP/harness-linux-amd64" \ + sh "$INSTALL" no-such-account >"$TMP/out-unknown" 2>&1 + [ "$?" -ne 0 ] && grep -q "not in" "$TMP/out-unknown" \ + && ok "T5 unknown account -> non-zero + message" \ + || bad "T5 unknown account" + + # --- T6 default binary lookup (staged bundle layout: ../bin/harness-linux-amd64) + STAGE="$TMP/stage"; mkdir -p "$STAGE/deploy" "$STAGE/bin" + cp "$TSV" "$TEMPLATE" "$INSTALL" "$STAGE/deploy/" + cp "$TMP/harness-linux-amd64" "$STAGE/bin/harness-linux-amd64" + FAKEHOME="$TMP/home-stage"; mkdir -p "$FAKEHOME" + HOME="$FAKEHOME" sh "$STAGE/deploy/install-account.sh" TSGCCO >"$TMP/out-stage" 2>&1 + rc=$? + [ "$rc" -eq 0 ] && [ -x "$FAKEHOME/.mopac/bin/harness" ] && \ + grep -q "vertical = \"cco\"" "$FAKEHOME/.mopac/harness.toml" \ + && ok "T6 staged-bundle lookup ../bin/harness-linux-amd64 works (no env override)" \ + || bad "T6 staged-bundle lookup (rc=$rc)" + + # --- T7 generated TOML loads (needs a built dev binary; skip otherwise) + if [ -x "$ROOT/bin/harness" ]; then + cfg="$TMP/home-TSGCOO/.mopac/harness.toml" + HOME="$TMP/home-TSGCOO" "$ROOT/bin/harness" once --dry-run --demo \ + -config "$cfg" >"$TMP/out-load" 2>&1 + rc=$? + grep -q "vertical: coo" "$TMP/out-load" && [ "$rc" -eq 0 ] \ + && ok "T7 rendered harness.toml loads (once --dry-run --demo, exit 0)" \ + || { bad "T7 rendered config loads (rc=$rc)"; sed 's/^/ /' "$TMP/out-load"; } + else + echo "skip T7 (bin/harness not built; run ./dev.sh check or make release first)" + fi +} + +t1 +t2_t6 + +echo +echo "pass=$PASS fail=$FAIL" +[ "$FAIL" -eq 0 ]