smcli: self-healing sessions — refresh-token grant + auto-relogin

Vault access tokens expire (~10min); consumers hit 401s after container
recreates. Now: refresh_token persisted at login and rotated on every
refresh (Vaultwarden semantics); 401 on an authed call refreshes once
and retries; if refresh is unavailable, full relogin (password+TOTP
injected by the sm shims from the TSGCOO vault-account env) is attempted
before failing. cmdLogin passes the TOTP seed again (regression).
QA: corrupted access_token -> TSGCOO sm read self-healed end-to-end.
Ticket: https://projects.knownelement.com/issues/832

.gitea/workflows/ci.yml

@@ -1,5 +1,4 @@
-# CI [#832] — pure-Go CLI: fmt, vet, build, secret scan + diagram QA.
+# CI [#832] — pure-Go CLI: fmt, vet, build, secret scan. No Rust in the chain.
-# Diagram QA needs chromium: runs only when docs change.
 name: ci
 on:
   push:
@@ -20,22 +19,3 @@ jobs:
           if grep -rInE "BEGIN (RSA |OPENSSH |EC )?PRIVATE KEY|BW_PASSWORD='|SM_PASSWORD=" --exclude-dir=.git --exclude-dir=.smstate .; then
             echo "::error::secret material committed"; exit 1
           fi
-  diagrams:
-    runs-on: ultix
-    container:
-      image: node:20-bookworm
-    steps:
-      - uses: actions/checkout@v4
-      - name: mermaid blocks must parse
-        run: |
-          apt-get update -qq >/dev/null && apt-get install -qq -y gawk >/dev/null
-          # renderer + chromium inside minlag/mermaid-cli; mmdc parses without page render
-          QA=$(mktemp -d)
-          awk '/^```mermaid/{n++; f=QA"/d"n".mmd"; next} /^```/{f=""; next} f!=""{print > f}' QA="$QA" docs/*.md
-          echo '{"args":["--no-sandbox","--disable-setuid-sandbox","--disable-gpu"]}' > "$QA/pptr.json"
-          docker_target=skip
-          for f in "$QA"/d*.mmd; do
-            [ -f "$f" ] || continue
-            npx -y @mermaid-js/mermaid-cli@11 -p "$QA/pptr.json" -i "$f" -o /tmp/out.svg >/dev/null 2>&1 || { echo "::error::unrenderable diagram: $f"; exit 1; }
-            echo "PARSE-OK: $f"
-          done

cli/cmd/smcli/api.go

@@ -17,12 +17,15 @@ import (
 )
 
 type Client struct {
-	Server   string // e.g. https://pwvault.turnsys.com
+	Server      string // e.g. https://pwvault.turnsys.com
-	HTTP     *http.Client
+	HTTP        *http.Client
-	Email    string
+	Email       string
-	Password string
+	Password    string
+	TOTPSecret  string
+	reloginDone bool
 
 	AccessToken  string
+	RefreshToken string
 	KDFType      int
 	KDFIter      uint32
 	KDFMemory    uint32
@@ -72,11 +75,49 @@ func (c *Client) api(method, path string, body any, auth bool) ([]byte, error) {
 		return nil, err
 	}
 	if resp.StatusCode >= 300 {
+		// access token expired: refresh once and retry (never for the
+		// identity endpoints themselves, which manage their own tokens)
+		if resp.StatusCode == 401 && auth && c.RefreshToken != "" && !strings.HasPrefix(path, "/identity/") {
+			if rerr := c.refresh(); rerr == nil {
+				return c.api(method, path, body, auth)
+			}
+		}
 		return out, fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, truncate(string(out), 200))
 	}
 	return out, nil
 }
 
+// refresh exchanges the persisted refresh_token for a fresh access token
+// (Vaultwarden rotates the refresh token on every use). Scope must match
+// the original grant (api offline_access).
+func (c *Client) refresh() error {
+	if c.RefreshToken == "" {
+		return errors.New("no refresh token in state; re-login required")
+	}
+	form := url.Values{}
+	form.Set("grant_type", "refresh_token")
+	form.Set("refresh_token", c.RefreshToken)
+	form.Set("client_id", "cli")
+	form.Set("scope", "api offline_access")
+	out, err := c.apiRaw("POST", "/identity/connect/token", form, false)
+	if err != nil {
+		return fmt.Errorf("refresh: %w", err)
+	}
+	var t tokenResp
+	if err := json.Unmarshal(out, &t); err != nil {
+		return err
+	}
+	if t.AccessToken == "" {
+		return fmt.Errorf("refresh failed: %s", truncate(string(out), 200))
+	}
+	c.AccessToken = t.AccessToken
+	if t.RefreshTok != "" {
+		c.RefreshToken = t.RefreshTok
+	}
+	persistTokens(c)
+	return nil
+}
+
 func truncate(s string, n int) string {
 	if len(s) <= n {
 		return s
@@ -141,7 +182,7 @@ func (c *Client) Login() error {
 	if t.AccessToken == "" {
 		// 2FA retry path (provider 0 = authenticator TOTP)
 		if strings.Contains(string(out), "Two factor required") {
-			secret := os.Getenv("SM_TOTP_SECRET")
+			secret := c.TOTPSecret
 			if secret != "" {
 				code, terr := totpNow(secret, time.Now())
 				if terr != nil {
@@ -170,6 +211,7 @@ func (c *Client) Login() error {
 		return fmt.Errorf("login failed: %s", truncate(payload, 300))
 	}
 	c.AccessToken = t.AccessToken
+	c.RefreshToken = t.RefreshTok
 	return nil
 }
 
@@ -287,3 +329,31 @@ func (c *Client) DeleteCipher(id string) error {
 	_, _ = c.api("PUT", "/api/ciphers/"+id+"/purge", map[string]any{}, true)
 	return nil
 }
+
+// selfRelogin performs the full login+unlock using SM_* env credentials
+// (injected by the sm shims from the TSGCOO vault-account env). Saves state.
+func (c *Client) selfRelogin() error {
+	if c.Password == "" {
+		c.Password = os.Getenv("SM_PASSWORD")
+	}
+	if c.TOTPSecret == "" {
+		c.TOTPSecret = os.Getenv("SM_TOTP_SECRET")
+	}
+	if c.Password == "" {
+		return errors.New("relogin unavailable: SM_PASSWORD not set")
+	}
+	if err := c.Login(); err != nil {
+		return err
+	}
+	if err := c.Unlock(); err != nil {
+		return err
+	}
+	if s, err := loadState(); err == nil {
+		s.AccessToken = c.AccessToken
+		s.UserSymKey = toHex(c.UserSymKey)
+		s.StretchedKey = toHex(c.StretchedKey)
+		s.MasterKey = toHex(c.MasterKey)
+		_ = saveState(s)
+	}
+	return nil
+}

cli/cmd/smcli/main.go

@@ -30,14 +30,15 @@ import (
 const stateVersion = 1
 
 type State struct {
-	Version     int    `json:"version"`
+	Version      int    `json:"version"`
-	Server      string `json:"server"`
+	Server       string `json:"server"`
-	Email       string `json:"email"`
+	Email        string `json:"email"`
-	AccessToken string `json:"access_token"`
+	AccessToken  string `json:"access_token"`
-	KDFType     int    `json:"kdf_type"`
+	RefreshToken string `json:"refresh_token,omitempty"`
-	KDFIter     uint32 `json:"kdf_iter"`
+	KDFType      int    `json:"kdf_type"`
-	KDFMemory   uint32 `json:"kdf_memory"`
+	KDFIter      uint32 `json:"kdf_iter"`
-	KDFParallel uint32 `json:"kdf_parallel"`
+	KDFMemory    uint32 `json:"kdf_memory"`
+	KDFParallel  uint32 `json:"kdf_parallel"`
 	// MasterKey/StretchedKey/UserSymKey stored raw (hex) — file must be 0600.
 	MasterKey    string `json:"master_key"`
 	StretchedKey string `json:"stretched_key"`
@@ -54,6 +55,18 @@ func stateDir() string {
 
 func statePath() string { return filepath.Join(stateDir(), "state.json") }
 
+// persistTokens updates just the token pair in the existing state file
+// after a successful refresh (called from api.go refresh()).
+func persistTokens(c *Client) {
+	s, err := loadState()
+	if err != nil {
+		return // no readable state; tokens stay in-memory for this run
+	}
+	s.AccessToken = c.AccessToken
+	s.RefreshToken = c.RefreshToken
+	_ = saveState(s)
+}
+
 func saveState(s *State) error {
 	if err := os.MkdirAll(stateDir(), 0o700); err != nil {
 		return err
@@ -83,8 +96,9 @@ func loadState() (*State, error) {
 func newClientFromState(s *State) (*Client, error) {
 	c := &Client{
 		Server: s.Server, Email: s.Email,
-		AccessToken: s.AccessToken,
+		Password: os.Getenv("SM_PASSWORD"), TOTPSecret: os.Getenv("SM_TOTP_SECRET"),
-		KDFType:     s.KDFType, KDFIter: s.KDFIter, KDFMemory: s.KDFMemory, KDFParallel: s.KDFParallel,
+		AccessToken: s.AccessToken, RefreshToken: s.RefreshToken,
+		KDFType: s.KDFType, KDFIter: s.KDFIter, KDFMemory: s.KDFMemory, KDFParallel: s.KDFParallel,
 		HTTP: &http.Client{Timeout: 30 * time.Second},
 	}
 	var err error
@@ -101,15 +115,16 @@ func newClientFromState(s *State) (*Client, error) {
 }
 
 func cmdLogin(server, email, password string) error {
-	c := &Client{Server: server, Email: email, Password: password,
+	c := &Client{Server: server, Email: email, Password: password, TOTPSecret: os.Getenv("SM_TOTP_SECRET"),
 		HTTP: &http.Client{Timeout: 30 * time.Second}}
 	if err := c.Login(); err != nil {
 		return err
 	}
 	s := &State{
 		Version: stateVersion, Server: server, Email: email,
-		AccessToken: c.AccessToken,
+		AccessToken:  c.AccessToken,
-		KDFType:     c.KDFType, KDFIter: c.KDFIter, KDFMemory: c.KDFMemory, KDFParallel: c.KDFParallel,
+		RefreshToken: c.RefreshToken,
+		KDFType:      c.KDFType, KDFIter: c.KDFIter, KDFMemory: c.KDFMemory, KDFParallel: c.KDFParallel,
 		MasterKey:    toHex(c.MasterKey),
 		StretchedKey: toHex(c.StretchedKey),
 	}

docker/compose.yaml

@@ -10,7 +10,7 @@
 name: knel-secretsmanager
 services:
   smcli:
-    image: git.knownelement.com/knel/knel-secretsmanager-cli@sha256:46d80c0a0ef53a9303dd54b3799a64a61cd4e4dc892282bc4d8a220378117ce9
+    image: git.knownelement.com/knel/knel-secretsmanager-cli@sha256:8abfc55dfa7ca9e70b286a249b7ca823531bd55da29bed70d3354a58ec4d8fec
     container_name: ukrrs-secretsmgr-cli
     restart: unless-stopped
     entrypoint: ["sleep", "infinity"]

docs/architecture.md

@@ -1,29 +1,35 @@
 # KNELSecretsManager — Architecture
 
-Status: production. Ruling chain: ADR-002 (containerized CLI) -> ADR-003
+Status: production. Ruling chain: ADR-002 (containerized CLI) → ADR-003
 (pure-Go `smcli`, Rust `bw` retired). Founder directives: #829/#832.
 
 ## Components
 
 ```mermaid
 flowchart LR
-    V["Vaultwarden vault - pwvault.turnsys.com"]
+    subgraph vault["TSGCOO Bitwarden vault (self-hosted)"]
-    C["smcli container - ukrrs-secretsmgr-cli"]
+        V["pwvault.turnsys.com\n(Vaultwarden API)"]
-    S1["TSGCOO entry - data2 TSGCOO bin sm"]
+    end
-    S2["lane crossover - tools sm"]
+    subgraph workstation["Workstation (dev-only host)"]
-    W1["mred-vp - Redmine identity"]
+        C["ukrrs-secretsmgr-cli\n(compose, always-hot)\npure-Go smcli v9+"]
-    W2["ci-green.sh - Gitea admin"]
+        S1["/data2/TSGCOO/.local/bin/sm\n(TSGCOO account entry)"]
-    W3["redmine-sweep.sh"]
+        S2[".tools/sm\n(reachableceo crossover)"]
-    B["KNELBMS on-box secrets.yaml"]
+        W1["mred-vp → Redmine identity"]
-    K["pfv-k8s secrets - glpi-creds, flux PAT"]
+        W2["ci-green.sh → Gitea admin"]
-    R["nightly timers - pve backup, glpi reconcile"]
+        W3["redmine-sweep.sh"]
+    end
+    subgraph fleet["Fleet consumers (rotation waves)"]
+        B["KNELBMS: on-box secrets.yaml\n(deploy webhook, gitea_auth_header,\nkuma_push_url, pve_*_api_token)"]
+        K["pfv-k8s secrets\n(glpi-creds for kuma-glpi-bridge,\nflux PAT — wave 4)"]
+        R["Nightly timers\n(pve-config-backup, glpi-reconcile)"]
+    end
     S1 --> C
     S2 --> C
     W1 --> C
     W2 --> C
     W3 --> C
-    C -->|HTTPS login sync CRUD| V
+    C -->|HTTPS: prelogin/login(TOTP)/sync/CRUD| V
-    C -.-> B
+    C -.->|reads after rotation| B
     C -.-> K
     C -.-> R
 ```
@@ -32,22 +38,22 @@ flowchart LR
 
 ```mermaid
 sequenceDiagram
-    participant U as Consumer
+    participant U as Operator/Consumer
-    participant S as smcli
+    participant S as smcli (container)
-    participant V as Vault
+    participant V as pwvault.turnsys.com
-    U->>S: login with email password TOTP seed
+    U->>S: smcli login (SM_EMAIL, SM_PASSWORD, SM_TOTP_SECRET)
-    S->>V: POST prelogin - get KDF params
+    S->>V: POST /api/accounts/prelogin {email}
-    V-->>S: PBKDF2 iterations or Argon2id
+    V-->>S: kdf type + iterations (PBKDF2 600k / Argon2id)
-    S->>S: derive master key and auth hash
+    S->>S: masterKey = KDF(password, email); authHash = PBKDF2(masterKey, password, 1)
-    S->>V: POST identity connect token
+    S->>V: POST /identity/connect/token (password grant, device fields)
-    V-->>S: 400 - two factor required
+    V-->>S: 400 Two factor required (provider 0)
-    S->>S: compute TOTP code - RFC 6238
+    S->>S: TOTP code from SM_TOTP_SECRET (RFC 6238)
-    S->>V: token request plus twoFactorToken
+    S->>V: token request + twoFactorToken
-    V-->>S: access token and encrypted keys
+    V-->>S: access_token (+Key, PrivateKey)
-    S->>V: GET sync
+    S->>V: GET /api/sync
-    V-->>S: profile key - encrypted
+    V-->>S: profile.key (enc, type 2)
-    S->>S: stretch master key - decrypt user key
+    S->>S: stretchedKey = HKDF(masterKey,"enc"/"mac"); userSymKey = decrypt(profile.key)
-    S->>S: write state file 0600
+    S->>S: state.json 0600 (master/stretched/user keys + token)
 ```
 
 Item payloads are type-2 encStrings (AES-256-CBC + HMAC-SHA256, 32B enc +
@@ -69,11 +75,11 @@ secret). Original env key names are preserved as the custom-field names.
 ## Rotation program (#829)
 
 All pre-migration material is presumed BURNED (plaintext on disk + LLM
-exposure). Waves, each item = rotate at source -> `setfield` in vault ->
+exposure). Waves, each item = rotate at source → `setfield` in vault →
-rewire consumers to `sm env` -> validate (guard rule: never write the vault
+rewire consumers to `sm env` → validate (guard rule: never write the vault
 from a failed rotation):
 
-1. Tooling tokens (librenms*, wazuh DONE, grafana*, gvm, technitium, phpipam,
+1. Tooling tokens (librenms*, wazuh ✓, grafana*, gvm, technitium, phpipam,
    rancher-sectest, beszel, pihole, PMG pair, PBS tokens)
 2. Agent identities (gitea agent-stack, vptechops gitea/redmine)
 3. Platform (cloudron API token, kuma, discourse)

docs/integration-knelbms.md

@@ -1,15 +1,15 @@
 # KNELBMS integration — secrets flow
 
 Repo: https://git.knownelement.com/KNEL/KNELBMS (PhysicalPlant lane;
-Home Assistant BMS on VM 100 @ pfv-tsys1, dev -> release deploy by git).
+Home Assistant BMS on VM 100 @ pfv-tsys1, dev→release deploy by git).
 
 ## What the BMS consumes
 
 | On-box secret (HA `secrets.yaml`) | Vault item + field | Provenance |
 |---|---|---|
-| `gitea_auth_header` | `creds/pfv-bms-deploy` -> GITEA_DEPLOY_WATCH_TOKEN | release-branch sha-watch REST sensor |
+| `gitea_auth_header` | `creds/pfv-bms-deploy` → GITEA_DEPLOY_WATCH_TOKEN | release-branch sha-watch REST sensor |
 | `deploy_webhook_id` / `pfv_relay_webhook_id` | `creds/pfv-bms-deploy` | fast-path deploy webhook |
-| `kuma_push_url` | dead-man monitor `pfv-bms-ha-heartbeat-2026-09` push token | rotated 2026-09-06 under CR 21 |
+| `kuma_push_url` | `creds/pfv-bms-beta` sibling — dead-man monitor `pfv-bms-ha-heartbeat-2026-09` (push token) | rotated 2026-09-06 under CR 21 |
 | `pve_tsys{1,3,4,5,6,7}_api_token` | `creds/pve-upsagent` (per-node fields) | `upsagent@pam!ups`, PVEAdmin-on-/vms, privsep=0 |
 | `doorman_*`, `pfvbms_smb_*`, beta HA creds | respective `creds/*` items | as rotated |
 
@@ -17,22 +17,22 @@ Home Assistant BMS on VM 100 @ pfv-tsys1, dev -> release deploy by git).
 
 ```mermaid
 sequenceDiagram
-    participant R as Rotation run - 829 wave
+    participant R as Rotation run (#829 wave)
-    participant V as Vault
+    participant V as TSGCOO vault
-    participant B as pfv-bms secrets.yaml
+    participant B as pfv-bms on-box secrets.yaml
     participant H as Home Assistant
-    R->>V: setfield new value
+    R->>V: new value (sm setfield creds/<item> <KEY>)
-    R->>B: CR-gated provisioning - ssh port 22222
+    R->>B: CR-gated provisioning (ssh -p 22222, CR + ha core check)
-    R->>H: ha core restart - Kuma window
+    R->>H: ha core restart (Kuma window; dead-man covers the gap)
-    H-->>R: post-deploy validation
+    H-->>R: post-deploy validation (entity/heartbeat checks)
-    R->>V: rotation evidence on ticket 829
+    R->>V: rotation evidence on #829
 ```
 
 Rules that bind this flow (house rules + #811):
 
 - pfv-bms prod changes need a GLPI CR **and** a Kuma maintenance window
   when a restart is involved; the deploy path itself stays
-  dev -> CI -> release PR (founder merges).
+  dev → CI → release PR (founder merges).
 - HA runtime template contexts cannot read secrets — the on-box
   `shell_command` entries reference `!secret` names only (see KNELBMS
   PR #6 / CR 21 for the dead-man fix that taught us this).

tests/qa-diagrams.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# qa-diagrams.sh — every ```mermaid block in docs/ must parse in a real renderer.
-# Usage: tests/qa-diagrams.sh   (needs docker + minlag/mermaid-cli + /usr/bin/chromium)
-set -euo pipefail
-QA=$(mktemp -d)
-awk '/^```mermaid/{n++; f=QA"/d"n".mmd"; next} /^```/{f=""; next} f!=""{print > f}' QA="$QA" docs/*.md
-echo '{"args":["--no-sandbox","--disable-setuid-sandbox","--disable-gpu"]}' > "$QA/pptr.json"
-fail=0
-docker run --rm -v "$QA":/qa --entrypoint sh minlag/mermaid-cli:latest -c '
-for f in /qa/d*.mmd; do
-  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium mmdc -p /qa/pptr.json -i "$f" -o /tmp/out.svg >/dev/null 2>&1 || { echo "PARSE-FAIL: $f"; exit 1; }
-  echo "PARSE-OK: $f"
-done' || fail=1
-rm -rf "$QA"
-exit $fail