feat: full app provisioning for all COO agents + race-hardened BW helper

App credentials now flow for every agent: shared handle_oidc_interaction
helper fills the per-app OIDC login form/TOTP/consent (the panel session
is not shared across app clients -- each may demand fresh credentials),
and the Discourse signup clears the prefilled username field before
typing (prefill+typed concatenation exceeded the 20-char cap and failed
validation silently).

BW helper hardened against the sync races observed across concurrent
containers: get_item_id re-syncs stale caches, create/edit retry with
backoff and post-write sync. This class of failure was mine -- the
login-path fix from session 3 left read paths on stale caches.

Final validated matrix (validate-all-logins.py, fresh-context logins
plus live API checks): 9/10 fully green; vp-compliance blocked on a
corrupt stored password (Cloudron admin reset needed). Redmine access
still Cloudron-denied for vp-secops, svp-knel, vp-techcompliance,
vp-facilities ("You do not have access" at the OIDC interaction).

bw_helper.py

@@ -24,6 +24,7 @@ import os
 import subprocess
 import sys
 import tempfile
+import time
 from typing import Optional
 
 
@@ -38,6 +39,7 @@ class BitwardenHelper:
         self.totp_secret = totp_secret
         self.server_url = server_url
         self.session: Optional[str] = None
+        self._last_sync: float = 0.0
 
     def _run_bw(self, args: list[str], capture: bool = True) -> str:
         """Run a bw CLI command with the active session."""
@@ -121,14 +123,59 @@ class BitwardenHelper:
         not appear in list/search results.
         """
         self._run_bw(["sync"])
+        self._last_sync = time.monotonic()
+
+    def _sync_if_stale(self, max_age_s: float = 30.0) -> None:
+        """Re-sync if the cache is older than max_age_s seconds.
+
+        Multiple containers share this vault (host wrapper, provisioner
+        runs). A read performed on a stale cache sees ghosts: items that
+        exist server-side appear missing (or vice versa). Cheap enough
+        to run before every read.
+        """
+        if time.monotonic() - self._last_sync > max_age_s:
+            self.sync()
+
+    def _run_bw_with_retry(self, args: list[str], retries: int = 3) -> str:
+        """Run a bw command, retrying on transient failures.
+
+        The bw CLI occasionally returns empty output or non-JSON errors
+        under load (observed: empty create response, 'Expecting value'
+        JSON decode upstream). Retry with backoff before failing.
+        """
+        last_err = None
+        for attempt in range(1, retries + 1):
+            try:
+                return self._run_bw(args)
+            except (RuntimeError, json.JSONDecodeError) as e:
+                last_err = e
+                msg = str(e)
+                # Non-retryable failures: re-raise immediately
+                if "not found" in msg.lower() or "already exists" in msg.lower():
+                    raise
+                if attempt < retries:
+                    delay = 2 * attempt
+                    time.sleep(delay)
+                    self.sync()
+        raise RuntimeError(
+            f"bw {' '.join(args)} failed after {retries} retries: {last_err}"
+        )
 
     def generate_password(self, length: int = 32) -> str:
         """Generate a strong password."""
         return self._run_bw(["generate", "-ulns", "--length", str(length)])
 
     def get_totp(self, item_name: str) -> str:
-        """Get the current TOTP code for a Bitwarden item."""
-        return self._run_bw(["get", "totp", item_name])
+        """Get the current TOTP code for a Bitwarden item.
+
+        Resolves via exact-name match first -- bw's search is fuzzy and
+        shared substrings (e.g. "coo" in every tsgstaff-coo-* username)
+        make name-based gets ambiguous.
+        """
+        item_id = self.get_item_id(item_name)
+        if not item_id:
+            raise RuntimeError(f"Item '{item_name}' not found")
+        return self._run_bw(["get", "totp", item_id])
 
     # -------------------------------------------------------------------
     # Item ID resolution -- the safe way to reference items
@@ -139,8 +186,11 @@ class BitwardenHelper:
 
         Returns the item ID if exactly one match exists, None if no match,
         and raises RuntimeError if multiple items share the name (ambiguous).
+        Always syncs if the cache is stale -- containers share this vault
+        and a stale cache sees ghosts.
         """
         try:
+            self._sync_if_stale()
             output = self._run_bw(["list", "items", "--search", name])
             items = json.loads(output)
             # Filter to exact name matches (bw search is fuzzy)
@@ -220,8 +270,9 @@ class BitwardenHelper:
             item["collectionIds"] = [collection_id]
 
         encoded_item = self._encode(item)
-        output = self._run_bw(["create", "item", encoded_item])
+        output = self._run_bw_with_retry(["create", "item", encoded_item])
         created = json.loads(output)
+        self.sync()
         return created.get("id", "")
 
     def update_item(
@@ -263,8 +314,9 @@ class BitwardenHelper:
             ]
 
         encoded_item = self._encode(current)
-        output = self._run_bw(["edit", "item", item_id, encoded_item])
+        output = self._run_bw_with_retry(["edit", "item", item_id, encoded_item])
         updated = json.loads(output)
+        self.sync()
         return updated.get("id", item_id)
 
     # -------------------------------------------------------------------

diagnose-sso-chain.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+diagnose-sso-chain.py -- Trace the full SSO navigation chain for one agent.
+
+Captures every navigation, page title, and visible messages during:
+  1. Cloudron panel login (establishes OIDC session)
+  2. Redmine SSO attempt (click -> ... -> final URL)
+  3. Discourse SSO attempt (modal -> OIDC -> signup/login -> final state)
+
+Usage:
+    docker compose run --rm --entrypoint python3 \
+        -e AGENT=vp-secops provision diagnose-sso-chain.py
+"""
+
+import os, sys, time
+from pathlib import Path
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = "https://my.knownelement.com"
+REDMINE_URL = "https://projects.knownelement.com"
+DISCOURSE_URL = "https://community.turnsys.com"
+STATE_DIR = Path("/app/state")
+
+AGENT = os.environ.get("AGENT", "vp-secops")
+
+
+def snapshot(page, label, trail):
+    body = ""
+    try:
+        body = page.evaluate("() => document.body.innerText.substring(0, 250)")
+    except Exception:
+        pass
+    entry = f"[{label}] {page.url}\n    body: {' | '.join(body.splitlines()[:6])}"
+    trail.append(entry)
+    print(entry)
+
+
+def cloudron_login(page, bw):
+    item = f"{AGENT} Cloudron"
+    email_map = {
+        "vp-secops": "tsgstaff-coo-vpsecops@turnsys.com",
+        "vp-techcompliance": "tsgstaff-coo-vptechcompliance@turnsys.com",
+    }
+    email = email_map.get(AGENT, f"{AGENT}@turnsys.com")
+    password = bw.get_item_password(item)
+    page.goto(f"{CLOUDRON_BASE}/login.html", wait_until="domcontentloaded", timeout=30000)
+    page.wait_for_timeout(2000)
+    page.wait_for_selector("#inputPassword", timeout=15000)
+    page.click("#inputUsername")
+    page.keyboard.type(email)
+    page.click("#inputPassword")
+    page.keyboard.type(password)
+    page.locator('[role="button"]:has-text("Log in")').first.click()
+    page.wait_for_timeout(4000)
+    totp = page.query_selector("#inputTotpToken")
+    if totp and totp.is_visible():
+        totp.click()
+        page.keyboard.type(bw.get_totp(item))
+        page.locator("#totpTokenSubmitButton").click()
+        page.wait_for_timeout(5000)
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+
+    trail = []
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+        page = context.new_page()
+
+        page.on("framenavigated", lambda frame: None)  # no-op; we poll manually
+
+        print(f"=== {AGENT}: Cloudron login ===")
+        cloudron_login(page, bw)
+        snapshot(page, "cloudron", trail)
+
+        print(f"\n=== {AGENT}: Redmine SSO chain ===")
+        page.goto(f"{REDMINE_URL}/login", wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(2000)
+        snapshot(page, "redmine-login", trail)
+        page.locator("#login-oauth-submit-1").first.click()
+        for i in range(6):
+            page.wait_for_timeout(1500)
+            snapshot(page, f"redmine+{(i+1)*1.5}s", trail)
+            if "projects.knownelement.com" in page.url and "/login" not in page.url:
+                break
+        # capture any flash error
+        err = page.evaluate("""() => {
+            const f = document.querySelector('#flash_notice, #flash_error, .flash, .error, .message');
+            return f ? f.textContent.trim() : '';
+        }""")
+        print(f"    flash/error element: {err!r}")
+
+        print(f"\n=== {AGENT}: Discourse SSO chain ===")
+        page.goto(DISCOURSE_URL, wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(3000)
+        snapshot(page, "discourse-home", trail)
+        login_btn = page.locator(".login-button")
+        if login_btn.count() > 0:
+            login_btn.first.click()
+            page.wait_for_timeout(2000)
+            snapshot(page, "discourse-modal", trail)
+            sso = page.locator('button:has-text("OpenID")')
+            if sso.count() > 0:
+                sso.first.click()
+                for i in range(6):
+                    page.wait_for_timeout(2000)
+                    snapshot(page, f"discourse+{(i+1)*2}s", trail)
+                    if "/signup" not in page.url and "/login" not in page.url:
+                        break
+                if "/signup" in page.url:
+                    body = page.evaluate("() => document.body.innerText")
+                    print(f"    SIGNUP PAGE body: {body[:300]!r}")
+
+        page.screenshot(path=str(STATE_DIR / f"sso-chain-{AGENT}-{time.strftime('%H%M%S')}.png"),
+                        full_page=True)
+        browser.close()
+
+    (STATE_DIR / f"sso-chain-{AGENT}-{time.strftime('%H%M%S')}.txt").write_text(
+        "\n".join(trail))
+
+
+if __name__ == "__main__":
+    main()

diagnose-vp-compliance.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+diagnose-vp-compliance.py -- Reproduce the vp-compliance login with full
+DOM dumps at every step to find where it diverges from the other agents.
+
+Usage:
+    docker compose run --rm --entrypoint python3 provision diagnose-vp-compliance.py
+"""
+
+import os, sys, time
+from pathlib import Path
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = "https://my.knownelement.com"
+STATE_DIR = Path("/app/state")
+ITEM = "vp-compliance Cloudron"
+EMAIL = "tsgstaff-coo-vpcompliance@turnsys.com"
+
+
+def dump(page, label):
+    ts = time.strftime("%H%M%S")
+    page.screenshot(path=str(STATE_DIR / f"diag-vpc-{label}-{ts}.png"), full_page=True)
+    body = page.evaluate("() => document.body.innerText.substring(0, 300)")
+    els = page.evaluate("""() => Array.from(document.querySelectorAll(
+        'input, [role="button"], button, .ui.message')).map(el => ({
+        tag: el.tagName, id: el.id, type: el.type||'',
+        vis: el.offsetParent !== null, text: (el.textContent||'').trim().substring(0,50)}))""")
+    print(f"--- {label} ---")
+    print(f"URL: {page.url}")
+    print(f"BODY: {body[:200]}")
+    for e in els:
+        print(f"  <{e['tag']}> id={e['id']} type={e['type']} vis={e['vis']} text={e['text']!r}")
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+    # Resolve by exact name to dodge the fuzzy-search ambiguity
+    item_id = bw.get_item_id(ITEM)
+    item = json.loads(bw._run_bw(["get", "item", item_id]))
+    password = item["login"]["password"]
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+        page = context.new_page()
+        page.goto(f"{CLOUDRON_BASE}/login.html", wait_until="networkidle", timeout=20000)
+        page.wait_for_timeout(2000)
+        dump(page, "01-login-page")
+
+        page.wait_for_selector("#inputPassword", timeout=15000)
+        page.click("#inputUsername")
+        page.keyboard.type(EMAIL)
+        page.click("#inputPassword")
+        page.keyboard.type(password)
+        dump(page, "02-filled")
+
+        btn = page.locator('[role="button"]:has-text("Log in")')
+        print(f"login button count={btn.count()} visible={btn.first.is_visible() if btn.count() else '?'}")
+        print(f"login button classes: {btn.first.get_attribute('class') if btn.count() else '?'}")
+        btn.first.click()
+        page.wait_for_timeout(5000)
+        dump(page, "03-after-submit")
+
+        totp = page.query_selector("#inputTotpToken")
+        print(f"TOTP field present={totp is not None} visible={totp.is_visible() if totp else False}")
+        if totp and totp.is_visible():
+            code = bw._run_bw(["get", "totp", item_id])
+            totp.click()
+            page.keyboard.type(code)
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+            dump(page, "04-after-totp")
+
+        browser.close()
+
+
+import json  # noqa: E402
+
+if __name__ == "__main__":
+    main()

docker-compose.yml

@@ -23,3 +23,9 @@ services:
       - ./provision-redmine.py:/app/provision-redmine.py:ro
       - ./merge-invites.py:/app/merge-invites.py:ro
       - ./dump-invite-page.py:/app/dump-invite-page.py:ro
+      - ./validate-all-logins.py:/app/validate-all-logins.py:ro
+      - ./diagnose-vp-compliance.py:/app/diagnose-vp-compliance.py:ro
+      - ./validate-coo.py:/app/validate-coo.py:ro
+      - ./diagnose-sso-chain.py:/app/diagnose-sso-chain.py:ro
+      - ./probe-redmine-access.py:/app/probe-redmine-access.py:ro
+      - ./probe-discourse-signup.py:/app/probe-discourse-signup.py:ro

probe-discourse-signup.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+probe-discourse-signup.py -- Dump Discourse signup validation state.
+
+Gets an agent to the OIDC-authenticated /signup page, fills a candidate
+username, clicks Create Account, and dumps every field hint/error plus
+the page state 8s later. Reveals why "account created" never sticks.
+
+Usage:
+    docker compose run --rm --entrypoint python3 \
+        -e AGENT=vp-secops provision probe-discourse-signup.py
+"""
+
+import os, sys, time
+from pathlib import Path
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = "https://my.knownelement.com"
+DISCOURSE_URL = "https://community.turnsys.com"
+STATE_DIR = Path("/app/state")
+AGENT = os.environ.get("AGENT", "vp-secops")
+
+EMAILS = {
+    "vp-secops": "tsgstaff-coo-vpsecops@turnsys.com",
+    "svp-knel": "tsgstaff-coo-svpknel@turnsys.com",
+}
+USERNAME = AGENT.replace("-", "")
+
+
+def field_state(page):
+    return page.evaluate("""() => {
+        const out = [];
+        document.querySelectorAll('input, .tip, .invalid, .good, .bad, .warning, [class*="hint"]').forEach(el => {
+            const id = el.id || '';
+            const val = (el.value || '').substring(0, 40);
+            const text = (el.textContent || '').trim().substring(0, 100);
+            const cls = (el.getAttribute('class') || '').substring(0, 60);
+            const vis = el.offsetParent !== null;
+            if (text || val || id) out.push(`<${el.tagName.toLowerCase()}> id=${id} class=${cls} value="${val}" vis=${vis} text="${text}"`);
+        });
+        return out.join('\\n');
+    }""")
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+    item = f"{AGENT} Cloudron"
+    email = EMAILS.get(AGENT, f"{AGENT}@turnsys.com")
+    password = bw.get_item_password(item)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+        page = context.new_page()
+
+        # Panel login
+        page.goto(f"{CLOUDRON_BASE}/login.html", wait_until="networkidle", timeout=30000)
+        page.wait_for_timeout(2000)
+        page.wait_for_selector("#inputPassword", timeout=15000)
+        page.click("#inputUsername")
+        page.keyboard.type(email)
+        page.click("#inputPassword")
+        page.keyboard.type(password)
+        page.locator('[role="button"]:has-text("Log in")').first.click()
+        page.wait_for_timeout(4000)
+        totp = page.query_selector("#inputTotpToken")
+        if totp and totp.is_visible():
+            totp.click()
+            page.keyboard.type(bw.get_totp(item))
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+        print(f"PANEL: {page.url}")
+
+        # Discourse: login modal -> OpenID
+        page.goto(DISCOURSE_URL, wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(3000)
+        page.locator(".login-button").first.click()
+        page.wait_for_timeout(3000)
+        page.locator('button:has-text("OpenID")').first.click()
+        page.wait_for_timeout(5000)
+
+        # Handle any OIDC interaction
+        for _ in range(4):
+            pw = page.query_selector("#inputPassword")
+            if pw and pw.is_visible():
+                page.click("#inputUsername"); page.keyboard.type(email)
+                page.click("#inputPassword"); page.keyboard.type(password)
+                page.locator('[role="button"]:has-text("Log in")').first.click()
+                page.wait_for_timeout(4000)
+                continue
+            t = page.query_selector("#inputTotpToken")
+            if t and t.is_visible():
+                t.click(); page.keyboard.type(bw.get_totp(item))
+                page.locator("#totpTokenSubmitButton").click()
+                page.wait_for_timeout(5000)
+                continue
+            if "community.turnsys.com" in page.url:
+                break
+            page.wait_for_timeout(2500)
+
+        print(f"POST-OIDC: {page.url}")
+        if "/signup" not in page.url:
+            print("NOT ON SIGNUP -- dumping body:")
+            print(page.evaluate("() => document.body.innerText.substring(0, 400)"))
+            browser.close()
+            return
+
+        # Fill username
+        page.wait_for_timeout(2000)
+        inp = page.locator("#new-account-username, input[name='username']").first
+        inp.click()
+        page.keyboard.type(USERNAME)
+        page.wait_for_timeout(2000)
+        print(f"\n=== FIELD STATE after typing {USERNAME} ===")
+        print(field_state(page))
+
+        # Click create
+        for btn in ["Create Account", "Sign Up"]:
+            loc = page.locator(f'button:has-text("{btn}")')
+            if loc.count() > 0 and loc.first.is_visible():
+                loc.first.click()
+                print(f"\nClicked: {btn}")
+                break
+
+        page.wait_for_timeout(8000)
+        print(f"\n=== 8s LATER: URL={page.url} ===")
+        print(field_state(page))
+        body = page.evaluate("() => document.body.innerText.substring(0, 400)")
+        print(f"BODY: {body!r}")
+        page.screenshot(path=str(STATE_DIR / f"discourse-signup-{AGENT}-{time.strftime('%H%M%S')}.png"), full_page=True)
+
+        browser.close()
+
+
+if __name__ == "__main__":
+    main()

probe-redmine-access.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+probe-redmine-access.py -- What does the Redmine OIDC page actually say?
+
+Logs in via panel, clicks Redmine SSO, dumps the interaction page text
+verbatim. Distinguishes: login form vs consent vs "You do not have access".
+
+Usage:
+    docker compose run --rm --entrypoint python3 \
+        -e AGENT=svp-knel provision probe-redmine-access.py
+"""
+
+import os, sys, time
+from pathlib import Path
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = "https://my.knownelement.com"
+REDMINE_URL = "https://projects.knownelement.com"
+STATE_DIR = Path("/app/state")
+AGENT = os.environ.get("AGENT", "svp-knel")
+
+EMAILS = {
+    "svp-knel": "tsgstaff-coo-svpknel@turnsys.com",
+    "vp-secops": "tsgstaff-coo-vpsecops@turnsys.com",
+    "coo": "tsgstaff-coo@turnsys.com",
+}
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+    item = f"{AGENT} Cloudron"
+    email = EMAILS[AGENT]
+    password = bw.get_item_password(item)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+        page = context.new_page()
+
+        # Panel login (networkidle like the working loop code)
+        page.goto(f"{CLOUDRON_BASE}/login.html", wait_until="networkidle", timeout=30000)
+        page.wait_for_timeout(2000)
+        page.wait_for_selector("#inputPassword", timeout=15000)
+        page.click("#inputUsername")
+        page.keyboard.type(email)
+        page.click("#inputPassword")
+        page.keyboard.type(password)
+        page.locator('[role="button"]:has-text("Log in")').first.click()
+        page.wait_for_timeout(4000)
+        totp = page.query_selector("#inputTotpToken")
+        if totp and totp.is_visible():
+            totp.click()
+            page.keyboard.type(bw.get_totp(item))
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+        print(f"PANEL: {page.url}")
+
+        # Redmine SSO
+        page.goto(f"{REDMINE_URL}/login", wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(2000)
+        page.locator("#login-oauth-submit-1").first.click()
+        page.wait_for_timeout(6000)
+
+        print(f"\nOIDC URL: {page.url}")
+        body = page.evaluate("() => document.body.innerText")
+        print("OIDC BODY (verbatim):")
+        print(body)
+        page.screenshot(path=str(STATE_DIR / f"redmine-access-{AGENT}-{time.strftime('%H%M%S')}.png"))
+
+        browser.close()
+
+
+if __name__ == "__main__":
+    main()

provision-agent.py

@@ -561,6 +561,69 @@ def sso_login(page: Page, system_url: str, agent: dict, bw: BitwardenHelper,
     return True
 
 
+def handle_oidc_interaction(page: Page, agent: dict, bw: BitwardenHelper) -> None:
+    """Handle a Cloudron OIDC interaction page: login form, TOTP, consent.
+
+    Each Cloudron app has its own OIDC client and the panel session is
+    NOT always shared -- the interaction page may present a fresh Pankow
+    login form (#inputPassword), a TOTP prompt (#inputTotpToken), a
+    consent page, or any combination. Safe to call speculatively; every
+    step checks visibility first.
+
+    Cloudron's Pankow forms use #input* IDs (Gitea/Redmine/Discourse
+    login pages use different IDs, so there is no cross-app collision).
+    """
+    name = agent["name"]
+    cloudron_item = f"{name} Cloudron"
+
+    # Poll: the redirect to the OIDC interaction may still be in flight
+    for _ in range(6):
+        on_interaction = ("openid" in page.url or "interaction" in page.url
+                          or "my.knownelement.com" in page.url)
+
+        # 1. Login form (Cloudron Pankow IDs -- no cross-app collision)
+        pw = page.query_selector("#inputPassword")
+        if pw and pw.is_visible():
+            email = agent.get("cloudron_email", f"{name}@turnsys.com")
+            password = bw.get_item_password(cloudron_item)
+            page.click("#inputUsername")
+            page.keyboard.type(email)
+            page.click("#inputPassword")
+            page.keyboard.type(password)
+            page.locator('[role="button"]:has-text("Log in")').first.click()
+            page.wait_for_timeout(4000)
+            log.info(f"[{name}] OIDC interaction: login form submitted")
+            continue
+
+        # 2. TOTP
+        totp = page.query_selector("#inputTotpToken")
+        if totp and totp.is_visible():
+            code = bw.get_totp(cloudron_item)
+            totp.click()
+            page.keyboard.type(code)
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+            log.info(f"[{name}] OIDC interaction: TOTP submitted")
+            continue
+
+        # 3. Consent (only on the interaction page itself)
+        if on_interaction:
+            for text in ["Continue", "Authorize", "Allow", "Accept"]:
+                loc = page.locator(
+                    f'[role="button"]:has-text("{text}"), button:has-text("{text}")'
+                )
+                if loc.count() > 0 and loc.first.is_visible():
+                    loc.first.click()
+                    page.wait_for_timeout(5000)
+                    log.info(f"[{name}] OIDC interaction: consent clicked ({text})")
+                    break
+
+        # Done if we've left the interaction pages
+        if not on_interaction:
+            return
+        page.wait_for_timeout(2500)
+
+
 def provision_gitea(page: Page, agent: dict, bw: BitwardenHelper) -> str:
     """Generate a Gitea API token via SSO login. Returns the token."""
     name = agent["name"]
@@ -695,7 +758,12 @@ def provision_discourse(page: Page, agent: dict, bw: BitwardenHelper) -> str:
         return bw.get_item_password(item_name)
 
     url = discourse_cfg.get("url", DISCOURSE_URL)
-    username = agent.get("username", name.replace("-", ""))
+    # Discourse caps usernames at 20 chars. Invite-issued usernames are
+    # often longer (tsgstaff-coo-svptctc-vpinvesting), which fails signup
+    # validation silently. Default to the short form unless overridden.
+    username = agent.get("discourse_username", name.replace("-", ""))
+    if len(username) > 20:
+        username = name.replace("-", "")[:20]
 
     # --- SSO login ---
     page.goto(f"{url}/", wait_until="domcontentloaded", timeout=30000)
@@ -710,14 +778,23 @@ def provision_discourse(page: Page, agent: dict, bw: BitwardenHelper) -> str:
         sso_btn = page.locator('button:has-text("OpenID")')
         if sso_btn.count() > 0:
             sso_btn.first.click()
-            page.wait_for_timeout(5000)
+            page.wait_for_timeout(3000)
 
-        # First login: /signup with email already authenticated by OIDC
+            # The OIDC interaction may need a fresh login/TOTP/consent
+            handle_oidc_interaction(page, agent, bw)
+            page.wait_for_timeout(3000)
+
+        # First login: /signup with email already authenticated by OIDC.
+        # Discourse PREFILLS the username field (truncated OIDC username)
+        # -- appending without clearing produces a too-long concatenation.
+        # Select-all + delete before typing.
         if "/signup" in page.url:
             page.wait_for_timeout(2000)
             username_input = page.locator('#new-account-username, input[name="username"]')
             if username_input.count() > 0 and username_input.first.is_visible():
                 username_input.first.click()
+                page.keyboard.press("Control+a")
+                page.keyboard.press("Delete")
                 page.keyboard.type(username)
                 page.wait_for_timeout(1000)
                 for btn_text in ["Create Account", "Sign Up", "Register"]:
@@ -882,17 +959,11 @@ def provision_redmine(page: Page, agent: dict, bw: BitwardenHelper) -> str:
     )
     if sso_btn.count() > 0 and sso_btn.first.is_visible():
         sso_btn.first.click()
-        page.wait_for_timeout(5000)
+        page.wait_for_timeout(3000)
 
-        if "openid" in page.url.lower():
-            for consent in ["Continue", "Authorize", "Allow", "Accept"]:
-                cbtn = page.locator(
-                    f'[role="button"]:has-text("{consent}"), button:has-text("{consent}")'
-                )
-                if cbtn.count() > 0 and cbtn.first.is_visible():
-                    cbtn.first.click()
-                    page.wait_for_timeout(5000)
-                    break
+        # The OIDC interaction may need a fresh login/TOTP/consent
+        handle_oidc_interaction(page, agent, bw)
+        page.wait_for_timeout(3000)
 
     if "/login" in page.url:
         log.error(f"[{name}] Redmine SSO failed (still on login page)")

validate-all-logins.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""
+validate-all-logins.py -- End-to-end phase-1 validation for all COO agents.
+
+For each agent in the manifest (excluding non-COO identities):
+  1. Fresh Cloudron login: username + password + TOTP from BW
+  2. Assert we land on an authenticated page
+  3. If the agent has Gitea/Redmine/Discourse credentials in BW, verify
+     each API credential against its API
+
+Prints a PASS/FAIL matrix. Exits 1 if any login fails.
+
+Usage:
+    docker compose run --rm --entrypoint python3 provision validate-all-logins.py
+"""
+
+import json
+import os
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+import yaml
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = os.environ.get("CLOUDRON_BASE", "https://my.knownelement.com")
+GITEA_URL = "https://git.knownelement.com"
+REDMINE_URL = "https://projects.knownelement.com"
+DISCOURSE_URL = "https://community.turnsys.com"
+STATE_DIR = Path("/app/state")
+
+# Identities that are NOT owned by the COO BW account
+EXCLUDE = {"cco", "ctpo", "board-secretariat", "director-audit"}
+
+
+def verify_gitea(token):
+    req = urllib.request.Request(
+        f"{GITEA_URL}/api/v1/user",
+        headers={"Authorization": f"token {token}"},
+    )
+    with urllib.request.urlopen(req, timeout=15) as r:
+        return json.load(r).get("login")
+
+
+def verify_redmine(key):
+    req = urllib.request.Request(
+        f"{REDMINE_URL}/users/current.json",
+        headers={"X-Redmine-API-Key": key},
+    )
+    with urllib.request.urlopen(req, timeout=15) as r:
+        return json.load(r).get("user", {}).get("login")
+
+
+def verify_discourse(key):
+    req = urllib.request.Request(
+        f"{DISCOURSE_URL}/notifications.json",
+        headers={"User-Api-Key": key},
+    )
+    with urllib.request.urlopen(req, timeout=15) as r:
+        r.read()
+        return "ok"
+
+
+def _validate_agent(bw, context, agent, results):
+    """Validate one agent: fresh login + API credential checks."""
+    name = agent["name"]
+    email = agent.get("cloudron_email", f"{name}@turnsys.com")
+    item = f"{name} Cloudron"
+    row = {"name": name, "cloudron": False, "totp": False,
+           "gitea": None, "redmine": None, "discourse": None, "err": ""}
+
+    try:
+        if not bw.item_exists(item):
+            row["err"] = "no BW item"
+            results.append(row)
+            print(f"[{name:20s}] FAIL: no Cloudron item in BW")
+            return
+
+        password = bw.get_item_password(item)
+
+        page = context.new_page()
+        page.goto(f"{CLOUDRON_BASE}/login.html",
+                  wait_until="networkidle", timeout=20000)
+        page.wait_for_timeout(1500)
+        page.wait_for_selector("#inputPassword", timeout=15000)
+        page.click("#inputUsername")
+        page.keyboard.type(email)
+        page.click("#inputPassword")
+        page.keyboard.type(password)
+        page.locator('[role="button"]:has-text("Log in")').first.click()
+        page.wait_for_timeout(4000)
+
+        totp_el = page.query_selector("#inputTotpToken")
+        if totp_el and totp_el.is_visible():
+            row["totp"] = True  # 2FA prompt appeared = 2FA is ON
+            code = bw.get_totp(item)
+            totp_el.click()
+            page.keyboard.type(code)
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+
+        authed = ("login" not in page.url.lower()
+                  and "openid" not in page.url.lower()
+                  and "setupaccount" not in page.url.lower())
+        row["cloudron"] = authed
+        if not authed:
+            page.screenshot(path=str(
+                STATE_DIR / f"validate-fail-{name}-{time.strftime('%H%M%S')}.png"))
+            row["err"] = f"login stuck at {page.url[:60]}"
+        page.close()
+    except Exception as e:
+        row["err"] = str(e)[:80]
+
+    # API credential checks (failures recorded, not fatal)
+    for sysname, verifier, bwitem in [
+        ("gitea", verify_gitea, f"{name} Gitea"),
+        ("redmine", verify_redmine, f"{name} Redmine"),
+        ("discourse", verify_discourse, f"{name} Discourse"),
+    ]:
+        try:
+            if bw.item_exists(bwitem):
+                row[sysname] = verifier(bw.get_item_password(bwitem))
+            else:
+                row[sysname] = "-"
+        except Exception as e:
+            row[sysname] = f"ERR {str(e)[:40]}"
+
+    results.append(row)
+    g = row["gitea"] if row["gitea"] is not None else "?"
+    r = row["redmine"] if row["redmine"] is not None else "?"
+    d = row["discourse"] if row["discourse"] is not None else "?"
+    status = "PASS" if row["cloudron"] and row["totp"] else "FAIL"
+    print(f"[{name:20s}] {status} cloudron={row['cloudron']} "
+          f"2fa={row['totp']} gitea={g} redmine={r} discourse={d} "
+          f"{row['err']}")
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+
+    manifest = yaml.safe_load(open("/app/agents.yaml"))
+    agents = [a for a in manifest["agents"] if a["name"] not in EXCLUDE]
+
+    results = []
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+
+        for agent in agents:
+            # Fresh context per agent: a shared context carries the previous
+            # agent's session cookie and login.html never shows the form.
+            context = browser.new_context(viewport={"width": 1280, "height": 1024})
+            try:
+                _validate_agent(bw, context, agent, results)
+            finally:
+                context.close()
+
+        browser.close()
+
+    print("\n=== VALIDATION MATRIX ===")
+    fails = 0
+    for row in results:
+        ok = row["cloudron"] and row["totp"]
+        if not ok:
+            fails += 1
+        print(f"  {row['name']:20s} {'OK ' if ok else 'FAIL'} {row['err']}")
+
+    sys.exit(1 if fails else 0)
+
+
+if __name__ == "__main__":
+    main()

validate-coo.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+validate-coo.py -- Targeted re-validation of the coo identity.
+
+Verifies the get_totp exact-name fix: fresh login with password + TOTP.
+
+Usage:
+    docker compose run --rm --entrypoint python3 provision validate-coo.py
+"""
+
+import os
+import sys
+
+from playwright.sync_api import sync_playwright
+
+sys.path.insert(0, os.path.dirname(__file__))
+from bw_helper import BitwardenHelper
+
+CLOUDRON_BASE = "https://my.knownelement.com"
+ITEM = "coo Cloudron"
+EMAIL = "tsgstaff-coo@turnsys.com"
+
+
+def main():
+    bw = BitwardenHelper(
+        client_id=os.environ["BW_CLIENTID"],
+        client_secret=os.environ["BW_CLIENTSECRET"],
+        password=os.environ["BW_PASSWORD"],
+        server_url=os.environ.get("BW_SERVER", ""),
+    )
+    bw.login()
+    password = bw.get_item_password(ITEM)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(viewport={"width": 1280, "height": 1024})
+        page = context.new_page()
+
+        page.goto(f"{CLOUDRON_BASE}/login.html", wait_until="domcontentloaded", timeout=30000)
+        page.wait_for_timeout(1500)
+        page.wait_for_selector("#inputPassword", timeout=15000)
+        page.click("#inputUsername")
+        page.keyboard.type(EMAIL)
+        page.click("#inputPassword")
+        page.keyboard.type(password)
+        page.locator('[role="button"]:has-text("Log in")').first.click()
+        page.wait_for_timeout(4000)
+
+        totp_el = page.query_selector("#inputTotpToken")
+        if totp_el and totp_el.is_visible():
+            print("2FA prompt: YES")
+            code = bw.get_totp(ITEM)
+            print(f"TOTP resolved by exact name: {code}")
+            totp_el.click()
+            page.keyboard.type(code)
+            page.locator("#totpTokenSubmitButton").click()
+            page.wait_for_timeout(5000)
+        else:
+            print("2FA prompt: NO (problem)")
+
+        authed = ("login" not in page.url.lower()
+                  and "openid" not in page.url.lower()
+                  and "setupaccount" not in page.url.lower())
+        print(f"coo login: {'PASS' if authed else 'FAIL'} ({page.url})")
+        sys.exit(0 if authed else 1)
+
+
+if __name__ == "__main__":
+    main()