From 6d89f166107b00b44da134d959df166659237ccc Mon Sep 17 00:00:00 2001 From: TSYS Group COO Date: Fri, 14 Aug 2026 08:23:37 -0500 Subject: [PATCH] feat: consolidate proven Discourse/Redmine flows into provision-agent.py Replaced the stale session-1 provision_discourse/provision_redmine with the flows proven on vp-techops this session. Both are now parameterized by the agent manifest (username derived from name, hyphens stripped, overridable via username: field). Discourse: login modal -> OpenID button -> signup on first login -> RSA User API key flow (PKCS1v15 decrypt, JSON payload). Redmine: KNEL Cloud SSO button -> consent -> Show/Reset on the API access key section via targeted DOM traversal. Added docs/JOURNAL.md with all working selectors, flows, gotchas, and verification results so future sessions do not rediscover them. --- agents.yaml.example | 2 + docs/JOURNAL.md | 84 +++++++++++ provision-agent.py | 361 ++++++++++++++++++++++++++++---------------- 3 files changed, 318 insertions(+), 129 deletions(-) create mode 100644 docs/JOURNAL.md diff --git a/agents.yaml.example b/agents.yaml.example index db194a5..af5ecd3 100644 --- a/agents.yaml.example +++ b/agents.yaml.example @@ -11,6 +11,8 @@ agents: - name: vp-techops display_name: "VP TechOps" cloudron_email: "vp-techops@turnsys.com" + # username defaults to name with hyphens stripped (vp-techops -> vptechops) + username: "vptechops" priority: Q3 cloudron_invite: "https://tsys-cloudron.knel.net/invitation/REPLACE_WITH_TOKEN" systems: diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md new file mode 100644 index 0000000..0e7f6b1 --- /dev/null +++ b/docs/JOURNAL.md @@ -0,0 +1,84 @@ +# JOURNAL.md — Agent Identity Provisioning + +> Append-only decision & pattern log. One section per change. Never delete or reorder. + +--- + +## 2026-08-13 — Session 2: BW sync fix, 2FA, all four systems proven + +**Commits:** f633a10, be2f607, 2d01a9f, 2258e1b, fcd484f, f414b0b + +### Decisions + +1. **BW sync lifecycle**: `BitwardenHelper.login()` must end with `bw sync`. + Root cause of session-1 "vanishing items": the container's local cache + was never synced after login. `list_items()` also syncs before reading. +2. **Container UID/GID**: provision user is 1002:1002, matching the host + TSGCOO account, so bind-mount state files are owned by the invoking user. + During rapid iteration `:latest` tagging with overwrite is acceptable. +3. **Source mounted read-only** into the container (bw_helper.py, + provision-agent.py, test files) so selector iterations do not require + image rebuilds. +4. **One-off scripts kept**: the exploration scripts (dump-cloudron-dom.py, + enable-cloudron-2fa.py, etc.) remain in the repo as proven references; + their flows have been consolidated into provision-agent.py. + +### Patterns (selectors and flows that WORK) + +**Cloudron panel (Pankow/Vue):** +- Login: `#inputUsername` / `#inputPassword`, type via `page.keyboard.type()` + (never `fill()`), submit via `[role="button"]:has-text("Log in")`. +- 2FA prompt on OIDC login: `#inputTotpToken` + `#totpTokenSubmitButton` + (NOT `#inputTotp`). +- 2FA enrollment: `#/profile` -> click `text=Setup` -> click + `text=switchToTotp` (Cloudron defaults to Passkey) -> secret is base32 + text on the page (regex `[A-Z2-7]{16,}`) -> enter code in + `#totpTokenInput` -> click Enable. + +**Gitea (proven session 1):** +- SSO button: `a[href*="oauth2/cloudron"]` at `/user/login`. +- Token page `/user/settings/applications`: fill `#name` via JS evaluate, + scopes are radio buttons, extract 40-hex from `.ui.info.message`. + +**Discourse:** +- Login modal via `.login-button`, then `button:has-text("OpenID")`. +- First SSO lands on `/signup` with email pre-authenticated: fill + `#new-account-username`, click Sign Up. +- User API key: RSA keypair -> `/user-api-key/new?...&public_key=` -> + click Authorize -> capture POST response -> decrypt with **PKCS1v15** + (not OAEP) -> payload JSON `{"key": "..."}`. +- API auth header is `User-Api-Key` (admin keys use `Api-Key`). + +**Redmine:** +- SSO button: `#login-oauth-submit-1` ("Continue with KNEL Cloud"). +- Prereq: Cloudron admin must grant the user access to the Redmine app, + otherwise OIDC shows "You do not have access" and redirects back. +- API key: `/my/account` -> click Show in `.api-key-actions` -> read + `#api-access-key` (40-hex). If absent, click the Reset link found by + DOM traversal from `#api-access-key` (generic `a:has-text("Reset")` + clicks the wrong section and logs you out). + +### Username derivation + +Manifest `name` is hyphenated (vp-techops); app usernames are not +(vptechops). Default: `agent.get("username", name.replace("-", ""))`. +Override with an explicit `username:` field in agents.yaml. + +### Gotchas + +- Em dashes (U+2014) break Python source; use `--`. +- Python f-string interpolation inside JS template literals does not work; + build JS strings with plain concatenation inside evaluate(). +- Discourse admin API keys page is admin-only; User API keys are the + self-service path. +- Gitea token page needs `wait_until="domcontentloaded"` (networkidle + times out). + +### Verification results (vp-techops) + +| System | Credential | Verified via | +|---|---|---| +| Cloudron | password + TOTP | full login round-trip | +| Gitea | 40-char token | `GET /api/v1/user` -> vptechops | +| Discourse | 32-char user key | `GET /latest.json` with User-Api-Key | +| Redmine | 40-char API key | `GET /users/current.json` -> id 11 | diff --git a/provision-agent.py b/provision-agent.py index 068abaf..2b14997 100644 --- a/provision-agent.py +++ b/provision-agent.py @@ -20,6 +20,7 @@ import argparse import json import logging import os +import re import sys import time from pathlib import Path @@ -647,187 +648,290 @@ def provision_gitea(page: Page, agent: dict, bw: BitwardenHelper) -> str: def provision_discourse(page: Page, agent: dict, bw: BitwardenHelper) -> str: """ - Generate a Discourse API key via SSO login. + Discourse SSO login + User API key generation. - Note: Discourse API keys typically require admin to create. - If the agent can't self-generate, this logs a warning. - Returns the API key (empty string if not possible). + Proven flow (session 2, verified on vp-techops): + 1. Click .login-button to open the login modal + 2. Click the OpenID Connect button inside the modal + 3. First login redirects to /signup with email pre-authenticated: + fill username, click Sign Up + 4. User API key via the RSA-encrypted flow: + - Generate RSA keypair, pass public key to /user-api-key/new + - Click Authorize + - Capture the POST response payload, decrypt (PKCS1v15), + parse JSON to extract the "key" field + + The resulting key authenticates via the User-Api-Key header + (NOT Api-Key -- that is for admin-created keys). + + Returns the API key (empty string on failure). """ name = agent["name"] systems = agent.get("systems", {}) discourse_cfg = systems.get("discourse", {}) if not discourse_cfg: - log.info(f"[{name}] No Discourse config — skipping") + log.info(f"[{name}] No Discourse config -- skipping") return "" item_name = f"{name} Discourse" if bw.item_exists(item_name): - log.info(f"[{name}] Discourse key already exists — skipping") + log.info(f"[{name}] Discourse key already exists -- skipping") return bw.get_item_password(item_name) url = discourse_cfg.get("url", DISCOURSE_URL) + username = agent.get("username", name.replace("-", "")) - # Discourse uses a login modal — can't use sso_login (it reloads the page). - # Handle the full flow inline. - page.goto(f"{url}/", wait_until="domcontentloaded", timeout=15000) + # --- SSO login --- + page.goto(f"{url}/", wait_until="domcontentloaded", timeout=30000) page.wait_for_timeout(3000) - # Already authenticated? - if not page.query_selector('input[type="password"]'): - # Check if we're on the homepage without login form - login_btn = page.locator('button:has-text("Log In")') - if login_btn.count() == 0 or not login_btn.first.is_visible(): - log.info(f"[{name}] Already authenticated at Discourse") - else: - # Open login modal and look for SSO + if not page.query_selector("#current-user, .current-user"): + login_btn = page.locator(".login-button, button:has-text('Log In')") + if login_btn.count() > 0: login_btn.first.click() page.wait_for_timeout(3000) - log.info(f"[{name}] Opened Discourse login modal") - - # Dump modal content for debugging - _debug_dump(page, f"discourse-modal-{name}") - - # Look for SSO/social login buttons in the modal - sso_clicked = False - for selector in [ - 'button.btn-social', - 'a[href*="auth/cloudron"]', - 'button:has-text("Cloud")', - 'button:has-text("KNEL")', - 'a:has-text("Cloud")', - '[data-login-name*="cloud"]', - 'button.social-buttons-button', - ]: - btn = page.locator(selector) - if btn.count() > 0 and btn.first.is_visible(): - btn.first.click() - sso_clicked = True - log.info(f"[{name}] Clicked Discourse SSO button: {selector}") - break - - if not sso_clicked: - log.warning(f"[{name}] No SSO button found in Discourse modal") - _debug_dump(page, f"discourse-no-sso-{name}") + sso_btn = page.locator('button:has-text("OpenID")') + if sso_btn.count() > 0: + sso_btn.first.click() page.wait_for_timeout(5000) - # Handle OIDC redirect if needed - if "openid" in page.url or "interaction" in page.url: - page.wait_for_timeout(2000) - has_login = page.query_selector("#inputPassword") - if has_login and has_login.is_visible(): - # Need to login on OIDC - cloudron_email = agent.get("cloudron_email", f"{name}@turnsys.com") - cloudron_item = f"{name} Cloudron" - password = bw.get_item_password(cloudron_item) - page.click("#inputUsername") - page.keyboard.type(cloudron_email) - page.click("#inputPassword") - page.keyboard.type(password) - for text in ["Log in", "Sign in", "Continue"]: - btn = page.locator(f'[role="button"]:has-text("{text}"), button:has-text("{text}")') - if btn.count() > 0 and btn.first.is_visible(): - btn.first.click() - break - page.wait_for_timeout(5000) - - # Consent - for consent_text in ["Continue", "Authorize", "Allow"]: - consent_btn = page.locator(f'[role="button"]:has-text("{consent_text}"), button:has-text("{consent_text}")') - if consent_btn.count() > 0: - consent_btn.first.click(force=True) + # First login: /signup with email already authenticated by OIDC + if "/signup" in page.url: + page.wait_for_timeout(2000) + username_input = page.locator('#new-account-username, input[name="username"]') + if username_input.count() > 0 and username_input.first.is_visible(): + username_input.first.click() + page.keyboard.type(username) + page.wait_for_timeout(1000) + for btn_text in ["Create Account", "Sign Up", "Register"]: + loc = page.locator(f'button:has-text("{btn_text}")') + if loc.count() > 0 and loc.first.is_visible(): + loc.first.click() page.wait_for_timeout(5000) break + log.info(f"[{name}] Discourse account created: {username}") - page.wait_for_timeout(3000) - log.info(f"[{name}] Discourse URL after SSO: {page.url}") + page.goto(f"{url}/", wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) + if not page.query_selector("#current-user, .current-user"): + log.error(f"[{name}] Discourse SSO login failed") + _debug_dump(page, f"discourse-login-failed-{name}") + return "" + log.info(f"[{name}] Discourse SSO login OK") - # Try to generate an API key from user preferences - # Note: In Discourse, only admin can create API keys via UI - # Non-admin users may not have this option - page.goto(f"{url}/u/{name}/preferences/account", wait_until="networkidle") + # --- User API key (RSA flow) --- + import base64 + import secrets + import uuid as uuid_lib + from urllib.parse import quote_plus - api_key_section = page.query_selector('.api-keys, [data-section="api-keys"]') + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding as asym_padding + from cryptography.hazmat.primitives.asymmetric import rsa as asym_rsa - if not api_key_section: - log.warning( - f"[{name}] Discourse API key self-generation not available. " - "An admin must create the key. The agent will need a manually-created key." - ) + private_key = asym_rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_pem = private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode("ascii") + + nonce = secrets.token_hex(16) + client_id = str(uuid_lib.uuid4()) + app_name = f"TSG-Agent-{agent.get('display_name', name)}" + params = ( + f"?application_name={quote_plus(app_name)}" + f"&client_id={client_id}" + f"&nonce={nonce}" + f"&scopes=read%2Cwrite" + f"&public_key={quote_plus(public_pem)}" + ) + + api_responses = [] + + def handle_response(response): + if "user-api-key" in response.url and response.request.method == "POST": + try: + api_responses.append(response.text()) + except Exception: + pass + + page.on("response", handle_response) + try: + page.goto(f"{url}/user-api-key/new{params}", wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) + + for btn_text in ["Authorize", "Approve", "Continue"]: + loc = page.locator(f'button:has-text("{btn_text}"), .btn-primary') + if loc.count() > 0 and loc.first.is_visible(): + loc.first.click() + break + page.wait_for_timeout(3000) + finally: + page.remove_listener("response", handle_response) + + api_key = "" + for resp_text in api_responses: + try: + data = json.loads(resp_text) + encrypted_raw = data.get("key") or data.get("payload") or "" + if not encrypted_raw: + continue + encrypted = base64.b64decode( + encrypted_raw.replace("\n", "").replace("\r", "").replace(" ", "") + ) + # Discourse uses PKCS1v15; try OAEP variants as fallback + paddings = [ + asym_padding.PKCS1v15(), + asym_padding.OAEP( + mgf=asym_padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), label=None), + asym_padding.OAEP( + mgf=asym_padding.MGF1(algorithm=hashes.SHA1()), + algorithm=hashes.SHA1(), label=None), + ] + for pad in paddings: + try: + decrypted = private_key.decrypt(encrypted, pad).decode("ascii") + try: + api_key = json.loads(decrypted).get("key", decrypted) + except json.JSONDecodeError: + api_key = decrypted + break + except Exception: + continue + if api_key: + break + except Exception: + continue + + if not api_key: + log.error(f"[{name}] Could not extract Discourse API key") + _debug_dump(page, f"discourse-no-key-{name}") return "" - # If the section exists, try to create a key - revoke_btn = page.query_selector('.api-keys button:has-text("Revoke")') - if not revoke_btn: - # No existing keys — create one - gen_btn = page.query_selector('button:has-text("New API Key"), button:has-text("Create")') - if gen_btn: - gen_btn.click() - page.wait_for_timeout(2000) + log.info(f"[{name}] Discourse API key obtained: {api_key[:8]}...") - # Read the key - key_el = page.query_selector('.api-key-value, code') - api_key = key_el.text_content().strip() if key_el else "" + bw.create_item( + name=item_name, + username=username, + password=api_key, + uris=[url], + collection_name=name, + ) + log.info(f"[{name}] Discourse API key stored in Bitwarden") - if api_key: - bw.create_item( - name=item_name, - username=name, - password=api_key, - uris=[url], - collection_name=name, - ) - log.info(f"[{name}] Discourse API key stored in Bitwarden") - return api_key - - log.warning(f"[{name}] Could not generate Discourse API key") - return "" + return api_key def provision_redmine(page: Page, agent: dict, bw: BitwardenHelper) -> str: - """Get the Redmine API access key via SSO login. Returns the key.""" + """ + Redmine SSO login + API access key extraction. + + Proven flow (session 2, verified on vp-techops): + 1. Click "Continue with KNEL Cloud" (#login-oauth-submit-1) + 2. OIDC may show a consent page -- click Continue + 3. On /my/account click "Show" in the .api-key-actions section + 4. Read the 40-hex key from the #api-access-key pre element + 5. If no key exists yet, click the API-key Reset link via targeted + DOM traversal (a generic "Reset" match clicks the wrong section) + + Prereq: the Cloudron user must have Redmine app access granted by + the Cloudron admin, else OIDC shows "You do not have access". + + Returns the API key (empty string on failure). + """ name = agent["name"] systems = agent.get("systems", {}) redmine_cfg = systems.get("redmine", {}) if not redmine_cfg: - log.info(f"[{name}] No Redmine config — skipping") + log.info(f"[{name}] No Redmine config -- skipping") return "" item_name = f"{name} Redmine" if bw.item_exists(item_name): - log.info(f"[{name}] Redmine key already exists — skipping") + log.info(f"[{name}] Redmine key already exists -- skipping") return bw.get_item_password(item_name) url = redmine_cfg.get("url", REDMINE_URL) + username = agent.get("username", name.replace("-", "")) - sso_login(page, f"{url}/login", agent, bw, - sso_button_selector='button[id*="login-oauth"]') + # --- SSO login --- + page.goto(f"{url}/login", wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) - # Navigate to account page where API key lives - page.goto(f"{url}/my/account", wait_until="networkidle") - page.wait_for_timeout(2000) + sso_btn = page.locator( + '#login-oauth-submit-1, button:has-text("KNEL"), button:has-text("Continue")' + ) + if sso_btn.count() > 0 and sso_btn.first.is_visible(): + sso_btn.first.click() + page.wait_for_timeout(5000) + + if "openid" in page.url.lower(): + for consent in ["Continue", "Authorize", "Allow", "Accept"]: + cbtn = page.locator( + f'[role="button"]:has-text("{consent}"), button:has-text("{consent}")' + ) + if cbtn.count() > 0 and cbtn.first.is_visible(): + cbtn.first.click() + page.wait_for_timeout(5000) + break + + if "/login" in page.url: + log.error(f"[{name}] Redmine SSO failed (still on login page)") + _debug_dump(page, f"redmine-sso-failed-{name}") + return "" + log.info(f"[{name}] Redmine SSO login OK") + + # --- API key --- + page.goto(f"{url}/my/account", wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) _debug_dump(page, f"redmine-account-{name}") - # The API key is in the right sidebar under "API access key" - show_link = page.query_selector('a:has-text("Show"), #api_access_key + a, a[href*="access_key"]') - if show_link: - show_link.click() - page.wait_for_timeout(1000) + api_key = "" - key_el = page.query_selector('#api_access_key, .api-key code, .api-access-key') - api_key = key_el.text_content().strip() if key_el else "" + # A key usually exists (auto-created); reveal it via "Show" + show_btn = page.locator('.api-key-actions a:has-text("Show"), a:has-text("Show")') + if show_btn.count() > 0 and show_btn.first.is_visible(): + show_btn.first.click() + page.wait_for_timeout(2000) + api_el = page.query_selector("#api-access-key") + if api_el: + matches = re.findall(r"[a-f0-9]{40}", api_el.text_content()) + if matches: + api_key = matches[0] + # No key yet: generate via the Reset link next to #api-access-key if not api_key: - reset_link = page.query_selector('a:has-text("Reset"), a:has-text("Generate")') - if reset_link: - reset_link.click() - page.wait_for_timeout(2000) - page.click('button:has-text("OK"), button:has-text("Confirm")') - page.wait_for_timeout(1000) - key_el = page.query_selector('#api_access_key, .api-key code') - api_key = key_el.text_content().strip() if key_el else "" + reset_clicked = page.evaluate("""() => { + const apiSection = document.querySelector('#api-access-key'); + if (!apiSection) return false; + let container = apiSection.closest('div, p, fieldset'); + while (container && container.parentElement) { + const reset = Array.from(container.querySelectorAll('a, button')).find(el => + el.textContent.trim().toLowerCase() === 'reset' && el.offsetParent !== null + ); + if (reset) { reset.click(); return true; } + container = container.parentElement; + if (container.tagName === 'FIELDSET' || container.tagName === 'FORM') break; + } + return false; + }""") + if reset_clicked: + page.wait_for_timeout(3000) + confirm = page.locator( + 'button:has-text("OK"), button:has-text("Confirm"), button:has-text("Yes")' + ) + if confirm.count() > 0 and confirm.first.is_visible(): + confirm.first.click() + page.wait_for_timeout(3000) + api_el = page.query_selector("#api-access-key") + if api_el: + matches = re.findall(r"[a-f0-9]{40}", api_el.text_content()) + if matches: + api_key = matches[0] if not api_key: log.error(f"[{name}] Could not get Redmine API key") @@ -836,10 +940,9 @@ def provision_redmine(page: Page, agent: dict, bw: BitwardenHelper) -> str: log.info(f"[{name}] Redmine API key obtained: {api_key[:8]}...") - # Store in Bitwarden bw.create_item( name=item_name, - username=name, + username=username, password=api_key, uris=[url], collection_name=name,