feat: consolidate proven Discourse/Redmine flows into provision-agent.py
Replaced the stale session-1 provision_discourse/provision_redmine with the flows proven on vp-techops this session. Both are now parameterized by the agent manifest (username derived from name, hyphens stripped, overridable via username: field). Discourse: login modal -> OpenID button -> signup on first login -> RSA User API key flow (PKCS1v15 decrypt, JSON payload). Redmine: KNEL Cloud SSO button -> consent -> Show/Reset on the API access key section via targeted DOM traversal. Added docs/JOURNAL.md with all working selectors, flows, gotchas, and verification results so future sessions do not rediscover them.
This commit is contained in:
TSYS Group COO
+232
-129
@@ -20,6 +20,7 @@ import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -647,187 +648,290 @@ def provision_gitea(page: Page, agent: dict, bw: BitwardenHelper) -> str:
|
||||
|
||||
def provision_discourse(page: Page, agent: dict, bw: BitwardenHelper) -> str:
|
||||
"""
|
||||
Generate a Discourse API key via SSO login.
|
||||
Discourse SSO login + User API key generation.
|
||||
|
||||
Note: Discourse API keys typically require admin to create.
|
||||
If the agent can't self-generate, this logs a warning.
|
||||
Returns the API key (empty string if not possible).
|
||||
Proven flow (session 2, verified on vp-techops):
|
||||
1. Click .login-button to open the login modal
|
||||
2. Click the OpenID Connect button inside the modal
|
||||
3. First login redirects to /signup with email pre-authenticated:
|
||||
fill username, click Sign Up
|
||||
4. User API key via the RSA-encrypted flow:
|
||||
- Generate RSA keypair, pass public key to /user-api-key/new
|
||||
- Click Authorize
|
||||
- Capture the POST response payload, decrypt (PKCS1v15),
|
||||
parse JSON to extract the "key" field
|
||||
|
||||
The resulting key authenticates via the User-Api-Key header
|
||||
(NOT Api-Key -- that is for admin-created keys).
|
||||
|
||||
Returns the API key (empty string on failure).
|
||||
"""
|
||||
name = agent["name"]
|
||||
systems = agent.get("systems", {})
|
||||
discourse_cfg = systems.get("discourse", {})
|
||||
|
||||
if not discourse_cfg:
|
||||
log.info(f"[{name}] No Discourse config — skipping")
|
||||
log.info(f"[{name}] No Discourse config -- skipping")
|
||||
return ""
|
||||
|
||||
item_name = f"{name} Discourse"
|
||||
if bw.item_exists(item_name):
|
||||
log.info(f"[{name}] Discourse key already exists — skipping")
|
||||
log.info(f"[{name}] Discourse key already exists -- skipping")
|
||||
return bw.get_item_password(item_name)
|
||||
|
||||
url = discourse_cfg.get("url", DISCOURSE_URL)
|
||||
username = agent.get("username", name.replace("-", ""))
|
||||
|
||||
# Discourse uses a login modal — can't use sso_login (it reloads the page).
|
||||
# Handle the full flow inline.
|
||||
page.goto(f"{url}/", wait_until="domcontentloaded", timeout=15000)
|
||||
# --- SSO login ---
|
||||
page.goto(f"{url}/", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
# Already authenticated?
|
||||
if not page.query_selector('input[type="password"]'):
|
||||
# Check if we're on the homepage without login form
|
||||
login_btn = page.locator('button:has-text("Log In")')
|
||||
if login_btn.count() == 0 or not login_btn.first.is_visible():
|
||||
log.info(f"[{name}] Already authenticated at Discourse")
|
||||
else:
|
||||
# Open login modal and look for SSO
|
||||
if not page.query_selector("#current-user, .current-user"):
|
||||
login_btn = page.locator(".login-button, button:has-text('Log In')")
|
||||
if login_btn.count() > 0:
|
||||
login_btn.first.click()
|
||||
page.wait_for_timeout(3000)
|
||||
log.info(f"[{name}] Opened Discourse login modal")
|
||||
|
||||
# Dump modal content for debugging
|
||||
_debug_dump(page, f"discourse-modal-{name}")
|
||||
|
||||
# Look for SSO/social login buttons in the modal
|
||||
sso_clicked = False
|
||||
for selector in [
|
||||
'button.btn-social',
|
||||
'a[href*="auth/cloudron"]',
|
||||
'button:has-text("Cloud")',
|
||||
'button:has-text("KNEL")',
|
||||
'a:has-text("Cloud")',
|
||||
'[data-login-name*="cloud"]',
|
||||
'button.social-buttons-button',
|
||||
]:
|
||||
btn = page.locator(selector)
|
||||
if btn.count() > 0 and btn.first.is_visible():
|
||||
btn.first.click()
|
||||
sso_clicked = True
|
||||
log.info(f"[{name}] Clicked Discourse SSO button: {selector}")
|
||||
break
|
||||
|
||||
if not sso_clicked:
|
||||
log.warning(f"[{name}] No SSO button found in Discourse modal")
|
||||
_debug_dump(page, f"discourse-no-sso-{name}")
|
||||
|
||||
sso_btn = page.locator('button:has-text("OpenID")')
|
||||
if sso_btn.count() > 0:
|
||||
sso_btn.first.click()
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
# Handle OIDC redirect if needed
|
||||
if "openid" in page.url or "interaction" in page.url:
|
||||
page.wait_for_timeout(2000)
|
||||
has_login = page.query_selector("#inputPassword")
|
||||
if has_login and has_login.is_visible():
|
||||
# Need to login on OIDC
|
||||
cloudron_email = agent.get("cloudron_email", f"{name}@turnsys.com")
|
||||
cloudron_item = f"{name} Cloudron"
|
||||
password = bw.get_item_password(cloudron_item)
|
||||
page.click("#inputUsername")
|
||||
page.keyboard.type(cloudron_email)
|
||||
page.click("#inputPassword")
|
||||
page.keyboard.type(password)
|
||||
for text in ["Log in", "Sign in", "Continue"]:
|
||||
btn = page.locator(f'[role="button"]:has-text("{text}"), button:has-text("{text}")')
|
||||
if btn.count() > 0 and btn.first.is_visible():
|
||||
btn.first.click()
|
||||
break
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
# Consent
|
||||
for consent_text in ["Continue", "Authorize", "Allow"]:
|
||||
consent_btn = page.locator(f'[role="button"]:has-text("{consent_text}"), button:has-text("{consent_text}")')
|
||||
if consent_btn.count() > 0:
|
||||
consent_btn.first.click(force=True)
|
||||
# First login: /signup with email already authenticated by OIDC
|
||||
if "/signup" in page.url:
|
||||
page.wait_for_timeout(2000)
|
||||
username_input = page.locator('#new-account-username, input[name="username"]')
|
||||
if username_input.count() > 0 and username_input.first.is_visible():
|
||||
username_input.first.click()
|
||||
page.keyboard.type(username)
|
||||
page.wait_for_timeout(1000)
|
||||
for btn_text in ["Create Account", "Sign Up", "Register"]:
|
||||
loc = page.locator(f'button:has-text("{btn_text}")')
|
||||
if loc.count() > 0 and loc.first.is_visible():
|
||||
loc.first.click()
|
||||
page.wait_for_timeout(5000)
|
||||
break
|
||||
log.info(f"[{name}] Discourse account created: {username}")
|
||||
|
||||
page.wait_for_timeout(3000)
|
||||
log.info(f"[{name}] Discourse URL after SSO: {page.url}")
|
||||
page.goto(f"{url}/", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
if not page.query_selector("#current-user, .current-user"):
|
||||
log.error(f"[{name}] Discourse SSO login failed")
|
||||
_debug_dump(page, f"discourse-login-failed-{name}")
|
||||
return ""
|
||||
log.info(f"[{name}] Discourse SSO login OK")
|
||||
|
||||
# Try to generate an API key from user preferences
|
||||
# Note: In Discourse, only admin can create API keys via UI
|
||||
# Non-admin users may not have this option
|
||||
page.goto(f"{url}/u/{name}/preferences/account", wait_until="networkidle")
|
||||
# --- User API key (RSA flow) ---
|
||||
import base64
|
||||
import secrets
|
||||
import uuid as uuid_lib
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
api_key_section = page.query_selector('.api-keys, [data-section="api-keys"]')
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding as asym_padding
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa as asym_rsa
|
||||
|
||||
if not api_key_section:
|
||||
log.warning(
|
||||
f"[{name}] Discourse API key self-generation not available. "
|
||||
"An admin must create the key. The agent will need a manually-created key."
|
||||
)
|
||||
private_key = asym_rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
public_pem = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PublicFormat.SubjectPublicKeyInfo,
|
||||
).decode("ascii")
|
||||
|
||||
nonce = secrets.token_hex(16)
|
||||
client_id = str(uuid_lib.uuid4())
|
||||
app_name = f"TSG-Agent-{agent.get('display_name', name)}"
|
||||
params = (
|
||||
f"?application_name={quote_plus(app_name)}"
|
||||
f"&client_id={client_id}"
|
||||
f"&nonce={nonce}"
|
||||
f"&scopes=read%2Cwrite"
|
||||
f"&public_key={quote_plus(public_pem)}"
|
||||
)
|
||||
|
||||
api_responses = []
|
||||
|
||||
def handle_response(response):
|
||||
if "user-api-key" in response.url and response.request.method == "POST":
|
||||
try:
|
||||
api_responses.append(response.text())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.on("response", handle_response)
|
||||
try:
|
||||
page.goto(f"{url}/user-api-key/new{params}", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
for btn_text in ["Authorize", "Approve", "Continue"]:
|
||||
loc = page.locator(f'button:has-text("{btn_text}"), .btn-primary')
|
||||
if loc.count() > 0 and loc.first.is_visible():
|
||||
loc.first.click()
|
||||
break
|
||||
page.wait_for_timeout(3000)
|
||||
finally:
|
||||
page.remove_listener("response", handle_response)
|
||||
|
||||
api_key = ""
|
||||
for resp_text in api_responses:
|
||||
try:
|
||||
data = json.loads(resp_text)
|
||||
encrypted_raw = data.get("key") or data.get("payload") or ""
|
||||
if not encrypted_raw:
|
||||
continue
|
||||
encrypted = base64.b64decode(
|
||||
encrypted_raw.replace("\n", "").replace("\r", "").replace(" ", "")
|
||||
)
|
||||
# Discourse uses PKCS1v15; try OAEP variants as fallback
|
||||
paddings = [
|
||||
asym_padding.PKCS1v15(),
|
||||
asym_padding.OAEP(
|
||||
mgf=asym_padding.MGF1(algorithm=hashes.SHA256()),
|
||||
algorithm=hashes.SHA256(), label=None),
|
||||
asym_padding.OAEP(
|
||||
mgf=asym_padding.MGF1(algorithm=hashes.SHA1()),
|
||||
algorithm=hashes.SHA1(), label=None),
|
||||
]
|
||||
for pad in paddings:
|
||||
try:
|
||||
decrypted = private_key.decrypt(encrypted, pad).decode("ascii")
|
||||
try:
|
||||
api_key = json.loads(decrypted).get("key", decrypted)
|
||||
except json.JSONDecodeError:
|
||||
api_key = decrypted
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if api_key:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not api_key:
|
||||
log.error(f"[{name}] Could not extract Discourse API key")
|
||||
_debug_dump(page, f"discourse-no-key-{name}")
|
||||
return ""
|
||||
|
||||
# If the section exists, try to create a key
|
||||
revoke_btn = page.query_selector('.api-keys button:has-text("Revoke")')
|
||||
if not revoke_btn:
|
||||
# No existing keys — create one
|
||||
gen_btn = page.query_selector('button:has-text("New API Key"), button:has-text("Create")')
|
||||
if gen_btn:
|
||||
gen_btn.click()
|
||||
page.wait_for_timeout(2000)
|
||||
log.info(f"[{name}] Discourse API key obtained: {api_key[:8]}...")
|
||||
|
||||
# Read the key
|
||||
key_el = page.query_selector('.api-key-value, code')
|
||||
api_key = key_el.text_content().strip() if key_el else ""
|
||||
bw.create_item(
|
||||
name=item_name,
|
||||
username=username,
|
||||
password=api_key,
|
||||
uris=[url],
|
||||
collection_name=name,
|
||||
)
|
||||
log.info(f"[{name}] Discourse API key stored in Bitwarden")
|
||||
|
||||
if api_key:
|
||||
bw.create_item(
|
||||
name=item_name,
|
||||
username=name,
|
||||
password=api_key,
|
||||
uris=[url],
|
||||
collection_name=name,
|
||||
)
|
||||
log.info(f"[{name}] Discourse API key stored in Bitwarden")
|
||||
return api_key
|
||||
|
||||
log.warning(f"[{name}] Could not generate Discourse API key")
|
||||
return ""
|
||||
return api_key
|
||||
|
||||
|
||||
def provision_redmine(page: Page, agent: dict, bw: BitwardenHelper) -> str:
|
||||
"""Get the Redmine API access key via SSO login. Returns the key."""
|
||||
"""
|
||||
Redmine SSO login + API access key extraction.
|
||||
|
||||
Proven flow (session 2, verified on vp-techops):
|
||||
1. Click "Continue with KNEL Cloud" (#login-oauth-submit-1)
|
||||
2. OIDC may show a consent page -- click Continue
|
||||
3. On /my/account click "Show" in the .api-key-actions section
|
||||
4. Read the 40-hex key from the #api-access-key pre element
|
||||
5. If no key exists yet, click the API-key Reset link via targeted
|
||||
DOM traversal (a generic "Reset" match clicks the wrong section)
|
||||
|
||||
Prereq: the Cloudron user must have Redmine app access granted by
|
||||
the Cloudron admin, else OIDC shows "You do not have access".
|
||||
|
||||
Returns the API key (empty string on failure).
|
||||
"""
|
||||
name = agent["name"]
|
||||
systems = agent.get("systems", {})
|
||||
redmine_cfg = systems.get("redmine", {})
|
||||
|
||||
if not redmine_cfg:
|
||||
log.info(f"[{name}] No Redmine config — skipping")
|
||||
log.info(f"[{name}] No Redmine config -- skipping")
|
||||
return ""
|
||||
|
||||
item_name = f"{name} Redmine"
|
||||
if bw.item_exists(item_name):
|
||||
log.info(f"[{name}] Redmine key already exists — skipping")
|
||||
log.info(f"[{name}] Redmine key already exists -- skipping")
|
||||
return bw.get_item_password(item_name)
|
||||
|
||||
url = redmine_cfg.get("url", REDMINE_URL)
|
||||
username = agent.get("username", name.replace("-", ""))
|
||||
|
||||
sso_login(page, f"{url}/login", agent, bw,
|
||||
sso_button_selector='button[id*="login-oauth"]')
|
||||
# --- SSO login ---
|
||||
page.goto(f"{url}/login", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
# Navigate to account page where API key lives
|
||||
page.goto(f"{url}/my/account", wait_until="networkidle")
|
||||
page.wait_for_timeout(2000)
|
||||
sso_btn = page.locator(
|
||||
'#login-oauth-submit-1, button:has-text("KNEL"), button:has-text("Continue")'
|
||||
)
|
||||
if sso_btn.count() > 0 and sso_btn.first.is_visible():
|
||||
sso_btn.first.click()
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
if "openid" in page.url.lower():
|
||||
for consent in ["Continue", "Authorize", "Allow", "Accept"]:
|
||||
cbtn = page.locator(
|
||||
f'[role="button"]:has-text("{consent}"), button:has-text("{consent}")'
|
||||
)
|
||||
if cbtn.count() > 0 and cbtn.first.is_visible():
|
||||
cbtn.first.click()
|
||||
page.wait_for_timeout(5000)
|
||||
break
|
||||
|
||||
if "/login" in page.url:
|
||||
log.error(f"[{name}] Redmine SSO failed (still on login page)")
|
||||
_debug_dump(page, f"redmine-sso-failed-{name}")
|
||||
return ""
|
||||
log.info(f"[{name}] Redmine SSO login OK")
|
||||
|
||||
# --- API key ---
|
||||
page.goto(f"{url}/my/account", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
_debug_dump(page, f"redmine-account-{name}")
|
||||
|
||||
# The API key is in the right sidebar under "API access key"
|
||||
show_link = page.query_selector('a:has-text("Show"), #api_access_key + a, a[href*="access_key"]')
|
||||
if show_link:
|
||||
show_link.click()
|
||||
page.wait_for_timeout(1000)
|
||||
api_key = ""
|
||||
|
||||
key_el = page.query_selector('#api_access_key, .api-key code, .api-access-key')
|
||||
api_key = key_el.text_content().strip() if key_el else ""
|
||||
# A key usually exists (auto-created); reveal it via "Show"
|
||||
show_btn = page.locator('.api-key-actions a:has-text("Show"), a:has-text("Show")')
|
||||
if show_btn.count() > 0 and show_btn.first.is_visible():
|
||||
show_btn.first.click()
|
||||
page.wait_for_timeout(2000)
|
||||
api_el = page.query_selector("#api-access-key")
|
||||
if api_el:
|
||||
matches = re.findall(r"[a-f0-9]{40}", api_el.text_content())
|
||||
if matches:
|
||||
api_key = matches[0]
|
||||
|
||||
# No key yet: generate via the Reset link next to #api-access-key
|
||||
if not api_key:
|
||||
reset_link = page.query_selector('a:has-text("Reset"), a:has-text("Generate")')
|
||||
if reset_link:
|
||||
reset_link.click()
|
||||
page.wait_for_timeout(2000)
|
||||
page.click('button:has-text("OK"), button:has-text("Confirm")')
|
||||
page.wait_for_timeout(1000)
|
||||
key_el = page.query_selector('#api_access_key, .api-key code')
|
||||
api_key = key_el.text_content().strip() if key_el else ""
|
||||
reset_clicked = page.evaluate("""() => {
|
||||
const apiSection = document.querySelector('#api-access-key');
|
||||
if (!apiSection) return false;
|
||||
let container = apiSection.closest('div, p, fieldset');
|
||||
while (container && container.parentElement) {
|
||||
const reset = Array.from(container.querySelectorAll('a, button')).find(el =>
|
||||
el.textContent.trim().toLowerCase() === 'reset' && el.offsetParent !== null
|
||||
);
|
||||
if (reset) { reset.click(); return true; }
|
||||
container = container.parentElement;
|
||||
if (container.tagName === 'FIELDSET' || container.tagName === 'FORM') break;
|
||||
}
|
||||
return false;
|
||||
}""")
|
||||
if reset_clicked:
|
||||
page.wait_for_timeout(3000)
|
||||
confirm = page.locator(
|
||||
'button:has-text("OK"), button:has-text("Confirm"), button:has-text("Yes")'
|
||||
)
|
||||
if confirm.count() > 0 and confirm.first.is_visible():
|
||||
confirm.first.click()
|
||||
page.wait_for_timeout(3000)
|
||||
api_el = page.query_selector("#api-access-key")
|
||||
if api_el:
|
||||
matches = re.findall(r"[a-f0-9]{40}", api_el.text_content())
|
||||
if matches:
|
||||
api_key = matches[0]
|
||||
|
||||
if not api_key:
|
||||
log.error(f"[{name}] Could not get Redmine API key")
|
||||
@@ -836,10 +940,9 @@ def provision_redmine(page: Page, agent: dict, bw: BitwardenHelper) -> str:
|
||||
|
||||
log.info(f"[{name}] Redmine API key obtained: {api_key[:8]}...")
|
||||
|
||||
# Store in Bitwarden
|
||||
bw.create_item(
|
||||
name=item_name,
|
||||
username=name,
|
||||
username=username,
|
||||
password=api_key,
|
||||
uris=[url],
|
||||
collection_name=name,
|
||||
|
||||
Reference in New Issue
Block a user