feat(awx): serve AWX over HTTPS via nginx TLS-termination proxy

Add playbooks/setup_awx_https.yml that deploys a host-level nginx
reverse proxy in front of the AWX LoadBalancer service, terminating
TLS on 443 and proxying to the existing HTTP service on 80.

Why a reverse proxy and not in-pod TLS: the AWX Operator only wires
nginx for HTTPS on the OpenShift Route passthrough code path
(ingress_type: route + route_tls_termination_mechanism: passthrough),
which requires the Route CRD and fails on k3s. The cluster has no
ingress controller either. A host nginx proxy is the lowest-risk
option and is trivially swappable when the internal CA / an ingress
controller + cert-manager arrive.

The self-signed cert (CN=tsys-awx.knel.net) carries SANs for the
FQDN, short hostname, and both LAN and Tailscale IPs. Re-run the
playbook to rotate the cert once the internal CA is rolled out.

Also update scripts/awx_create_job_templates.py to use https by
default and add AWX_VERIFY_TLS / AWX_CA_BUNDLE env vars so it works
with the self-signed cert now and verifies properly once the internal
CA bundle is distributed.

🤖 Generated with [Crush](https://github.com/charmassociates/crush)

Assisted-by: GLM-5 via Crush <crush@charm.land>

playbooks/setup_awx_https.yml

@@ -0,0 +1,132 @@
+---
+# Put AWX behind HTTPS with a host-level nginx TLS-termination reverse proxy.
+#
+# Why a reverse proxy instead of in-pod TLS:
+#   The AWX Operator only wires nginx for HTTPS on the OpenShift Route
+#   "passthrough" code path (ingress_type: route + route_tls_termination_mechanism:
+#   passthrough), which requires the Route CRD and breaks on k3s. There is also no
+#   ingress controller on the cluster. Terminating TLS at a host nginx proxy in
+#   front of the existing LoadBalancer service is the lowest-risk option and is
+#   trivially swappable once an internal CA / ingress controller + cert-manager
+#   arrive.
+#
+# Topology after this playbook:
+#   client --https:443--> host nginx (TLS terminate) --http:80--> k3s ServiceLB
+#                                                                   |
+#                                                                   v
+#                                            tsys-awx-service (LB) -> awx-web:8052
+#
+# The cert is self-signed (CN=tsys-awx.knel.net) with SANs covering the FQDN,
+# the short hostname, and both the LAN and Tailscale IPs. Re-run this playbook
+# after rotating the cert (e.g. when the internal CA is rolled out).
+
+- name: AWX HTTPS reverse proxy
+  hosts: tsys-awx
+  gather_facts: true
+  become: true
+
+  vars:
+    # Cert subject / SANs. Extend these lists as more names are needed.
+    awx_tls_cn: tsys-awx.knel.net
+    awx_tls_sans:
+      - "DNS:tsys-awx.knel.net"
+      - "DNS:tsys-awx"
+      - "IP:192.168.3.200"
+      - "IP:100.91.39.53"
+    awx_tls_dir: /etc/nginx/ssl
+    awx_tls_cert: "{{ awx_tls_dir }}/awx-self-signed.crt"
+    awx_tls_key: "{{ awx_tls_dir }}/awx-self-signed.key"
+
+  tasks:
+    - name: Install nginx
+      ansible.builtin.apt:
+        name: nginx
+        state: present
+        update_cache: true
+
+    - name: Ensure TLS cert directory exists
+      ansible.builtin.file:
+        path: "{{ awx_tls_dir }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Generate self-signed certificate (only if missing)  # noqa no-changed-when
+      ansible.builtin.shell: |
+        set -o pipefail
+        openssl req -x509 -newkey rsa:2048 -nodes -days 825 \
+          -keyout {{ awx_tls_key }} -out {{ awx_tls_cert }} \
+          -subj "/CN={{ awx_tls_cn }}" \
+          -addext "subjectAltName={{ awx_tls_sans | join(',') }}"
+      args:
+        creates: "{{ awx_tls_cert }}"
+
+    - name: Lock down private key permissions
+      ansible.builtin.file:
+        path: "{{ awx_tls_key }}"
+        owner: root
+        group: root
+        mode: "0600"
+
+    - name: Deploy nginx TLS proxy site config
+      ansible.builtin.copy:
+        dest: /etc/nginx/sites-available/awx-tls.conf
+        owner: root
+        group: root
+        mode: "0644"
+        content: |
+          # Ansible-managed: TLS termination for AWX.
+          # Proxies https://tsys-awx.knel.net -> http://127.0.0.1:80 (k3s ServiceLB -> AWX).
+          # map must live in the http{} context; sites-enabled/* are included inside http.
+          map $http_upgrade $awx_connection_upgrade {
+              default upgrade;
+              ''      close;
+          }
+
+          server {
+              listen 443 ssl;
+              listen [::]:443 ssl;
+              http2 on;
+
+              server_name tsys-awx.knel.net tsys-awx 192.168.3.200 100.91.39.53;
+
+              ssl_certificate     /etc/nginx/ssl/awx-self-signed.crt;
+              ssl_certificate_key /etc/nginx/ssl/awx-self-signed.key;
+              ssl_protocols TLSv1.2 TLSv1.3;
+              ssl_ciphers HIGH:!aNULL:!MD5;
+              ssl_session_cache shared:SSL:10m;
+              ssl_session_timeout 1d;
+
+              client_max_body_size 50M;
+
+              location / {
+                  proxy_pass http://127.0.0.1:80;
+                  proxy_http_version 1.1;
+                  proxy_set_header Host $http_host;
+                  proxy_set_header X-Real-IP $remote_addr;
+                  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                  proxy_set_header X-Forwarded-Proto https;
+                  proxy_set_header Upgrade $http_upgrade;
+                  proxy_set_header Connection $awx_connection_upgrade;
+                  proxy_read_timeout 300s;
+                  proxy_redirect off;
+              }
+          }
+      notify: Reload nginx
+
+    - name: Enable the AWX TLS site
+      ansible.builtin.file:
+        src: /etc/nginx/sites-available/awx-tls.conf
+        dest: /etc/nginx/sites-enabled/awx-tls.conf
+        state: link
+
+    - name: Test nginx configuration  # noqa no-changed-when
+      ansible.builtin.command: nginx -t
+
+  handlers:
+    - name: Reload nginx
+      ansible.builtin.systemd:
+        name: nginx
+        state: reloaded
+        enabled: true

scripts/awx_create_job_templates.py

@@ -14,17 +14,21 @@ Usage:
   python3 awx_create_job_templates.py
 
 Configuration (environment variables):
-  AWX_URL       AWX API base URL (default: http://100.91.39.53/api/v2)
+  AWX_URL       AWX API base URL (default: https://tsys-awx.knel.net/api/v2)
   AWX_USER      AWX admin username   (default: admin)
   AWX_PASSWORD  AWX admin password   (REQUIRED - never committed to the repo)
   AWX_PROJECT_ID, AWX_INVENTORY_ID, AWX_SCM_CRED_ID,
   AWX_MACHINE_CRED_ID, AWX_ORG_ID  resource IDs (sensible defaults below)
+  AWX_VERIFY_TLS  "false" to skip TLS verification (for self-signed certs).
+                  Default: verify against the system trust store.
+  AWX_CA_BUNDLE   Path to a CA bundle file (e.g. internal CA) to verify against.
 """
 
 import os
+import ssl
 import urllib.request, urllib.parse, json, base64, time, sys
 
-API = os.environ.get("AWX_URL", "http://100.91.39.53/api/v2")
+API = os.environ.get("AWX_URL", "https://tsys-awx.knel.net/api/v2")
 _AWX_USER = os.environ.get("AWX_USER", "admin")
 _AWX_PASS = os.environ.get("AWX_PASSWORD")
 if not _AWX_PASS:
@@ -37,6 +41,16 @@ SCM_CRED_ID = int(os.environ.get("AWX_SCM_CRED_ID", "3"))
 MACHINE_CRED_ID = int(os.environ.get("AWX_MACHINE_CRED_ID", "4"))
 ORG_ID = int(os.environ.get("AWX_ORG_ID", "1"))
 
+# TLS verification context. With a self-signed cert (until the internal CA
+# bundle is distributed), set AWX_VERIFY_TLS=false. With an internal CA, point
+# AWX_CA_BUNDLE at it and leave verification enabled.
+if os.environ.get("AWX_VERIFY_TLS", "true").lower() == "false":
+    _SSL_CONTEXT = ssl._create_unverified_context()
+elif os.environ.get("AWX_CA_BUNDLE"):
+    _SSL_CONTEXT = ssl.create_default_context(cafile=os.environ["AWX_CA_BUNDLE"])
+else:
+    _SSL_CONTEXT = None  # use urllib's default (system trust store)
+
 def awx_req(method, endpoint, data=None):
     url = f"{API}/{endpoint}"
     body = json.dumps(data).encode() if data else None
@@ -44,7 +58,7 @@ def awx_req(method, endpoint, data=None):
     req.add_header("Authorization", f"Basic {AUTH}")
     req.add_header("Content-Type", "application/json")
     try:
-        with urllib.request.urlopen(req, timeout=30) as resp:
+        with urllib.request.urlopen(req, timeout=30, context=_SSL_CONTEXT) as resp:
             raw = resp.read()
             return json.loads(raw) if raw else {}
     except urllib.error.HTTPError as e:
@@ -157,8 +171,10 @@ print("\n" + "=" * 60)
 print("ALL DONE - AWX is ready!")
 print("=" * 60)
 print("""
-AWX URL: http://tsys-awx.knel.net (or http://100.91.39.53)
+AWX URL: https://tsys-awx.knel.net (or https://100.91.39.53)
 Login:   $AWX_USER (password supplied via $AWX_PASSWORD env var)
+Note:    Self-signed cert for now — set AWX_VERIFY_TLS=false until the
+         internal CA bundle is distributed (then use AWX_CA_BUNDLE).
 
 Resources created:
   Credentials: