From 6e4f794b03b4f4124f9103e3d71940e872c55957 Mon Sep 17 00:00:00 2001
From: reachableceo <charles@turnsys.com>
Date: Sat, 5 Sep 2026 09:35:25 -0500
Subject: [PATCH] [#800] prod step-ca ACME endpoint LIVE on tsys-ca (fleet-root
 chain); http-01 validation fetch open
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

compose.yaml: host networking (bridge+embedded-DNS broke rooted-name
lookups), :8443 via ca.json address, image digest-pinned; init-stepca.sh:
scaffold + ECDSA-P256 intermediate signed by /root/ca-root fleet root
(verified), ACME provisioner, managed password. Directory serving
RFC8555 on tailnet. OPEN: step-ca validation GET fails instantly with
zero dial attempts (authz says could-not-connect) — bisected caps,
read_only, bridge vs host, container DNS. Debug continues next run
(GODEBUG=netdns=2 + strace plan).
https://projects.knownelement.com/issues/800
---
 .crush/active-ticket              |  2 +-
 docker/stepca-prod/compose.yaml   | 26 ++++++++++
 docker/stepca-prod/init-stepca.sh | 83 +++++++++++++++++++++++++++++++
 3 files changed, 110 insertions(+), 1 deletion(-)
 create mode 100644 docker/stepca-prod/compose.yaml
 create mode 100755 docker/stepca-prod/init-stepca.sh

diff --git a/.crush/active-ticket b/.crush/active-ticket
index f334266..f8c1dbb 100644
--- a/.crush/active-ticket
+++ b/.crush/active-ticket
@@ -1 +1 @@
-#697
+#800
diff --git a/docker/stepca-prod/compose.yaml b/docker/stepca-prod/compose.yaml
new file mode 100644
index 0000000..5fe0a50
--- /dev/null
+++ b/docker/stepca-prod/compose.yaml
@@ -0,0 +1,26 @@
+# step-ca PROD ACME endpoint on tsys-ca [#800 #697]
+#
+# Bring-your-own-chain: intermediate signed by the fleet root (offline
+# /root/ca-root), served on the TAILNET only. HTTP-01 challenges do NOT hit
+# this box — ACME clients serve them on the target host's port 80.
+#
+# Files expected next to this compose (created by init-stepca.sh):
+#   data/          step home (config, certs, secrets; uid 1000)
+# Init (first deploy only): bash init-stepca.sh   (run on tsys-ca as root)
+# Verify:  curl -k https://100.102.96.24:8443/acme/acme/directory
+#
+# PROVISIONER: ACME provisioner named "acme" is added by init-stepca.sh
+# (step ca provisioner add acme --type ACME) after scaffolding.
+
+services:
+  stepca:
+    image: smallstep/step-ca@sha256:e9e8fa3262bf37b130962ffddbf6a64ac188f0bbb80959cf3ddc04c6bf294c3d
+    container_name: ukrrs-oam-ca-stepca
+    restart: unless-stopped
+    # step-ca validates http-01 by dialing the ROOTED target name; docker's
+    # embedded DNS on tsys-ca cannot answer absolute tailnet FQDNs, so pin
+    # the tailscale resolver (MagicDNS) explicitly.
+    network_mode: host
+    volumes:
+      - ./data:/home/step
+    # hardening (read_only+caps) re-enabled after validation debugging
diff --git a/docker/stepca-prod/init-stepca.sh b/docker/stepca-prod/init-stepca.sh
new file mode 100755
index 0000000..069a6e7
--- /dev/null
+++ b/docker/stepca-prod/init-stepca.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+# init-stepca.sh — provision the PROD step-ca ACME endpoint on tsys-ca [#800]
+#
+# Run ON tsys-ca as root, from /opt/stepca-prod (this script + compose.yaml
+# live there; data/ is created). Refuses to touch an existing data/ unless
+# --reinit.
+#
+# What it does:
+#   1. scaffolds a step home (docker one-shot init; throwaway self-signed CA)
+#   2. generates an ECDSA P-256 ACME intermediate and signs it with the
+#      FLEET ROOT (/root/ca-root) — same constraints as ca-init.sh
+#   3. swaps in the fleet chain (root cert + intermediate), adds an ACME
+#      provisioner, writes the runtime password
+# The root KEY never leaves /root/ca-root. The ACME intermediate key and the
+# runtime password live only in /opt/stepca-prod (0700, root-owned).
+set -euo pipefail
+
+BASE="/opt/stepca-prod"
+DATA="$BASE/data"
+ROOT_CRT="/root/ca-root/root.crt"
+ROOT_KEY="/root/ca-root/root.key"
+IMAGE="smallstep/step-ca@sha256:e9e8fa3262bf37b130962ffddbf6a64ac188f0bbb80959cf3ddc04c6bf294c3d"
+TAILNET_IP="$(tailscale ip -4)"
+
+[ -f "$ROOT_CRT" ] && [ -f "$ROOT_KEY" ] || { echo "FAIL: fleet root not found at $ROOT_CRT/$ROOT_KEY" >&2; exit 1; }
+[ -n "$TAILNET_IP" ] || { echo "FAIL: no tailscale IP" >&2; exit 1; }
+if [ -d "$DATA" ] && [ "${1:-}" != "--reinit" ]; then
+  echo "FAIL: $DATA exists (pass --reinit to wipe and redo)" >&2; exit 1
+fi
+
+mkdir -p "$BASE" "$BASE/secrets"
+chmod 700 "$BASE"; chmod 711 "$BASE/secrets"
+rm -rf "$DATA"; mkdir -p "$DATA/certs" "$DATA/secrets" "$DATA/config" "$DATA/db"
+# scaffold + runtime run as container uid 1000 (step) — tree must be writable by it
+chown -R 1000:1000 "$DATA"
+
+# runtime + provisioner password (generated once; container uid 1000 must read it)
+if [ ! -s "$BASE/secrets/password" ]; then
+  openssl rand -base64 18 > "$BASE/secrets/password"
+fi
+chown 1000:1000 "$BASE/secrets/password"
+chmod 600 "$BASE/secrets/password"
+
+# 1. scaffold (throwaway self-signed CA; replaced below)
+docker run --rm -v "$DATA:/home/step" -v "$BASE/secrets:/secrets:ro" \
+  --entrypoint /bin/sh "$IMAGE" -c '
+  step ca init --name "KNEL Fleet CA" \
+    --dns "localhost" --dns "tsys-ca.knel.net" --dns "'"$TAILNET_IP"'" \
+    --address ":9000" --provisioner "admin" \
+    --password-file /secrets/password --provisioner-password-file /secrets/password \
+    --deployment-type standalone >/dev/null'
+echo "scaffold done"
+
+# 2. ACME intermediate signed by the fleet root (ECDSA P-256, pathlen 0)
+openssl ecparam -name prime256v1 -genkey -noout -out /tmp/acme-int.key
+chmod 400 /tmp/acme-int.key
+openssl req -new -key /tmp/acme-int.key -out /tmp/acme-int.csr -sha256 \
+  -subj "/C=US/ST=Texas/O=Known Element Enterprises/OU=TechOps/CN=KNEL Fleet ACME Intermediate"
+openssl x509 -req -in /tmp/acme-int.csr -CA "$ROOT_CRT" -CAkey "$ROOT_KEY" \
+  -CAcreateserial -days 1825 -sha256 -out /tmp/acme-int.crt \
+  -extfile <(printf 'basicConstraints=critical,CA:TRUE,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always')
+openssl verify -CAfile "$ROOT_CRT" /tmp/acme-int.crt
+rm -f /tmp/acme-int.csr
+
+# 3. swap in the fleet chain (root KEY stays offline in /root/ca-root)
+cp "$ROOT_CRT" "$DATA/certs/root_ca.crt"
+cp /tmp/acme-int.crt "$DATA/certs/intermediate_ca.crt"
+mv /tmp/acme-int.key "$DATA/secrets/intermediate_ca_key"
+rm -f "$DATA/secrets/root_ca_key"
+
+# 4. ACME provisioner (JWK password re-read from the same file)
+docker run --rm -v "$DATA:/home/step" -v "$BASE/secrets:/secrets:ro" \
+  --entrypoint /bin/sh "$IMAGE" -c '
+  step ca provisioner add acme --type ACME --password-file /secrets/password >/dev/null'
+
+# 5. runtime password where the container entrypoint expects it
+cp "$BASE/secrets/password" "$DATA/secrets/password"
+
+# ownership to uid 1000 (step)
+chown -R 1000:1000 "$DATA"
+chmod 400 "$DATA/secrets/intermediate_ca_key"
+
+echo "init complete: $DATA ready — bring up with: cd $BASE && docker compose -f compose.yaml up -d"