chore: initial import from KNEL/PFVCluster@041d311 [#769]

Split per O&M lane work order. Full history: KNEL/PFVCluster.
https://projects.knownelement.com/issues/769#note-4152

dns-cluster-setup/README.md

@@ -0,0 +1,10 @@
+# netinfra/dns-cluster-setup/README.md
+
+> **Documentation moved to Discourse — the canonical source of truth.**
+>
+> **Technitium DNS cluster setup**
+>
+> **Read it here:** https://community.turnsys.com/t/306
+>
+> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
+> Do not update content here — edit the Discourse wiki topic instead.*

dns-cluster-setup/remote-dns.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/bash
+#
+# remote-dns.sh
+#
+# Single chokepoint for ALL ssh/scp access to the DNS infrastructure hosts.
+# Every other script in dns-cluster-setup/ MUST route through this wrapper.
+# Never call ssh/scp directly.
+#
+# WHY: one place to configure host aliases/users/keys, one place to audit,
+# and the command scanner only permits ssh when invoked indirectly via a
+# script. Mirrors the pattern of tests/remote.sh.
+#
+# HOSTS (override IPs via env if needed):
+#   netinfra01     pfv-netinfra-01.knel.net   (Technitium primary target)
+#   netinfra02     pfv-netinfra-02.knel.net   (Technitium secondary target)
+#   netboot        pfv-netboot.knel.net       (reference / validation client)
+#   sandbox        sectestbed-sandbox.knel.net (validation client)
+#
+# NOTE: the former tsrouter alias (tailscale-router.knel.net) was removed
+# 2026-09-02 — that host is retired; subnet routing is now the netinfra pair.
+#
+# All hosts are accessed as $VM_USER (default: localuser) over SSH with key auth
+# and passwordless sudo.
+#
+# USAGE:
+#   remote-dns.sh <host-alias> <cmd...>          run command on host
+#   remote-dns.sh <host-alias>-root <cmd...>     run command on host as root (sudo)
+#   remote-dns.sh <host-alias>-file <script>     run a local script file on host (bash -s)
+#   remote-dns.sh <host-alias>-copy <local> <remote-dest>   copy a file to host
+#
+#   e.g.
+#   remote-dns.sh netinfra01-root 'systemctl status dnsServer'
+#   remote-dns.sh netboot-file ./probe.sh
+#
+set -uo pipefail
+
+VM_USER="${VM_USER:-localuser}"
+# Hostname -> FQDN map. Override individual IPs via env if a host moves.
+NETINFRA01_HOST="${NETINFRA01_HOST:-pfv-netinfra-01.knel.net}"
+NETINFRA02_HOST="${NETINFRA02_HOST:-pfv-netinfra-02.knel.net}"
+NETBOOT_HOST="${NETBOOT_HOST:-pfv-netboot.knel.net}"
+SANDBOX_HOST="${SANDBOX_HOST:-sectestbed-sandbox.knel.net}"
+
+SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
+
+die() { echo "remote-dns.sh: $*" >&2; exit 1; }
+
+host_fqdn() {
+  case "$1" in
+    netinfra01)  printf '%s' "$NETINFRA01_HOST" ;;
+    netinfra02)  printf '%s' "$NETINFRA02_HOST" ;;
+    netboot)     printf '%s' "$NETBOOT_HOST" ;;
+    sandbox)     printf '%s' "$SANDBOX_HOST" ;;
+    *) return 1 ;;
+  esac
+}
+
+_run() { ssh "${SSH_OPTS[@]}" "${VM_USER}@$1" "$2"; }
+_run_root() { ssh "${SSH_OPTS[@]}" "${VM_USER}@$1" "sudo -n bash -c $(printf '%q' "$2")"; }
+_run_file() { ssh "${SSH_OPTS[@]}" "${VM_USER}@$1" "bash -s" < "$2"; }
+_copy() {
+  local fqdn="$1" local="$2" dest="$3"
+  if command -v rsync >/dev/null 2>&1 \
+     && ssh "${SSH_OPTS[@]}" "${VM_USER}@${fqdn}" 'command -v rsync' >/dev/null 2>&1; then
+    rsync -az -e "ssh ${SSH_OPTS[*]}" "$local" "${VM_USER}@${fqdn}:${dest}"
+  else
+    ssh "${SSH_OPTS[@]}" "${VM_USER}@${fqdn}" "cat > '$dest'" < "$local"
+  fi
+}
+
+spec="${1:-}"; shift || true
+# Split host alias from mode: "netinfra01", "netinfra01-root", "netinfra01-file", "netinfra01-copy"
+mode="run"
+alias="$spec"
+case "$spec" in
+  *-root) mode="root"; alias="${spec%-root}" ;;
+  *-file) mode="file"; alias="${spec%-file}" ;;
+  *-copy) mode="copy"; alias="${spec%-copy}" ;;
+esac
+
+fqdn="$(host_fqdn "$alias")" || die "unknown host alias '$alias' (try: netinfra01|netinfra02|netboot|sandbox)"
+
+case "$mode" in
+  run)  _run "$fqdn" "$*" ;;
+  root) [ "$#" -ge 1 ] || die "need command"; _run_root "$fqdn" "$*" ;;
+  file) [ -f "${1:-}" ] || die "need local script file"; _run_file "$fqdn" "$1" ;;
+  copy) [ -f "${1:-}" ] || die "need local file"; _copy "$fqdn" "$1" "${2:-}" ;;
+  *) die "bad mode" ;;
+esac

dns-cluster-setup/setup.sh

@@ -0,0 +1,477 @@
+#!/usr/bin/bash
+#
+# setup.sh — Technitium DNS Cluster Setup
+#
+# BOOTSTRAP (historical): originally exported the production config from the
+# retired tailscale-router host to the pfv-netinfra-01/02 pair. Since
+# 2026-09-02 (router retired) the export source is pfv-netinfra-01 itself,
+# the live primary and SoR for knel.net zones.
+#
+# Configures 01 as primary and 02 as secondary with automatic zone
+# transfers (AXFR).
+#
+# PRODUCTION SAFETY: the export source is accessed READ-ONLY. No file on it
+# is modified. The only operation is a docker cp (read) to export the config.
+#
+# ARCHITECTURE AFTER SETUP:
+#
+#   pfv-netinfra-01 (192.168.3.252) — PRIMARY
+#     Pi-hole (:53) → Technitium (:5300 inside container)
+#     All zones are Primary; zone transfer allowed from 02
+#
+#   pfv-netinfra-02 (192.168.3.253) — SECONDARY
+#     Pi-hole (:53) → Technitium (:5300 inside container)
+#     All zones are Secondary; AXFR from 01 on changes
+#
+# CLUSTERING MECHANISM:
+#   Technitium primary/secondary via DNS zone transfers (AXFR/IXFR + NOTIFY).
+#   01 serves all zones as Primary. 02 fetches them as Secondary from
+#   01's address (192.168.3.252:5300). When a record changes on 01, it sends
+#   a DNS NOTIFY to 02, which immediately pulls the update via IXFR.
+#
+# CREDENTIALS:
+#   The production auth.config (users + 2FA) is copied to both targets, so
+#   the existing admin username, password, and 2FA device work identically on
+#   all three servers.
+#
+# USAGE:
+#   ./setup.sh export     # Step 1: read-only export from primary (netinfra-01)
+#   ./setup.sh deploy01   # Step 2: deploy config to netinfra-01 (primary)
+#   ./setup.sh deploy02   # Step 3: deploy config to netinfra-02 (secondary)
+#   ./setup.sh cluster    # Step 4: configure clustering (01 primary, 02 secondary)
+#   ./setup.sh verify     # Step 5: test everything
+#   ./setup.sh all        # Steps 1-5 in sequence
+#
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REMOTE="$HERE/remote-dns.sh"
+
+# Host aliases (defined in remote-dns.sh)
+PROD="$PRIMARY"        # export source: primary (was tailscale-router, retired 2026-09-02)
+PRIMARY="netinfra01"   # pfv-netinfra-01
+SECONDARY="netinfra02" # pfv-netinfra-02
+
+# Network addresses for zone transfer
+PRIMARY_IP="${PRIMARY_IP:-192.168.3.252}"
+SECONDARY_IP="${SECONDARY_IP:-192.168.3.253}"
+# Technitium DNS port on the host (from docker-compose port mapping)
+TECH_PORT="${TECH_PORT:-5300}"
+
+# Config directory on the netinfra hosts (bind mount target)
+CONFIG_DIR="${CONFIG_DIR:-/home/localuser/services/technitium/config}"
+COMPOSE_FILE="${COMPOSE_FILE:-/home/localuser/services/technitium/docker-compose.yml}"
+
+# Temporary admin password used ONLY during clustering API calls.
+# After configuration, the production auth.config (with 2FA) is restored.
+TEMP_ADMIN_PW="${TEMP_ADMIN_PW:-KnelCluster2026}"
+
+# Local working directory for exports
+WORK_DIR="$HERE/.export"
+mkdir -p "$WORK_DIR"
+
+# Files/dirs to EXCLUDE from the config copy (runtime data, not configuration)
+EXCLUDE_PATTERNS=(cache.bin stats logs)
+
+log()  { printf '\033[0;36m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*"; }
+die()  { log "ERROR: $*"; exit 1; }
+
+# -----------------------------------------------------------------------------
+# Helpers
+# -----------------------------------------------------------------------------
+
+# Build an exclude-args string for tar
+exclude_args() {
+  local args=""
+  for p in "${EXCLUDE_PATTERNS[@]}"; do
+    args+=" --exclude=$p"
+  done
+  printf '%s' "$args"
+}
+
+# Run a command on a host as root via the wrapper
+run_root() { bash "$REMOTE" "$1-root" "${@:2}"; }
+run()      { bash "$REMOTE" "$1" "${@:2}"; }
+
+# Get a Technitium API token on a host (temporary admin, no 2FA)
+# Uses root to avoid PATH issues with non-interactive SSH sessions.
+# Usage: get_token <host-alias>
+get_token() {
+  local host="$1"
+  local resp
+  resp=$(run_root "$host" "curl -sk --max-time 10 -X POST http://127.0.0.1:5380/api/user/login -d 'user=admin&pass=${TEMP_ADMIN_PW}'" 2>/dev/null || true)
+  local token
+  token=$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || true)
+  printf '%s' "$token"
+}
+
+# API call helper (uses root for reliable curl access)
+# Usage: api_call <host> <token> <endpoint> [param=value ...]
+api_call() {
+  local host="$1" token="$2" endpoint="$3"; shift 3
+  local url="http://127.0.0.1:5380/api/${endpoint}?token=${token}"
+  local p
+  for p in "$@"; do url+="&${p}"; done
+  run_root "$host" "curl -sk --max-time 10 '$url'" 2>/dev/null || true
+}
+
+# -----------------------------------------------------------------------------
+# Step 1: Export production config (READ-ONLY on the primary; historically
+# the retired tailscale-router)
+# -----------------------------------------------------------------------------
+do_export() {
+  log "=== STEP 1: Exporting production config from $PROD (READ-ONLY) ==="
+
+  local export_tar="$WORK_DIR/technitium-production-config.tar.gz"
+
+  log "Exporting config volume from $PROD (piped, no disk writes on prod)..."
+  # Read the Docker volume directory directly from the host filesystem.
+  # No docker exec needed (avoids /tmp space issues on the prod host).
+  # Pipe tar → ssh → local file. Nothing is written on production's disk.
+  local vol_path
+  vol_path=$(bash "$REMOTE" "$PROD-root" \
+    "docker volume inspect -f '{{.Mountpoint}}' dns_tsys-dns-config 2>/dev/null" \
+    | tr -d '[:space:]')
+  [ -n "$vol_path" ] || die "Could not find Docker volume path on $PROD."
+  log "Volume path: $vol_path"
+
+  bash "$REMOTE" "$PROD-root" \
+    "tar czf - -C '$vol_path' --exclude=cache.bin --exclude=stats --exclude=logs ." \
+    > "$export_tar" 2>/dev/null || die "Export pipe failed."
+
+  [ -s "$export_tar" ] || die "Export tarball is empty."
+
+  # Inspect
+  local zone_count
+  zone_count=$(tar tzf "$export_tar" | grep -c '\.zone$' || true)
+  log "Export complete: $(du -h "$export_tar" | cut -f1), $zone_count zones."
+
+  # Save the zone name list for clustering
+  tar tzf "$export_tar" | grep '\.zone$' | sed 's|^\./||; s|^zones/||; s|\.zone$||' | sort > "$WORK_DIR/zones.txt"
+  log "Zone list saved ($zone_count zones): $(head -5 "$WORK_DIR/zones.txt" | tr '\n' ' ')..."
+}
+
+# -----------------------------------------------------------------------------
+# Step 2: Deploy to netinfra-01 (PRIMARY)
+# -----------------------------------------------------------------------------
+do_deploy_primary() {
+  log "=== STEP 2: Deploying PRIMARY to $PRIMARY ==="
+  _deploy "$PRIMARY" "primary"
+}
+
+# -----------------------------------------------------------------------------
+# Step 3: Deploy to netinfra-02 (SECONDARY — initial clone, clustering in step 4)
+# -----------------------------------------------------------------------------
+do_deploy_secondary() {
+  log "=== STEP 3: Deploying SECONDARY to $SECONDARY ==="
+  _deploy "$SECONDARY" "secondary"
+}
+
+# Shared deploy logic
+# Usage: _deploy <host-alias> <role>
+_deploy() {
+  local host="$1" role="$2"
+  local export_tar="$WORK_DIR/technitium-production-config.tar.gz"
+  [ -f "$export_tar" ] || die "No export found. Run '$0 export' first."
+
+  log "Stopping Technitium on $host..."
+  run_root "$host" "cd $CONFIG_DIR/.. && docker compose down" 2>/dev/null \
+    || run_root "$host" "docker stop tsys-dns" 2>/dev/null || true
+
+  log "Backing up existing config on $host..."
+  run_root "$host" "
+    if [ -d '$CONFIG_DIR' ]; then
+      mv '$CONFIG_DIR' '${CONFIG_DIR}.backup-$(date +%Y%m%d-%H%M%S)'
+    fi
+    mkdir -p '$CONFIG_DIR'
+  " || die "Backup failed."
+
+  log "Uploading production config to $host..."
+  bash "$REMOTE" "$host-root" "cat > /tmp/technitium-config.tar.gz" < "$export_tar" \
+    || die "Upload failed."
+
+  log "Extracting config on $host..."
+  run_root "$host" "
+    cd '$CONFIG_DIR'
+    tar xzf /tmp/technitium-config.tar.gz
+    rm -f /tmp/technitium-config.tar.gz
+    chown -R 1654:1654 '$CONFIG_DIR' 2>/dev/null || true
+    ls -la '$CONFIG_DIR/' | head -20
+  " || die "Extract failed."
+
+  # Update compose with production env vars
+  log "Updating docker-compose env on $host ($role)..."
+  run_root "$host" "
+    cat > /tmp/compose-patch.py << 'PYEOF'
+import re, sys
+f = sys.argv[1]
+with open(f) as fh: c = fh.read()
+# Ensure DNS_SERVER_DOMAIN and web service env vars are set
+if 'DNS_SERVER_DOMAIN' not in c:
+    c = re.sub(r'(image:.*\n)', r'\1    environment:\n      - DNS_SERVER_DOMAIN=knel.net\n', c, count=1)
+print(c)
+PYEOF
+    python3 /tmp/compose-patch.py '$COMPOSE_FILE' > '${COMPOSE_FILE}.new' 2>/dev/null && mv '${COMPOSE_FILE}.new' '$COMPOSE_FILE' || true
+    rm -f /tmp/compose-patch.py
+  " || log "WARN: compose patch skipped (non-critical)."
+
+  log "Starting Technitium on $host..."
+  run_root "$host" "cd $CONFIG_DIR/.. && docker compose up -d" 2>/dev/null \
+    || run_root "$host" "docker start tsys-dns" || die "Start failed."
+
+  log "Waiting for Technitium to come up on $host..."
+  local i
+  for i in $(seq 1 20); do
+    if run "$host" "curl -sk --max-time 3 http://127.0.0.1:5380/api/config/getVersion 2>/dev/null | head -c 50" 2>/dev/null | grep -qE 'token|error'; then
+      log "Technitium is up on $host (after ${i}s)."
+      return 0
+    fi
+    sleep 2
+  done
+  die "Technitium did not come up on $host within 40s."
+}
+
+# -----------------------------------------------------------------------------
+# Step 4: Configure clustering
+#
+# On PRIMARY (01): enable zone transfer for SECONDARY's IP on all zones.
+# On SECONDARY (02): replace all primary zones with secondary zones pointing
+#   to PRIMARY's address. Uses a temporary admin (no 2FA) for API access,
+#   then restores the production auth.config.
+# -----------------------------------------------------------------------------
+do_cluster() {
+  log "=== STEP 4: Configuring clustering ($PRIMARY → $SECONDARY) ==="
+
+  # --- 4a: On PRIMARY, enable zone transfer (for manual AXFR if needed) ---
+  log "4a: Enabling zone transfer on $PRIMARY..."
+  _with_temp_admin "$PRIMARY" "_cluster_enable_transfer"
+  log "Zone transfers enabled on primary."
+
+  # --- 4b: Install rsync-based zone replication on SECONDARY ---
+  log "4b: Installing rsync-based zone replication on $SECONDARY..."
+  _install_rsync_replication
+  log "Replication installed."
+}
+
+# Install rsync-based zone sync on the secondary as a systemd timer.
+_install_rsync_replication() {
+  local sync_script="$HERE/sync-zones.sh"
+  [ -f "$sync_script" ] || die "sync-zones.sh not found."
+
+  # Upload the sync script (copy to /tmp first, then move as root since
+  # the services dir may be root-owned from docker operations)
+  bash "$REMOTE" "$SECONDARY-copy" "$sync_script" "/tmp/sync-zones.sh" \
+    || die "Could not copy sync-zones.sh to /tmp."
+  run_root "$SECONDARY" "cp /tmp/sync-zones.sh /home/localuser/services/technitium/sync-zones.sh && chmod +x /home/localuser/services/technitium/sync-zones.sh && chown localuser:localuser /home/localuser/services/technitium/sync-zones.sh && rm /tmp/sync-zones.sh" \
+    || die "Could not install sync-zones.sh."
+
+  # Set up SSH key for rsync from secondary → primary (passwordless)
+  log "Setting up SSH key for rsync (secondary → primary)..."
+  run_root "$SECONDARY" "
+    if [ ! -f /home/localuser/.ssh/id_ed25519 ]; then
+      sudo -u localuser ssh-keygen -t ed25519 -N '' -f /home/localuser/.ssh/id_ed25519 -q
+    fi
+    cat /home/localuser/.ssh/id_ed25519.pub
+  " 2>/dev/null | grep -E 'ssh-ed25519' | while read -r pubkey; do
+    log "Adding secondary's SSH key to primary's authorized_keys..."
+    run_root "$PRIMARY" "mkdir -p /home/localuser/.ssh && echo '$pubkey' >> /home/localuser/.ssh/authorized_keys && chmod 600 /home/localuser/.ssh/authorized_keys" \
+      2>/dev/null || log "WARN: could not add key to primary"
+  done
+
+  # Install systemd timer for periodic sync
+  run_root "$SECONDARY" "
+    cat > /etc/systemd/system/technitium-zone-sync.service << 'SVCEOF'
+[Unit]
+Description=Technitium Zone Sync (primary → secondary)
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=localuser
+ExecStart=/home/localuser/services/technitium/sync-zones.sh
+SVCEOF
+
+    cat > /etc/systemd/system/technitium-zone-sync.timer << 'TMREOF'
+[Unit]
+Description=Run Technitium Zone Sync every minute
+
+[Timer]
+OnBootSec=30
+OnUnitActiveSec=60
+AccuracySec=10
+
+[Install]
+WantedBy=timers.target
+TMREOF
+
+    systemctl daemon-reload
+    systemctl enable --now technitium-zone-sync.timer
+    echo 'timer installed'
+  " 2>/dev/null || die "Could not install systemd timer."
+
+  # Trigger an immediate sync
+  log "Triggering initial sync..."
+  run_root "$SECONDARY" "sudo -u localuser /home/localuser/services/technitium/sync-zones.sh 2>&1" 2>/dev/null || true
+  sleep 3
+
+  # Check result
+  local zones
+  zones=$(run_root "$SECONDARY" "ls /home/localuser/services/technitium/config/zones/ 2>/dev/null | wc -l" 2>/dev/null | tr -d '[:space:]')
+  log "Secondary now has $zones zones."
+}
+
+# Enable zone transfer for the secondary IP on all primary zones.
+# Runs inside _with_temp_admin, so $1 = host.
+_cluster_enable_transfer() {
+  local host="$1"
+  local token; token="$(get_token "$host")"
+  [ -n "$token" ] || die "Cannot get API token on $host."
+
+  # Set global zone transfer allow list to include the secondary.
+  # Technitium per-zone "allow zone transfer" — use the API to set it.
+  local zone
+  while IFS= read -r zone <&3; do
+    [ -z "$zone" ] && continue
+    # Set zone transfer to AllowAnyone so the secondary can AXFR.
+    # Technitium API param: zoneTransfer (not allowZoneTransfer).
+    api_call "$host" "$token" "zones/options/set" \
+      "zone=$zone" "zoneTransfer=Allow" \
+      >/dev/null 2>&1 || true
+  done 3< "$WORK_DIR/zones.txt"
+  log "Zone transfer set to AllowAnyone for ${SECONDARY_IP} on all zones."
+}
+
+# Delete all primary zones and recreate as secondary zones.
+# Runs inside _with_temp_admin, so $1 = host.
+_cluster_make_secondary() {
+  local host="$1"
+  local token; token="$(get_token "$host")"
+  [ -n "$token" ] || die "Cannot get API token on $host."
+
+  local zone total
+  total=$(wc -l < "$WORK_DIR/zones.txt")
+  local n=0
+  # Use FD 3 so SSH (called by api_call/run_root) doesn't consume the loop's
+  # stdin (a classic bash pitfall: ssh inherits and reads from FD 0).
+  while IFS= read -r zone <&3; do
+    [ -z "$zone" ] && continue
+    n=$((n + 1))
+    # Delete the existing (primary) zone
+    api_call "$host" "$token" "zones/delete" "zone=$zone" >/dev/null 2>&1 || true
+    # Create as secondary zone pointing to primary
+    api_call "$host" "$token" "zones/create" \
+      "zone=$zone" "type=Secondary" "primaryServer=${PRIMARY_IP}%3A${TECH_PORT}" \
+      >/dev/null 2>&1 || true
+    [ $((n % 20)) -eq 0 ] && log "  ...converted $n/$total zones"
+  done 3< "$WORK_DIR/zones.txt"
+  log "Converted $n zones to secondary (AXFR from ${PRIMARY_IP}:${TECH_PORT})."
+
+  # Give Technitium a moment to AXFR
+  log "Waiting 10s for initial zone transfer..."
+  sleep 10
+}
+
+# Helper: temporarily replace auth.config with a fresh admin (no 2FA),
+# run a function, then restore the original auth.config.
+# Uses a docker-compose.override.yml (auto-merged by compose) so the original
+# compose file is never modified.
+# Usage: _with_temp_admin <host> <function_name>
+_with_temp_admin() {
+  local host="$1" func="$2"
+  log "Temporarily resetting admin on $host for API access (will restore after)..."
+
+  local svc_dir; svc_dir="$(dirname "$CONFIG_DIR")"
+
+  # Stop the container FIRST (otherwise it recreates auth.config from memory
+  # before we can delete it), then back up + delete auth.config, then create
+  # the override file, then restart.
+  log "Stopping Technitium on $host..."
+  run_root "$host" "cd '$svc_dir' && docker compose down 2>/dev/null || docker stop tsys-dns 2>/dev/null || true" \
+    || die "Could not stop Technitium on $host."
+
+  # Back up production auth.config, then remove it so Technitium creates a
+  # fresh admin on next start.
+  run_root "$host" "
+    cp '$CONFIG_DIR/auth.config' '$CONFIG_DIR/auth.config.production'
+    rm -f '$CONFIG_DIR/auth.config'
+  " || die "Could not back up/remove auth.config on $host."
+
+  # Create a compose override that injects the temp admin password.
+  run_root "$host" "
+    printf 'services:\\n  technitium:\\n    environment:\\n      - DNS_SERVER_ADMIN_PASSWORD=${TEMP_ADMIN_PW}\\n' \
+      > '$svc_dir/docker-compose.override.yml'
+  " || die "Could not create compose override on $host."
+
+  # Restart with override in effect
+  run_root "$host" "cd '$svc_dir' && docker compose up -d" \
+    2>/dev/null || die "Could not restart with temp admin on $host."
+
+  # Wait for API to come up (check with root to avoid PATH issues)
+  local i
+  for i in $(seq 1 20); do
+    if run_root "$host" "curl -sk --max-time 3 http://127.0.0.1:5380/api/config/getVersion 2>/dev/null" 2>/dev/null | grep -q .; then
+      log "Temp admin API is up on $host."
+      # Give the auth subsystem a few seconds to finish creating the admin user.
+      sleep 5
+      break
+    fi
+    sleep 2
+  done
+
+  # Debug: show what login returns
+  local login_resp
+  login_resp=$(run_root "$host" "curl -sk --max-time 10 -X POST http://127.0.0.1:5380/api/user/login -d 'user=admin&pass=${TEMP_ADMIN_PW}'" 2>/dev/null || true)
+  log "Login response: $(echo "$login_resp" | head -c 200)"
+
+  # Run the configuration function
+  "$func" "$host" || die "Configuration function $func failed on $host."
+
+  # Restore: production auth.config + remove override + restart
+  log "Restoring production auth.config (with 2FA) on $host..."
+  run_root "$host" "
+    cd '$svc_dir'
+    docker compose down 2>/dev/null || true
+    cp '$CONFIG_DIR/auth.config.production' '$CONFIG_DIR/auth.config'
+    rm -f '$CONFIG_DIR/auth.config.production'
+    chown 1654:1654 '$CONFIG_DIR/auth.config' 2>/dev/null || true
+    rm -f docker-compose.override.yml
+    docker compose up -d 2>/dev/null || true
+  " || die "Could not restore auth.config on $host."
+
+  sleep 3
+  log "Production auth restored on $host."
+}
+
+# -----------------------------------------------------------------------------
+# Step 5: Verify
+# -----------------------------------------------------------------------------
+do_verify() {
+  log "=== STEP 5: Verification ==="
+  bash "$HERE/verify.sh"
+}
+
+# -----------------------------------------------------------------------------
+# Dispatch
+# -----------------------------------------------------------------------------
+subcmd="${1:-}"
+case "$subcmd" in
+  export)    do_export ;;
+  deploy01)  do_deploy_primary ;;
+  deploy02)  do_deploy_secondary ;;
+  cluster)   do_cluster ;;
+  verify)    do_verify ;;
+  all)
+    do_export
+    do_deploy_primary
+    do_deploy_secondary
+    do_cluster
+    do_verify
+    ;;
+  ""|-h|--help|help)
+    sed -n '2,60p' "${BASH_SOURCE[0]}" >&2
+    exit 0
+    ;;
+  *) die "Unknown command '$subcmd'. Run '$0 help'." ;;
+esac
+
+log "=== DONE: $subcmd ==="

dns-cluster-setup/sync-zones.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/bash
+#
+# sync-zones.sh — rsync-based zone replication from primary to secondary
+#
+# Runs on the SECONDARY (netinfra-02). Syncs the zones/ directory from the
+# primary (netinfra-01) every 60 seconds. When rsync changes any zone file,
+# Technitium is reloaded (container restart) so it serves the fresh zones —
+# Technitium does NOT reliably auto-detect externally modified zone files,
+# which left the secondary serving stale records (seen 2026-09-01 [#344]).
+#
+# This is used instead of AXFR-based zone transfer because Technitium's zone
+# transfer mechanism uses port 53 (standard DNS), but on the netinfra hosts
+# port 53 is Pi-hole and Technitium is on port 5300. rsync-based replication
+# avoids the port conflict entirely. Native clustering tracked in [#469].
+#
+# Install as a systemd service/timer or run via cron:
+#   * * * * * /home/localuser/services/technitium/sync-zones.sh
+#
+set -uo pipefail
+
+PRIMARY_HOST="${PRIMARY_HOST:-pfv-netinfra-01.knel.net}"
+CONFIG_DIR="${CONFIG_DIR:-/home/localuser/services/technitium/config}"
+ZONE_DIR="$CONFIG_DIR/zones"
+LOCK_FILE="/tmp/technitium-zone-sync.lock"
+LOG_FILE="${LOG_FILE:-/home/localuser/services/technitium/sync.log}"
+# Reload command; array form so tests can substitute a fake.
+read -r -a RELOAD_CMD <<< "${RELOAD_CMD:-docker restart tsys-dns}"
+export RELOAD_CMD
+
+log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" >> "$LOG_FILE"; }
+
+# Sync zones from the primary and reload Technitium when anything changed.
+# Returns non-zero when rsync fails (reload failures are logged, not fatal).
+sync_and_reload() {
+  local changed rc zone_count changed_count
+  # --temp-dir avoids partial writes; --delete removes zones deleted on
+  # primary; --itemize-changes reports exactly what changed.
+  changed=$(rsync -az --delete --itemize-changes --out-format='%n' --temp-dir=/tmp \
+    "${PRIMARY_HOST}:$ZONE_DIR/" "$ZONE_DIR/" 2>> "$LOG_FILE")
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    return "$rc"
+  fi
+  zone_count=$(find "$ZONE_DIR" -maxdepth 1 -type f | wc -l)
+  if [ -n "$changed" ]; then
+    changed_count=$(printf '%s\n' "$changed" | wc -l)
+    log "Sync complete: $zone_count zones, $changed_count changed — reloading Technitium"
+    if "${RELOAD_CMD[@]}" >> "$LOG_FILE" 2>&1; then
+      log "Technitium reload triggered"
+    else
+      log "WARN: reload command failed (rc=$?) — records may be stale until next change"
+    fi
+  else
+    log "Sync complete: $zone_count zones (no changes)"
+  fi
+}
+
+main() {
+  # Prevent overlapping runs
+  exec 9>"$LOCK_FILE" || exit 0
+  flock -n 9 || { log "another sync is running; skipping"; exit 0; }
+
+  mkdir -p "$ZONE_DIR"
+
+  log "Syncing zones from $PRIMARY_HOST..."
+  sync_and_reload || { log "ERROR: rsync failed (rc=$?)"; exit 1; }
+}
+
+# Allow sourcing (unit tests) without executing the sync
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  main
+fi

dns-cluster-setup/verify.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/bash
+#
+# verify.sh — Comprehensive Technitium DNS Cluster Verification
+#
+# Tests that the primary/secondary DNS cluster is correctly configured and
+# functioning: zones present on both servers, zone transfers working, records
+# resolve identically, failover works, and credentials are replicated.
+#
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REMOTE="$HERE/remote-dns.sh"
+
+PRIMARY="netinfra01"
+SECONDARY="netinfra02"
+# The retired tailscale-router was the old PROD comparison target; the
+# primary itself is production since 2026-09-02.
+PROD="$PRIMARY"
+
+PRIMARY_IP="${PRIMARY_IP:-192.168.3.252}"
+SECONDARY_IP="${SECONDARY_IP:-192.168.3.253}"
+TECH_PORT="${TECH_PORT:-5300}"
+
+PASS=0; FAIL=0; WARN=0
+ok()   { echo "✅ $*"; PASS=$((PASS+1)); }
+fail() { echo "❌ $*"; FAIL=$((FAIL+1)); }
+warn() { echo "⚠️  $*"; WARN=$((WARN+1)); }
+section() { echo ""; echo "=== $* ==="; }
+
+run()      { bash "$REMOTE" "$1" "${@:2}"; }
+run_root() { bash "$REMOTE" "$1-root" "${@:2}"; }
+
+# =============================================================================
+section "1. Container health on both nodes"
+
+for h in "$PRIMARY" "$SECONDARY"; do
+  status=$(run_root "$h" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
+  if echo "$status" | grep -qi 'Up'; then
+    ok "Technitium container running on $h ($status)"
+  else
+    fail "Technitium container NOT running on $h (status: ${status:-none})"
+  fi
+done
+
+# =============================================================================
+section "2. Technitium API responds on both nodes"
+
+for h in "$PRIMARY" "$SECONDARY"; do
+  resp=$(run "$h" "curl -sk --max-time 5 http://127.0.0.1:5380/api/config/getVersion 2>/dev/null" || true)
+  if echo "$resp" | grep -qE 'token|error|invalid'; then
+    ok "API responds on $h"
+  else
+    fail "API not responding on $h"
+  fi
+done
+
+# =============================================================================
+section "3. Zone count matches between primary and production"
+
+# Count zones from the container on each host
+count_zones() {
+  local host="$1"
+  run_root "$host" "docker exec tsys-dns sh -c 'ls /etc/dns/zones/ 2>/dev/null | wc -l'" 2>/dev/null | tr -d '[:space:]'
+}
+
+prod_zones=$(count_zones "$PROD")
+pri_zones=$(count_zones "$PRIMARY")
+sec_zones=$(count_zones "$SECONDARY")
+
+echo "  Production zones: $prod_zones"
+echo "  Primary (01) zones: $pri_zones"
+echo "  Secondary (02) zones: $sec_zones"
+
+if [ "$prod_zones" -gt 0 ] 2>/dev/null; then ok "Production has $prod_zones zones"; else fail "Production zone count invalid"; fi
+if [ "$pri_zones" -gt 0 ] 2>/dev/null; then ok "Primary has $pri_zones zones"; else fail "Primary zone count invalid"; fi
+if [ "$sec_zones" -gt 0 ] 2>/dev/null; then ok "Secondary has $sec_zones zones"; else fail "Secondary zone count invalid"; fi
+
+if [ "$pri_zones" = "$prod_zones" ]; then
+  ok "Primary zone count matches production ($pri_zones)"
+else
+  warn "Primary zone count ($pri_zones) differs from production ($prod_zones)"
+fi
+
+if [ "$sec_zones" = "$pri_zones" ]; then
+  ok "Secondary zone count matches primary ($sec_zones)"
+else
+  warn "Secondary zone count ($sec_zones) differs from primary ($pri_zones) — may still be transferring"
+fi
+
+# =============================================================================
+section "4. knel.net zone resolves identically on primary and secondary"
+
+# Query a known record on both servers directly via Technitium's port
+for name in pfv-netinfra-01 pfv-netinfra-02 tsys-cloudron tsys-siem tsys-wazuh; do
+  fqdn="${name}.knel.net"
+  # Query via dig against each Technitium instance (through Pi-hole on :53)
+  pri_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)
+  sec_ans=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)
+
+  if [ -n "$pri_ans" ] && [ "$pri_ans" = "$sec_ans" ]; then
+    ok "$fqdn resolves identically: $pri_ans"
+  elif [ -n "$pri_ans" ] && [ -z "$sec_ans" ]; then
+    warn "$fqdn: primary=$pri_ans secondary=<no answer> (may still be syncing)"
+  elif [ -z "$pri_ans" ] && [ -z "$sec_ans" ]; then
+    warn "$fqdn: no answer on either server"
+  else
+    fail "$fqdn MISMATCH: primary=$pri_ans secondary=$sec_ans"
+  fi
+done
+
+# =============================================================================
+section "5. External DNS resolution works on both nodes"
+
+for h in "$PRIMARY" "$SECONDARY"; do
+  ans=$(run "$h" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 github.com A 2>/dev/null | head -1" 2>/dev/null || true)
+  if [ -n "$ans" ]; then
+    ok "$h resolves github.com → $ans"
+  else
+    fail "$h cannot resolve github.com"
+  fi
+done
+
+# =============================================================================
+section "6. Zone transfer (AXFR) from primary to secondary"
+
+# Test AXFR of knel.net from the primary
+axfr=$(run "$SECONDARY" "dig +short +time=5 +tries=1 @${PRIMARY_IP} -p ${TECH_PORT} knel.net AXFR 2>/dev/null | wc -l" 2>/dev/null || echo "0")
+if [ "$axfr" -gt 1 ] 2>/dev/null; then
+  ok "AXFR of knel.net from primary succeeds ($axfr records transferred)"
+else
+  warn "AXFR test returned $axfr records — zone transfer may be restricted or in progress"
+fi
+
+# =============================================================================
+section "7. Reverse DNS works"
+
+# Pick a known reverse zone and test PTR resolution
+ptr_test="181.103.100.in-addr.arpa"
+ptr_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
+if [ -n "$ptr_ans" ]; then
+  ok "Reverse zone $ptr_test has SOA on primary"
+else
+  warn "Reverse zone $ptr_test: no SOA on primary"
+fi
+
+ptr_ans2=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
+if [ -n "$ptr_ans2" ]; then
+  ok "Reverse zone $ptr_test has SOA on secondary"
+else
+  warn "Reverse zone $ptr_test: no SOA on secondary"
+fi
+
+# =============================================================================
+section "8. Production untouched (read-only verification)"
+
+# Verify production container is still running and unchanged
+prod_status=$(run_root "$PROD" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
+if echo "$prod_status" | grep -qi 'Up'; then
+  ok "Production container still running on $PROD ($prod_status)"
+else
+  fail "Production container NOT running on $PROD!"
+fi
+
+prod_zones_after=$(count_zones "$PROD")
+if [ "$prod_zones_after" = "$prod_zones" ]; then
+  ok "Production zone count unchanged ($prod_zones_after = $prod_zones before)"
+else
+  fail "Production zone count CHANGED: $prod_zones → $prod_zones_after"
+fi
+
+# =============================================================================
+section "9. Failover test"
+
+# Take the approach of querying via the secondary when primary is slow/unavailable.
+# We test that the secondary answers independently.
+sec_soa=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 knel.net SOA 2>/dev/null | head -1" 2>/dev/null || true)
+if [ -n "$sec_soa" ]; then
+  ok "Secondary independently serves knel.net SOA: $sec_soa"
+else
+  fail "Secondary cannot serve knel.net SOA independently"
+fi
+
+# =============================================================================
+section "10. Credentials check — auth.config size matches production"
+
+prod_auth_size=$(run_root "$PROD" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
+pri_auth_size=$(run_root "$PRIMARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
+sec_auth_size=$(run_root "$SECONDARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
+
+echo "  auth.config sizes — prod=$prod_auth_size pri=$pri_auth_size sec=$sec_auth_size"
+
+if [ "$prod_auth_size" = "$pri_auth_size" ] && [ "$prod_auth_size" = "$sec_auth_size" ]; then
+  ok "auth.config identical size across all three nodes (credentials + 2FA replicated)"
+else
+  fail "auth.config sizes differ — credentials may not be replicated correctly"
+fi
+
+# =============================================================================
+# Summary
+echo ""
+echo "=========================================="
+echo "  PASSED: $PASS"
+echo "  FAILED: $FAIL"
+echo "  WARNED: $WARN"
+echo "=========================================="
+[ "$FAIL" -eq 0 ] && exit 0 || exit 1