KNELServerBuild/dns-cluster-setup/verify.sh

#!/usr/bin/bash
#
# verify.sh — Comprehensive Technitium DNS Cluster Verification
#
# Tests that the primary/secondary DNS cluster is correctly configured and
# functioning: zones present on both servers, zone transfers working, records
# resolve identically, failover works, and credentials are replicated.
#
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REMOTE="$HERE/remote-dns.sh"

PRIMARY="netinfra01"
SECONDARY="netinfra02"
PROD="tsrouter"

PRIMARY_IP="${PRIMARY_IP:-192.168.3.252}"
SECONDARY_IP="${SECONDARY_IP:-192.168.3.253}"
TECH_PORT="${TECH_PORT:-5300}"

PASS=0; FAIL=0; WARN=0
ok()   { echo "✅ $*"; PASS=$((PASS+1)); }
fail() { echo "❌ $*"; FAIL=$((FAIL+1)); }
warn() { echo "⚠️  $*"; WARN=$((WARN+1)); }
section() { echo ""; echo "=== $* ==="; }

run()      { bash "$REMOTE" "$1" "${@:2}"; }
run_root() { bash "$REMOTE" "$1-root" "${@:2}"; }

# =============================================================================
section "1. Container health on both nodes"

for h in "$PRIMARY" "$SECONDARY"; do
  status=$(run_root "$h" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
  if echo "$status" | grep -qi 'Up'; then
    ok "Technitium container running on $h ($status)"
  else
    fail "Technitium container NOT running on $h (status: ${status:-none})"
  fi
done

# =============================================================================
section "2. Technitium API responds on both nodes"

for h in "$PRIMARY" "$SECONDARY"; do
  resp=$(run "$h" "curl -sk --max-time 5 http://127.0.0.1:5380/api/config/getVersion 2>/dev/null" || true)
  if echo "$resp" | grep -qE 'token|error|invalid'; then
    ok "API responds on $h"
  else
    fail "API not responding on $h"
  fi
done

# =============================================================================
section "3. Zone count matches between primary and production"

# Count zones from the container on each host
count_zones() {
  local host="$1"
  run_root "$host" "docker exec tsys-dns sh -c 'ls /etc/dns/zones/ 2>/dev/null | wc -l'" 2>/dev/null | tr -d '[:space:]'
}

prod_zones=$(count_zones "$PROD")
pri_zones=$(count_zones "$PRIMARY")
sec_zones=$(count_zones "$SECONDARY")

echo "  Production zones: $prod_zones"
echo "  Primary (01) zones: $pri_zones"
echo "  Secondary (02) zones: $sec_zones"

[ "$prod_zones" -gt 0 ] 2>/dev/null && ok "Production has $prod_zones zones" || fail "Production zone count invalid"
[ "$pri_zones" -gt 0 ] 2>/dev/null && ok "Primary has $pri_zones zones" || fail "Primary zone count invalid"
[ "$sec_zones" -gt 0 ] 2>/dev/null && ok "Secondary has $sec_zones zones" || fail "Secondary zone count invalid"

if [ "$pri_zones" = "$prod_zones" ]; then
  ok "Primary zone count matches production ($pri_zones)"
else
  warn "Primary zone count ($pri_zones) differs from production ($prod_zones)"
fi

if [ "$sec_zones" = "$pri_zones" ]; then
  ok "Secondary zone count matches primary ($sec_zones)"
else
  warn "Secondary zone count ($sec_zones) differs from primary ($pri_zones) — may still be transferring"
fi

# =============================================================================
section "4. knel.net zone resolves identically on primary and secondary"

# Query a known record on both servers directly via Technitium's port
for name in pfv-netinfra-01 pfv-netinfra-02 tailscale-router tsys-cloudron tsys-nsm; do
  fqdn="${name}.knel.net"
  # Query via dig against each Technitium instance (through Pi-hole on :53)
  pri_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)
  sec_ans=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)

  if [ -n "$pri_ans" ] && [ "$pri_ans" = "$sec_ans" ]; then
    ok "$fqdn resolves identically: $pri_ans"
  elif [ -n "$pri_ans" ] && [ -z "$sec_ans" ]; then
    warn "$fqdn: primary=$pri_ans secondary=<no answer> (may still be syncing)"
  elif [ -z "$pri_ans" ] && [ -z "$sec_ans" ]; then
    warn "$fqdn: no answer on either server"
  else
    fail "$fqdn MISMATCH: primary=$pri_ans secondary=$sec_ans"
  fi
done

# =============================================================================
section "5. External DNS resolution works on both nodes"

for h in "$PRIMARY" "$SECONDARY"; do
  ans=$(run "$h" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 github.com A 2>/dev/null | head -1" 2>/dev/null || true)
  if [ -n "$ans" ]; then
    ok "$h resolves github.com → $ans"
  else
    fail "$h cannot resolve github.com"
  fi
done

# =============================================================================
section "6. Zone transfer (AXFR) from primary to secondary"

# Test AXFR of knel.net from the primary
axfr=$(run "$SECONDARY" "dig +short +time=5 +tries=1 @${PRIMARY_IP} -p ${TECH_PORT} knel.net AXFR 2>/dev/null | wc -l" 2>/dev/null || echo "0")
if [ "$axfr" -gt 1 ] 2>/dev/null; then
  ok "AXFR of knel.net from primary succeeds ($axfr records transferred)"
else
  warn "AXFR test returned $axfr records — zone transfer may be restricted or in progress"
fi

# =============================================================================
section "7. Reverse DNS works"

# Pick a known reverse zone and test PTR resolution
ptr_test="181.103.100.in-addr.arpa"
ptr_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
if [ -n "$ptr_ans" ]; then
  ok "Reverse zone $ptr_test has SOA on primary"
else
  warn "Reverse zone $ptr_test: no SOA on primary"
fi

ptr_ans2=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
if [ -n "$ptr_ans2" ]; then
  ok "Reverse zone $ptr_test has SOA on secondary"
else
  warn "Reverse zone $ptr_test: no SOA on secondary"
fi

# =============================================================================
section "8. Production untouched (read-only verification)"

# Verify production container is still running and unchanged
prod_status=$(run_root "$PROD" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
if echo "$prod_status" | grep -qi 'Up'; then
  ok "Production container still running on $PROD ($prod_status)"
else
  fail "Production container NOT running on $PROD!"
fi

prod_zones_after=$(count_zones "$PROD")
if [ "$prod_zones_after" = "$prod_zones" ]; then
  ok "Production zone count unchanged ($prod_zones_after = $prod_zones before)"
else
  fail "Production zone count CHANGED: $prod_zones → $prod_zones_after"
fi

# =============================================================================
section "9. Failover test"

# Take the approach of querying via the secondary when primary is slow/unavailable.
# We test that the secondary answers independently.
sec_soa=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 knel.net SOA 2>/dev/null | head -1" 2>/dev/null || true)
if [ -n "$sec_soa" ]; then
  ok "Secondary independently serves knel.net SOA: $sec_soa"
else
  fail "Secondary cannot serve knel.net SOA independently"
fi

# =============================================================================
section "10. Credentials check — auth.config size matches production"

prod_auth_size=$(run_root "$PROD" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
pri_auth_size=$(run_root "$PRIMARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
sec_auth_size=$(run_root "$SECONDARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')

echo "  auth.config sizes — prod=$prod_auth_size pri=$pri_auth_size sec=$sec_auth_size"

if [ "$prod_auth_size" = "$pri_auth_size" ] && [ "$prod_auth_size" = "$sec_auth_size" ]; then
  ok "auth.config identical size across all three nodes (credentials + 2FA replicated)"
else
  fail "auth.config sizes differ — credentials may not be replicated correctly"
fi

# =============================================================================
# Summary
echo ""
echo "=========================================="
echo "  PASSED: $PASS"
echo "  FAILED: $FAIL"
echo "  WARNED: $WARN"
echo "=========================================="
[ "$FAIL" -eq 0 ] && exit 0 || exit 1