mrcharles
0fa0692c37
Establish shellcheck as a mandatory pre-commit quality gate and bring all 93
shell scripts to a clean state.
- tests/shellcheck.sh: wrapper that runs koalaman/shellcheck:stable via Docker
(no native binary needed), skips vendored + upstream librenms-agent scripts.
- .shellcheckrc: documents intentional codebase-wide disables (dynamic source
paths SC1090/SC1091, client-side ssh expansion SC2029).
- AGENTS.md: new Git Policy rule mandating clean shellcheck for every shell
script before commit.
Fixes applied (real bugs + quality): missing quote in netinfra/gather-configs.sh
(caused cascading parse errors), unquoted expansions, declare-and-assign masking,
egrep -> grep -E, $FUNCNAME array indexing, unused variable removal, cd || exit.
Intentional patterns (sourced config, sysfs/ps diagnostics, ssh heredocs that
expand local config) get justified targeted disables.
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
205 lines
8.1 KiB
Bash
Executable File
205 lines
8.1 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
#
|
|
# verify.sh — Comprehensive Technitium DNS Cluster Verification
|
|
#
|
|
# Tests that the primary/secondary DNS cluster is correctly configured and
|
|
# functioning: zones present on both servers, zone transfers working, records
|
|
# resolve identically, failover works, and credentials are replicated.
|
|
#
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REMOTE="$HERE/remote-dns.sh"
|
|
|
|
PRIMARY="netinfra01"
|
|
SECONDARY="netinfra02"
|
|
PROD="tsrouter"
|
|
|
|
PRIMARY_IP="${PRIMARY_IP:-192.168.3.252}"
|
|
SECONDARY_IP="${SECONDARY_IP:-192.168.3.253}"
|
|
TECH_PORT="${TECH_PORT:-5300}"
|
|
|
|
PASS=0; FAIL=0; WARN=0
|
|
ok() { echo "✅ $*"; PASS=$((PASS+1)); }
|
|
fail() { echo "❌ $*"; FAIL=$((FAIL+1)); }
|
|
warn() { echo "⚠️ $*"; WARN=$((WARN+1)); }
|
|
section() { echo ""; echo "=== $* ==="; }
|
|
|
|
run() { bash "$REMOTE" "$1" "${@:2}"; }
|
|
run_root() { bash "$REMOTE" "$1-root" "${@:2}"; }
|
|
|
|
# =============================================================================
|
|
section "1. Container health on both nodes"
|
|
|
|
for h in "$PRIMARY" "$SECONDARY"; do
|
|
status=$(run_root "$h" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
|
|
if echo "$status" | grep -qi 'Up'; then
|
|
ok "Technitium container running on $h ($status)"
|
|
else
|
|
fail "Technitium container NOT running on $h (status: ${status:-none})"
|
|
fi
|
|
done
|
|
|
|
# =============================================================================
|
|
section "2. Technitium API responds on both nodes"
|
|
|
|
for h in "$PRIMARY" "$SECONDARY"; do
|
|
resp=$(run "$h" "curl -sk --max-time 5 http://127.0.0.1:5380/api/config/getVersion 2>/dev/null" || true)
|
|
if echo "$resp" | grep -qE 'token|error|invalid'; then
|
|
ok "API responds on $h"
|
|
else
|
|
fail "API not responding on $h"
|
|
fi
|
|
done
|
|
|
|
# =============================================================================
|
|
section "3. Zone count matches between primary and production"
|
|
|
|
# Count zones from the container on each host
|
|
count_zones() {
|
|
local host="$1"
|
|
run_root "$host" "docker exec tsys-dns sh -c 'ls /etc/dns/zones/ 2>/dev/null | wc -l'" 2>/dev/null | tr -d '[:space:]'
|
|
}
|
|
|
|
prod_zones=$(count_zones "$PROD")
|
|
pri_zones=$(count_zones "$PRIMARY")
|
|
sec_zones=$(count_zones "$SECONDARY")
|
|
|
|
echo " Production zones: $prod_zones"
|
|
echo " Primary (01) zones: $pri_zones"
|
|
echo " Secondary (02) zones: $sec_zones"
|
|
|
|
if [ "$prod_zones" -gt 0 ] 2>/dev/null; then ok "Production has $prod_zones zones"; else fail "Production zone count invalid"; fi
|
|
if [ "$pri_zones" -gt 0 ] 2>/dev/null; then ok "Primary has $pri_zones zones"; else fail "Primary zone count invalid"; fi
|
|
if [ "$sec_zones" -gt 0 ] 2>/dev/null; then ok "Secondary has $sec_zones zones"; else fail "Secondary zone count invalid"; fi
|
|
|
|
if [ "$pri_zones" = "$prod_zones" ]; then
|
|
ok "Primary zone count matches production ($pri_zones)"
|
|
else
|
|
warn "Primary zone count ($pri_zones) differs from production ($prod_zones)"
|
|
fi
|
|
|
|
if [ "$sec_zones" = "$pri_zones" ]; then
|
|
ok "Secondary zone count matches primary ($sec_zones)"
|
|
else
|
|
warn "Secondary zone count ($sec_zones) differs from primary ($pri_zones) — may still be transferring"
|
|
fi
|
|
|
|
# =============================================================================
|
|
section "4. knel.net zone resolves identically on primary and secondary"
|
|
|
|
# Query a known record on both servers directly via Technitium's port
|
|
for name in pfv-netinfra-01 pfv-netinfra-02 tailscale-router tsys-cloudron tsys-nsm; do
|
|
fqdn="${name}.knel.net"
|
|
# Query via dig against each Technitium instance (through Pi-hole on :53)
|
|
pri_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)
|
|
sec_ans=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $fqdn A 2>/dev/null | head -1" 2>/dev/null || true)
|
|
|
|
if [ -n "$pri_ans" ] && [ "$pri_ans" = "$sec_ans" ]; then
|
|
ok "$fqdn resolves identically: $pri_ans"
|
|
elif [ -n "$pri_ans" ] && [ -z "$sec_ans" ]; then
|
|
warn "$fqdn: primary=$pri_ans secondary=<no answer> (may still be syncing)"
|
|
elif [ -z "$pri_ans" ] && [ -z "$sec_ans" ]; then
|
|
warn "$fqdn: no answer on either server"
|
|
else
|
|
fail "$fqdn MISMATCH: primary=$pri_ans secondary=$sec_ans"
|
|
fi
|
|
done
|
|
|
|
# =============================================================================
|
|
section "5. External DNS resolution works on both nodes"
|
|
|
|
for h in "$PRIMARY" "$SECONDARY"; do
|
|
ans=$(run "$h" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 github.com A 2>/dev/null | head -1" 2>/dev/null || true)
|
|
if [ -n "$ans" ]; then
|
|
ok "$h resolves github.com → $ans"
|
|
else
|
|
fail "$h cannot resolve github.com"
|
|
fi
|
|
done
|
|
|
|
# =============================================================================
|
|
section "6. Zone transfer (AXFR) from primary to secondary"
|
|
|
|
# Test AXFR of knel.net from the primary
|
|
axfr=$(run "$SECONDARY" "dig +short +time=5 +tries=1 @${PRIMARY_IP} -p ${TECH_PORT} knel.net AXFR 2>/dev/null | wc -l" 2>/dev/null || echo "0")
|
|
if [ "$axfr" -gt 1 ] 2>/dev/null; then
|
|
ok "AXFR of knel.net from primary succeeds ($axfr records transferred)"
|
|
else
|
|
warn "AXFR test returned $axfr records — zone transfer may be restricted or in progress"
|
|
fi
|
|
|
|
# =============================================================================
|
|
section "7. Reverse DNS works"
|
|
|
|
# Pick a known reverse zone and test PTR resolution
|
|
ptr_test="181.103.100.in-addr.arpa"
|
|
ptr_ans=$(run "$PRIMARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
|
|
if [ -n "$ptr_ans" ]; then
|
|
ok "Reverse zone $ptr_test has SOA on primary"
|
|
else
|
|
warn "Reverse zone $ptr_test: no SOA on primary"
|
|
fi
|
|
|
|
ptr_ans2=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 $ptr_test SOA 2>/dev/null | head -1" 2>/dev/null || true)
|
|
if [ -n "$ptr_ans2" ]; then
|
|
ok "Reverse zone $ptr_test has SOA on secondary"
|
|
else
|
|
warn "Reverse zone $ptr_test: no SOA on secondary"
|
|
fi
|
|
|
|
# =============================================================================
|
|
section "8. Production untouched (read-only verification)"
|
|
|
|
# Verify production container is still running and unchanged
|
|
prod_status=$(run_root "$PROD" "docker ps --format '{{.Status}}' tsys-dns 2>/dev/null" | head -1)
|
|
if echo "$prod_status" | grep -qi 'Up'; then
|
|
ok "Production container still running on $PROD ($prod_status)"
|
|
else
|
|
fail "Production container NOT running on $PROD!"
|
|
fi
|
|
|
|
prod_zones_after=$(count_zones "$PROD")
|
|
if [ "$prod_zones_after" = "$prod_zones" ]; then
|
|
ok "Production zone count unchanged ($prod_zones_after = $prod_zones before)"
|
|
else
|
|
fail "Production zone count CHANGED: $prod_zones → $prod_zones_after"
|
|
fi
|
|
|
|
# =============================================================================
|
|
section "9. Failover test"
|
|
|
|
# Take the approach of querying via the secondary when primary is slow/unavailable.
|
|
# We test that the secondary answers independently.
|
|
sec_soa=$(run "$SECONDARY" "dig +short +time=3 +tries=1 @127.0.0.1 -p 53 knel.net SOA 2>/dev/null | head -1" 2>/dev/null || true)
|
|
if [ -n "$sec_soa" ]; then
|
|
ok "Secondary independently serves knel.net SOA: $sec_soa"
|
|
else
|
|
fail "Secondary cannot serve knel.net SOA independently"
|
|
fi
|
|
|
|
# =============================================================================
|
|
section "10. Credentials check — auth.config size matches production"
|
|
|
|
prod_auth_size=$(run_root "$PROD" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
|
|
pri_auth_size=$(run_root "$PRIMARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
|
|
sec_auth_size=$(run_root "$SECONDARY" "docker exec tsys-dns wc -c < /etc/dns/auth.config 2>/dev/null" | tr -d '[:space:]')
|
|
|
|
echo " auth.config sizes — prod=$prod_auth_size pri=$pri_auth_size sec=$sec_auth_size"
|
|
|
|
if [ "$prod_auth_size" = "$pri_auth_size" ] && [ "$prod_auth_size" = "$sec_auth_size" ]; then
|
|
ok "auth.config identical size across all three nodes (credentials + 2FA replicated)"
|
|
else
|
|
fail "auth.config sizes differ — credentials may not be replicated correctly"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Summary
|
|
echo ""
|
|
echo "=========================================="
|
|
echo " PASSED: $PASS"
|
|
echo " FAILED: $FAIL"
|
|
echo " WARNED: $WARN"
|
|
echo "=========================================="
|
|
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|