feat(dns): git SoR + drift-check for pihole/dhcpd; serial-restart rule [#469][#420]

2026-09-01 17:07:23 -05:00
parent e2d6e5c52f
commit adfdcafeab
6 changed files with 3664 additions and 0 deletions
@@ -0,0 +1,87 @@
+#!/usr/bin/bash
+#
+# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469]
+#
+# Verifies that the running configuration on netinfra-01/02 matches the
+# files tracked in this repo. Founded after the 2026-09-01 DNS incident:
+# the live systems are production; git is the source of truth; drift is
+# a defect.
+#
+# Checks per node:
+#   - /etc/dhcp/dhcpd.conf        vs netinfra/dhcp/dhcpd-{primary,secondary}.conf
+#   - /etc/pihole/pihole.toml     vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml
+#     (secrets redacted on both sides before compare; "Last updated" line ignored)
+#
+# Usage:
+#   drift-check.sh [--node 01|02|all]   (default: all)
+# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure
+#
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DNS_SETUP="$HERE/dns-cluster-setup"
+DHCP_DIR="$HERE/dhcp"
+PIHOLE_DIR="$HERE/dns/pihole"
+
+REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/'
+
+# redact_config <stdin> <stdout> — strip secrets from pihole.toml content
+redact_config() { sed -E "$REDACT"; }
+
+# normalize_toml <stdin> <stdout> — redact + drop churn lines (timestamps)
+normalize_toml() { redact_config | grep -v "Last updated on"; }
+
+# fetch <node> <remote-cmd> — run via the remote-dns.sh chokepoint (env IPs honored)
+fetch() {
+  local node="$1" cmd="$2"
+  bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null
+}
+
+# check_file <label> <node> <remote-cat-cmd> <local-file> <normalize-fn>
+check_file() {
+  local label="$1" node="$2" rcmd="$3" local_file="$4" norm="$5" tmp rc
+  tmp="$(mktemp)"
+  fetch "$node" "$rcmd" | "$norm" > "$tmp"
+  if [ ! -s "$tmp" ]; then
+    echo "DRIFT-ERROR: $label: live fetch empty (node $node unreachable?)"
+    rm -f "$tmp"
+    return 2
+  fi
+  "$norm" < "$local_file" | diff -q - "$tmp" >/dev/null 2>&1
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    echo "DRIFT: $label (node $node) differs from git: $local_file"
+    "$norm" < "$local_file" | diff - "$tmp" | head -10
+  else
+    echo "OK: $label (node $node) in sync"
+  fi
+  rm -f "$tmp"
+  return "$rc"
+}
+
+main() {
+  local nodes="${1:-all}" node rc_total=0 rc
+  [ "$nodes" = "all" ] && nodes="01 02"
+  for node in $nodes; do
+    local_dhcp="$DHCP_DIR/dhcpd-primary.conf"
+    [ "$node" = "02" ] && local_dhcp="$DHCP_DIR/dhcpd-secondary.conf"
+    local_pihole="$PIHOLE_DIR/netinfra-01.pihole.toml"
+    [ "$node" = "02" ] && local_pihole="$PIHOLE_DIR/netinfra-02.pihole.toml"
+
+    check_file "dhcpd.conf" "$node" "cat /etc/dhcp/dhcpd.conf" "$local_dhcp" cat || rc_total=1
+    check_file "pihole.toml" "$node" \
+      "docker exec pihole cat /etc/pihole/pihole.toml" "$local_pihole" normalize_toml || rc_total=1
+  done
+  if [ "$rc_total" -eq 0 ]; then
+    echo "drift-check: ALL IN SYNC"
+  else
+    echo "drift-check: DRIFT DETECTED — reconcile git <-> live before any change"
+  fi
+  return "$rc_total"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  node="${2:-all}"
+  [ "${1:-}" = "--node" ] && node="${2:-}" || node="all"
+  main "$node"
+fi