feat(dns): git SoR + drift-check for pihole/dhcpd; serial-restart rule [#469][#420]

AGENTS.md

@@ -288,6 +288,19 @@ Hard-won operational gotchas. One line each; keep them short and load-bearing.
 - **Supervisor add-on options via API:** docker cp a script into
   hassio_supervisor, read token from /data/cli.json `access_token`, POST
   /addons/<slug>/options (the /apps/ spelling 404s).
+- **NEVER restart/kill BOTH members of a redundant pair simultaneously**
+  (founder ruling 2026-09-01 after the DNS outage): restart/redeploy ONE
+  node, verify service health from an independent vantage, only then the
+  second. Blue/green with a health gate between hops — always serial.
+- **Pi-hole v6: `dnsmasq_lines` in pihole.toml are passed to FTL's embedded
+  dnsmasq, which rejects some valid-dnsmasq options (e.g. `no-negcache`)
+  with "bad option" and dnsmasq then never starts — the container looks
+  "Up" but serves nothing on :53. Use first-class toml settings instead;
+  never sed-edit pihole.toml without a single-node health-gated rollout.
+- **DNS outage recovery path:** workstation resolv.conf dies with the LAN
+  Pi-holes; use `tailscale status` peer IPs + the chokepoint env overrides
+  (NETINFRA01_HOST/NETINFRA02_HOST, VM_IP) to reach hosts by Tailscale
+  while names are unresolvable. Fix one node, verify, then the other.
 
 ## Questions (NON-NEGOTIABLE)
 

WORKING.md

@@ -19,3 +19,6 @@ replies 46-48, and git PFVCluster + KNEL/pfv-bms)
 - [x] HomeKit VLAN-segmentation finding + vNIC proposal posted [#619]
 - [x] BTU watts source decision: Emporia CT Friday (APC lacks load vars) [#614]
 - [x] Roadmap ticketed #618-#628; backfill parked per founder [#622]
+- [x] DNS incident recovered (serial, health-gated); both Pi-holes serving fresh [#469]
+- [x] AGENTS.md: never dual-kill redundant pairs + blue/green rule + FTL dnsmasq_lines trap
+- [x] Git SoR: pihole.toml (both nodes, secrets redacted) + drift-check.sh (TDD green) — live==git verified [#420][#469]

netinfra/dns/drift-check.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/bash
+#
+# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469]
+#
+# Verifies that the running configuration on netinfra-01/02 matches the
+# files tracked in this repo. Founded after the 2026-09-01 DNS incident:
+# the live systems are production; git is the source of truth; drift is
+# a defect.
+#
+# Checks per node:
+#   - /etc/dhcp/dhcpd.conf        vs netinfra/dhcp/dhcpd-{primary,secondary}.conf
+#   - /etc/pihole/pihole.toml     vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml
+#     (secrets redacted on both sides before compare; "Last updated" line ignored)
+#
+# Usage:
+#   drift-check.sh [--node 01|02|all]   (default: all)
+# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure
+#
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DNS_SETUP="$HERE/dns-cluster-setup"
+DHCP_DIR="$HERE/dhcp"
+PIHOLE_DIR="$HERE/dns/pihole"
+
+REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/'
+
+# redact_config <stdin> <stdout> — strip secrets from pihole.toml content
+redact_config() { sed -E "$REDACT"; }
+
+# normalize_toml <stdin> <stdout> — redact + drop churn lines (timestamps)
+normalize_toml() { redact_config | grep -v "Last updated on"; }
+
+# fetch <node> <remote-cmd> — run via the remote-dns.sh chokepoint (env IPs honored)
+fetch() {
+  local node="$1" cmd="$2"
+  bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null
+}
+
+# check_file <label> <node> <remote-cat-cmd> <local-file> <normalize-fn>
+check_file() {
+  local label="$1" node="$2" rcmd="$3" local_file="$4" norm="$5" tmp rc
+  tmp="$(mktemp)"
+  fetch "$node" "$rcmd" | "$norm" > "$tmp"
+  if [ ! -s "$tmp" ]; then
+    echo "DRIFT-ERROR: $label: live fetch empty (node $node unreachable?)"
+    rm -f "$tmp"
+    return 2
+  fi
+  "$norm" < "$local_file" | diff -q - "$tmp" >/dev/null 2>&1
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    echo "DRIFT: $label (node $node) differs from git: $local_file"
+    "$norm" < "$local_file" | diff - "$tmp" | head -10
+  else
+    echo "OK: $label (node $node) in sync"
+  fi
+  rm -f "$tmp"
+  return "$rc"
+}
+
+main() {
+  local nodes="${1:-all}" node rc_total=0 rc
+  [ "$nodes" = "all" ] && nodes="01 02"
+  for node in $nodes; do
+    local_dhcp="$DHCP_DIR/dhcpd-primary.conf"
+    [ "$node" = "02" ] && local_dhcp="$DHCP_DIR/dhcpd-secondary.conf"
+    local_pihole="$PIHOLE_DIR/netinfra-01.pihole.toml"
+    [ "$node" = "02" ] && local_pihole="$PIHOLE_DIR/netinfra-02.pihole.toml"
+
+    check_file "dhcpd.conf" "$node" "cat /etc/dhcp/dhcpd.conf" "$local_dhcp" cat || rc_total=1
+    check_file "pihole.toml" "$node" \
+      "docker exec pihole cat /etc/pihole/pihole.toml" "$local_pihole" normalize_toml || rc_total=1
+  done
+  if [ "$rc_total" -eq 0 ]; then
+    echo "drift-check: ALL IN SYNC"
+  else
+    echo "drift-check: DRIFT DETECTED — reconcile git <-> live before any change"
+  fi
+  return "$rc_total"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  node="${2:-all}"
+  [ "${1:-}" = "--node" ] && node="${2:-}" || node="all"
+  main "$node"
+fi

tests/unit/dns-drift-check.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Unit tests for netinfra/dns/drift-check.sh [#420][#469]
+# Tests the redaction and normalization functions without touching hosts.
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SCRIPT="$PROJECT_ROOT/netinfra/dns/drift-check.sh"
+
+failed=0
+
+if [[ -f "$SCRIPT" ]] && (source "$SCRIPT"); then
+    echo "✅ script exists + sources cleanly"
+else
+    echo "❌ script missing or fails to source: $SCRIPT"
+    exit 1
+fi
+# shellcheck disable=SC1091
+source "$SCRIPT"
+
+test_redact_strips_secrets() {
+    local out
+    out=$(printf '  pwhash = "SECRETBALLOONHASH"\n  totp_secret = "ABC234"\n  password = "hunter2"\n  port = 53\n' | redact_config)
+    if grep -qE "SECRETBALLOON|ABC234|hunter2" <<<"$out"; then
+        echo "❌ redact_config leaked a secret"
+        ((++failed))
+    elif grep -q '"REDACTED"' <<<"$out" && grep -q "port = 53" <<<"$out"; then
+        echo "✅ redact_config strips secrets, keeps config"
+    else
+        echo "❌ redact_config mangled output"
+        ((++failed))
+    fi
+}
+
+test_normalize_drops_timestamp() {
+    local out
+    out=$(printf '# Last updated on 2026-09-01 17:01:15 CDT\n  dnsmasq_lines = []\n' | normalize_toml)
+    if grep -q "Last updated" <<<"$out"; then
+        echo "❌ normalize_toml kept the timestamp churn line"
+        ((++failed))
+    elif grep -q "dnsmasq_lines" <<<"$out"; then
+        echo "✅ normalize_toml drops timestamp, keeps config"
+    else
+        echo "❌ normalize_toml dropped config lines"
+        ((++failed))
+    fi
+}
+
+test_normalize_implies_redaction() {
+    local out
+    out=$(printf '  totp_secret = "XYZ789"\n  a = 1\n' | normalize_toml)
+    if grep -q XYZ789 <<<"$out"; then
+        echo "❌ normalize_toml leaked a secret"
+        ((++failed))
+    else
+        echo "✅ normalize_toml redacts secrets"
+    fi
+}
+
+test_redact_strips_secrets
+test_normalize_drops_timestamp
+test_normalize_implies_redaction
+
+if ((failed > 0)); then
+    echo "❌ $failed drift-check test(s) failed"
+    exit 1
+fi
+echo "✅ all drift-check tests passed"