From adfdcafeab5acc0599da7ac33153cff6b88024b1 Mon Sep 17 00:00:00 2001 From: reachableceo Date: Tue, 1 Sep 2026 17:07:23 -0500 Subject: [PATCH] feat(dns): git SoR + drift-check for pihole/dhcpd; serial-restart rule [#469][#420] --- AGENTS.md | 13 + WORKING.md | 3 + netinfra/dns/drift-check.sh | 87 + netinfra/dns/pihole/netinfra-01.pihole.toml | 1747 +++++++++++++++++++ netinfra/dns/pihole/netinfra-02.pihole.toml | 1747 +++++++++++++++++++ tests/unit/dns-drift-check.sh | 67 + 6 files changed, 3664 insertions(+) create mode 100755 netinfra/dns/drift-check.sh create mode 100644 netinfra/dns/pihole/netinfra-01.pihole.toml create mode 100644 netinfra/dns/pihole/netinfra-02.pihole.toml create mode 100755 tests/unit/dns-drift-check.sh diff --git a/AGENTS.md b/AGENTS.md index 12bd347..8b7a90c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,6 +288,19 @@ Hard-won operational gotchas. One line each; keep them short and load-bearing. - **Supervisor add-on options via API:** docker cp a script into hassio_supervisor, read token from /data/cli.json `access_token`, POST /addons//options (the /apps/ spelling 404s). +- **NEVER restart/kill BOTH members of a redundant pair simultaneously** + (founder ruling 2026-09-01 after the DNS outage): restart/redeploy ONE + node, verify service health from an independent vantage, only then the + second. Blue/green with a health gate between hops — always serial. +- **Pi-hole v6: `dnsmasq_lines` in pihole.toml are passed to FTL's embedded + dnsmasq, which rejects some valid-dnsmasq options (e.g. `no-negcache`) + with "bad option" and dnsmasq then never starts — the container looks + "Up" but serves nothing on :53. Use first-class toml settings instead; + never sed-edit pihole.toml without a single-node health-gated rollout. +- **DNS outage recovery path:** workstation resolv.conf dies with the LAN + Pi-holes; use `tailscale status` peer IPs + the chokepoint env overrides + (NETINFRA01_HOST/NETINFRA02_HOST, VM_IP) to reach hosts by Tailscale + while names are unresolvable. Fix one node, verify, then the other. ## Questions (NON-NEGOTIABLE) diff --git a/WORKING.md b/WORKING.md index b9612d1..f3d00bb 100644 --- a/WORKING.md +++ b/WORKING.md @@ -19,3 +19,6 @@ replies 46-48, and git PFVCluster + KNEL/pfv-bms) - [x] HomeKit VLAN-segmentation finding + vNIC proposal posted [#619] - [x] BTU watts source decision: Emporia CT Friday (APC lacks load vars) [#614] - [x] Roadmap ticketed #618-#628; backfill parked per founder [#622] +- [x] DNS incident recovered (serial, health-gated); both Pi-holes serving fresh [#469] +- [x] AGENTS.md: never dual-kill redundant pairs + blue/green rule + FTL dnsmasq_lines trap +- [x] Git SoR: pihole.toml (both nodes, secrets redacted) + drift-check.sh (TDD green) — live==git verified [#420][#469] diff --git a/netinfra/dns/drift-check.sh b/netinfra/dns/drift-check.sh new file mode 100755 index 0000000..8df27cd --- /dev/null +++ b/netinfra/dns/drift-check.sh @@ -0,0 +1,87 @@ +#!/usr/bin/bash +# +# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469] +# +# Verifies that the running configuration on netinfra-01/02 matches the +# files tracked in this repo. Founded after the 2026-09-01 DNS incident: +# the live systems are production; git is the source of truth; drift is +# a defect. +# +# Checks per node: +# - /etc/dhcp/dhcpd.conf vs netinfra/dhcp/dhcpd-{primary,secondary}.conf +# - /etc/pihole/pihole.toml vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml +# (secrets redacted on both sides before compare; "Last updated" line ignored) +# +# Usage: +# drift-check.sh [--node 01|02|all] (default: all) +# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure +# +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DNS_SETUP="$HERE/dns-cluster-setup" +DHCP_DIR="$HERE/dhcp" +PIHOLE_DIR="$HERE/dns/pihole" + +REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/' + +# redact_config — strip secrets from pihole.toml content +redact_config() { sed -E "$REDACT"; } + +# normalize_toml — redact + drop churn lines (timestamps) +normalize_toml() { redact_config | grep -v "Last updated on"; } + +# fetch — run via the remote-dns.sh chokepoint (env IPs honored) +fetch() { + local node="$1" cmd="$2" + bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null +} + +# check_file