mrcharles
ebefbe47a7
ntp.conf captured (nodes byte-identical); 171 Technitium DZ zone files captured from the primary; drift-check now covers dhcpd/pihole/ntp on both nodes plus a zones md5-manifest check. Live run caught real drift: node-02 dhcpd.conf was a stale primary copy missing the #614 minisplit reservation — reconciled git->02 with dhcpd -t gate, serial restart, failover "Both servers normal", all 7 checks in sync. Red-green unit tests extended (gen_manifest, tracked-file invariants). Results: https://projects.knownelement.com/issues/630#note-2
116 lines
4.4 KiB
Bash
Executable File
116 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
#
|
|
# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469]
|
|
#
|
|
# Verifies that the running configuration on netinfra-01/02 matches the
|
|
# files tracked in this repo. Founded after the 2026-09-01 DNS incident:
|
|
# the live systems are production; git is the source of truth; drift is
|
|
# a defect.
|
|
#
|
|
# Checks per node:
|
|
# - /etc/dhcp/dhcpd.conf vs netinfra/dhcp/dhcpd-{primary,secondary}.conf
|
|
# - /etc/pihole/pihole.toml vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml
|
|
# (secrets redacted on both sides before compare; "Last updated" line ignored)
|
|
# - /etc/ntpsec/ntp.conf vs netinfra/ntp/ntp.conf (nodes are identical)
|
|
# - Technitium zones (node 01, the replication primary): md5 manifest of the
|
|
# binary DZ store vs netinfra/dns/technitium/zones/ [#630]
|
|
#
|
|
# Usage:
|
|
# drift-check.sh [--node 01|02|all] (default: all)
|
|
# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure
|
|
#
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DNS_SETUP="$HERE/dns-cluster-setup"
|
|
DHCP_DIR="$HERE/dhcp"
|
|
PIHOLE_DIR="$HERE/dns/pihole"
|
|
NTP_FILE="$HERE/ntp/ntp.conf"
|
|
ZONES_DIR="$HERE/dns/technitium/zones"
|
|
REMOTE_ZONE_DIR="/home/localuser/services/technitium/config/zones"
|
|
|
|
REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/'
|
|
|
|
# redact_config <stdin> <stdout> — strip secrets from pihole.toml content
|
|
redact_config() { sed -E "$REDACT"; }
|
|
|
|
# normalize_toml <stdin> <stdout> — redact + drop churn lines (timestamps)
|
|
normalize_toml() { redact_config | grep -v "Last updated on"; }
|
|
|
|
# gen_manifest <dir> — sorted 'md5 name' manifest of *.zone files (name-sorted)
|
|
gen_manifest() { (cd "$1" && md5sum -- *.zone 2>/dev/null | sort -k2); }
|
|
|
|
# fetch <node> <remote-cmd> — run via the remote-dns.sh chokepoint (env IPs honored)
|
|
fetch() {
|
|
local node="$1" cmd="$2"
|
|
bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null
|
|
}
|
|
|
|
# check_file <label> <node> <remote-cat-cmd> <local-file> <normalize-fn>
|
|
check_file() {
|
|
local label="$1" node="$2" rcmd="$3" local_file="$4" norm="$5" tmp rc
|
|
tmp="$(mktemp)"
|
|
fetch "$node" "$rcmd" | "$norm" > "$tmp"
|
|
if [ ! -s "$tmp" ]; then
|
|
echo "DRIFT-ERROR: $label: live fetch empty (node $node unreachable?)"
|
|
rm -f "$tmp"
|
|
return 2
|
|
fi
|
|
"$norm" < "$local_file" | diff -q - "$tmp" >/dev/null 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "DRIFT: $label (node $node) differs from git: $local_file"
|
|
"$norm" < "$local_file" | diff - "$tmp" | head -10
|
|
else
|
|
echo "OK: $label (node $node) in sync"
|
|
fi
|
|
rm -f "$tmp"
|
|
return "$rc"
|
|
}
|
|
|
|
main() {
|
|
local nodes="${1:-all}" node rc_total=0 rc
|
|
[ "$nodes" = "all" ] && nodes="01 02"
|
|
for node in $nodes; do
|
|
local_dhcp="$DHCP_DIR/dhcpd-primary.conf"
|
|
[ "$node" = "02" ] && local_dhcp="$DHCP_DIR/dhcpd-secondary.conf"
|
|
local_pihole="$PIHOLE_DIR/netinfra-01.pihole.toml"
|
|
[ "$node" = "02" ] && local_pihole="$PIHOLE_DIR/netinfra-02.pihole.toml"
|
|
|
|
check_file "dhcpd.conf" "$node" "cat /etc/dhcp/dhcpd.conf" "$local_dhcp" cat || rc_total=1
|
|
check_file "pihole.toml" "$node" \
|
|
"docker exec pihole cat /etc/pihole/pihole.toml" "$local_pihole" normalize_toml || rc_total=1
|
|
check_file "ntp.conf" "$node" "cat /etc/ntpsec/ntp.conf" "$NTP_FILE" cat || rc_total=1
|
|
done
|
|
|
|
# Technitium zones: binary DZ files, compared by md5 manifest against the
|
|
# primary (netinfra-01) — the replication source for -02.
|
|
if [[ "$nodes" == *01* ]]; then
|
|
remote_manifest="$(fetch 01 "cd $REMOTE_ZONE_DIR && md5sum -- *.zone | sort -k2")"
|
|
if [ -z "$remote_manifest" ]; then
|
|
echo "DRIFT-ERROR: technitium zones: live fetch empty (node 01 unreachable?)"
|
|
rc_total=1
|
|
else
|
|
if diff -q <(gen_manifest "$ZONES_DIR") <(printf '%s\n' "$remote_manifest") >/dev/null 2>&1; then
|
|
echo "OK: technitium zones (node 01) in sync"
|
|
else
|
|
echo "DRIFT: technitium zones (node 01) manifest differs from git: $ZONES_DIR"
|
|
diff <(gen_manifest "$ZONES_DIR") <(printf '%s\n' "$remote_manifest") | head -10
|
|
rc_total=1
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$rc_total" -eq 0 ]; then
|
|
echo "drift-check: ALL IN SYNC"
|
|
else
|
|
echo "drift-check: DRIFT DETECTED — reconcile git <-> live before any change"
|
|
fi
|
|
return "$rc_total"
|
|
}
|
|
|
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
node="${2:-all}"
|
|
[ "${1:-}" = "--node" ] && node="${2:-}" || node="all"
|
|
main "$node"
|
|
fi
|