chore: initial import from KNEL/PFVCluster@041d311 [#769]

Split per O&M lane work order. Full history: KNEL/PFVCluster. https://projects.knownelement.com/issues/769#note-4152
2026-09-03 21:30:32 -05:00
commit 33827de0dc
236 changed files with 9587 additions and 0 deletions
@@ -0,0 +1,115 @@
+#!/usr/bin/bash
+#
+# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469]
+#
+# Verifies that the running configuration on netinfra-01/02 matches the
+# files tracked in this repo. Founded after the 2026-09-01 DNS incident:
+# the live systems are production; git is the source of truth; drift is
+# a defect.
+#
+# Checks per node:
+#   - /etc/dhcp/dhcpd.conf        vs netinfra/dhcp/dhcpd-{primary,secondary}.conf
+#   - /etc/pihole/pihole.toml     vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml
+#     (secrets redacted on both sides before compare; "Last updated" line ignored)
+#   - /etc/ntpsec/ntp.conf        vs netinfra/ntp/ntp.conf (nodes are identical)
+#   - Technitium zones (node 01, the replication primary): md5 manifest of the
+#     binary DZ store vs netinfra/dns/technitium/zones/ [#630]
+#
+# Usage:
+#   drift-check.sh [--node 01|02|all]   (default: all)
+# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure
+#
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DNS_SETUP="$HERE/dns-cluster-setup"
+DHCP_DIR="$HERE/dhcp"
+PIHOLE_DIR="$HERE/dns/pihole"
+NTP_FILE="$HERE/ntp/ntp.conf"
+ZONES_DIR="$HERE/dns/technitium/zones"
+REMOTE_ZONE_DIR="/home/localuser/services/technitium/config/zones"
+
+REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/'
+
+# redact_config <stdin> <stdout> — strip secrets from pihole.toml content
+redact_config() { sed -E "$REDACT"; }
+
+# normalize_toml <stdin> <stdout> — redact + drop churn lines (timestamps)
+normalize_toml() { redact_config | grep -v "Last updated on"; }
+
+# gen_manifest <dir> — sorted 'md5  name' manifest of *.zone files (name-sorted)
+gen_manifest() { (cd "$1" && md5sum -- *.zone 2>/dev/null | sort -k2); }
+
+# fetch <node> <remote-cmd> — run via the remote-dns.sh chokepoint (env IPs honored)
+fetch() {
+  local node="$1" cmd="$2"
+  bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null
+}
+
+# check_file <label> <node> <remote-cat-cmd> <local-file> <normalize-fn>
+check_file() {
+  local label="$1" node="$2" rcmd="$3" local_file="$4" norm="$5" tmp rc
+  tmp="$(mktemp)"
+  fetch "$node" "$rcmd" | "$norm" > "$tmp"
+  if [ ! -s "$tmp" ]; then
+    echo "DRIFT-ERROR: $label: live fetch empty (node $node unreachable?)"
+    rm -f "$tmp"
+    return 2
+  fi
+  "$norm" < "$local_file" | diff -q - "$tmp" >/dev/null 2>&1
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    echo "DRIFT: $label (node $node) differs from git: $local_file"
+    "$norm" < "$local_file" | diff - "$tmp" | head -10
+  else
+    echo "OK: $label (node $node) in sync"
+  fi
+  rm -f "$tmp"
+  return "$rc"
+}
+
+main() {
+  local nodes="${1:-all}" node rc_total=0 rc
+  [ "$nodes" = "all" ] && nodes="01 02"
+  for node in $nodes; do
+    local_dhcp="$DHCP_DIR/dhcpd-primary.conf"
+    [ "$node" = "02" ] && local_dhcp="$DHCP_DIR/dhcpd-secondary.conf"
+    local_pihole="$PIHOLE_DIR/netinfra-01.pihole.toml"
+    [ "$node" = "02" ] && local_pihole="$PIHOLE_DIR/netinfra-02.pihole.toml"
+
+    check_file "dhcpd.conf" "$node" "cat /etc/dhcp/dhcpd.conf" "$local_dhcp" cat || rc_total=1
+    check_file "pihole.toml" "$node" \
+      "docker exec pihole cat /etc/pihole/pihole.toml" "$local_pihole" normalize_toml || rc_total=1
+    check_file "ntp.conf" "$node" "cat /etc/ntpsec/ntp.conf" "$NTP_FILE" cat || rc_total=1
+  done
+
+  # Technitium zones: binary DZ files, compared by md5 manifest against the
+  # primary (netinfra-01) — the replication source for -02.
+  if [[ "$nodes" == *01* ]]; then
+    remote_manifest="$(fetch 01 "cd $REMOTE_ZONE_DIR && md5sum -- *.zone | sort -k2")"
+    if [ -z "$remote_manifest" ]; then
+      echo "DRIFT-ERROR: technitium zones: live fetch empty (node 01 unreachable?)"
+      rc_total=1
+    else
+      if diff -q <(gen_manifest "$ZONES_DIR") <(printf '%s\n' "$remote_manifest") >/dev/null 2>&1; then
+        echo "OK: technitium zones (node 01) in sync"
+      else
+        echo "DRIFT: technitium zones (node 01) manifest differs from git: $ZONES_DIR"
+        diff <(gen_manifest "$ZONES_DIR") <(printf '%s\n' "$remote_manifest") | head -10
+        rc_total=1
+      fi
+    fi
+  fi
+  if [ "$rc_total" -eq 0 ]; then
+    echo "drift-check: ALL IN SYNC"
+  else
+    echo "drift-check: DRIFT DETECTED — reconcile git <-> live before any change"
+  fi
+  return "$rc_total"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  node="${2:-all}"
+  [ "${1:-}" = "--node" ] && node="${2:-}" || node="all"
+  main "$node"
+fi