This commit is contained in:
@@ -288,6 +288,19 @@ Hard-won operational gotchas. One line each; keep them short and load-bearing.
|
||||
- **Supervisor add-on options via API:** docker cp a script into
|
||||
hassio_supervisor, read token from /data/cli.json `access_token`, POST
|
||||
/addons/<slug>/options (the /apps/ spelling 404s).
|
||||
- **NEVER restart/kill BOTH members of a redundant pair simultaneously**
|
||||
(founder ruling 2026-09-01 after the DNS outage): restart/redeploy ONE
|
||||
node, verify service health from an independent vantage, only then the
|
||||
second. Blue/green with a health gate between hops — always serial.
|
||||
- **Pi-hole v6: `dnsmasq_lines` in pihole.toml are passed to FTL's embedded
|
||||
dnsmasq, which rejects some valid-dnsmasq options (e.g. `no-negcache`)
|
||||
with "bad option" and dnsmasq then never starts — the container looks
|
||||
"Up" but serves nothing on :53. Use first-class toml settings instead;
|
||||
never sed-edit pihole.toml without a single-node health-gated rollout.
|
||||
- **DNS outage recovery path:** workstation resolv.conf dies with the LAN
|
||||
Pi-holes; use `tailscale status` peer IPs + the chokepoint env overrides
|
||||
(NETINFRA01_HOST/NETINFRA02_HOST, VM_IP) to reach hosts by Tailscale
|
||||
while names are unresolvable. Fix one node, verify, then the other.
|
||||
|
||||
## Questions (NON-NEGOTIABLE)
|
||||
|
||||
|
||||
@@ -19,3 +19,6 @@ replies 46-48, and git PFVCluster + KNEL/pfv-bms)
|
||||
- [x] HomeKit VLAN-segmentation finding + vNIC proposal posted [#619]
|
||||
- [x] BTU watts source decision: Emporia CT Friday (APC lacks load vars) [#614]
|
||||
- [x] Roadmap ticketed #618-#628; backfill parked per founder [#622]
|
||||
- [x] DNS incident recovered (serial, health-gated); both Pi-holes serving fresh [#469]
|
||||
- [x] AGENTS.md: never dual-kill redundant pairs + blue/green rule + FTL dnsmasq_lines trap
|
||||
- [x] Git SoR: pihole.toml (both nodes, secrets redacted) + drift-check.sh (TDD green) — live==git verified [#420][#469]
|
||||
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/bash
|
||||
#
|
||||
# drift-check.sh — compare live netinfra DNS/DHCP config against git [#420][#469]
|
||||
#
|
||||
# Verifies that the running configuration on netinfra-01/02 matches the
|
||||
# files tracked in this repo. Founded after the 2026-09-01 DNS incident:
|
||||
# the live systems are production; git is the source of truth; drift is
|
||||
# a defect.
|
||||
#
|
||||
# Checks per node:
|
||||
# - /etc/dhcp/dhcpd.conf vs netinfra/dhcp/dhcpd-{primary,secondary}.conf
|
||||
# - /etc/pihole/pihole.toml vs netinfra/dns/pihole/netinfra-0{1,2}.pihole.toml
|
||||
# (secrets redacted on both sides before compare; "Last updated" line ignored)
|
||||
#
|
||||
# Usage:
|
||||
# drift-check.sh [--node 01|02|all] (default: all)
|
||||
# Exit: 0 = in sync, 1 = drift detected, 2 = fetch failure
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DNS_SETUP="$HERE/dns-cluster-setup"
|
||||
DHCP_DIR="$HERE/dhcp"
|
||||
PIHOLE_DIR="$HERE/dns/pihole"
|
||||
|
||||
REDACT='s/^( *pwhash *=).*/\1 "REDACTED"/; s/^( *totp_secret *=).*/\1 "REDACTED"/; s/^( *password *=).*/\1 "REDACTED"/'
|
||||
|
||||
# redact_config <stdin> <stdout> — strip secrets from pihole.toml content
|
||||
redact_config() { sed -E "$REDACT"; }
|
||||
|
||||
# normalize_toml <stdin> <stdout> — redact + drop churn lines (timestamps)
|
||||
normalize_toml() { redact_config | grep -v "Last updated on"; }
|
||||
|
||||
# fetch <node> <remote-cmd> — run via the remote-dns.sh chokepoint (env IPs honored)
|
||||
fetch() {
|
||||
local node="$1" cmd="$2"
|
||||
bash "$DNS_SETUP/remote-dns.sh" "netinfra${node}-root" "$cmd" 2>/dev/null
|
||||
}
|
||||
|
||||
# check_file <label> <node> <remote-cat-cmd> <local-file> <normalize-fn>
|
||||
check_file() {
|
||||
local label="$1" node="$2" rcmd="$3" local_file="$4" norm="$5" tmp rc
|
||||
tmp="$(mktemp)"
|
||||
fetch "$node" "$rcmd" | "$norm" > "$tmp"
|
||||
if [ ! -s "$tmp" ]; then
|
||||
echo "DRIFT-ERROR: $label: live fetch empty (node $node unreachable?)"
|
||||
rm -f "$tmp"
|
||||
return 2
|
||||
fi
|
||||
"$norm" < "$local_file" | diff -q - "$tmp" >/dev/null 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "DRIFT: $label (node $node) differs from git: $local_file"
|
||||
"$norm" < "$local_file" | diff - "$tmp" | head -10
|
||||
else
|
||||
echo "OK: $label (node $node) in sync"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
main() {
|
||||
local nodes="${1:-all}" node rc_total=0 rc
|
||||
[ "$nodes" = "all" ] && nodes="01 02"
|
||||
for node in $nodes; do
|
||||
local_dhcp="$DHCP_DIR/dhcpd-primary.conf"
|
||||
[ "$node" = "02" ] && local_dhcp="$DHCP_DIR/dhcpd-secondary.conf"
|
||||
local_pihole="$PIHOLE_DIR/netinfra-01.pihole.toml"
|
||||
[ "$node" = "02" ] && local_pihole="$PIHOLE_DIR/netinfra-02.pihole.toml"
|
||||
|
||||
check_file "dhcpd.conf" "$node" "cat /etc/dhcp/dhcpd.conf" "$local_dhcp" cat || rc_total=1
|
||||
check_file "pihole.toml" "$node" \
|
||||
"docker exec pihole cat /etc/pihole/pihole.toml" "$local_pihole" normalize_toml || rc_total=1
|
||||
done
|
||||
if [ "$rc_total" -eq 0 ]; then
|
||||
echo "drift-check: ALL IN SYNC"
|
||||
else
|
||||
echo "drift-check: DRIFT DETECTED — reconcile git <-> live before any change"
|
||||
fi
|
||||
return "$rc_total"
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
node="${2:-all}"
|
||||
[ "${1:-}" = "--node" ] && node="${2:-}" || node="all"
|
||||
main "$node"
|
||||
fi
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/bin/bash
|
||||
# Unit tests for netinfra/dns/drift-check.sh [#420][#469]
|
||||
# Tests the redaction and normalization functions without touching hosts.
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
SCRIPT="$PROJECT_ROOT/netinfra/dns/drift-check.sh"
|
||||
|
||||
failed=0
|
||||
|
||||
if [[ -f "$SCRIPT" ]] && (source "$SCRIPT"); then
|
||||
echo "✅ script exists + sources cleanly"
|
||||
else
|
||||
echo "❌ script missing or fails to source: $SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT"
|
||||
|
||||
test_redact_strips_secrets() {
|
||||
local out
|
||||
out=$(printf ' pwhash = "SECRETBALLOONHASH"\n totp_secret = "ABC234"\n password = "hunter2"\n port = 53\n' | redact_config)
|
||||
if grep -qE "SECRETBALLOON|ABC234|hunter2" <<<"$out"; then
|
||||
echo "❌ redact_config leaked a secret"
|
||||
((++failed))
|
||||
elif grep -q '"REDACTED"' <<<"$out" && grep -q "port = 53" <<<"$out"; then
|
||||
echo "✅ redact_config strips secrets, keeps config"
|
||||
else
|
||||
echo "❌ redact_config mangled output"
|
||||
((++failed))
|
||||
fi
|
||||
}
|
||||
|
||||
test_normalize_drops_timestamp() {
|
||||
local out
|
||||
out=$(printf '# Last updated on 2026-09-01 17:01:15 CDT\n dnsmasq_lines = []\n' | normalize_toml)
|
||||
if grep -q "Last updated" <<<"$out"; then
|
||||
echo "❌ normalize_toml kept the timestamp churn line"
|
||||
((++failed))
|
||||
elif grep -q "dnsmasq_lines" <<<"$out"; then
|
||||
echo "✅ normalize_toml drops timestamp, keeps config"
|
||||
else
|
||||
echo "❌ normalize_toml dropped config lines"
|
||||
((++failed))
|
||||
fi
|
||||
}
|
||||
|
||||
test_normalize_implies_redaction() {
|
||||
local out
|
||||
out=$(printf ' totp_secret = "XYZ789"\n a = 1\n' | normalize_toml)
|
||||
if grep -q XYZ789 <<<"$out"; then
|
||||
echo "❌ normalize_toml leaked a secret"
|
||||
((++failed))
|
||||
else
|
||||
echo "✅ normalize_toml redacts secrets"
|
||||
fi
|
||||
}
|
||||
|
||||
test_redact_strips_secrets
|
||||
test_normalize_drops_timestamp
|
||||
test_normalize_implies_redaction
|
||||
|
||||
if ((failed > 0)); then
|
||||
echo "❌ $failed drift-check test(s) failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ all drift-check tests passed"
|
||||
Reference in New Issue
Block a user