feat(network): use pfv-netinfra-01/02 as redundant DNS and NTP

Route every host built by this project through the new
pfv-netinfra-01 (192.168.3.252) / pfv-netinfra-02 (192.168.3.253)
pair for both name resolution and time, with automatic failover.

- NTP: replace the single pfv-netboot.knel.net upstream with both
  netinfra servers (iburst) so time sync survives either one failing.
- DNS: add a managed static /etc/resolv.conf (new ConfigFiles/Resolv/).
  The repo previously had no resolver configuration at all. Both servers
  are listed so glibc falls through to the secondary on failure.
- DHCP: request domain-name-servers/domain-search/ntp-servers and
  supersede them to the netinfra pair, so a DHCP renew can't silently
  revert to whatever the DHCP server advertises.
- SetupNewSystem.sh: deploy resolv.conf (robustly replacing any
  systemd-resolved/NetworkManager symlink) and add pfv-netinfra to the
  NTP-server self-exclusion guard so those boxes don't client off
  themselves.

LAN IPs are used throughout (not the knel.net hostnames) because those
hostnames resolve to Tailscale CGNAT addresses, not the LAN addresses,
and NTP must come up before DNS. Add a validation test asserting the
config is present and both servers actually answer DNS and NTP queries.

🤖 Generated with [Crush](https://github.com/charmassociates/crush)

Assisted-by: GLM-5 via Crush <crush@charm.land>

Project-Tests/validation/dns-ntp-redundancy.sh

@@ -0,0 +1,202 @@
+#!/bin/bash
+
+# Redundant DNS/NTP Validation Test
+# Validates that the host is configured to use the redundant pfv-netinfra-01/02
+# pair for name resolution and time, and that both servers actually answer.
+
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+# The authoritative pair (pfv-netinfra-01 / pfv-netinfra-02).
+DNS_PRIMARY="192.168.3.252"
+DNS_SECONDARY="192.168.3.253"
+NTP_PRIMARY="192.168.3.252"
+NTP_SECONDARY="192.168.3.253"
+
+RESOLV_CONF="/etc/resolv.conf"
+NTP_CONF="/etc/ntpsec/ntp.conf"
+
+# A name every recursive resolver must be able to resolve.
+DNS_PROBE_NAME="github.com"
+
+failed=0
+have() { command -v "$1" >/dev/null 2>&1; }
+
+# --- Configuration assertions -------------------------------------------------
+
+function test_dns_config_present() {
+    echo "🔍 Checking $RESOLV_CONF ..."
+    local problems=0
+
+    if [[ -L "$RESOLV_CONF" ]]; then
+        echo "❌ $RESOLV_CONF is a symlink (would be overwritten by a resolver manager)"
+        ((++problems))
+    elif [[ ! -f "$RESOLV_CONF" ]]; then
+        echo "❌ $RESOLV_CONF missing"
+        ((++problems))
+    fi
+
+    for ns in "$DNS_PRIMARY" "$DNS_SECONDARY"; do
+        if grep -Eq "^[[:space:]]*nameserver[[:space:]]+$ns" "$RESOLV_CONF" 2>/dev/null; then
+            echo "✅ nameserver $ns configured"
+        else
+            echo "❌ nameserver $ns NOT in $RESOLV_CONF"
+            ((++problems))
+        fi
+    done
+
+    return $problems
+}
+
+function test_ntp_config_present() {
+    echo "🔍 Checking $NTP_CONF ..."
+    if [[ ! -f "$NTP_CONF" ]]; then
+        echo "❌ $NTP_CONF missing (is ntpsec installed?)"
+        return 1
+    fi
+    local problems=0
+    for s in "$NTP_PRIMARY" "$NTP_SECONDARY"; do
+        if grep -Eq "^[[:space:]]*(server|pool)[[:space:]]+$s" "$NTP_CONF"; then
+            echo "✅ NTP server $s configured"
+        else
+            echo "❌ NTP server $s NOT in $NTP_CONF"
+            ((++problems))
+        fi
+    done
+    return $problems
+}
+
+# --- Functional assertions: each server actually answers ----------------------
+
+function _dns_resolves() {
+    # $1 = server ip. Returns 0 if it resolves DNS_PROBE_NAME.
+    local server="$1"
+    if have dig; then
+        dig @"$server" +short +time=4 +tries=1 "$DNS_PROBE_NAME" A 2>/dev/null | grep -Eq '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'
+    elif have nslookup; then
+        nslookup "$DNS_PROBE_NAME" "$server" 2>/dev/null | grep -Eq 'Address:[[:space:]]*[0-9]'
+    elif have host; then
+        host "$DNS_PROBE_NAME" "$server" 2>/dev/null | grep -Eq 'has address'
+    else
+        # Last resort: the resolver itself.
+        getent ahostsv4 "$DNS_PROBE_NAME" >/dev/null 2>&1
+    fi
+}
+
+function test_dns_servers_answer() {
+    echo "🔍 Probing DNS servers ..."
+    local problems=0
+    for ns in "$DNS_PRIMARY" "$DNS_SECONDARY"; do
+        if _dns_resolves "$ns"; then
+            echo "✅ $ns resolves $DNS_PROBE_NAME"
+        else
+            echo "❌ $ns did not resolve $DNS_PROBE_NAME"
+            ((++problems))
+        fi
+    done
+    return $problems
+}
+
+function _ntp_answers() {
+    # $1 = server ip. Returns 0 if it responds to a time query.
+    local server="$1"
+    if have ntpdate; then
+        timeout 8 ntpdate -q "$server" 2>/dev/null | grep -Eq 'no-leap|leap'
+    elif have sntp; then
+        timeout 8 sntp -t 4 "$server" >/dev/null 2>&1
+    elif have chronyc; then
+        # NTS/chrony not expected here, but be tolerant.
+        chronyc -n -h "$server" tracking >/dev/null 2>&1
+    else
+        return 2  # cannot test
+    fi
+}
+
+function test_ntp_servers_answer() {
+    echo "🔍 Probing NTP servers ..."
+    local problems=0
+    for s in "$NTP_PRIMARY" "$NTP_SECONDARY"; do
+        if _ntp_answers "$s"; then
+            echo "✅ NTP $s responds to time query"
+        else
+            echo "❌ $s did not respond to NTP query"
+            ((++problems))
+        fi
+    done
+    return $problems
+}
+
+# --- End-to-end: the host is actually USING the pair --------------------------
+
+function test_resolver_endtoend() {
+    echo "🔍 End-to-end resolution via $RESOLV_CONF ..."
+    if getent ahostsv4 "$DNS_PROBE_NAME" >/dev/null 2>&1; then
+        echo "✅ Host resolves $DNS_PROBE_NAME via configured resolver"
+        return 0
+    else
+        echo "❌ Host cannot resolve $DNS_PROBE_NAME via configured resolver"
+        return 1
+    fi
+}
+
+function test_ntp_daemon_peers() {
+    echo "🔍 NTP daemon peer list ..."
+    local peers
+    if have ntpq; then
+        peers="$(ntpq -pn 2>/dev/null || true)"
+    elif have chronyc; then
+        peers="$(chronyc -n sources 2>/dev/null || true)"
+    else
+        echo "⚠️  No ntpq/chronyc available; skipping daemon peer check"
+        return 0
+    fi
+
+    local problems=0
+    for s in "$NTP_PRIMARY" "$NTP_SECONDARY"; do
+        if echo "$peers" | grep -Eq "^\\s*${s//./\\.}"; then
+            echo "✅ NTP daemon has peer $s"
+        else
+            echo "❌ NTP daemon is NOT tracking $s"
+            ((++problems))
+        fi
+    done
+
+    # Sync status is informational only: a freshly started daemon needs several
+    # polls before the reach counter stabilises, so we warn rather than fail.
+    if echo "$peers" | grep -Eq '\*'; then
+        echo "✅ NTP daemon reports a synced peer"
+    else
+        echo "⚠️  NTP daemon not yet synced (normal for a few minutes after restart)"
+    fi
+    return $problems
+}
+
+# --- Main ---------------------------------------------------------------------
+
+function main() {
+    echo "🛰️  Running Redundant DNS/NTP Validation Tests"
+    echo "================================================"
+
+    local total_failures=0
+
+    test_dns_config_present        || ((++total_failures))
+    test_ntp_config_present        || ((++total_failures))
+    test_dns_servers_answer        || ((++total_failures))
+    test_ntp_servers_answer        || ((++total_failures))
+    test_resolver_endtoend         || ((++total_failures))
+    test_ntp_daemon_peers          || ((++total_failures))
+
+    echo "================================================"
+    if [[ $total_failures -eq 0 ]]; then
+        echo "✅ All redundant DNS/NTP validation tests passed"
+        exit 0
+    else
+        echo "❌ $total_failures redundant DNS/NTP tests failed"
+        exit 1
+    fi
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main "$@"
+fi

ProjectCode/ConfigFiles/DHCP/dhclient.conf

@@ -3,4 +3,12 @@ option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
 send host-name = gethostname();
 request subnet-mask, broadcast-address, time-offset, routers,
         domain-name, host-name,
+        domain-name-servers, domain-search, ntp-servers,
         rfc3442-classless-static-routes;
+
+# Pin DNS and NTP to the redundant pfv-netinfra-01/02 pair regardless of what
+# the DHCP server advertises, so every host on this build uses the same
+# authoritative recursive resolvers and time sources.
+supersede domain-name-servers 192.168.3.252, 192.168.3.253;
+supersede domain-search "knel.net";
+supersede ntp-servers 192.168.3.252, 192.168.3.253;

ProjectCode/ConfigFiles/NTP/ntp.conf

@@ -1,6 +1,13 @@
 driftfile /var/lib/ntp/ntp.drift
 leapfile /usr/share/zoneinfo/leap-seconds.list
-server pfv-netboot.knel.net
+
+# Redundant upstream time sources: pfv-netinfra-01/02 (Technitium/Pi-hole hosts
+# also serving NTP). IPs are used (not hostnames) because the knel.net name for
+# these hosts resolves to a Tailscale CGNAT address, not the LAN address, and
+# because NTP must come up before DNS is available. iburst speeds initial sync.
+server 192.168.3.252 iburst
+server 192.168.3.253 iburst
+
 restrict 127.0.0.1
 restrict ::1
 interface ignore wildcard

ProjectCode/ConfigFiles/Resolv/resolv.conf

@@ -0,0 +1,11 @@
+# Managed by KNELServerBuild — do not edit; changes will be overwritten.
+#
+# Redundant recursive DNS via pfv-netinfra-01/02 (Technitium + Pi-hole).
+# IPs are used (required: nameserver directives must be addresses, and the
+# knel.net name for these hosts resolves to a Tailscale CGNAT address rather
+# than the LAN address). If the primary is unreachable, glibc's resolver
+# automatically falls through to the secondary.
+domain knel.net
+search knel.net
+nameserver 192.168.3.252
+nameserver 192.168.3.253

ProjectCode/SetupNewSystem.sh

@@ -267,6 +267,14 @@ function global-postPackageConfiguration() {
 
   cat "$CONFIGFILES_PATH/DHCP/dhclient.conf" >/etc/dhcp/dhclient.conf
 
+  # Authoritative recursive DNS via the redundant pfv-netinfra-01/02 pair.
+  # Replace whatever is at /etc/resolv.conf (including a systemd-resolved or
+  # NetworkManager symlink) with the managed static file so every lookup goes
+  # to our servers and nothing else rewrites it behind our backs.
+  rm -f /etc/resolv.conf
+  cat "$CONFIGFILES_PATH/Resolv/resolv.conf" >/etc/resolv.conf
+  chmod 644 /etc/resolv.conf
+
   systemctl stop snmpd && /etc/init.d/snmpd stop
 
   cat "$CONFIGFILES_PATH/SNMP/snmp-sudo.conf" >/etc/sudoers.d/Debian-snmp
@@ -304,7 +312,7 @@ function global-postPackageConfiguration() {
   fi
 
   export NTP_SERVER_CHECK
-  NTP_SERVER_CHECK="$(hostname | egrep -c 'pfv-netboot|pfvsvrpi' || true)"
+  NTP_SERVER_CHECK="$(hostname | egrep -c 'pfv-netboot|pfvsvrpi|pfv-netinfra' || true)"
 
   if [ "$NTP_SERVER_CHECK" -eq 0 ]; then