PFVCluster/scripts/audit-fleet.sh

#!/usr/bin/bash
#
# audit-fleet.sh — fleet-wide Linux access + naming audit [#446]
#
# For every Linux peer in the tailnet, verify (read-only, changes nothing):
#   1. knel.net DNS name resolves to the peer's Tailscale IP
#   2. Proxmox VM name matches the Tailscale hostname (VMs only)
#   3. SSH login works via the chokepoint scripts with the expected user
#      (root on Proxmox, subodev on Pis/Jetson, localuser elsewhere)
#   4. Passwordless root escalation works (sudo -n; root logins pass trivially)
#
# Emits pipe-delimited results on stdout (one line per host), progress on
# stderr, per-host SSH stderr into $1 (default /tmp/fleet-audit-errors.log).
# Scope: tsys-cloudron-new (Reston prod VPS) is excluded entirely;
# homeassistant/umbrel get DNS/name checks but no SSH probe (appliance OSes).
# Example: bash scripts/audit-fleet.sh > /tmp/fleet-audit.tsv
#
set -uo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REMOTE="$REPO_ROOT/tests/remote.sh"
REMOTE_DNS="$HOME/projects/KNEL/netinfra/dns-cluster-setup/remote-dns.sh"  # moved to KNEL/netinfra [#769]
DNS_SUFFIX="knel.net"
DNS_SERVER="pfv-netinfra-01.knel.net"
PROX_NODES=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9)
PROBE='id -un; hostname; sudo -n id -un 2>/dev/null || echo SUDO_FAIL'

# Canonical knel.net name for Tailscale hostnames that drifted from their
# DNS/VM names. Anything not listed is assumed to match its Tailscale name.
knel_alias() {
    case "$1" in
        *)                 printf '%s' "$1" ;;
    esac
}

# Expected SSH user + chokepoint mode per system class.
# Modes: prox (tests/remote.sh prox, root) | vm (tests/remote.sh vm)
#        dns1/dns2 (remote-dns.sh netinfra01/netinfra02)
classify() {
    case "$1" in
        pfv-tsys1|pfv-tsys3|pfv-tsys4|pfv-tsys5|pfv-tsys6|pfv-tsys7|pfv-tsys9)
            printf '%s' "root prox" ;;
        pfv-proxmox-backup-server|sectestbed-proxmox-pve)
            printf '%s' "root vm" ;;
        *-proxmox-datacenter|*-proxmox-mailgw*|*-proxmox-pbs)
            printf '%s' "root vm" ;;
        stlpc-*)
            printf '%s' "root vm" ;;
        tsys-ucs-01|tsys-ucs-02)
            printf '%s' "root vm" ;;
        ultix-field)
            printf '%s' "ultixfield vm" ;;
        pfvsvrpi)
            printf '%s' "localuser vm" ;;
        subopi3|subopi-dev-3|subopi-dev-4)
            printf '%s' "subodev vm" ;;
        pfv-netinfra-01) printf '%s' "localuser dns1" ;;
        pfv-netinfra-02) printf '%s' "localuser dns2" ;;
        *)               printf '%s' "localuser vm" ;;
    esac
}

dns_lookup() {
    local fqdn="$1" want="$2" got
    got="$(dig +short "@$DNS_SERVER" "$fqdn" A 2>/dev/null | tail -n 1)"
    if [ -z "$got" ]; then
        printf '%s' "NXDOMAIN"
    elif [ "$got" = "$want" ]; then
        printf '%s' "OK"
    else
        printf '%s' "WRONG_IP:${got}"
    fi
}

echo "ts_host|ts_ip|online|dns_name|dns_result|vm_name|ssh_user|ssh_result|os_hostname|sudo_result"

VM_CACHE="$(mktemp)"
TS_CACHE="$(mktemp)"
ERR_LOG="${1:-/tmp/fleet-audit-errors.log}"
: > "$ERR_LOG"
trap 'rm -f "$VM_CACHE" "$TS_CACHE"' EXIT

for node in "${PROX_NODES[@]}"; do
    printf '[cache] qm list %s\n' "$node" >&2
    PROX_HOST="$node" timeout 60 bash "$REMOTE" prox \
        "qm list 2>/dev/null | tail -n +2 | awk '{print \$2}'" \
        </dev/null \
        | grep -vE '^(KNELTemplate|sectestbed-template|RestoreTemplate)$' \
        | sed "s/^/${node} /" >> "$VM_CACHE" 2>/dev/null || true
done

tailscale status --json 2>/dev/null \
    | jq -r '.Peer[] | select(.OS == "linux") | [(.DNSName | split(".")[0]), .TailscaleIPs[0], (.Online|tostring)] | @tsv' \
    | sort > "$TS_CACHE"

total="$(wc -l < "$TS_CACHE")"
n=0
while IFS=$'\t' read -r host ts_ip online; do
    n=$((n + 1))
    printf '[%d/%d] %s\n' "$n" "$total" "$host" >&2

    knel="$(knel_alias "$host")"
    fqdn="${knel}.${DNS_SUFFIX}"
    dns_result="$(dns_lookup "$fqdn" "$ts_ip")"

    vm_entry="$(awk -v h="$knel" '$2 == h {print $1; exit}' "$VM_CACHE")"
    if [ -n "$vm_entry" ]; then
        vm_name="${knel}@${vm_entry}"
    elif grep -qw "$host" "$VM_CACHE" 2>/dev/null; then
        vm_name="${host}"
    else
        vm_name="PHYSICAL_OR_MISSING"
    fi

    case "$host" in
        tsys-cloudron)     continue ;;  # Reston prod VPS (OS hostname tsys-cloudron-new) — out of scope
        netbird)           continue ;;  # Reston VPS (NetBird controller) — name-checked only
        stlp-3dscanner)   continue ;;   # offline ~7mo; user pulled from audit scope
        sectestbed-sandbox) continue ;; # disposable test VM — frequently broken by design
    esac

    if [ "$online" != "true" ]; then
        printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
            "$host" "$ts_ip" "$online" "$fqdn" "$dns_result" "$vm_name" "-" "OFFLINE" "-" "-"
        continue
    fi

    case "$host" in
        pfv-bms|tsys-umbrel)
            printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
                "$host" "$ts_ip" "$online" "$fqdn" "$dns_result" "$vm_name" "-" "BY_DESIGN_NO_SSH" "-" "-"
            continue ;;
    esac

    read -r user mode <<< "$(classify "$host")"

    case "$mode" in
        prox)
            printf '===== %s =====\n' "$host" >> "$ERR_LOG"
            out="$(PROX_HOST="$fqdn" timeout 45 bash "$REMOTE" prox "$PROBE" </dev/null 2>>"$ERR_LOG")" || out=""
            ;;
        vm)
            printf '===== %s =====\n' "$host" >> "$ERR_LOG"
            out="$(VM_IP="$fqdn" VM_USER="$user" timeout 45 bash "$REMOTE" vm "$PROBE" </dev/null 2>>"$ERR_LOG")" || out=""
            ;;
        dns1)
            printf '===== %s =====\n' "$host" >> "$ERR_LOG"
            out="$(timeout 45 bash "$REMOTE_DNS" netinfra01 "$PROBE" </dev/null 2>>"$ERR_LOG")" || out=""
            ;;
        dns2)
            printf '===== %s =====\n' "$host" >> "$ERR_LOG"
            out="$(timeout 45 bash "$REMOTE_DNS" netinfra02 "$PROBE" </dev/null 2>>"$ERR_LOG")" || out=""
            ;;
    esac

    if [ -n "$out" ]; then
        login_user="$(printf '%s\n' "$out" | sed -n '1p')"
        os_hostname="$(printf '%s\n' "$out" | sed -n '2p')"
        sudo_result="$(printf '%s\n' "$out" | sed -n '3p')"
        if [ "$mode" = "prox" ]; then sudo_result="n/a_root_login"; fi
        if [ "$login_user" = "$user" ]; then ssh_result="OK"; else ssh_result="AUTH_MISMATCH:${login_user:-none}"; fi
    else
        os_hostname="-"; sudo_result="-"; ssh_result="UNREACHABLE_OR_AUTH_FAIL"
    fi

    printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
        "$host" "$ts_ip" "$online" "$fqdn" "$dns_result" "$vm_name" "$user" "$ssh_result" "$os_hostname" "$sudo_result"
done < "$TS_CACHE"