diff --git a/scripts/audit-fleet.sh b/scripts/audit-fleet.sh new file mode 100755 index 0000000..4934cc4 --- /dev/null +++ b/scripts/audit-fleet.sh @@ -0,0 +1,146 @@ +#!/usr/bin/bash +# +# audit-fleet.sh — fleet-wide Linux access + naming audit [#446] +# +# For every Linux peer in the tailnet, verify (read-only, changes nothing): +# 1. knel.net DNS name resolves to the peer's Tailscale IP +# 2. Proxmox VM name matches the Tailscale hostname (VMs only) +# 3. SSH login works via the chokepoint scripts with the expected user +# (root on Proxmox, subodev on Pis/Jetson, localuser elsewhere) +# 4. Passwordless root escalation works (sudo -n; root logins pass trivially) +# +# Emits pipe-delimited results on stdout (one line per host), progress on +# stderr, per-host SSH stderr into $1 (default /tmp/fleet-audit-errors.log). +# Example: bash scripts/audit-fleet.sh > /tmp/fleet-audit.tsv +# +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REMOTE="$REPO_ROOT/tests/remote.sh" +REMOTE_DNS="$REPO_ROOT/netinfra/dns-cluster-setup/remote-dns.sh" +DNS_SUFFIX="knel.net" +PROX_NODES=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9) +PROBE='id -un; hostname; sudo -n id -un 2>/dev/null || echo SUDO_FAIL' + +# Canonical knel.net name for Tailscale hostnames that drifted from their +# DNS/VM names. Anything not listed is assumed to match its Tailscale name. +knel_alias() { + case "$1" in + homeassistant) printf '%s' "pfv-bms" ;; + umbrel) printf '%s' "tsys-umbrel" ;; + ultixfield) printf '%s' "ultix-field" ;; + sectestbed-hfnoc) printf '%s' "sectestbed-hfnoc-uisp" ;; + tsys-cloudron-new) printf '%s' "tsys-cloudron" ;; + *) printf '%s' "$1" ;; + esac +} + +# Expected SSH user + chokepoint mode per system class. +# Modes: prox (tests/remote.sh prox, root) | vm (tests/remote.sh vm) +# dns1/dns2 (remote-dns.sh netinfra01/netinfra02) +classify() { + case "$1" in + pfv-tsys1|pfv-tsys3|pfv-tsys4|pfv-tsys5|pfv-tsys6|pfv-tsys7|pfv-tsys9) + printf '%s' "root prox" ;; + pfv-proxmox-backup-server|sectestbed-proxmox-pve) + printf '%s' "root vm" ;; + pfvsvrpi|subopi3|subopi-dev-3|subopi-dev-4|pfv-jetson-nano-1) + printf '%s' "subodev vm" ;; + pfv-netinfra-01) printf '%s' "localuser dns1" ;; + pfv-netinfra-02) printf '%s' "localuser dns2" ;; + *) printf '%s' "localuser vm" ;; + esac +} + +dns_lookup() { + local fqdn="$1" want="$2" got + got="$(getent ahostsv4 "$fqdn" 2>/dev/null | awk 'NR==1 {print $1}')" + if [ -z "$got" ]; then + printf '%s' "NXDOMAIN" + elif [ "$got" = "$want" ]; then + printf '%s' "OK" + else + printf '%s' "WRONG_IP:${got}" + fi +} + +echo "ts_host|ts_ip|online|dns_name|dns_result|vm_name|ssh_user|ssh_result|os_hostname|sudo_result" + +VM_CACHE="$(mktemp)" +TS_CACHE="$(mktemp)" +ERR_LOG="${1:-/tmp/fleet-audit-errors.log}" +: > "$ERR_LOG" +trap 'rm -f "$VM_CACHE" "$TS_CACHE"' EXIT + +for node in "${PROX_NODES[@]}"; do + printf '[cache] qm list %s\n' "$node" >&2 + PROX_HOST="$node" timeout 60 bash "$REMOTE" prox \ + "qm list 2>/dev/null | tail -n +2 | awk '{print \$2}'" \ + > "$VM_CACHE" 2>/dev/null || true +done + +tailscale status --json 2>/dev/null \ + | jq -r '.Peer[] | select(.OS == "linux") | [.HostName, .TailscaleIPs[0], (.Online|tostring)] | @tsv' \ + | sort > "$TS_CACHE" + +total="$(wc -l < "$TS_CACHE")" +n=0 +while IFS=$'\t' read -r host ts_ip online; do + n=$((n + 1)) + printf '[%d/%d] %s\n' "$n" "$total" "$host" >&2 + + knel="$(knel_alias "$host")" + fqdn="${knel}.${DNS_SUFFIX}" + dns_result="$(dns_lookup "$fqdn" "$ts_ip")" + + vm_entry="$(awk -v h="$knel" '$2 == h {print $1; exit}' "$VM_CACHE")" + if [ -n "$vm_entry" ]; then + vm_name="${knel}@${vm_entry}" + elif grep -qw "$host" "$VM_CACHE" 2>/dev/null; then + vm_name="${host}" + else + vm_name="PHYSICAL_OR_MISSING" + fi + + if [ "$online" != "true" ]; then + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$host" "$ts_ip" "$online" "$fqdn" "$dns_result" "$vm_name" "-" "OFFLINE" "-" "-" + continue + fi + + read -r user mode <<< "$(classify "$host")" + + case "$mode" in + prox) + printf '===== %s =====\n' "$host" >> "$ERR_LOG" + out="$(PROX_HOST="$fqdn" timeout 45 bash "$REMOTE" prox "$PROBE" >"$ERR_LOG")" || out="" + ;; + vm) + printf '===== %s =====\n' "$host" >> "$ERR_LOG" + out="$(VM_IP="$fqdn" VM_USER="$user" timeout 45 bash "$REMOTE" vm "$PROBE" >"$ERR_LOG")" || out="" + ;; + dns1) + printf '===== %s =====\n' "$host" >> "$ERR_LOG" + out="$(timeout 45 bash "$REMOTE_DNS" netinfra01 "$PROBE" >"$ERR_LOG")" || out="" + ;; + dns2) + printf '===== %s =====\n' "$host" >> "$ERR_LOG" + out="$(timeout 45 bash "$REMOTE_DNS" netinfra02 "$PROBE" >"$ERR_LOG")" || out="" + ;; + esac + + if [ -n "$out" ]; then + login_user="$(printf '%s\n' "$out" | sed -n '1p')" + os_hostname="$(printf '%s\n' "$out" | sed -n '2p')" + sudo_result="$(printf '%s\n' "$out" | sed -n '3p')" + if [ "$mode" = "prox" ]; then sudo_result="n/a_root_login"; fi + if [ "$login_user" = "$user" ]; then ssh_result="OK"; else ssh_result="AUTH_MISMATCH:${login_user:-none}"; fi + else + os_hostname="-"; sudo_result="-"; ssh_result="UNREACHABLE_OR_AUTH_FAIL" + fi + + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$host" "$ts_ip" "$online" "$fqdn" "$dns_result" "$vm_name" "$user" "$ssh_result" "$os_hostname" "$sudo_result" +done < "$TS_CACHE"