#!/bin/bash
###############################################################################
# check.sh
#
# READ-ONLY, comprehensive Proxmox host assessment.
#   - Installs NOTHING. Modifies NOTHING. Writes only its own log.
#   - Detects what is/is isn't installed so the next step can be informed.
#   - Runtime ~30-60 seconds on a busy node.
#
# Output:  /root/<short-hostname>.log
#
# Usage (as root):
#     bash check.sh
###############################################################################
set -u
umask 077

HOST="$(hostname -s)"
OUT="/root/${HOST}.log"
: > "$OUT"
TS="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
START_EPOCH="$(date +%s)"

# ===========================================================================
# Helpers (pattern from project memory)
# ===========================================================================
blank(){ printf '\n' >> "$OUT"; }

hdr() {
    {
        printf '\n'
        printf '###############################################################################\n'
        printf '# %s\n' "$1"
        printf '###############################################################################\n'
    } >> "$OUT"
}

# cmd LABEL CMD [ARGS...] - simple commands only (no pipes/loops/&&)
cmd() {
    local label="$1"; shift
    {
        printf '\n--- %s\n' "$label"
        printf '    $ %s\n' "$*"
    } >> "$OUT"
    "$@" >> "$OUT" 2>&1 || printf '    [exit=%s / not installed or not permitted]\n' "$?" >> "$OUT"
}

# cmdsh LABEL <<'EOF' ... EOF - arbitrary shell snippet (pipes, loops OK).
# Heredoc body MUST use quoted 'EOF' so $vars pass through to eval literally;
# export any outer-scope vars the body needs before the call.
# shellcheck disable=SC2310
cmdsh() {
    local label="$1" snippet
    snippet=$(cat)
    {
        printf '\n--- %s\n' "$label"
        printf '    $ %s\n' "$snippet"
    } >> "$OUT"
    eval "$snippet" >> "$OUT" 2>&1 || printf '    [exit=%s]\n' "$?" >> "$OUT"
}

# filecat LABEL PATH [PATH...] - dump file contents (cap 300 lines each)
filecat() {
    local label="$1"; shift
    printf '\n--- %s\n' "$label" >> "$OUT"
    local f
    for f in "$@"; do
        if [ -r "$f" ]; then
            printf '    --- %s ---\n' "$f" >> "$OUT"
            sed -n '1,300p' "$f" >> "$OUT"
        else
            printf '    [missing: %s]\n' "$f" >> "$OUT"
        fi
    done
}

# dirlist LABEL PATH - list directory contents (existence + sizes)
dirlist() {
    local label="$1" d="$2"
    printf '\n--- %s\n' "$label" >> "$OUT"
    if [ -d "$d" ]; then
        ls -la "$d" >> "$OUT" 2>&1
    else
        printf '    [missing: %s]\n' "$d" >> "$OUT"
    fi
}

# ===========================================================================
# Header
# ===========================================================================
{
    printf '###############################################################################\n'
    printf '# Proxmox VE host performance assessment  (READ-ONLY)\n'
    printf '# Host:      %s (%s)\n' "$HOST" "$(hostname -f 2>/dev/null)"
    printf '# FQDN:      %s\n' "$(hostname -f 2>/dev/null)"
    printf '# All IPs:   %s\n' "$(hostname -I 2>/dev/null)"
    printf '# Collected: %s (UTC)\n' "$TS"
    printf '# Kernel:    %s\n' "$(uname -r)"
    printf '# Machine:   %s %s\n' "$(uname -m)" "$(uname -o)"
    printf '# Caller:    %s (uid=%s)\n' "$(whoami)" "$(id -u)"
    printf '# Script:    check.sh  -  strictly READ-ONLY\n'
    printf '###############################################################################\n'
} >> "$OUT"

# ===========================================================================
# 1. TOOL AVAILABILITY MATRIX
#    (most important section: tells us what is/isn't installed)
# ===========================================================================
hdr "1. TOOL AVAILABILITY MATRIX"

cmdsh "command presence + path + version" <<'EOF'
# Each line: name | present? | path | version
printf '%-22s %-8s %-30s %s\n' "TOOL" "PRESENT" "PATH" "VERSION/NOTE"
printf '%-22s %-8s %-30s %s\n' "----" "-------" "----" "------------"
for t in \
    pveversion pvecm qm pct pvesm pvesh pveperf proxmox-backup-client \
    ip ss ethtool bridge brctl nmcli \
    tcpdump mtr nstat netstat nicstat ifstat \
    dmidecode lscpu lshw hwinfo inxi lspci lsusb numactl numastat \
    cpupower turbostat x86_energy_perf_policy rdmsr \
    tuned-adm irqbalance \
    zpool zfs zdb arcstat arc_summary \
    lsblk blkid blockdev findmnt smartctl nvme hdparm \
    storcli storcli64 MegaCli MegaCli64 tw-cli arcconf \
    sas2ircu sas3ircu hpssacli ssacli perccli \
    multipath pvs vgs lvs lvdisplay \
    xfs_info dumpe2fs btrfs \
    fio iperf iperf3 sysbench bonnie++ iozone \
    iostat mpstat pidstat sar vmstat \
    top htop atop glances dstat nethogs iftop iotop smem pmap slabtop \
    perf strace lsof fatrace \
    dig host nslookup \
    chronyc ntpq timedatectl \
    nft iptables iptables-save ipset ufw \
    jq bc awk smartctl nvme \
    apt dpkg
do
    p="$(command -v "$t" 2>/dev/null)"
    if [ -n "$p" ]; then
        v=""
        case "$t" in
            zpool|zfs)        v="$(timeout 3 "$t" version 2>/dev/null | head -n 2 | tr '\n' ' ')" ;;
            iperf3|fio|sysbench|smartctl|nvme|tuned-adm|ethtool|cpupower)
                v="$(timeout 3 "$t" --version 2>&1 | head -n 1)" ;;
            pveversion)       v="$(timeout 3 "$t" 2>/dev/null)" ;;
            storcli64|storcli|MegaCli64|MegaCli|arcconf|sas2ircu|sas3ircu|ssacli|hpssacli)
                v="(storage HBA CLI - present)" ;;
        esac
        printf '%-22s %-8s %-30s %s\n' "$t" "yes" "$p" "$v"
    else
        printf '%-22s %-8s %-30s %s\n' "$t" "no" "-" "-"
    fi
done
EOF

cmdsh "Debian packages relevant to perf/storage/net" <<'EOF'
dpkg -l 2>/dev/null | awk '/^ii/ && ($2 ~ /^(proxmox-|pve-|zfs|zfsutils|nfs|nfs-kernel-server|nfs-common|open-iscsi|ceph|tuned|irqbalance|sysstat|smartmontools|nvme-cli|ethtool|bridge-utils|ifenslave|fio|iperf3|sysbench|glances|htop|atop|lm-sensors|numactl|cpufrequtils|linux-image-|pve-kernel|qemu|libvirt|corosync|pacemaker)/) {print $2"  "$3}' | head -n 80
EOF

cmdsh "kernel modules available (relevant to perf)" <<'EOF'
for m in kvm_intel kvm_amd \
         ixgbe i40e i40iw ice igb igc mlx4_en mlx5_core mlx4_core bnx2x bnx2 tg3 be2net qed qede virtio_net \
         nvme nvme-tcp nvme-fabrics sd_mod sr_mod \
         zfs spl znvpair zcommon zunicode zavl icp zlua \
         megaraid_sas mpt3sas hpsa aacraid smartpqi isci \
         dm_multipath multipath \
         tcp_dctcp tcp_bbr tcp_cubic tcp_htcp \
         8021q bonding bridge br_netfilter \
         vhost_net vhost_scsi target_core_mod \
         nfsv4 nfsv3 rpcsec_gss_krb5
do
    modinfo "$m" >/dev/null 2>&1 && printf 'available : %s (%s)\n' "$m" "$(modinfo -F version "$m" 2>/dev/null || echo built-in)"
done
EOF

cmdsh "currently loaded modules (filtered)" <<'EOF'
lsmod | awk 'NR==1 || $1 ~ /^(kvm|virtio|vhost|ixgb|i40e|ice|igb|mlx|bnx|tg3|be2|qed|nvme|zfs|spl|z|dm_|megaraid|mpt|hpsa|aacraid|smartpqi|8021q|bonding|bridge|nfsv|tcp_|rpc|sunrpc|loop)/{print}'
EOF

# ===========================================================================
# 2. OS / DISTRO / UPTIME
# ===========================================================================
hdr "2. OS / DISTRO / UPTIME"
filecat "os-release"             /etc/os-release
cmd "debian version"             cat /etc/debian_version
cmd "uptime"                     uptime
cmd "uptime raw (since boot)"    cat /proc/uptime
cmd "last reboot"                last -x reboot 2>/dev/null | head -n 5
cmd "last logins"                last 2>/dev/null | head -n 10
cmd "uname"                      uname -a
cmd "/proc/cmdline (boot args)"  cat /proc/cmdline

# ===========================================================================
# 3. HARDWARE INVENTORY
# ===========================================================================
hdr "3. HARDWARE INVENTORY"
cmd "dmidecode -t system"        dmidecode -t system
cmd "dmidecode -t baseboard"     dmidecode -t baseboard
cmd "dmidecode -t bios"          dmidecode -t bios
cmd "dmidecode -t chassis"       dmidecode -t chassis
cmd "dmidecode -t processor"     dmidecode -t processor
cmd "dmidecode -t memory"        dmidecode -t memory
cmd "dmidecode -t cache"         dmidecode -t cache
cmd "dmidecode -t connector"     dmidecode -t connector
cmd "dmidecode -t slot"          dmidecode -t slot
cmd "lscpu"                      lscpu
cmd "lscpu cache"                lscpu -C
cmd "numactl --hardware"         numactl --hardware
cmd "numactl --show"             numactl --show
cmd "lspci -nnvv (verbose)"      lspci -nnvv
cmd "lspci -tv (tree)"           lspci -tv
cmd "lspci numa nodes"           cat /sys/bus/pci/devices/*/numa_node 2>/dev/null | sort | uniq -c
cmd "lsusb"                      lsusb
cmd "lshw -short"                lshw -short 2>/dev/null
dirlist "PCIe AER errors"        /sys/bus/pci/devices/*/aer_dev_correctable
cmdsh "PCIe AER summary (errors matter for NIC/NVMe stability)" <<'EOF'
found=0
for d in /sys/bus/pci/devices/*; do
    [ -d "$d" ] || continue
    addr="$(basename "$d")"
    class="$(cat "$d/class" 2>/dev/null)"
    case "$class" in
        0x020000|0x010802|0x010700) ;;  # ethernet, nvme, raid
        *) continue ;;
    esac
    for f in "$d"/aer_dev_*; do
        [ -r "$f" ] || continue
        val="$(cat "$f" 2>/dev/null)"
        [ "$val" = "0" ] && continue
        [ "$val" = "" ] && continue
        printf '%s %s = %s\n' "$addr" "$(basename "$f")" "$val"
        found=1
    done
done
[ "$found" = "0" ] && echo "  no AER errors on NIC/NVMe/RAID devices"
EOF

# ===========================================================================
# 4. CPU POWER / IDLE / MICROCODE
# ===========================================================================
hdr "4. CPU POWER / FREQUENCY / IDLE STATES"

cmdsh "scaling_governor per cpu (counts)" <<'EOF'
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
    [ -e "$c" ] || continue
    cat "$c"
done | sort | uniq -c
EOF

cmdsh "scaling_driver / available governors / freq limits" <<'EOF'
for f in /sys/devices/system/cpu/cpu0/cpufreq/*; do
    [ -r "$f" ] || continue
    [ -d "$f" ] && continue
    printf '%-40s = %s\n' "$(basename "$f")" "$(cat "$f" 2>/dev/null)"
done
EOF

cmdsh "EPP / EPB / intel_pstate" <<'EOF'
echo "intel_pstate status: $(cat /sys/devices/system/cpu/intel_pstate/status 2>/dev/null)"
echo "intel_pstate no_turbo: $(cat /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null)"
echo "intel_pstate turbo_pct: $(cat /sys/devices/system/cpu/intel_pstate/turbo_pct 2>/dev/null)"
echo "intel_pstate max_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/max_perf_pct 2>/dev/null)"
echo "intel_pstate min_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/min_perf_pct 2>/dev/null)"
echo "EPB (energy_perf_bias) per cpu:"
for c in /sys/devices/system/cpu/cpu*/power/energy_perf_bias; do
    [ -r "$c" ] || continue
    printf '  %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
done | sort | uniq -c
echo "EPP (energy_performance_preference) per cpu:"
for c in /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference; do
    [ -r "$c" ] || continue
    printf '  %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
done | sort | uniq -c
echo "available EPP:"
cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_available_preferences 2>/dev/null
EOF

cmd "cpupower frequency-info"    cpupower frequency-info 2>/dev/null
cmd "turbostat snapshot"         timeout 3 turbostat --quiet --show CPU,Core,Bzy_MHz,IRQ,Busy%,Bzy_MHz,TSC_MHz,CoreTmp,PkgTmp sleep 1 2>/dev/null
cmd "x86_energy_perf_policy -r"  x86_energy_perf_policy -r 2>/dev/null

cmdsh "C-states (idle) enabled per cpu0" <<'EOF'
for s in /sys/devices/system/cpu/cpu0/cpuidle/state*; do
    [ -d "$s" ] || continue
    printf '  %s : %s disabled=%s  latency=%sus  residency=%sus\n' \
        "$(cat "$s/name" 2>/dev/null)" \
        "$(cat "$s/desc" 2>/dev/null)" \
        "$(cat "$s/disable" 2>/dev/null)" \
        "$(cat "$s/latency" 2>/dev/null)" \
        "$(cat "$s/time" 2>/dev/null)"
done
EOF

cmd "THP enabled"                cat /sys/kernel/mm/transparent_hugepage/enabled
cmd "THP defrag"                 cat /sys/kernel/mm/transparent_hugepage/defrag
cmd "THP khugepaged pages_to_scan" cat /sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
cmd "THP khugepaged defrag"      cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
cmd "microcode"                  dmesg 2>/dev/null | grep -i microcode | tail -n 5
cmd "cpu MHz (live snapshot)"    grep -c '^processor' /proc/cpuinfo; head -n 30 /proc/cpuinfo

# ===========================================================================
# 5. MEMORY / NUMA / SWAP
# ===========================================================================
hdr "5. MEMORY / NUMA / SWAP"
cmd "free -h"                    free -h
cmd "free -b"                    free -b
cmd "swapon"                     swapon --show
cmd "/proc/meminfo highlights"   grep -E 'MemTotal|MemFree|MemAvailable|Buffers|^Cached|SwapTotal|SwapFree|Hugepagesize|HugePages_Total|HugePages_Free|AnonHugePages|DirectMap|Slab|SReclaimable|SUnreclaim|Dirty|Writeback|AnonPages|Mapped|Shmem|KernelStack|PageTables|NFS_Unstable' /proc/meminfo
cmd "numa stat"                  cat /proc/sys/vm/numa_stat 2>/dev/null
cmd "numa_balancing"             cat /proc/sys/kernel/numa_balancing 2>/dev/null
cmd "zone reclaim"               cat /proc/sys/vm/zone_reclaim_mode 2>/dev/null
cmd "slab top (top 20)"          slabtop -o -s c 2>/dev/null | head -n 25
cmd "kswapd stats"               grep -E 'kswapd|pgmigrate|pgmigrate_failed|compact_' /proc/vmstat 2>/dev/null | head -n 30

# ===========================================================================
# 6. KERNEL TUNABLES
# ===========================================================================
hdr "6. KERNEL TUNABLES / SYSCTL"

cmdsh "vm.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^vm\.(dirty_(ratio|background_ratio|bytes|background_bytes|expire_centisecs|writeback_centisecs)|swappiness|vfs_cache_pressure|min_free_kbytes|watermark_scale_factor|overcommit_memory|overcommit_ratio|nr_hugepages|nr_overcommit_hugepages|zone_reclaim_mode|compact_unevictable_allowed|stat_interval|admin_reserve_kbytes|user_reserve_kbytes)'
EOF

cmdsh "fs.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^fs\.(file-max|file-nr|aio-max-nr|aio-nr|nr_open|pipe-max-size|epoll/max_user_watches|lease-break-time|mqueue|max_user_ns)'
EOF

cmdsh "kernel.* perf-relevant" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^kernel\.(sched_(child_runs_first|migration_cost|domain|rt_period|rt_runtime|idle)|numa_balancing|watchdog|nmi_watchdog|hung_task|randomize_va_space|max_pid|threads-max|sched_time_avg)'
EOF

# ===========================================================================
# 7. NETWORK SYSCTL / TCP
# ===========================================================================
hdr "7. NETWORK SYSCTL / TCP"
cmdsh "net.core.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.core\.(rmem_default|rmem_max|wmem_default|wmem_max|optmem_max|somaxconn|netdev_max_backlog|netdev_budget|netdev_budget_usecs|netdev_tstamp_prequeue|rps_flow_cnt|busy_poll|busy_read|default_qdisc|dev_weight|xfrm)'
EOF

cmdsh "net.ipv4.tcp.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.ipv4\.tcp_(rmem|wmem|congestion_control|available_congestion_control|mtu_probing|slow_start_after_idle|no_metrics_save|timestamps|window_scaling|sack|low_latency|fastopen|ecn|ecn_fallback|syncookies|max_syn_backlog|tw_reuse|fin_timeout|keepalive_time|keepalive_probes|keepalive_intvl|thp_linear_timeouts|pacing|pacing_ss_ratio|pacing_ca_ratio|no_ssthresh_metrics_save|md5sig|abort_on_overflow|buffer_split)'
EOF

cmdsh "other net.* sysctls" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.(ipv4\.ip_(local_port_range|local_reserved_ports|no_pmtu_disc|forwarding)|ipv4\.conf\.all\.(rp_filter|accept_redirects|send_redirects)|ipv4\.conf\.default\.forwarding|core\.netdev_budget|ipv6\.conf\.all\.disable_ipv6)'
EOF

dirlist "sysctl.d"               /etc/sysctl.d
filecat "/etc/sysctl.conf"       /etc/sysctl.conf
cmdsh "non-default sysctl values (sorted, filtered)" <<'EOF'
# Show only sysctls that differ from running state vs an empty config;
# too noisy to dump all of /proc/sys. Just show /etc/sysctl.d content.
for f in /etc/sysctl.d/*.conf /etc/sysctl.conf; do
    [ -r "$f" ] || continue
    echo "=== $f ==="
    grep -vE '^\s*#|^\s*$' "$f" 2>/dev/null
done
EOF

# ===========================================================================
# 8. tuned / irqbalance / udev
# ===========================================================================
hdr "8. tuned / irqbalance / udev"
cmd "tuned-adm active"           tuned-adm active 2>/dev/null
cmd "tuned-adm list"             tuned-adm list 2>/dev/null
cmd "systemctl is-active tuned"  systemctl is-active tuned 2>/dev/null
cmd "irqbalance status"          systemctl is-active irqbalance 2>/dev/null
filecat "irqbalance config"      /etc/default/irqbalance /etc/irqbalance.conf
dirlist "modprobe.d"             /etc/modprobe.d
dirlist "udev rules.d"           /etc/udev/rules.d
filecat "modprobe.d files"       /etc/modprobe.d/*.conf
cmdsh "module params: kvm_intel" <<'EOF'
for p in /sys/module/kvm_intel/parameters/*; do
    [ -e "$p" ] && printf '  %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
done
EOF
cmdsh "module params: kvm_amd" <<'EOF'
for p in /sys/module/kvm_amd/parameters/*; do
    [ -e "$p" ] && printf '  %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
done
EOF
cmdsh "module params: relevant net drivers" <<'EOF'
for mod in i40e ixgbe mlx5_core igb ice bnx2x virtio_net; do
    [ -d "/sys/module/$mod/parameters" ] || continue
    echo "=== $mod ==="
    for p in /sys/module/$mod/parameters/*; do
        [ -e "$p" ] && printf '  %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
    done
done
EOF

# ===========================================================================
# 9. PROXMOX / CLUSTER
# ===========================================================================
hdr "9. PROXMOX VERSION & CLUSTER"
cmd "pveversion"                 pveversion
cmd "pveversion -v"              pveversion -v
cmd "pvecm status"               pvecm status
cmd "pvecm nodes"                pvecm nodes
cmd "pvecm expected"             pvecm expected 2>/dev/null
cmd "pveperf"                    timeout 15 pveperf 2>/dev/null
filecat "corosync.conf"          /etc/pve/corosync.conf
filecat "datacenter.cfg"         /etc/pve/datacenter.cfg
filecat "storage.cfg"            /etc/pve/storage.cfg
filecat "user.cfg"               /etc/pve/user.cfg 2>/dev/null
filecat "domains.cfg"            /etc/pve/domains.cfg 2>/dev/null
cmdsh "cluster resources (compact)" <<'EOF'
pvesh get /cluster/resources --type node --output-format json 2>/dev/null \
  | jq -r '.[] | "\(.node)\tcpu=\(.cpu)\tmem=\(.mem)\tmaxmem=\(.maxmem)\tmaxdisk=\(.maxdisk)\tstatus=\(.status)\tssl_fingerprint=\(.ssl_fingerprint)"' 2>/dev/null \
  || pvesh get /cluster/resources --type node 2>/dev/null
EOF

# ===========================================================================
# 10. NETWORK: INTERFACES
# ===========================================================================
hdr "10. NETWORK: INTERFACES"
cmd "ip -d link"                 ip -d link
cmd "ip -o link"                 ip -o link
cmd "ip -o -4 addr"              ip -o -4 addr
cmd "ip -o -6 addr (link-local filter)" ip -o -6 addr 2>/dev/null | grep -v 'scope link'
cmd "ip addr"                    ip addr
cmd "ip route"                   ip route
cmd "ip rule"                    ip rule
cmd "ip -s -s link (counters/errors)" ip -s -s link
cmd "ip neigh"                   ip neigh
cmd "/proc/net/dev (counters)"   cat /proc/net/dev
cmd "ip -6 route (brief)"        ip -6 route 2>/dev/null | head -n 20

# Build NETIFS list once, export for use in heredoc snippets
NETIFS=""
for _n in /sys/class/net/*; do
    _n="$(basename "$_n")"
    case "$_n" in
        lo|sit0|ip6tnl0|gre0|gretap0|erspan0|bonding_masters|tailscale0|wg0|tunl0) continue ;;
    esac
    NETIFS="${NETIFS:+$NETIFS }$_n"
done
export NETIFS

cmdsh "sysfs interface attributes" <<'EOF'
for ifc in $NETIFS; do
    d="/sys/class/net/$ifc"
    [ -d "$d" ] || continue
    printf '%-16s speed=%-8s duplex=%-8s mtu=%-6s carrier=%-3s tx_queue_len=%-5s qlen=%-5s type=%s\n' \
        "$ifc" \
        "$(cat "$d/speed" 2>/dev/null)" \
        "$(cat "$d/duplex" 2>/dev/null)" \
        "$(cat "$d/mtu" 2>/dev/null)" \
        "$(cat "$d/carrier" 2>/dev/null)" \
        "$(cat "$d/tx_queue_len" 2>/dev/null)" \
        "$(cat "$d/tx_queue_len" 2>/dev/null)" \
        "$(cat "$d/type" 2>/dev/null)"
done
EOF

# Per-interface deep ethtool dump
cmdsh "ethtool per-interface deep dump" <<'EOF'
for ifc in $NETIFS; do
    [ -e "/sys/class/net/$ifc" ] || continue
    echo ""
    echo "----- ethtool: $ifc -----"
    echo "[driver/version/firmware/bus-info -i]"
    ethtool -i "$ifc" 2>&1
    echo "[settings - speed/duplex/autoneg/port]"
    ethtool "$ifc" 2>&1
    echo "[channels -l (RX/TX/combined queue counts)]"
    ethtool -l "$ifc" 2>&1
    echo "[coalesce -c (interrupt moderation)]"
    ethtool -c "$ifc" 2>&1
    echo "[ring -g (descriptor counts)]"
    ethtool -g "$ifc" 2>&1
    echo "[pause / flow-control -a]"
    ethtool -a "$ifc" 2>&1
    echo "[offload features -k]"
    ethtool -k "$ifc" 2>&1
    echo "[private flags --show-priv-flags]"
    ethtool --show-priv-flags "$ifc" 2>&1
    echo "[ntuple/RSS -n]"
    ethtool -n "$ifc" 2>&1 | head -n 40
    echo "[SFP/DOM transceiver -m]"
    ethtool -m "$ifc" 2>&1
    echo "[statistics -S (key error/drop counters only)]"
    ethtool -S "$ifc" 2>/dev/null \
        | grep -iE 'drop|discard|error|crc|miss|pause|fifo|no_buff|rx_no_|_err|fw_|_bad|oversiz|undersize|frag|jabber|length_err|alignment' \
        | head -n 60
    echo "[statistics -S (throughput counters)]"
    ethtool -S "$ifc" 2>/dev/null \
        | grep -iE 'rx_(octets|bytes|packets)|tx_(octets|bytes|packets)|rx_(unicast|multicast|broadcast)|tx_(unicast|multicast|broadcast)' \
        | head -n 30
done
EOF

# ===========================================================================
# 11. NETWORK: BRIDGES / BONDS / VLANS
# ===========================================================================
hdr "11. NETWORK: BRIDGES / BONDS / VLANS"
cmd "bridge link"                bridge link 2>/dev/null
cmd "bridge -d link"             bridge -d link 2>/dev/null
cmd "bridge vlan"                bridge vlan 2>/dev/null
cmd "bridge vlan show"           bridge vlan show 2>/dev/null
cmd "brctl show"                 brctl show 2>/dev/null
filecat "bonding state"          /proc/net/bonding/bond*
cmd "vlan config"                cat /proc/net/vlan/config 2>/dev/null
dirlist "/proc/net/vlan"         /proc/net/vlan
filecat "/etc/network/interfaces" /etc/network/interfaces
cmdsh "interfaces.d (per file)" <<'EOF'
for f in /etc/network/interfaces.d/*; do
    [ -r "$f" ] || continue
    echo "--- $f ---"
    cat "$f"
done
EOF

# ===========================================================================
# 12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET
# ===========================================================================
hdr "12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET"
cmd "/proc/interrupts"           cat /proc/interrupts

cmdsh "NIC IRQ affinities (which CPUs handle which queue)" <<'EOF'
for ifc in $NETIFS; do
    [ -d "/sys/class/net/$ifc/device/msi_irqs" ] || continue
    echo "=== $ifc IRQ list (first 40) ==="
    ls "/sys/class/net/$ifc/device/msi_irqs" 2>/dev/null | head -n 40
done
EOF

cmdsh "RPS masks (non-zero only)" <<'EOF'
for q in /sys/class/net/*/queues/rx-*/rps_cpus; do
    [ -e "$q" ] || continue
    val=$(cat "$q")
    [ "$val" != "0" ] && printf '  %s = %s\n' "$q" "$val"
done | head -n 60
EOF

cmdsh "XPS masks (non-zero only)" <<'EOF'
for q in /sys/class/net/*/queues/tx-*/xps_cpus; do
    [ -e "$q" ] || continue
    val=$(cat "$q")
    [ "$val" != "0" ] && printf '  %s = %s\n' "$q" "$val"
done | head -n 60
EOF

cmdsh "RFS flow counts" <<'EOF'
for q in /sys/class/net/*/queues/rx-*/rps_flow_cnt; do
    [ -e "$q" ] || continue
    printf '  %s = %s\n' "$q" "$(cat "$q")"
done | head -n 60
EOF

cmd "/proc/net/softnet_stat (drops/backlog)"  cat /proc/net/softnet_stat
cmdsh "softnet_stat decoded" <<'EOF'
# columns: TOTAL DROPPED DROPPED_BY_NETRX TIME_SQUEEZED CPU_COLLISIONS RPS_RECEIVED_FLOW_LIMIT
awk '{ printf "cpu%-2d total=%-8s dropped=%-6s time_squeezed=%-6s cpu_collision=%-4s rps_received=%-6s flow_limit=%s\n",
       NR-1, $1, $2, $4, $5, $6, $7 }' /proc/net/softnet_stat
EOF

cmdsh "netstat drop counters (significant ones)" <<'EOF'
nstat -az 2>/dev/null | grep -iE '(Drop|Retrans|Fail|Abort|Overflow|Loss|ListenDrop|EmbryonicRsts|InErrs|OutErrs)' || \
netstat -s 2>/dev/null | grep -iE '(drop|retrans|fail|abort|overflow|loss|listen|error)' | head -n 60
EOF

# ===========================================================================
# 13. NETWORK: SOCKET / TCP STATE
# ===========================================================================
hdr "13. NETWORK: SOCKET / TCP STATE"
cmd "ss -s"                      ss -s
cmd "ss -t -m (top TCP with skmem/cwnd/rto)"  ss -tno state established '( dport != :22 and sport != :22 )' -m 2>/dev/null | head -n 40
cmd "ss -ti (TCP internals)"     ss -ti 2>/dev/null | head -n 80
cmd "/proc/net/snmp (TCP summary)" grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null
cmd "TCP retrans stats"          grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null | tail -n 1 | awk '{print "RetransSegs="$13}'
cmd "nstat TCP summary"          nstat -az Tcp_RetransSegs Tcp_InErrs Tcp_OutRsts Tcp_ActiveOpens Tcp_PassiveOpens 2>/dev/null

# ===========================================================================
# 14. STORAGE: BLOCK DEVICES
# ===========================================================================
hdr "14. STORAGE: BLOCK DEVICES"
cmd "lsblk (key cols)"           lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL,STATE,ROTA,DISC-GRAN,DISC-MAX
cmd "lsblk -t"                   lsblk -t
cmd "lsblk -O (everything, capped)" lsblk -O | head -n 200
cmd "blkid"                      blkid 2>/dev/null
cmd "/proc/partitions"           cat /proc/partitions
cmd "fdisk -l"                   fdisk -l 2>/dev/null
cmd "parted -l"                  parted -l 2>/dev/null

cmdsh "block device queue settings (per device)" <<'EOF'
for d in $(ls /sys/block/ 2>/dev/null | grep -vE '^(loop|ram|md|dm-|sr|fd)'); do
    q="/sys/block/$d/queue"
    [ -d "$q" ] || continue
    printf '----- %s -----\n' "$d"
    for k in scheduler rotational nr_requests read_ahead_kb max_sectors_kb max_hw_sectors_kb \
             logical_block_size physical_block_size discard_granularity discard_max_hw_bytes \
             rq_affinity nomerges wbt_lat_usec write_cache stable_writes io_poll io_poll_delay \
             fua rot_stable_writes add_random zoned dma_alignment iostats;
    do
        f="$q/$k"
        [ -r "$f" ] && printf '  %-22s = %s\n' "$k" "$(cat "$f" 2>/dev/null)"
    done
    # NVMe-specific
    for f in /sys/block/$d/device/model /sys/block/$d/device/serial /sys/block/$d/device/transport /sys/block/$d/device/queue_count; do
        [ -r "$f" ] && printf '  %s = %s\n' "$(echo "$f" | sed "s|/sys/block/$d/device/||")" "$(cat "$f")"
    done
done
EOF

cmd "df -h (filesystem usage)"   df -h
cmd "df -i (inode usage)"        df -i

# ===========================================================================
# 15. STORAGE: MOUNTS & FILESYSTEMS
# ===========================================================================
hdr "15. STORAGE: MOUNTS & FILESYSTEMS"
cmd "mount (filtered)"           mount | grep -vE 'proc|sysfs|cgroup|tmpfs|devpts|mqueue|pstore|bpf|tracefs|debugfs|configfs|fusectl|securityfs|hugetlbfs|autofs|systemd|mtd|rpc_pipefs|binfmt'
cmd "findmnt (tree)"             findmnt -t nfs,nfs4,zfs,ext4,xfs,btrfs 2>/dev/null
filecat "/etc/fstab"             /etc/fstab
cmdsh "ext4 reserved block % (per fs)" <<'EOF'
for dev in $(awk '$3=="ext4"{print $1}' /proc/mounts 2>/dev/null); do
    echo "=== $dev ==="
    dumpe2fs -h "$dev" 2>/dev/null | grep -E 'Reserved|Block count|Free blocks|Block size|Mount count|Maximum mount|Last'
done
EOF
cmdsh "xfs info (per fs)" <<'EOF'
for mp in $(awk '$3=="xfs"{print $2}' /proc/mounts 2>/dev/null); do
    echo "=== $mp ==="
    xfs_info "$mp" 2>/dev/null
done
EOF

# ===========================================================================
# 16. STORAGE: NFS (client + server) - critical for this cluster
# ===========================================================================
hdr "16. STORAGE: NFS (CLIENT + SERVER)"
cmd "nfsstat -m (mount options)"  nfsstat -m 2>/dev/null
cmd "nfsstat -c (client stats)"   nfsstat -c 2>/dev/null
cmd "nfsstat -s (server stats)"   nfsstat -s 2>/dev/null
cmd "nfsstat -234 (version stats)" nfsstat -234 2>/dev/null
cmd "showmount -e (exports)"      showmount -e 2>/dev/null
cmd "/proc/net/rpc/nfs"           cat /proc/net/rpc/nfs 2>/dev/null | head -n 60
cmd "/proc/net/rpc/nfsd"          cat /proc/net/rpc/nfsd 2>/dev/null | head -n 60
filecat "exports"                 /etc/exports /etc/exports.d/*.exports
filecat "nfs.conf"                /etc/nfs.conf /etc/nfs.conf.d/*.conf
filecat "nfs defaults"            /etc/default/nfs-kernel-server /etc/default/nfs-common
filecat "idmapd.conf"             /etc/idmapd.conf

cmd "pvesm status"                pvesm status
cmdsh "pvesm list per storage" <<'EOF'
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3=="active"{print $1}'); do
    echo "--- storage: $s ---"
    pvesm list "$s" 2>/dev/null | head -n 30
    echo "--- pvesm config $s ---"
    pvesm config "$s" 2>/dev/null
done
EOF

# ===========================================================================
# 17. STORAGE: LVM
# ===========================================================================
hdr "17. STORAGE: LVM"
cmd "pvs"                        pvs -v
cmd "vgs"                        vgs -v
cmd "lvs"                        lvs -o +lv_layout,role,pool_lv,data_percent,metadata_percent --units b
cmd "pvdisplay"                  pvdisplay
cmd "vgdisplay"                  vgdisplay
cmd "lvdisplay"                  lvdisplay
filecat "lvm/lvm.conf (filters only)" /etc/lvm/lvm.conf
cmdsh "lvm.conf active filter" <<'EOF'
grep -E '^\s*(filter|global_filter|use_devicesfile|obtain_device_list_from_udev)' /etc/lvm/lvm.conf 2>/dev/null
EOF

# ===========================================================================
# 18. STORAGE: MULTIPATH
# ===========================================================================
hdr "18. STORAGE: MULTIPATH"
cmd "multipath -ll"              multipath -ll
cmd "multipath -v3 -ll (verbose, capped)" multipath -v3 -ll 2>/dev/null | head -n 100
filecat "multipath.conf"         /etc/multipath.conf
cmd "systemctl is-active multipathd"  systemctl is-active multipathd 2>/dev/null

# ===========================================================================
# 19. STORAGE: ZFS
# ===========================================================================
hdr "19. STORAGE: ZFS"
if command -v zpool >/dev/null 2>&1; then
    cmd "zfs version (userland)"   zfs version 2>/dev/null
    cmd "zpool version"            zpool version 2>/dev/null
    cmd "zpool list -v"            zpool list -v
    cmd "zpool status -v"          zpool status -v
    cmdsh "zpool key properties" <<'EOF'
zpool get name,size,alloc,free,expandsize,fragmentation,capacity,health,ashift,autotrim,autoreplace,cachefile,comment,listsnapshots,multihost,version,bootfs,delegation,failmode,freeing,leaked,readonly
EOF
    cmdsh "zfs per-dataset key properties" <<'EOF'
zfs get -r -o name,property,value,source \
    compression,atime,recordsize,volblocksize,sync,primarycache,secondarycache,checksum,xattr,acltype,aclinherit,copies,logbias,refreservation,reservation,quota,used,avail,refer,mountpoint,mounted,canmount,relative_path,snapdir,version,primarycache,redundant_metadata,encryption 2>/dev/null
EOF
    cmd "ARC stats"                cat /proc/spl/kstat/zfs/arcstats
    cmd "dmu_tx"                   cat /proc/spl/kstat/zfs/dmu_tx 2>/dev/null
    cmd "zfetchstats"              cat /proc/spl/kstat/zfs/zfetchstats 2>/dev/null
    cmd "vdev_cache_stats"         cat /proc/spl/kstat/zfs/vdev_cache_stats 2>/dev/null
    cmd "abdstats"                 cat /proc/spl/kstat/zfs/abdstats 2>/dev/null
    cmdsh "ZFS module parameters (perf-relevant)" <<'EOF'
for f in /sys/module/zfs/parameters/*; do
    [ -r "$f" ] || continue
    name="$(basename "$f")"
    case "$name" in
        zfs_arc_*|zfs_abd_*|zfs_dbuf_*|zfs_txg_*|zfs_vdev_*|zfs_sync_*|zfs_dirty_*|zfs_*io_*|zvol_*|l2arc_*|zfs_prefetch_*|zfs_*max_active|zfs_*_taskq*) ;;
        *) continue ;;
    esac
    printf '  %-50s = %s\n' "$name" "$(cat "$f")"
done
EOF
    cmd "zpool iostat snapshot"    zpool iostat 1 1 2>/dev/null
    cmd "arc_summary (if present)" arc_summary 2>/dev/null | head -n 80
    cmd "zpool events -v (recent)"  zpool events -v 2>/dev/null | tail -n 50
    cmdsh "zpool history (last 30 events)" <<'EOF'
zpool history -il 2>/dev/null | tail -n 30
EOF
else
    printf '\nZFS not present on this host.\n' >> "$OUT"
fi

# ===========================================================================
# 20. STORAGE: DRIVE HEALTH (SMART / NVMe)
# ===========================================================================
hdr "20. STORAGE: DRIVE HEALTH"
cmd "nvme list"                  nvme list 2>/dev/null
cmdsh "nvme smart-log per device" <<'EOF'
for n in /dev/nvme[0-9]*; do
    [ -b "$n" ] || continue
    echo "--- $n smart-log ---"
    nvme smart-log "$n" 2>&1
    echo "--- $n error-log (first 5) ---"
    nvme error-log "$n" 2>&1 | head -n 80
done
EOF
cmdsh "nvme id-ctrl per device (key fields)" <<'EOF'
for n in /dev/nvme[0-9]*; do
    [ -b "$n" ] || continue
    echo "--- $n id-ctrl (filtered) ---"
    nvme id-ctrl "$n" 2>&1 | grep -E '^(mn|sn|fn|fr|tnvmcap|vid|IEEE|cntlid|ver|rab|lpa|elpe|npdg|mdts|hmpre|hmmin|cap|frmw|lba|ws|awun|awupf|acwu|apsta)'
done
EOF
if command -v smartctl >/dev/null 2>&1; then
    cmdsh "smartctl -a per disk" <<'EOF'
for d in $(ls /sys/block/ 2>/dev/null | grep -E '^(sd|vd)'); do
    dev="/dev/$d"
    [ -b "$dev" ] || continue
    echo "--- smartctl -a $dev ---"
    smartctl -a "$dev" 2>&1 | head -n 200
done
EOF
fi

# ===========================================================================
# 21. STORAGE: HBA / RAID CONTROLLERS
# ===========================================================================
hdr "21. STORAGE: HBA / RAID CONTROLLERS"
for cli in storcli64 storcli MegaCli64 MegaCli perccli; do
    if command -v "$cli" >/dev/null 2>&1; then
        printf '\n--- %s found ---\n' "$cli" >> "$OUT"
        case "$cli" in
            storcli*|perccli)
                cmd "$cli /call show all"      "$cli" /call show all
                cmd "$cli /call/vall show all" "$cli" /call/vall show all 2>/dev/null
                ;;
            MegaCli*)
                cmd "MegaCli adapter info"     "$cli" -AdpAllInfo -aALL
                cmd "MegaCli physical drives"  "$cli" -PDList -aALL
                cmd "MegaCli logical drives"   "$cli" -LDInfo -Lall -aALL
                cmd "MegaCli BBUs"             "$cli" -AdpBbuCmd -aALL
                ;;
        esac
        break
    fi
done
for cli in sas2ircu sas3ircu; do
    if command -v "$cli" >/dev/null 2>&1; then
        cmd "$cli list"    "$cli" list
        cmd "$cli 0 display" "$cli" 0 display 2>/dev/null
    fi
done
command -v arcconf >/dev/null 2>&1 && cmd "arcconf list" arcconf list
command -v ssacli >/dev/null 2>&1 && cmd "ssacli ctrl all show config" ssacli ctrl all show config
command -v hpssacli >/dev/null 2>&1 && cmd "hpssacli ctrl all show config" hpssacli ctrl all show config

# ===========================================================================
# 22. VIRTUAL MACHINES (qm)
# ===========================================================================
hdr "22. VIRTUAL MACHINES (qm)"
cmd "qm list"                    qm list
cmdsh "qm config per VM" <<'EOF'
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
    echo "--- VM $vmid ---"
    qm config "$vmid" 2>&1
    echo ""
done
EOF
cmdsh "VM runtime status (qm guest agent if running)" <<'EOF'
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
    echo "--- VM $vmid runtime ---"
    qm guest cmd "$vmid" network-get-interfaces 2>/dev/null | head -n 40 || echo "  [no agent or VM not running]"
done
EOF
dirlist "/etc/pve/qemu-server"   /etc/pve/qemu-server
dirlist "/etc/pve/firewall"      /etc/pve/firewall

# ===========================================================================
# 23. CONTAINERS (pct)
# ===========================================================================
hdr "23. CONTAINERS (pct)"
cmd "pct list"                   pct list
cmdsh "pct config per CT" <<'EOF'
for ctid in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
    echo "--- CT $ctid ---"
    pct config "$ctid" 2>&1
    echo ""
done
EOF

# ===========================================================================
# 24. SERVICES / FAILED UNITS
# ===========================================================================
hdr "24. SERVICES / FAILED UNITS"
cmd "running services"           systemctl list-units --type=service --state=running --no-pager
cmd "failed services"            systemctl --failed --no-pager
cmd "enabled services"           systemctl list-unit-files --state=enabled --no-pager | head -n 80
cmdsh "pve-* service status" <<'EOF'
systemctl status pve-cluster pvedaemon pveproxy pve-firewall pve-ha-crm pve-ha-lrm pvestatd pvebanner --no-pager 2>&1 | head -n 120
EOF
cmdsh "storage-related service status" <<'EOF'
systemctl status nfs-kernel-server nfs-server rpcbind rpc-statd zfs-import zfs-mount zfs-target smartd nvme-stat multipathd --no-pager 2>&1 | head -n 80
EOF

# ===========================================================================
# 25. TIMERS / CRON (could cause load spikes during work hours)
# ===========================================================================
hdr "25. TIMERS / CRON"
cmd "systemctl list-timers"      systemctl list-timers --all --no-pager | head -n 40
dirlist "/etc/cron.d"            /etc/cron.d
dirlist "/etc/cron.daily"        /etc/cron.daily
dirlist "/etc/cron.hourly"       /etc/cron.hourly
dirlist "/etc/cron.weekly"       /etc/cron.weekly
filecat "crontabs"               /etc/crontab
cmdsh "root crontab" <<'EOF'
crontab -l 2>/dev/null || echo "  [no root crontab]"
EOF

# ===========================================================================
# 26. LIVE PERFORMANCE SNAPSHOT
# ===========================================================================
hdr "26. LIVE PERFORMANCE SNAPSHOT"
cmd "/proc/loadavg"              cat /proc/loadavg
cmd "vmstat 1 3"                 vmstat 1 3
cmd "mpstat 1 3 (per-cpu)"       mpstat 1 3 2>/dev/null
cmd "iostat -xz 1 3"             iostat -xz 1 3 2>/dev/null
cmd "sar -dp 1 2"                sar -dp 1 2 2>/dev/null
cmd "sar -n DEV 1 2"             sar -n DEV 1 2 2>/dev/null
cmd "sar -n NFS,NFSD 1 2"        sar -n NFS,NFSD 1 2 2>/dev/null
cmd "sar -n SOCK 1 1"            sar -n SOCK 1 1 2>/dev/null
cmd "sar -n TCP,ETCP 1 2"        sar -n TCP,ETCP 1 2 2>/dev/null
cmd "pidstat -d 1 2 (top disk)"  pidstat -d 1 2 2>/dev/null | head -n 40
cmd "pidstat -u 1 2 (top cpu)"   pidstat -u 1 2 2>/dev/null | head -n 40
cmd "top (single snapshot, top 30)" top -b -n 1 | head -n 35
cmd "zpool iostat 1 2"           zpool iostat 1 2 2>/dev/null
cmd "arcstat 1 2"                arcstat 1 2 2>/dev/null

# /proc/net/dev snapshot twice for delta
cmdsh "/proc/net/dev delta (1s)" <<'EOF'
awk '{print $1}' /proc/net/dev
echo "--- t0 ---"
cat /proc/net/dev
sleep 1
echo "--- t1 (1s later) ---"
cat /proc/net/dev
EOF

# ===========================================================================
# 27. DNS / NTP / TIME
# ===========================================================================
hdr "27. DNS / NTP / TIME"
filecat "/etc/resolv.conf"       /etc/resolv.conf
filecat "/etc/hosts"             /etc/hosts
filecat "/etc/ntp.conf"          /etc/ntp.conf 2>/dev/null
filecat "/etc/chrony/chrony.conf" /etc/chrony/chrony.conf 2>/dev/null
cmd "timedatectl"                timedatectl 2>/dev/null
cmd "chronyc tracking"           chronyc tracking 2>/dev/null
cmd "chronyc sources"            chronyc sources 2>/dev/null
cmd "ntpq peers"                 ntpq -pn 2>/dev/null

# ===========================================================================
# 28. FIREWALL
# ===========================================================================
hdr "28. FIREWALL / SECURITY"
cmd "pve-firewall status"        pve-firewall status 2>/dev/null
cmd "nft ruleset (count)"        nft list ruleset 2>/dev/null | wc -l
cmd "nft ruleset (head)"         nft list ruleset 2>/dev/null | head -n 60
cmd "iptables-save (legacy)"     iptables-save 2>/dev/null | head -n 60
cmd "ipset list (names)"         ipset list -n 2>/dev/null
cmd "apparmor status"            aa-status 2>/dev/null | head -n 20

# ===========================================================================
# 29. TOP CONSUMERS (memory + disk + file handles)
# ===========================================================================
hdr "29. TOP CONSUMERS"
cmdsh "top 10 memory hogs" <<'EOF'
ps -eo pid,user,rss,vsz,pcpu,pmem,comm --sort=-rss | head -n 11
EOF
cmdsh "top 10 CPU hogs" <<'EOF'
ps -eo pid,user,pcpu,pmem,rss,comm --sort=-pcpu | head -n 11
EOF
cmdsh "top file-handle holders" <<'EOF'
for pid in $(ps -e -o pid= | head -n 200); do
    n="$(ls /proc/$pid/fd 2>/dev/null | wc -l)"
    [ "$n" -gt 50 ] && printf '%6d fd=%-6s %s\n' "$pid" "$n" "$(cat /proc/$pid/comm 2>/dev/null)"
done | sort -k2 -t= -nr | head -n 15
EOF
cmdsh "largest files in /var/log (top 10)" <<'EOF'
find /var/log -type f -size +10M -exec ls -lh {} \; 2>/dev/null | sort -k5 -h -r | head -n 10
EOF

# ===========================================================================
# 30. QUICK-LOOK SUSPECT SUMMARY (auto-detect common issues)
# ===========================================================================
hdr "30. QUICK-LOOK SUSPECT SUMMARY"
blank
printf 'Auto-detected performance anti-patterns (verify against full data above):\n' >> "$OUT"
blank

cmdsh "[A] Network interfaces < 10 Gbps and up" <<'EOF'
hits=0
for ifc in $NETIFS; do
    s=$(cat "/sys/class/net/$ifc/speed" 2>/dev/null)
    c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
    case "$s" in
        10|100|1000)
            if [ "$c" = "1" ]; then
                printf '  %s : %s Mbps (UP)  <-- sub-10GbE; suspect if this carries storage/migration\n' "$ifc" "$s"
                hits=$((hits+1))
            fi
            ;;
    esac
done
[ "$hits" = "0" ] && echo "  no <10GbE interfaces are UP (or speed unreadable)"
EOF

blank
cmdsh "[B] MTU not jumbo on carrier-up interfaces" <<'EOF'
hits=0
for ifc in $NETIFS; do
    m=$(cat "/sys/class/net/$ifc/mtu" 2>/dev/null)
    c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
    [ "$c" = "1" ] || continue
    if [ -n "$m" ] && [ "$m" != "9000" ]; then
        printf '  %s : mtu=%s\n' "$ifc" "$m"
        hits=$((hits+1))
    fi
done
[ "$hits" = "0" ] && echo "  all carrier-up interfaces have MTU 9000 (or none are up)"
EOF

blank
cmdsh "[C] CPU governor != performance" <<'EOF'
g=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)
if [ -z "$g" ]; then
    echo "  cpufreq driver not loaded (governor unknown)"
elif [ "$g" = "performance" ]; then
    echo "  OK: performance governor"
else
    echo "  <-- governor=$g  (recommend 'performance')"
fi
EOF

blank
cmdsh "[D] EPB / EPP not performance-bias" <<'EOF'
epb=$(cat /sys/devices/system/cpu/cpu0/power/energy_perf_bias 2>/dev/null)
epp=$(cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference 2>/dev/null)
echo "  EPB=$epb  EPP=$epp  (0=performance, 15=max-power on EPB; 'performance' on EPP)"
EOF

blank
cmdsh "[E] vm.swappiness and dirty ratios" <<'EOF'
sw=$(sysctl -n vm.swappiness 2>/dev/null)
dr=$(sysctl -n vm.dirty_ratio 2>/dev/null)
dbr=$(sysctl -n vm.dirty_background_ratio 2>/dev/null)
echo "  swappiness=$sw  dirty_ratio=$dr  dirty_background_ratio=$dbr"
[ "$sw" != "10" ] && [ -n "$sw" ] && echo "    <-- swappiness not 10 (Proxmox default)"
EOF

blank
cmdsh "[F] TCP congestion control" <<'EOF'
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null)
echo "  active: $cc"
echo "  available: $avail"
[ "$cc" = "cubic" ] && echo "    <-- consider bbr (if available) for high-BDP NFS paths"
EOF

blank
cmdsh "[G] ZFS dataset red flags" <<'EOF'
if ! command -v zfs >/dev/null 2>&1; then
    echo "  ZFS not present"
else
    zfs get -r -o name,property,value compression,atime,sync,recordsize,xattr 2>/dev/null \
      | awk '$2=="compression" && $3=="off" {print "  <-- "$1" has compression=off"}
             $2=="atime" && $3=="on"        {print "  <-- "$1" has atime=on"}
             $2=="sync" && $3=="always"     {print "  <-- "$1" has sync=always (slow without SLOG)"}
             $2=="xattr" && $3=="on"        {print "  <-- "$1" has xattr=on (use sa)"}'
fi
EOF

blank
cmdsh "[H] ARC size vs RAM" <<'EOF'
if [ -r /proc/spl/kstat/zfs/arcstats ]; then
    arc=$(awk '/^size /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
    cmax=$(awk '/^c_max /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
    ram=$(awk '/MemTotal/{print $2*1024}' /proc/meminfo)
    echo "  ARC current=$arc bytes  ARC max=$cmax bytes  RAM total=$ram bytes"
    if [ -n "$cmax" ] && [ -n "$ram" ] && [ "$ram" -gt 0 ]; then
        pct=$((cmax * 100 / ram))
        echo "  ARC max is ${pct}% of RAM (Proxmox default ~50%)"
    fi
    hits=$(awk '/^hits /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
    miss=$(awk '/^misses /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
    if [ -n "$hits" ] && [ -n "$miss" ] && [ "$((hits+miss))" -gt 0 ]; then
        hr=$((hits * 100 / (hits + miss)))
        echo "  ARC hit rate since boot: ${hr}% (hits=$hits misses=$miss)"
    fi
else
    echo "  ZFS ARC stats not available (ZFS not loaded?)"
fi
EOF

blank
cmdsh "[I] VMs with non-optimal storage/NIC models" <<'EOF'
hits=0
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
    cfg=$(qm config "$vmid" 2>/dev/null)
    flags=""
    echo "$cfg" | grep -qE '^ide[0-9]'                 && flags="${flags}ide-disk "
    echo "$cfg" | grep -qE '^sata[0-9]'                && flags="${flags}sata-disk "
    echo "$cfg" | grep -qE '^scsi[0-9].*:,'            && echo "$cfg" | grep -E '^scsi[0-9]' | grep -qv 'virtio-scsi' && flags="${flags}scsi-no-virtio-scsi "
    echo "$cfg" | grep -qE '^scsihw: ' && ! echo "$cfg" | grep -qE '^scsihw: virtio-scsi-single' && flags="${flags}scsihw!=virtio-scsi-single "
    echo "$cfg" | grep -qE '^cpu: .*kvm64'             && flags="${flags}cpu=kvm64 "
    echo "$cfg" | grep -q '^iothread: 1'               || echo "$cfg" | grep -qE '^scsi[0-9].*iothread=1' || flags="${flags}no-iothread "
    echo "$cfg" | grep -qE '^net[0-9].*model=(e1000|rtl8139|vmxnet)' && flags="${flags}legacy-nic "
    echo "$cfg" | grep -qE '^net[0-9].*model=virtio' && ! echo "$cfg" | grep -qE 'multiqueue=|queues=' && flags="${flags}virtio-no-multiqueue "
    if [ -n "$flags" ]; then
        echo "  VM $vmid: $flags"
        hits=$((hits+1))
    fi
done
[ "$hits" = "0" ] && echo "  no obvious VM storage/NIC anti-patterns"
EOF

blank
cmdsh "[J] NFS client mount option red flags" <<'EOF'
if ! command -v nfsstat >/dev/null 2>&1; then
    echo "  nfsstat not available"
else
    nfsstat -m 2>/dev/null | awk '
        /^.*:/ {mp=$0; next}
        /Flags:/ {
            gsub(/Flags:|,/,"",$0)
            line=$0
            if (line !~ /noatime/)        print "  "mp" : missing noatime"
            if (line !~ /nconnect/)       print "  "mp" : missing nconnect (kernel >=5.3)"
            if (line !~ /rsize=1048576/)  print "  "mp" : rsize<1M"
            if (line !~ /wsize=1048576/)  print "  "mp" : wsize<1M"
            if (line ~ /sync/)            print "  "mp" : sync mount (slow)"
        }
    '
fi
EOF

blank
cmdsh "[K] Softnet / NIC drops (non-zero)" <<'EOF'
awk 'NR<=32 && ($2+0 > 0 || $4+0 > 0) {printf "  cpu%d: dropped=%d time_squeezed=%d\n", NR-1, $2, $4}' /proc/net/softnet_stat
echo "  (also check ethtool -S sections above for rx_dropped/rx_missed_errors)"
EOF

blank
cmdsh "[L] Failed systemd services" <<'EOF'
failed=$(systemctl --failed --no-legend 2>/dev/null | head -n 10)
if [ -n "$failed" ]; then
    echo "$failed" | sed 's/^/  /'
else
    echo "  no failed services"
fi
EOF

blank
cmdsh "[M] Filesystem usage > 80%" <<'EOF'
df -h 2>/dev/null | awk 'NR>1 && $5+0 > 80 {print "  "$0}'
EOF

blank
cmdsh "[N] SMART / NVMe health warnings" <<'EOF'
if command -v nvme >/dev/null 2>&1; then
    for n in /dev/nvme[0-9]*; do
        [ -b "$n" ] || continue
        crit=$(nvme smart-log "$n" 2>/dev/null | grep -iE 'critical_warning|media_and_data_integrity|available_spare' | head -n 5)
        [ -n "$crit" ] && echo "  $n: $crit" | tr '\n' ' '
    done
fi
if command -v smartctl >/dev/null 2>&1; then
    for d in $(ls /sys/block/ 2>/dev/null | grep -E '^sd'); do
        dev="/dev/$d"
        [ -b "$dev" ] || continue
        chk=$(smartctl -H "$dev" 2>/dev/null | grep -iE 'result|health|passed|failed')
        echo "$chk" | grep -qi fail && echo "  $dev: $chk"
    done
fi
echo "  (see section 20 for full smartctl/nvme output)"
EOF

# ===========================================================================
# Footer
# ===========================================================================
END_EPOCH="$(date +%s)"
DUR=$((END_EPOCH - START_EPOCH))
BYTES="$(wc -c < "$OUT")"
LINES="$(wc -l < "$OUT")"
{
    printf '\n###############################################################################\n'
    printf '# Collection complete: %s\n' "$HOST"
    printf '# Output:  %s\n' "$OUT"
    printf '# Size:    %s bytes / %s lines\n' "$BYTES" "$LINES"
    printf '# Elapsed: %s seconds\n' "$DUR"
    printf '###############################################################################\n'
} >> "$OUT"

chmod 600 "$OUT"
echo
echo "================================================================"
echo " check.sh done. Wrote: ${OUT}"
echo " Size:    $(du -h "$OUT" | cut -f1)  (${LINES} lines)"
echo " Elapsed: ${DUR}s"
echo " Please copy ${OUT} back for analysis."
echo "================================================================"