KNEL/PFVCluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
422999bf3c4fa43ec6952c42c6a6f2ba4ff4fcdb
PFVCluster/scripts/check.sh
T
mrcharles 422999bf3c chore: initialize repo with full project state 
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.

Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
  nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
  hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
  storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
  distribution, StorageClass design, and ETL/HPC workload planning

💘 Generated with Crush

Assisted-by: Crush:glm-5.2
2026-07-27 11:31:29 -05:00

1139 lines
50 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
###############################################################################
# check.sh
#
# READ-ONLY, comprehensive Proxmox host assessment.
# - Installs NOTHING. Modifies NOTHING. Writes only its own log.
# - Detects what is/is isn't installed so the next step can be informed.
# - Runtime ~30-60 seconds on a busy node.
#
# Output: /root/<short-hostname>.log
#
# Usage (as root):
# bash check.sh
###############################################################################
set -u
umask 077
HOST="$(hostname -s)"
OUT="/root/${HOST}.log"
: > "$OUT"
TS="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
START_EPOCH="$(date +%s)"
# ===========================================================================
# Helpers (pattern from project memory)
# ===========================================================================
blank(){ printf '\n' >> "$OUT"; }
hdr() {
{
printf '\n'
printf '###############################################################################\n'
printf '# %s\n' "$1"
printf '###############################################################################\n'
} >> "$OUT"
}
# cmd LABEL CMD [ARGS...] - simple commands only (no pipes/loops/&&)
cmd() {
local label="$1"; shift
{
printf '\n--- %s\n' "$label"
printf ' $ %s\n' "$*"
} >> "$OUT"
"$@" >> "$OUT" 2>&1 || printf ' [exit=%s / not installed or not permitted]\n' "$?" >> "$OUT"
}
# cmdsh LABEL <<'EOF' ... EOF - arbitrary shell snippet (pipes, loops OK).
# Heredoc body MUST use quoted 'EOF' so $vars pass through to eval literally;
# export any outer-scope vars the body needs before the call.
# shellcheck disable=SC2310
cmdsh() {
local label="$1" snippet
snippet=$(cat)
{
printf '\n--- %s\n' "$label"
printf ' $ %s\n' "$snippet"
} >> "$OUT"
eval "$snippet" >> "$OUT" 2>&1 || printf ' [exit=%s]\n' "$?" >> "$OUT"
}
# filecat LABEL PATH [PATH...] - dump file contents (cap 300 lines each)
filecat() {
local label="$1"; shift
printf '\n--- %s\n' "$label" >> "$OUT"
local f
for f in "$@"; do
if [ -r "$f" ]; then
printf ' --- %s ---\n' "$f" >> "$OUT"
sed -n '1,300p' "$f" >> "$OUT"
else
printf ' [missing: %s]\n' "$f" >> "$OUT"
fi
done
}
# dirlist LABEL PATH - list directory contents (existence + sizes)
dirlist() {
local label="$1" d="$2"
printf '\n--- %s\n' "$label" >> "$OUT"
if [ -d "$d" ]; then
ls -la "$d" >> "$OUT" 2>&1
else
printf ' [missing: %s]\n' "$d" >> "$OUT"
fi
}
# ===========================================================================
# Header
# ===========================================================================
{
printf '###############################################################################\n'
printf '# Proxmox VE host performance assessment (READ-ONLY)\n'
printf '# Host: %s (%s)\n' "$HOST" "$(hostname -f 2>/dev/null)"
printf '# FQDN: %s\n' "$(hostname -f 2>/dev/null)"
printf '# All IPs: %s\n' "$(hostname -I 2>/dev/null)"
printf '# Collected: %s (UTC)\n' "$TS"
printf '# Kernel: %s\n' "$(uname -r)"
printf '# Machine: %s %s\n' "$(uname -m)" "$(uname -o)"
printf '# Caller: %s (uid=%s)\n' "$(whoami)" "$(id -u)"
printf '# Script: check.sh - strictly READ-ONLY\n'
printf '###############################################################################\n'
} >> "$OUT"
# ===========================================================================
# 1. TOOL AVAILABILITY MATRIX
# (most important section: tells us what is/isn't installed)
# ===========================================================================
hdr "1. TOOL AVAILABILITY MATRIX"
cmdsh "command presence + path + version" <<'EOF'
# Each line: name | present? | path | version
printf '%-22s %-8s %-30s %s\n' "TOOL" "PRESENT" "PATH" "VERSION/NOTE"
printf '%-22s %-8s %-30s %s\n' "----" "-------" "----" "------------"
for t in \
pveversion pvecm qm pct pvesm pvesh pveperf proxmox-backup-client \
ip ss ethtool bridge brctl nmcli \
tcpdump mtr nstat netstat nicstat ifstat \
dmidecode lscpu lshw hwinfo inxi lspci lsusb numactl numastat \
cpupower turbostat x86_energy_perf_policy rdmsr \
tuned-adm irqbalance \
zpool zfs zdb arcstat arc_summary \
lsblk blkid blockdev findmnt smartctl nvme hdparm \
storcli storcli64 MegaCli MegaCli64 tw-cli arcconf \
sas2ircu sas3ircu hpssacli ssacli perccli \
multipath pvs vgs lvs lvdisplay \
xfs_info dumpe2fs btrfs \
fio iperf iperf3 sysbench bonnie++ iozone \
iostat mpstat pidstat sar vmstat \
top htop atop glances dstat nethogs iftop iotop smem pmap slabtop \
perf strace lsof fatrace \
dig host nslookup \
chronyc ntpq timedatectl \
nft iptables iptables-save ipset ufw \
jq bc awk smartctl nvme \
apt dpkg
do
p="$(command -v "$t" 2>/dev/null)"
if [ -n "$p" ]; then
v=""
case "$t" in
zpool|zfs) v="$(timeout 3 "$t" version 2>/dev/null | head -n 2 | tr '\n' ' ')" ;;
iperf3|fio|sysbench|smartctl|nvme|tuned-adm|ethtool|cpupower)
v="$(timeout 3 "$t" --version 2>&1 | head -n 1)" ;;
pveversion) v="$(timeout 3 "$t" 2>/dev/null)" ;;
storcli64|storcli|MegaCli64|MegaCli|arcconf|sas2ircu|sas3ircu|ssacli|hpssacli)
v="(storage HBA CLI - present)" ;;
esac
printf '%-22s %-8s %-30s %s\n' "$t" "yes" "$p" "$v"
else
printf '%-22s %-8s %-30s %s\n' "$t" "no" "-" "-"
fi
done
EOF
cmdsh "Debian packages relevant to perf/storage/net" <<'EOF'
dpkg -l 2>/dev/null | awk '/^ii/ && ($2 ~ /^(proxmox-|pve-|zfs|zfsutils|nfs|nfs-kernel-server|nfs-common|open-iscsi|ceph|tuned|irqbalance|sysstat|smartmontools|nvme-cli|ethtool|bridge-utils|ifenslave|fio|iperf3|sysbench|glances|htop|atop|lm-sensors|numactl|cpufrequtils|linux-image-|pve-kernel|qemu|libvirt|corosync|pacemaker)/) {print $2" "$3}' | head -n 80
EOF
cmdsh "kernel modules available (relevant to perf)" <<'EOF'
for m in kvm_intel kvm_amd \
ixgbe i40e i40iw ice igb igc mlx4_en mlx5_core mlx4_core bnx2x bnx2 tg3 be2net qed qede virtio_net \
nvme nvme-tcp nvme-fabrics sd_mod sr_mod \
zfs spl znvpair zcommon zunicode zavl icp zlua \
megaraid_sas mpt3sas hpsa aacraid smartpqi isci \
dm_multipath multipath \
tcp_dctcp tcp_bbr tcp_cubic tcp_htcp \
8021q bonding bridge br_netfilter \
vhost_net vhost_scsi target_core_mod \
nfsv4 nfsv3 rpcsec_gss_krb5
do
modinfo "$m" >/dev/null 2>&1 && printf 'available : %s (%s)\n' "$m" "$(modinfo -F version "$m" 2>/dev/null || echo built-in)"
done
EOF
cmdsh "currently loaded modules (filtered)" <<'EOF'
lsmod | awk 'NR==1 || $1 ~ /^(kvm|virtio|vhost|ixgb|i40e|ice|igb|mlx|bnx|tg3|be2|qed|nvme|zfs|spl|z|dm_|megaraid|mpt|hpsa|aacraid|smartpqi|8021q|bonding|bridge|nfsv|tcp_|rpc|sunrpc|loop)/{print}'
EOF
# ===========================================================================
# 2. OS / DISTRO / UPTIME
# ===========================================================================
hdr "2. OS / DISTRO / UPTIME"
filecat "os-release" /etc/os-release
cmd "debian version" cat /etc/debian_version
cmd "uptime" uptime
cmd "uptime raw (since boot)" cat /proc/uptime
cmd "last reboot" last -x reboot 2>/dev/null | head -n 5
cmd "last logins" last 2>/dev/null | head -n 10
cmd "uname" uname -a
cmd "/proc/cmdline (boot args)" cat /proc/cmdline
# ===========================================================================
# 3. HARDWARE INVENTORY
# ===========================================================================
hdr "3. HARDWARE INVENTORY"
cmd "dmidecode -t system" dmidecode -t system
cmd "dmidecode -t baseboard" dmidecode -t baseboard
cmd "dmidecode -t bios" dmidecode -t bios
cmd "dmidecode -t chassis" dmidecode -t chassis
cmd "dmidecode -t processor" dmidecode -t processor
cmd "dmidecode -t memory" dmidecode -t memory
cmd "dmidecode -t cache" dmidecode -t cache
cmd "dmidecode -t connector" dmidecode -t connector
cmd "dmidecode -t slot" dmidecode -t slot
cmd "lscpu" lscpu
cmd "lscpu cache" lscpu -C
cmd "numactl --hardware" numactl --hardware
cmd "numactl --show" numactl --show
cmd "lspci -nnvv (verbose)" lspci -nnvv
cmd "lspci -tv (tree)" lspci -tv
cmd "lspci numa nodes" cat /sys/bus/pci/devices/*/numa_node 2>/dev/null | sort | uniq -c
cmd "lsusb" lsusb
cmd "lshw -short" lshw -short 2>/dev/null
dirlist "PCIe AER errors" /sys/bus/pci/devices/*/aer_dev_correctable
cmdsh "PCIe AER summary (errors matter for NIC/NVMe stability)" <<'EOF'
found=0
for d in /sys/bus/pci/devices/*; do
[ -d "$d" ] || continue
addr="$(basename "$d")"
class="$(cat "$d/class" 2>/dev/null)"
case "$class" in
0x020000|0x010802|0x010700) ;; # ethernet, nvme, raid
*) continue ;;
esac
for f in "$d"/aer_dev_*; do
[ -r "$f" ] || continue
val="$(cat "$f" 2>/dev/null)"
[ "$val" = "0" ] && continue
[ "$val" = "" ] && continue
printf '%s %s = %s\n' "$addr" "$(basename "$f")" "$val"
found=1
done
done
[ "$found" = "0" ] && echo " no AER errors on NIC/NVMe/RAID devices"
EOF
# ===========================================================================
# 4. CPU POWER / IDLE / MICROCODE
# ===========================================================================
hdr "4. CPU POWER / FREQUENCY / IDLE STATES"
cmdsh "scaling_governor per cpu (counts)" <<'EOF'
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
[ -e "$c" ] || continue
cat "$c"
done | sort | uniq -c
EOF
cmdsh "scaling_driver / available governors / freq limits" <<'EOF'
for f in /sys/devices/system/cpu/cpu0/cpufreq/*; do
[ -r "$f" ] || continue
[ -d "$f" ] && continue
printf '%-40s = %s\n' "$(basename "$f")" "$(cat "$f" 2>/dev/null)"
done
EOF
cmdsh "EPP / EPB / intel_pstate" <<'EOF'
echo "intel_pstate status: $(cat /sys/devices/system/cpu/intel_pstate/status 2>/dev/null)"
echo "intel_pstate no_turbo: $(cat /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null)"
echo "intel_pstate turbo_pct: $(cat /sys/devices/system/cpu/intel_pstate/turbo_pct 2>/dev/null)"
echo "intel_pstate max_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/max_perf_pct 2>/dev/null)"
echo "intel_pstate min_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/min_perf_pct 2>/dev/null)"
echo "EPB (energy_perf_bias) per cpu:"
for c in /sys/devices/system/cpu/cpu*/power/energy_perf_bias; do
[ -r "$c" ] || continue
printf ' %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
done | sort | uniq -c
echo "EPP (energy_performance_preference) per cpu:"
for c in /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference; do
[ -r "$c" ] || continue
printf ' %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
done | sort | uniq -c
echo "available EPP:"
cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_available_preferences 2>/dev/null
EOF
cmd "cpupower frequency-info" cpupower frequency-info 2>/dev/null
cmd "turbostat snapshot" timeout 3 turbostat --quiet --show CPU,Core,Bzy_MHz,IRQ,Busy%,Bzy_MHz,TSC_MHz,CoreTmp,PkgTmp sleep 1 2>/dev/null
cmd "x86_energy_perf_policy -r" x86_energy_perf_policy -r 2>/dev/null
cmdsh "C-states (idle) enabled per cpu0" <<'EOF'
for s in /sys/devices/system/cpu/cpu0/cpuidle/state*; do
[ -d "$s" ] || continue
printf ' %s : %s disabled=%s latency=%sus residency=%sus\n' \
"$(cat "$s/name" 2>/dev/null)" \
"$(cat "$s/desc" 2>/dev/null)" \
"$(cat "$s/disable" 2>/dev/null)" \
"$(cat "$s/latency" 2>/dev/null)" \
"$(cat "$s/time" 2>/dev/null)"
done
EOF
cmd "THP enabled" cat /sys/kernel/mm/transparent_hugepage/enabled
cmd "THP defrag" cat /sys/kernel/mm/transparent_hugepage/defrag
cmd "THP khugepaged pages_to_scan" cat /sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
cmd "THP khugepaged defrag" cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
cmd "microcode" dmesg 2>/dev/null | grep -i microcode | tail -n 5
cmd "cpu MHz (live snapshot)" grep -c '^processor' /proc/cpuinfo; head -n 30 /proc/cpuinfo
# ===========================================================================
# 5. MEMORY / NUMA / SWAP
# ===========================================================================
hdr "5. MEMORY / NUMA / SWAP"
cmd "free -h" free -h
cmd "free -b" free -b
cmd "swapon" swapon --show
cmd "/proc/meminfo highlights" grep -E 'MemTotal|MemFree|MemAvailable|Buffers|^Cached|SwapTotal|SwapFree|Hugepagesize|HugePages_Total|HugePages_Free|AnonHugePages|DirectMap|Slab|SReclaimable|SUnreclaim|Dirty|Writeback|AnonPages|Mapped|Shmem|KernelStack|PageTables|NFS_Unstable' /proc/meminfo
cmd "numa stat" cat /proc/sys/vm/numa_stat 2>/dev/null
cmd "numa_balancing" cat /proc/sys/kernel/numa_balancing 2>/dev/null
cmd "zone reclaim" cat /proc/sys/vm/zone_reclaim_mode 2>/dev/null
cmd "slab top (top 20)" slabtop -o -s c 2>/dev/null | head -n 25
cmd "kswapd stats" grep -E 'kswapd|pgmigrate|pgmigrate_failed|compact_' /proc/vmstat 2>/dev/null | head -n 30
# ===========================================================================
# 6. KERNEL TUNABLES
# ===========================================================================
hdr "6. KERNEL TUNABLES / SYSCTL"
cmdsh "vm.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^vm\.(dirty_(ratio|background_ratio|bytes|background_bytes|expire_centisecs|writeback_centisecs)|swappiness|vfs_cache_pressure|min_free_kbytes|watermark_scale_factor|overcommit_memory|overcommit_ratio|nr_hugepages|nr_overcommit_hugepages|zone_reclaim_mode|compact_unevictable_allowed|stat_interval|admin_reserve_kbytes|user_reserve_kbytes)'
EOF
cmdsh "fs.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^fs\.(file-max|file-nr|aio-max-nr|aio-nr|nr_open|pipe-max-size|epoll/max_user_watches|lease-break-time|mqueue|max_user_ns)'
EOF
cmdsh "kernel.* perf-relevant" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^kernel\.(sched_(child_runs_first|migration_cost|domain|rt_period|rt_runtime|idle)|numa_balancing|watchdog|nmi_watchdog|hung_task|randomize_va_space|max_pid|threads-max|sched_time_avg)'
EOF
# ===========================================================================
# 7. NETWORK SYSCTL / TCP
# ===========================================================================
hdr "7. NETWORK SYSCTL / TCP"
cmdsh "net.core.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.core\.(rmem_default|rmem_max|wmem_default|wmem_max|optmem_max|somaxconn|netdev_max_backlog|netdev_budget|netdev_budget_usecs|netdev_tstamp_prequeue|rps_flow_cnt|busy_poll|busy_read|default_qdisc|dev_weight|xfrm)'
EOF
cmdsh "net.ipv4.tcp.* tunables" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.ipv4\.tcp_(rmem|wmem|congestion_control|available_congestion_control|mtu_probing|slow_start_after_idle|no_metrics_save|timestamps|window_scaling|sack|low_latency|fastopen|ecn|ecn_fallback|syncookies|max_syn_backlog|tw_reuse|fin_timeout|keepalive_time|keepalive_probes|keepalive_intvl|thp_linear_timeouts|pacing|pacing_ss_ratio|pacing_ca_ratio|no_ssthresh_metrics_save|md5sig|abort_on_overflow|buffer_split)'
EOF
cmdsh "other net.* sysctls" <<'EOF'
sysctl -a 2>/dev/null | grep -E '^net\.(ipv4\.ip_(local_port_range|local_reserved_ports|no_pmtu_disc|forwarding)|ipv4\.conf\.all\.(rp_filter|accept_redirects|send_redirects)|ipv4\.conf\.default\.forwarding|core\.netdev_budget|ipv6\.conf\.all\.disable_ipv6)'
EOF
dirlist "sysctl.d" /etc/sysctl.d
filecat "/etc/sysctl.conf" /etc/sysctl.conf
cmdsh "non-default sysctl values (sorted, filtered)" <<'EOF'
# Show only sysctls that differ from running state vs an empty config;
# too noisy to dump all of /proc/sys. Just show /etc/sysctl.d content.
for f in /etc/sysctl.d/*.conf /etc/sysctl.conf; do
[ -r "$f" ] || continue
echo "=== $f ==="
grep -vE '^\s*#|^\s*$' "$f" 2>/dev/null
done
EOF
# ===========================================================================
# 8. tuned / irqbalance / udev
# ===========================================================================
hdr "8. tuned / irqbalance / udev"
cmd "tuned-adm active" tuned-adm active 2>/dev/null
cmd "tuned-adm list" tuned-adm list 2>/dev/null
cmd "systemctl is-active tuned" systemctl is-active tuned 2>/dev/null
cmd "irqbalance status" systemctl is-active irqbalance 2>/dev/null
filecat "irqbalance config" /etc/default/irqbalance /etc/irqbalance.conf
dirlist "modprobe.d" /etc/modprobe.d
dirlist "udev rules.d" /etc/udev/rules.d
filecat "modprobe.d files" /etc/modprobe.d/*.conf
cmdsh "module params: kvm_intel" <<'EOF'
for p in /sys/module/kvm_intel/parameters/*; do
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
done
EOF
cmdsh "module params: kvm_amd" <<'EOF'
for p in /sys/module/kvm_amd/parameters/*; do
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
done
EOF
cmdsh "module params: relevant net drivers" <<'EOF'
for mod in i40e ixgbe mlx5_core igb ice bnx2x virtio_net; do
[ -d "/sys/module/$mod/parameters" ] || continue
echo "=== $mod ==="
for p in /sys/module/$mod/parameters/*; do
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
done
done
EOF
# ===========================================================================
# 9. PROXMOX / CLUSTER
# ===========================================================================
hdr "9. PROXMOX VERSION & CLUSTER"
cmd "pveversion" pveversion
cmd "pveversion -v" pveversion -v
cmd "pvecm status" pvecm status
cmd "pvecm nodes" pvecm nodes
cmd "pvecm expected" pvecm expected 2>/dev/null
cmd "pveperf" timeout 15 pveperf 2>/dev/null
filecat "corosync.conf" /etc/pve/corosync.conf
filecat "datacenter.cfg" /etc/pve/datacenter.cfg
filecat "storage.cfg" /etc/pve/storage.cfg
filecat "user.cfg" /etc/pve/user.cfg 2>/dev/null
filecat "domains.cfg" /etc/pve/domains.cfg 2>/dev/null
cmdsh "cluster resources (compact)" <<'EOF'
pvesh get /cluster/resources --type node --output-format json 2>/dev/null \
| jq -r '.[] | "\(.node)\tcpu=\(.cpu)\tmem=\(.mem)\tmaxmem=\(.maxmem)\tmaxdisk=\(.maxdisk)\tstatus=\(.status)\tssl_fingerprint=\(.ssl_fingerprint)"' 2>/dev/null \
|| pvesh get /cluster/resources --type node 2>/dev/null
EOF
# ===========================================================================
# 10. NETWORK: INTERFACES
# ===========================================================================
hdr "10. NETWORK: INTERFACES"
cmd "ip -d link" ip -d link
cmd "ip -o link" ip -o link
cmd "ip -o -4 addr" ip -o -4 addr
cmd "ip -o -6 addr (link-local filter)" ip -o -6 addr 2>/dev/null | grep -v 'scope link'
cmd "ip addr" ip addr
cmd "ip route" ip route
cmd "ip rule" ip rule
cmd "ip -s -s link (counters/errors)" ip -s -s link
cmd "ip neigh" ip neigh
cmd "/proc/net/dev (counters)" cat /proc/net/dev
cmd "ip -6 route (brief)" ip -6 route 2>/dev/null | head -n 20
# Build NETIFS list once, export for use in heredoc snippets
NETIFS=""
for _n in /sys/class/net/*; do
_n="$(basename "$_n")"
case "$_n" in
lo|sit0|ip6tnl0|gre0|gretap0|erspan0|bonding_masters|tailscale0|wg0|tunl0) continue ;;
esac
NETIFS="${NETIFS:+$NETIFS }$_n"
done
export NETIFS
cmdsh "sysfs interface attributes" <<'EOF'
for ifc in $NETIFS; do
d="/sys/class/net/$ifc"
[ -d "$d" ] || continue
printf '%-16s speed=%-8s duplex=%-8s mtu=%-6s carrier=%-3s tx_queue_len=%-5s qlen=%-5s type=%s\n' \
"$ifc" \
"$(cat "$d/speed" 2>/dev/null)" \
"$(cat "$d/duplex" 2>/dev/null)" \
"$(cat "$d/mtu" 2>/dev/null)" \
"$(cat "$d/carrier" 2>/dev/null)" \
"$(cat "$d/tx_queue_len" 2>/dev/null)" \
"$(cat "$d/tx_queue_len" 2>/dev/null)" \
"$(cat "$d/type" 2>/dev/null)"
done
EOF
# Per-interface deep ethtool dump
cmdsh "ethtool per-interface deep dump" <<'EOF'
for ifc in $NETIFS; do
[ -e "/sys/class/net/$ifc" ] || continue
echo ""
echo "----- ethtool: $ifc -----"
echo "[driver/version/firmware/bus-info -i]"
ethtool -i "$ifc" 2>&1
echo "[settings - speed/duplex/autoneg/port]"
ethtool "$ifc" 2>&1
echo "[channels -l (RX/TX/combined queue counts)]"
ethtool -l "$ifc" 2>&1
echo "[coalesce -c (interrupt moderation)]"
ethtool -c "$ifc" 2>&1
echo "[ring -g (descriptor counts)]"
ethtool -g "$ifc" 2>&1
echo "[pause / flow-control -a]"
ethtool -a "$ifc" 2>&1
echo "[offload features -k]"
ethtool -k "$ifc" 2>&1
echo "[private flags --show-priv-flags]"
ethtool --show-priv-flags "$ifc" 2>&1
echo "[ntuple/RSS -n]"
ethtool -n "$ifc" 2>&1 | head -n 40
echo "[SFP/DOM transceiver -m]"
ethtool -m "$ifc" 2>&1
echo "[statistics -S (key error/drop counters only)]"
ethtool -S "$ifc" 2>/dev/null \
| grep -iE 'drop|discard|error|crc|miss|pause|fifo|no_buff|rx_no_|_err|fw_|_bad|oversiz|undersize|frag|jabber|length_err|alignment' \
| head -n 60
echo "[statistics -S (throughput counters)]"
ethtool -S "$ifc" 2>/dev/null \
| grep -iE 'rx_(octets|bytes|packets)|tx_(octets|bytes|packets)|rx_(unicast|multicast|broadcast)|tx_(unicast|multicast|broadcast)' \
| head -n 30
done
EOF
# ===========================================================================
# 11. NETWORK: BRIDGES / BONDS / VLANS
# ===========================================================================
hdr "11. NETWORK: BRIDGES / BONDS / VLANS"
cmd "bridge link" bridge link 2>/dev/null
cmd "bridge -d link" bridge -d link 2>/dev/null
cmd "bridge vlan" bridge vlan 2>/dev/null
cmd "bridge vlan show" bridge vlan show 2>/dev/null
cmd "brctl show" brctl show 2>/dev/null
filecat "bonding state" /proc/net/bonding/bond*
cmd "vlan config" cat /proc/net/vlan/config 2>/dev/null
dirlist "/proc/net/vlan" /proc/net/vlan
filecat "/etc/network/interfaces" /etc/network/interfaces
cmdsh "interfaces.d (per file)" <<'EOF'
for f in /etc/network/interfaces.d/*; do
[ -r "$f" ] || continue
echo "--- $f ---"
cat "$f"
done
EOF
# ===========================================================================
# 12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET
# ===========================================================================
hdr "12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET"
cmd "/proc/interrupts" cat /proc/interrupts
cmdsh "NIC IRQ affinities (which CPUs handle which queue)" <<'EOF'
for ifc in $NETIFS; do
[ -d "/sys/class/net/$ifc/device/msi_irqs" ] || continue
echo "=== $ifc IRQ list (first 40) ==="
ls "/sys/class/net/$ifc/device/msi_irqs" 2>/dev/null | head -n 40
done
EOF
cmdsh "RPS masks (non-zero only)" <<'EOF'
for q in /sys/class/net/*/queues/rx-*/rps_cpus; do
[ -e "$q" ] || continue
val=$(cat "$q")
[ "$val" != "0" ] && printf ' %s = %s\n' "$q" "$val"
done | head -n 60
EOF
cmdsh "XPS masks (non-zero only)" <<'EOF'
for q in /sys/class/net/*/queues/tx-*/xps_cpus; do
[ -e "$q" ] || continue
val=$(cat "$q")
[ "$val" != "0" ] && printf ' %s = %s\n' "$q" "$val"
done | head -n 60
EOF
cmdsh "RFS flow counts" <<'EOF'
for q in /sys/class/net/*/queues/rx-*/rps_flow_cnt; do
[ -e "$q" ] || continue
printf ' %s = %s\n' "$q" "$(cat "$q")"
done | head -n 60
EOF
cmd "/proc/net/softnet_stat (drops/backlog)" cat /proc/net/softnet_stat
cmdsh "softnet_stat decoded" <<'EOF'
# columns: TOTAL DROPPED DROPPED_BY_NETRX TIME_SQUEEZED CPU_COLLISIONS RPS_RECEIVED_FLOW_LIMIT
awk '{ printf "cpu%-2d total=%-8s dropped=%-6s time_squeezed=%-6s cpu_collision=%-4s rps_received=%-6s flow_limit=%s\n",
NR-1, $1, $2, $4, $5, $6, $7 }' /proc/net/softnet_stat
EOF
cmdsh "netstat drop counters (significant ones)" <<'EOF'
nstat -az 2>/dev/null | grep -iE '(Drop|Retrans|Fail|Abort|Overflow|Loss|ListenDrop|EmbryonicRsts|InErrs|OutErrs)' || \
netstat -s 2>/dev/null | grep -iE '(drop|retrans|fail|abort|overflow|loss|listen|error)' | head -n 60
EOF
# ===========================================================================
# 13. NETWORK: SOCKET / TCP STATE
# ===========================================================================
hdr "13. NETWORK: SOCKET / TCP STATE"
cmd "ss -s" ss -s
cmd "ss -t -m (top TCP with skmem/cwnd/rto)" ss -tno state established '( dport != :22 and sport != :22 )' -m 2>/dev/null | head -n 40
cmd "ss -ti (TCP internals)" ss -ti 2>/dev/null | head -n 80
cmd "/proc/net/snmp (TCP summary)" grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null
cmd "TCP retrans stats" grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null | tail -n 1 | awk '{print "RetransSegs="$13}'
cmd "nstat TCP summary" nstat -az Tcp_RetransSegs Tcp_InErrs Tcp_OutRsts Tcp_ActiveOpens Tcp_PassiveOpens 2>/dev/null
# ===========================================================================
# 14. STORAGE: BLOCK DEVICES
# ===========================================================================
hdr "14. STORAGE: BLOCK DEVICES"
cmd "lsblk (key cols)" lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL,STATE,ROTA,DISC-GRAN,DISC-MAX
cmd "lsblk -t" lsblk -t
cmd "lsblk -O (everything, capped)" lsblk -O | head -n 200
cmd "blkid" blkid 2>/dev/null
cmd "/proc/partitions" cat /proc/partitions
cmd "fdisk -l" fdisk -l 2>/dev/null
cmd "parted -l" parted -l 2>/dev/null
cmdsh "block device queue settings (per device)" <<'EOF'
for d in $(ls /sys/block/ 2>/dev/null | grep -vE '^(loop|ram|md|dm-|sr|fd)'); do
q="/sys/block/$d/queue"
[ -d "$q" ] || continue
printf '----- %s -----\n' "$d"
for k in scheduler rotational nr_requests read_ahead_kb max_sectors_kb max_hw_sectors_kb \
logical_block_size physical_block_size discard_granularity discard_max_hw_bytes \
rq_affinity nomerges wbt_lat_usec write_cache stable_writes io_poll io_poll_delay \
fua rot_stable_writes add_random zoned dma_alignment iostats;
do
f="$q/$k"
[ -r "$f" ] && printf ' %-22s = %s\n' "$k" "$(cat "$f" 2>/dev/null)"
done
# NVMe-specific
for f in /sys/block/$d/device/model /sys/block/$d/device/serial /sys/block/$d/device/transport /sys/block/$d/device/queue_count; do
[ -r "$f" ] && printf ' %s = %s\n' "$(echo "$f" | sed "s|/sys/block/$d/device/||")" "$(cat "$f")"
done
done
EOF
cmd "df -h (filesystem usage)" df -h
cmd "df -i (inode usage)" df -i
# ===========================================================================
# 15. STORAGE: MOUNTS & FILESYSTEMS
# ===========================================================================
hdr "15. STORAGE: MOUNTS & FILESYSTEMS"
cmd "mount (filtered)" mount | grep -vE 'proc|sysfs|cgroup|tmpfs|devpts|mqueue|pstore|bpf|tracefs|debugfs|configfs|fusectl|securityfs|hugetlbfs|autofs|systemd|mtd|rpc_pipefs|binfmt'
cmd "findmnt (tree)" findmnt -t nfs,nfs4,zfs,ext4,xfs,btrfs 2>/dev/null
filecat "/etc/fstab" /etc/fstab
cmdsh "ext4 reserved block % (per fs)" <<'EOF'
for dev in $(awk '$3=="ext4"{print $1}' /proc/mounts 2>/dev/null); do
echo "=== $dev ==="
dumpe2fs -h "$dev" 2>/dev/null | grep -E 'Reserved|Block count|Free blocks|Block size|Mount count|Maximum mount|Last'
done
EOF
cmdsh "xfs info (per fs)" <<'EOF'
for mp in $(awk '$3=="xfs"{print $2}' /proc/mounts 2>/dev/null); do
echo "=== $mp ==="
xfs_info "$mp" 2>/dev/null
done
EOF
# ===========================================================================
# 16. STORAGE: NFS (client + server) - critical for this cluster
# ===========================================================================
hdr "16. STORAGE: NFS (CLIENT + SERVER)"
cmd "nfsstat -m (mount options)" nfsstat -m 2>/dev/null
cmd "nfsstat -c (client stats)" nfsstat -c 2>/dev/null
cmd "nfsstat -s (server stats)" nfsstat -s 2>/dev/null
cmd "nfsstat -234 (version stats)" nfsstat -234 2>/dev/null
cmd "showmount -e (exports)" showmount -e 2>/dev/null
cmd "/proc/net/rpc/nfs" cat /proc/net/rpc/nfs 2>/dev/null | head -n 60
cmd "/proc/net/rpc/nfsd" cat /proc/net/rpc/nfsd 2>/dev/null | head -n 60
filecat "exports" /etc/exports /etc/exports.d/*.exports
filecat "nfs.conf" /etc/nfs.conf /etc/nfs.conf.d/*.conf
filecat "nfs defaults" /etc/default/nfs-kernel-server /etc/default/nfs-common
filecat "idmapd.conf" /etc/idmapd.conf
cmd "pvesm status" pvesm status
cmdsh "pvesm list per storage" <<'EOF'
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3=="active"{print $1}'); do
echo "--- storage: $s ---"
pvesm list "$s" 2>/dev/null | head -n 30
echo "--- pvesm config $s ---"
pvesm config "$s" 2>/dev/null
done
EOF
# ===========================================================================
# 17. STORAGE: LVM
# ===========================================================================
hdr "17. STORAGE: LVM"
cmd "pvs" pvs -v
cmd "vgs" vgs -v
cmd "lvs" lvs -o +lv_layout,role,pool_lv,data_percent,metadata_percent --units b
cmd "pvdisplay" pvdisplay
cmd "vgdisplay" vgdisplay
cmd "lvdisplay" lvdisplay
filecat "lvm/lvm.conf (filters only)" /etc/lvm/lvm.conf
cmdsh "lvm.conf active filter" <<'EOF'
grep -E '^\s*(filter|global_filter|use_devicesfile|obtain_device_list_from_udev)' /etc/lvm/lvm.conf 2>/dev/null
EOF
# ===========================================================================
# 18. STORAGE: MULTIPATH
# ===========================================================================
hdr "18. STORAGE: MULTIPATH"
cmd "multipath -ll" multipath -ll
cmd "multipath -v3 -ll (verbose, capped)" multipath -v3 -ll 2>/dev/null | head -n 100
filecat "multipath.conf" /etc/multipath.conf
cmd "systemctl is-active multipathd" systemctl is-active multipathd 2>/dev/null
# ===========================================================================
# 19. STORAGE: ZFS
# ===========================================================================
hdr "19. STORAGE: ZFS"
if command -v zpool >/dev/null 2>&1; then
cmd "zfs version (userland)" zfs version 2>/dev/null
cmd "zpool version" zpool version 2>/dev/null
cmd "zpool list -v" zpool list -v
cmd "zpool status -v" zpool status -v
cmdsh "zpool key properties" <<'EOF'
zpool get name,size,alloc,free,expandsize,fragmentation,capacity,health,ashift,autotrim,autoreplace,cachefile,comment,listsnapshots,multihost,version,bootfs,delegation,failmode,freeing,leaked,readonly
EOF
cmdsh "zfs per-dataset key properties" <<'EOF'
zfs get -r -o name,property,value,source \
compression,atime,recordsize,volblocksize,sync,primarycache,secondarycache,checksum,xattr,acltype,aclinherit,copies,logbias,refreservation,reservation,quota,used,avail,refer,mountpoint,mounted,canmount,relative_path,snapdir,version,primarycache,redundant_metadata,encryption 2>/dev/null
EOF
cmd "ARC stats" cat /proc/spl/kstat/zfs/arcstats
cmd "dmu_tx" cat /proc/spl/kstat/zfs/dmu_tx 2>/dev/null
cmd "zfetchstats" cat /proc/spl/kstat/zfs/zfetchstats 2>/dev/null
cmd "vdev_cache_stats" cat /proc/spl/kstat/zfs/vdev_cache_stats 2>/dev/null
cmd "abdstats" cat /proc/spl/kstat/zfs/abdstats 2>/dev/null
cmdsh "ZFS module parameters (perf-relevant)" <<'EOF'
for f in /sys/module/zfs/parameters/*; do
[ -r "$f" ] || continue
name="$(basename "$f")"
case "$name" in
zfs_arc_*|zfs_abd_*|zfs_dbuf_*|zfs_txg_*|zfs_vdev_*|zfs_sync_*|zfs_dirty_*|zfs_*io_*|zvol_*|l2arc_*|zfs_prefetch_*|zfs_*max_active|zfs_*_taskq*) ;;
*) continue ;;
esac
printf ' %-50s = %s\n' "$name" "$(cat "$f")"
done
EOF
cmd "zpool iostat snapshot" zpool iostat 1 1 2>/dev/null
cmd "arc_summary (if present)" arc_summary 2>/dev/null | head -n 80
cmd "zpool events -v (recent)" zpool events -v 2>/dev/null | tail -n 50
cmdsh "zpool history (last 30 events)" <<'EOF'
zpool history -il 2>/dev/null | tail -n 30
EOF
else
printf '\nZFS not present on this host.\n' >> "$OUT"
fi
# ===========================================================================
# 20. STORAGE: DRIVE HEALTH (SMART / NVMe)
# ===========================================================================
hdr "20. STORAGE: DRIVE HEALTH"
cmd "nvme list" nvme list 2>/dev/null
cmdsh "nvme smart-log per device" <<'EOF'
for n in /dev/nvme[0-9]*; do
[ -b "$n" ] || continue
echo "--- $n smart-log ---"
nvme smart-log "$n" 2>&1
echo "--- $n error-log (first 5) ---"
nvme error-log "$n" 2>&1 | head -n 80
done
EOF
cmdsh "nvme id-ctrl per device (key fields)" <<'EOF'
for n in /dev/nvme[0-9]*; do
[ -b "$n" ] || continue
echo "--- $n id-ctrl (filtered) ---"
nvme id-ctrl "$n" 2>&1 | grep -E '^(mn|sn|fn|fr|tnvmcap|vid|IEEE|cntlid|ver|rab|lpa|elpe|npdg|mdts|hmpre|hmmin|cap|frmw|lba|ws|awun|awupf|acwu|apsta)'
done
EOF
if command -v smartctl >/dev/null 2>&1; then
cmdsh "smartctl -a per disk" <<'EOF'
for d in $(ls /sys/block/ 2>/dev/null | grep -E '^(sd|vd)'); do
dev="/dev/$d"
[ -b "$dev" ] || continue
echo "--- smartctl -a $dev ---"
smartctl -a "$dev" 2>&1 | head -n 200
done
EOF
fi
# ===========================================================================
# 21. STORAGE: HBA / RAID CONTROLLERS
# ===========================================================================
hdr "21. STORAGE: HBA / RAID CONTROLLERS"
for cli in storcli64 storcli MegaCli64 MegaCli perccli; do
if command -v "$cli" >/dev/null 2>&1; then
printf '\n--- %s found ---\n' "$cli" >> "$OUT"
case "$cli" in
storcli*|perccli)
cmd "$cli /call show all" "$cli" /call show all
cmd "$cli /call/vall show all" "$cli" /call/vall show all 2>/dev/null
;;
MegaCli*)
cmd "MegaCli adapter info" "$cli" -AdpAllInfo -aALL
cmd "MegaCli physical drives" "$cli" -PDList -aALL
cmd "MegaCli logical drives" "$cli" -LDInfo -Lall -aALL
cmd "MegaCli BBUs" "$cli" -AdpBbuCmd -aALL
;;
esac
break
fi
done
for cli in sas2ircu sas3ircu; do
if command -v "$cli" >/dev/null 2>&1; then
cmd "$cli list" "$cli" list
cmd "$cli 0 display" "$cli" 0 display 2>/dev/null
fi
done
command -v arcconf >/dev/null 2>&1 && cmd "arcconf list" arcconf list
command -v ssacli >/dev/null 2>&1 && cmd "ssacli ctrl all show config" ssacli ctrl all show config
command -v hpssacli >/dev/null 2>&1 && cmd "hpssacli ctrl all show config" hpssacli ctrl all show config
# ===========================================================================
# 22. VIRTUAL MACHINES (qm)
# ===========================================================================
hdr "22. VIRTUAL MACHINES (qm)"
cmd "qm list" qm list
cmdsh "qm config per VM" <<'EOF'
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
echo "--- VM $vmid ---"
qm config "$vmid" 2>&1
echo ""
done
EOF
cmdsh "VM runtime status (qm guest agent if running)" <<'EOF'
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
echo "--- VM $vmid runtime ---"
qm guest cmd "$vmid" network-get-interfaces 2>/dev/null | head -n 40 || echo " [no agent or VM not running]"
done
EOF
dirlist "/etc/pve/qemu-server" /etc/pve/qemu-server
dirlist "/etc/pve/firewall" /etc/pve/firewall
# ===========================================================================
# 23. CONTAINERS (pct)
# ===========================================================================
hdr "23. CONTAINERS (pct)"
cmd "pct list" pct list
cmdsh "pct config per CT" <<'EOF'
for ctid in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
echo "--- CT $ctid ---"
pct config "$ctid" 2>&1
echo ""
done
EOF
# ===========================================================================
# 24. SERVICES / FAILED UNITS
# ===========================================================================
hdr "24. SERVICES / FAILED UNITS"
cmd "running services" systemctl list-units --type=service --state=running --no-pager
cmd "failed services" systemctl --failed --no-pager
cmd "enabled services" systemctl list-unit-files --state=enabled --no-pager | head -n 80
cmdsh "pve-* service status" <<'EOF'
systemctl status pve-cluster pvedaemon pveproxy pve-firewall pve-ha-crm pve-ha-lrm pvestatd pvebanner --no-pager 2>&1 | head -n 120
EOF
cmdsh "storage-related service status" <<'EOF'
systemctl status nfs-kernel-server nfs-server rpcbind rpc-statd zfs-import zfs-mount zfs-target smartd nvme-stat multipathd --no-pager 2>&1 | head -n 80
EOF
# ===========================================================================
# 25. TIMERS / CRON (could cause load spikes during work hours)
# ===========================================================================
hdr "25. TIMERS / CRON"
cmd "systemctl list-timers" systemctl list-timers --all --no-pager | head -n 40
dirlist "/etc/cron.d" /etc/cron.d
dirlist "/etc/cron.daily" /etc/cron.daily
dirlist "/etc/cron.hourly" /etc/cron.hourly
dirlist "/etc/cron.weekly" /etc/cron.weekly
filecat "crontabs" /etc/crontab
cmdsh "root crontab" <<'EOF'
crontab -l 2>/dev/null || echo " [no root crontab]"
EOF
# ===========================================================================
# 26. LIVE PERFORMANCE SNAPSHOT
# ===========================================================================
hdr "26. LIVE PERFORMANCE SNAPSHOT"
cmd "/proc/loadavg" cat /proc/loadavg
cmd "vmstat 1 3" vmstat 1 3
cmd "mpstat 1 3 (per-cpu)" mpstat 1 3 2>/dev/null
cmd "iostat -xz 1 3" iostat -xz 1 3 2>/dev/null
cmd "sar -dp 1 2" sar -dp 1 2 2>/dev/null
cmd "sar -n DEV 1 2" sar -n DEV 1 2 2>/dev/null
cmd "sar -n NFS,NFSD 1 2" sar -n NFS,NFSD 1 2 2>/dev/null
cmd "sar -n SOCK 1 1" sar -n SOCK 1 1 2>/dev/null
cmd "sar -n TCP,ETCP 1 2" sar -n TCP,ETCP 1 2 2>/dev/null
cmd "pidstat -d 1 2 (top disk)" pidstat -d 1 2 2>/dev/null | head -n 40
cmd "pidstat -u 1 2 (top cpu)" pidstat -u 1 2 2>/dev/null | head -n 40
cmd "top (single snapshot, top 30)" top -b -n 1 | head -n 35
cmd "zpool iostat 1 2" zpool iostat 1 2 2>/dev/null
cmd "arcstat 1 2" arcstat 1 2 2>/dev/null
# /proc/net/dev snapshot twice for delta
cmdsh "/proc/net/dev delta (1s)" <<'EOF'
awk '{print $1}' /proc/net/dev
echo "--- t0 ---"
cat /proc/net/dev
sleep 1
echo "--- t1 (1s later) ---"
cat /proc/net/dev
EOF
# ===========================================================================
# 27. DNS / NTP / TIME
# ===========================================================================
hdr "27. DNS / NTP / TIME"
filecat "/etc/resolv.conf" /etc/resolv.conf
filecat "/etc/hosts" /etc/hosts
filecat "/etc/ntp.conf" /etc/ntp.conf 2>/dev/null
filecat "/etc/chrony/chrony.conf" /etc/chrony/chrony.conf 2>/dev/null
cmd "timedatectl" timedatectl 2>/dev/null
cmd "chronyc tracking" chronyc tracking 2>/dev/null
cmd "chronyc sources" chronyc sources 2>/dev/null
cmd "ntpq peers" ntpq -pn 2>/dev/null
# ===========================================================================
# 28. FIREWALL
# ===========================================================================
hdr "28. FIREWALL / SECURITY"
cmd "pve-firewall status" pve-firewall status 2>/dev/null
cmd "nft ruleset (count)" nft list ruleset 2>/dev/null | wc -l
cmd "nft ruleset (head)" nft list ruleset 2>/dev/null | head -n 60
cmd "iptables-save (legacy)" iptables-save 2>/dev/null | head -n 60
cmd "ipset list (names)" ipset list -n 2>/dev/null
cmd "apparmor status" aa-status 2>/dev/null | head -n 20
# ===========================================================================
# 29. TOP CONSUMERS (memory + disk + file handles)
# ===========================================================================
hdr "29. TOP CONSUMERS"
cmdsh "top 10 memory hogs" <<'EOF'
ps -eo pid,user,rss,vsz,pcpu,pmem,comm --sort=-rss | head -n 11
EOF
cmdsh "top 10 CPU hogs" <<'EOF'
ps -eo pid,user,pcpu,pmem,rss,comm --sort=-pcpu | head -n 11
EOF
cmdsh "top file-handle holders" <<'EOF'
for pid in $(ps -e -o pid= | head -n 200); do
n="$(ls /proc/$pid/fd 2>/dev/null | wc -l)"
[ "$n" -gt 50 ] && printf '%6d fd=%-6s %s\n' "$pid" "$n" "$(cat /proc/$pid/comm 2>/dev/null)"
done | sort -k2 -t= -nr | head -n 15
EOF
cmdsh "largest files in /var/log (top 10)" <<'EOF'
find /var/log -type f -size +10M -exec ls -lh {} \; 2>/dev/null | sort -k5 -h -r | head -n 10
EOF
# ===========================================================================
# 30. QUICK-LOOK SUSPECT SUMMARY (auto-detect common issues)
# ===========================================================================
hdr "30. QUICK-LOOK SUSPECT SUMMARY"
blank
printf 'Auto-detected performance anti-patterns (verify against full data above):\n' >> "$OUT"
blank
cmdsh "[A] Network interfaces < 10 Gbps and up" <<'EOF'
hits=0
for ifc in $NETIFS; do
s=$(cat "/sys/class/net/$ifc/speed" 2>/dev/null)
c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
case "$s" in
10|100|1000)
if [ "$c" = "1" ]; then
printf ' %s : %s Mbps (UP) <-- sub-10GbE; suspect if this carries storage/migration\n' "$ifc" "$s"
hits=$((hits+1))
fi
;;
esac
done
[ "$hits" = "0" ] && echo " no <10GbE interfaces are UP (or speed unreadable)"
EOF
blank
cmdsh "[B] MTU not jumbo on carrier-up interfaces" <<'EOF'
hits=0
for ifc in $NETIFS; do
m=$(cat "/sys/class/net/$ifc/mtu" 2>/dev/null)
c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
[ "$c" = "1" ] || continue
if [ -n "$m" ] && [ "$m" != "9000" ]; then
printf ' %s : mtu=%s\n' "$ifc" "$m"
hits=$((hits+1))
fi
done
[ "$hits" = "0" ] && echo " all carrier-up interfaces have MTU 9000 (or none are up)"
EOF
blank
cmdsh "[C] CPU governor != performance" <<'EOF'
g=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)
if [ -z "$g" ]; then
echo " cpufreq driver not loaded (governor unknown)"
elif [ "$g" = "performance" ]; then
echo " OK: performance governor"
else
echo " <-- governor=$g (recommend 'performance')"
fi
EOF
blank
cmdsh "[D] EPB / EPP not performance-bias" <<'EOF'
epb=$(cat /sys/devices/system/cpu/cpu0/power/energy_perf_bias 2>/dev/null)
epp=$(cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference 2>/dev/null)
echo " EPB=$epb EPP=$epp (0=performance, 15=max-power on EPB; 'performance' on EPP)"
EOF
blank
cmdsh "[E] vm.swappiness and dirty ratios" <<'EOF'
sw=$(sysctl -n vm.swappiness 2>/dev/null)
dr=$(sysctl -n vm.dirty_ratio 2>/dev/null)
dbr=$(sysctl -n vm.dirty_background_ratio 2>/dev/null)
echo " swappiness=$sw dirty_ratio=$dr dirty_background_ratio=$dbr"
[ "$sw" != "10" ] && [ -n "$sw" ] && echo " <-- swappiness not 10 (Proxmox default)"
EOF
blank
cmdsh "[F] TCP congestion control" <<'EOF'
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null)
echo " active: $cc"
echo " available: $avail"
[ "$cc" = "cubic" ] && echo " <-- consider bbr (if available) for high-BDP NFS paths"
EOF
blank
cmdsh "[G] ZFS dataset red flags" <<'EOF'
if ! command -v zfs >/dev/null 2>&1; then
echo " ZFS not present"
else
zfs get -r -o name,property,value compression,atime,sync,recordsize,xattr 2>/dev/null \
| awk '$2=="compression" && $3=="off" {print " <-- "$1" has compression=off"}
$2=="atime" && $3=="on" {print " <-- "$1" has atime=on"}
$2=="sync" && $3=="always" {print " <-- "$1" has sync=always (slow without SLOG)"}
$2=="xattr" && $3=="on" {print " <-- "$1" has xattr=on (use sa)"}'
fi
EOF
blank
cmdsh "[H] ARC size vs RAM" <<'EOF'
if [ -r /proc/spl/kstat/zfs/arcstats ]; then
arc=$(awk '/^size /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
cmax=$(awk '/^c_max /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
ram=$(awk '/MemTotal/{print $2*1024}' /proc/meminfo)
echo " ARC current=$arc bytes ARC max=$cmax bytes RAM total=$ram bytes"
if [ -n "$cmax" ] && [ -n "$ram" ] && [ "$ram" -gt 0 ]; then
pct=$((cmax * 100 / ram))
echo " ARC max is ${pct}% of RAM (Proxmox default ~50%)"
fi
hits=$(awk '/^hits /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
miss=$(awk '/^misses /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
if [ -n "$hits" ] && [ -n "$miss" ] && [ "$((hits+miss))" -gt 0 ]; then
hr=$((hits * 100 / (hits + miss)))
echo " ARC hit rate since boot: ${hr}% (hits=$hits misses=$miss)"
fi
else
echo " ZFS ARC stats not available (ZFS not loaded?)"
fi
EOF
blank
cmdsh "[I] VMs with non-optimal storage/NIC models" <<'EOF'
hits=0
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
cfg=$(qm config "$vmid" 2>/dev/null)
flags=""
echo "$cfg" | grep -qE '^ide[0-9]' && flags="${flags}ide-disk "
echo "$cfg" | grep -qE '^sata[0-9]' && flags="${flags}sata-disk "
echo "$cfg" | grep -qE '^scsi[0-9].*:,' && echo "$cfg" | grep -E '^scsi[0-9]' | grep -qv 'virtio-scsi' && flags="${flags}scsi-no-virtio-scsi "
echo "$cfg" | grep -qE '^scsihw: ' && ! echo "$cfg" | grep -qE '^scsihw: virtio-scsi-single' && flags="${flags}scsihw!=virtio-scsi-single "
echo "$cfg" | grep -qE '^cpu: .*kvm64' && flags="${flags}cpu=kvm64 "
echo "$cfg" | grep -q '^iothread: 1' || echo "$cfg" | grep -qE '^scsi[0-9].*iothread=1' || flags="${flags}no-iothread "
echo "$cfg" | grep -qE '^net[0-9].*model=(e1000|rtl8139|vmxnet)' && flags="${flags}legacy-nic "
echo "$cfg" | grep -qE '^net[0-9].*model=virtio' && ! echo "$cfg" | grep -qE 'multiqueue=|queues=' && flags="${flags}virtio-no-multiqueue "
if [ -n "$flags" ]; then
echo " VM $vmid: $flags"
hits=$((hits+1))
fi
done
[ "$hits" = "0" ] && echo " no obvious VM storage/NIC anti-patterns"
EOF
blank
cmdsh "[J] NFS client mount option red flags" <<'EOF'
if ! command -v nfsstat >/dev/null 2>&1; then
echo " nfsstat not available"
else
nfsstat -m 2>/dev/null | awk '
/^.*:/ {mp=$0; next}
/Flags:/ {
gsub(/Flags:|,/,"",$0)
line=$0
if (line !~ /noatime/) print " "mp" : missing noatime"
if (line !~ /nconnect/) print " "mp" : missing nconnect (kernel >=5.3)"
if (line !~ /rsize=1048576/) print " "mp" : rsize<1M"
if (line !~ /wsize=1048576/) print " "mp" : wsize<1M"
if (line ~ /sync/) print " "mp" : sync mount (slow)"
}
'
fi
EOF
blank
cmdsh "[K] Softnet / NIC drops (non-zero)" <<'EOF'
awk 'NR<=32 && ($2+0 > 0 || $4+0 > 0) {printf " cpu%d: dropped=%d time_squeezed=%d\n", NR-1, $2, $4}' /proc/net/softnet_stat
echo " (also check ethtool -S sections above for rx_dropped/rx_missed_errors)"
EOF
blank
cmdsh "[L] Failed systemd services" <<'EOF'
failed=$(systemctl --failed --no-legend 2>/dev/null | head -n 10)
if [ -n "$failed" ]; then
echo "$failed" | sed 's/^/ /'
else
echo " no failed services"
fi
EOF
blank
cmdsh "[M] Filesystem usage > 80%" <<'EOF'
df -h 2>/dev/null | awk 'NR>1 && $5+0 > 80 {print " "$0}'
EOF
blank
cmdsh "[N] SMART / NVMe health warnings" <<'EOF'
if command -v nvme >/dev/null 2>&1; then
for n in /dev/nvme[0-9]*; do
[ -b "$n" ] || continue
crit=$(nvme smart-log "$n" 2>/dev/null | grep -iE 'critical_warning|media_and_data_integrity|available_spare' | head -n 5)
[ -n "$crit" ] && echo " $n: $crit" | tr '\n' ' '
done
fi
if command -v smartctl >/dev/null 2>&1; then
for d in $(ls /sys/block/ 2>/dev/null | grep -E '^sd'); do
dev="/dev/$d"
[ -b "$dev" ] || continue
chk=$(smartctl -H "$dev" 2>/dev/null | grep -iE 'result|health|passed|failed')
echo "$chk" | grep -qi fail && echo " $dev: $chk"
done
fi
echo " (see section 20 for full smartctl/nvme output)"
EOF
# ===========================================================================
# Footer
# ===========================================================================
END_EPOCH="$(date +%s)"
DUR=$((END_EPOCH - START_EPOCH))
BYTES="$(wc -c < "$OUT")"
LINES="$(wc -l < "$OUT")"
{
printf '\n###############################################################################\n'
printf '# Collection complete: %s\n' "$HOST"
printf '# Output: %s\n' "$OUT"
printf '# Size: %s bytes / %s lines\n' "$BYTES" "$LINES"
printf '# Elapsed: %s seconds\n' "$DUR"
printf '###############################################################################\n'
} >> "$OUT"
chmod 600 "$OUT"
echo
echo "================================================================"
echo " check.sh done. Wrote: ${OUT}"
echo " Size: $(du -h "$OUT" | cut -f1) (${LINES} lines)"
echo " Elapsed: ${DUR}s"
echo " Please copy ${OUT} back for analysis."
echo "================================================================"
Reference in New Issue View Git Blame Copy Permalink